Update dependencies from https://github.com/dotnet/arcade build 20200…

…924.4 (dotnet#1510)

[release/5.0] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,13 +3,13 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20431.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20474.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4be47e467013f8a07a1ed7b6e49e39c8150bde54</Sha>
+      <Sha>61cde6e8fb9d5c9790867b279deb41783a780cd8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.ApiCompat" Version="5.0.0-beta.20431.1">
+    <Dependency Name="Microsoft.DotNet.ApiCompat" Version="5.0.0-beta.20474.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4be47e467013f8a07a1ed7b6e49e39c8150bde54</Sha>
+      <Sha>61cde6e8fb9d5c9790867b279deb41783a780cd8</Sha>
     </Dependency>
     <Dependency Name="Microsoft.NET.Sdk.IL" Version="5.0.0-rc.1.20416.7">
       <Uri>https://github.com/dotnet/runtime</Uri>

eng/Versions.props

@@ -20,7 +20,7 @@
     <SystemReflectionMetadataVersion>1.5.0</SystemReflectionMetadataVersion>
     <MicrosoftBuildFrameworkVersion>15.4.8</MicrosoftBuildFrameworkVersion>
     <MicrosoftBuildUtilitiesCoreVersion>15.4.8</MicrosoftBuildUtilitiesCoreVersion>
-    <MicrosoftDotNetApiCompatVersion>5.0.0-beta.20431.1</MicrosoftDotNetApiCompatVersion>
+    <MicrosoftDotNetApiCompatVersion>5.0.0-beta.20474.4</MicrosoftDotNetApiCompatVersion>
     <!-- This controls the version of the cecil package, or the version of cecil in the project graph
          when we build the cecil submodule. The reference assembly package will depend on this version of cecil.
          Keep this in sync with ProjectInfo.cs in the submodule. -->

eng/common/internal/Directory.Build.props

@@ -1,4 +1,4 @@
-<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
+<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
 <Project>
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 </Project>

eng/common/performance/performance-setup.sh

@@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then
 fi
 
 if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
-    configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
 fi
 
@@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then
 fi
 
 if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
+    configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
 fi
 

eng/common/post-build/publish-using-darc.ps1

@@ -8,15 +8,15 @@ param(
   [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
   [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
   [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
-  [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums,
+  [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
   [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
 )
 
 try {
   . $PSScriptRoot\post-build-utils.ps1
   # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
-  . $PSScriptRoot\..\darc-init.ps1 -darcVersion "1.1.0-beta.20418.1"
+  $darc = Get-Darc "1.1.0-beta.20418.1"
 
   $optionalParams = [System.Collections.ArrayList]::new()
 
@@ -29,7 +29,7 @@ try {
     $optionalParams.Add("--no-wait") | Out-Null
   }
 
-  if ("true" -eq $PublishInstallersAndChecksums) {
+  if ("false" -ne $PublishInstallersAndChecksums) {
     $optionalParams.Add("--publish-installers-and-checksums") | Out-Null
   }
 
@@ -50,7 +50,7 @@ try {
     }
   }
 
-  & darc add-build-to-channel `
+  & $darc add-build-to-channel `
   --id $buildId `
   --publishing-infra-version $PublishingInfraVersion `
   --default-channels `

eng/common/post-build/sourcelink-validation.ps1

@@ -144,11 +144,27 @@ $ValidatePackage = {
 
   if ($FailedFiles -eq 0) {
     Write-Host 'Passed.'
-    return 0
+    return [pscustomobject]@{
+      result = 0
+      packagePath = $PackagePath
+    }
   }
   else {
     Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
-    return 1
+    return [pscustomobject]@{
+      result = 1
+      packagePath = $PackagePath
+    }
+  }
+}
+
+function CheckJobResult(
+    $result, 
+    $packagePath,
+    [ref]$ValidationFailures) {
+  if ($jobResult.result -ne '0') {
+    Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
+    $ValidationFailures.Value++
   }
 }
 
@@ -211,10 +227,8 @@ function ValidateSourceLinkLinks {
       }
 
       foreach ($Job in @(Get-Job -State 'Completed')) {
-        $jobResult = Receive-Job -Id $Job.Id
-        if ($jobResult -ne '0') {
-          $ValidationFailures++
-        }
+        $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+        CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
         Remove-Job -Id $Job.Id
       }
     }

eng/common/post-build/symbols-validation.ps1

@@ -141,11 +141,6 @@ $CountMissingSymbols = {
   if ($using:Clean) {
     Remove-Item $ExtractPath -Recurse -Force
   }
-
-  if ($MissingSymbols -ne 0)
-  {
-    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath"
-  }
 
   Pop-Location
 
@@ -165,6 +160,7 @@ function CheckJobResult(
     $DupedSymbols.Value++
   } 
   elseif ($jobResult.result -ne '0') {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
     $TotalFailures.Value++
   }
 }
@@ -201,7 +197,6 @@ function CheckSymbolsAvailable {
       Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null
 
       $NumJobs = @(Get-Job -State 'Running').Count
-      Write-Host $NumJobs
 
       while ($NumJobs -ge $MaxParallelJobs) {
         Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."

eng/common/sdk-task.ps1

@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/templates/job/source-build.yml

@@ -0,0 +1,49 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. The template produces a server job with a
+  # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+  # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+  jobNamePrefix: 'Source_Build'
+
+  # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+  # managed-only repositories. This is an object with these properties:
+  #
+  # name: ''
+  #   The name of the job. This is included in the job ID.
+  # targetRID: ''
+  #   The name of the target RID to use, instead of the one auto-detected by Arcade.
+  # nonPortable: false
+  #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  # container: ''
+  #   A container to use. Runs in docker.
+  # pool: {}
+  #   A pool to use. Runs directly on an agent.
+  # buildScript: ''
+  #   Specifies the build script to invoke to perform the build in the repo. The default
+  #   './build.sh' should work for typical Arcade repositories, but this is customizable for
+  #   difficult situations.
+  # jobProperties: {}
+  #   A list of job properties to inject at the top level, for potential extensibility beyond
+  #   container and pool.
+  platform: {}
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+  displayName: Source-Build (${{ parameters.platform.name }})
+
+  ${{ each property in parameters.platform.jobProperties }}:
+    ${{ property.key }}: ${{ property.value }}
+
+  ${{ if ne(parameters.platform.container, '') }}:
+    container: ${{ parameters.platform.container }}
+  ${{ if ne(parameters.platform.pool, '') }}:
+    pool: ${{ parameters.platform.pool }}
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: /eng/common/templates/steps/source-build.yml
+    parameters:
+      platform: ${{ parameters.platform }}

eng/common/templates/jobs/jobs.yml

@@ -24,6 +24,13 @@ parameters:
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
 
+  # Optional: Enable running the source-build jobs to build repo from source
+  runSourceBuild: false
+
+  # Optional: Parameters for source-build template.
+  #           See /eng/common/templates/jobs/source-build.yml for options
+  sourceBuildParameters: []
+
 # Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
 # and some (Microbuild) should only be applied to non-PR cases for internal builds.
 
@@ -43,6 +50,13 @@ jobs:
 
       name: ${{ job.job }}
 
+- ${{ if eq(parameters.runSourceBuild, true) }}:
+  - template: /eng/common/templates/jobs/source-build.yml
+    parameters:
+      allCompletedJobId: Source_Build_Complete
+      ${{ each parameter in parameters.sourceBuildParameters }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+
 - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
     - template: ../job/publish-build-assets.yml
@@ -55,6 +69,8 @@ jobs:
         - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
           - ${{ each job in parameters.jobs }}:
             - ${{ job.job }}
+        - ${{ if eq(parameters.runSourceBuild, true) }}:
+          - Source_Build_Complete
         pool:
           vmImage: vs2017-win2016
         runAsPublic: ${{ parameters.runAsPublic }}

eng/common/templates/jobs/source-build.yml

@@ -0,0 +1,48 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+  # well as an optional server job that completes when all platform jobs complete.
+
+  # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+  # not included. Existing repo pipelines can use this job depend on all source-build jobs
+  # completing without maintaining a separate list of every single job ID: just depend on this one
+  # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+  allCompletedJobId: ''
+
+  # See /eng/common/templates/job/source-build.yml
+  jobNamePrefix: 'Source_Build'
+
+  # If changed to true, causes this template to include the default platform for a managed-only
+  # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
+  # but since the repo is supposed to be managed-only, the risk should be very low.
+  includeDefaultManagedPlatform: false
+  defaultManagedPlatform:
+    name: 'Managed'
+    container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+
+  # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+  # object in this array is sent to the job template as 'platform'.
+  platforms: []
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+  - job: ${{ parameters.allCompletedJobId }}
+    displayName: Source-Build Complete
+    pool: server
+    dependsOn:
+    - ${{ each platform in parameters.platforms }}:
+      - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+    - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+      - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ platform }}
+
+- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ parameters.defaultManagedPlatform }}

eng/common/templates/post-build/channels/generic-internal-channel.yml

@@ -4,7 +4,7 @@ parameters:
   artifactsPublishingAdditionalParameters: ''
   dependsOn:
   - Validate
-  publishInstallersAndChecksums: false
+  publishInstallersAndChecksums: true
   symbolPublishingAdditionalParameters: ''
   stageName: ''
   channelName: ''
@@ -158,7 +158,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
             /p:Configuration=Release
-            /p:PublishInstallersAndChecksums=true
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
             /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
             /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)

eng/common/templates/post-build/channels/generic-public-channel.yml

@@ -4,7 +4,7 @@ parameters:
   artifactsPublishingAdditionalParameters: ''
   dependsOn:
   - Validate
-  publishInstallersAndChecksums: false
+  publishInstallersAndChecksums: true
   symbolPublishingAdditionalParameters: ''
   stageName: ''
   channelName: ''

eng/common/templates/post-build/post-build.yml

@@ -19,7 +19,7 @@ parameters:
   enableSigningValidation: true
   enableSymbolValidation: false
   enableNugetValidation: true
-  publishInstallersAndChecksums: false
+  publishInstallersAndChecksums: true
   SDLValidationParameters:
     enable: false
     continueOnError: false
@@ -78,21 +78,22 @@ stages:
       BARBuildId: ${{ parameters.BARBuildId }}
       PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
 
-  - job:
-    displayName: Post-build Checks
-    dependsOn: setupMaestroVars
-    variables:
-      - name: TargetChannels
-        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
-    pool:
-      vmImage: 'windows-2019'
-    steps:
-      - task: PowerShell@2
-        displayName: Maestro Channels Consistency
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
-          arguments: -PromoteToChannels "$(TargetChannels)"
-            -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
+  - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+    - job:
+      displayName: Post-build Checks
+      dependsOn: setupMaestroVars
+      variables:
+        - name: TargetChannels
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: PowerShell@2
+          displayName: Maestro Channels Consistency
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
+            arguments: -PromoteToChannels "$(TargetChannels)"
+              -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
 
   - job:
     displayName: NuGet Validation