Merge pull request #7399 from dotnet/darc-master-8677af5b-d393-44df-8…

…734-473c17102eb2

[master] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19412.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19420.8">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>ec2dd5b3e7d11b88b2ca0688bb1685836cfad20a</Sha>
+      <Sha>fa168dd690798a683ef0d1e65d60ce5d6918d987</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/performance/performance-setup.ps1

@@ -2,7 +2,7 @@ Param(
     [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
     [string] $CoreRootDirectory,
     [string] $Architecture="x64",
-    [string] $Framework="netcoreapp3.0",
+    [string] $Framework="netcoreapp5.0",
     [string] $CompilationMode="Tiered",
     [string] $Repository=$env:BUILD_REPOSITORY_NAME,
     [string] $Branch=$env:BUILD_SOURCEBRANCH,
@@ -33,7 +33,7 @@ if ($Framework.StartsWith("netcoreapp")) {
 }
 
 if ($Internal) {
-    $Queue = "Windows.10.Amd64.ClientRS5.Perf"
+    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
     $PerfLabArguments = "--upload-to-perflab-container"
     $ExtraBenchmarkDotNetArguments = ""
     $Creator = ""

eng/common/performance/performance-setup.sh

@@ -3,7 +3,7 @@
 source_directory=$BUILD_SOURCESDIRECTORY
 core_root_directory=
 architecture=x64
-framework=netcoreapp3.0
+framework=netcoreapp5.0
 compilation_mode=tiered
 repository=$BUILD_REPOSITORY_NAME
 branch=$BUILD_SOURCEBRANCH
@@ -132,7 +132,7 @@ if [[ "$internal" == true ]]; then
     if [[ "$architecture" = "arm64" ]]; then
         queue=Ubuntu.1804.Arm64.Perf
     else
-        queue=Ubuntu.1804.Amd64.Perf
+        queue=Ubuntu.1804.Amd64.Tiger.Perf
     fi
 fi
 
@@ -156,21 +156,26 @@ if [[ "$use_core_run" = true ]]; then
     mv $core_root_directory $new_core_root
 fi
 
+ci=true
+
+_script_dir=$(pwd)/eng/common
+. "$_script_dir/pipeline-logging-functions.sh"
+
 # Make sure all of our variables are available for future steps
-echo "##vso[task.setvariable variable=UseCoreRun]$use_core_run"
-echo "##vso[task.setvariable variable=Architecture]$architecture"
-echo "##vso[task.setvariable variable=PayloadDirectory]$payload_directory"
-echo "##vso[task.setvariable variable=PerformanceDirectory]$performance_directory"
-echo "##vso[task.setvariable variable=WorkItemDirectory]$workitem_directory"
-echo "##vso[task.setvariable variable=Queue]$queue"
-echo "##vso[task.setvariable variable=SetupArguments]$setup_arguments"
-echo "##vso[task.setvariable variable=Python]python3"
-echo "##vso[task.setvariable variable=PerfLabArguments]$perflab_arguments"
-echo "##vso[task.setvariable variable=ExtraBenchmarkDotNetArguments]$extra_benchmark_dotnet_arguments"
-echo "##vso[task.setvariable variable=BDNCategories]$run_categories"
-echo "##vso[task.setvariable variable=TargetCsproj]$csproj"
-echo "##vso[task.setvariable variable=RunFromPerfRepo]$run_from_perf_repo"
-echo "##vso[task.setvariable variable=Creator]$creator"
-echo "##vso[task.setvariable variable=HelixSourcePrefix]$helix_source_prefix"
-echo "##vso[task.setvariable variable=Kind]$kind"
-echo "##vso[task.setvariable variable=_BuildConfig]$architecture.$kind.$framework"
+Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false
+Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Python" -value "$python3" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false
+Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false
+Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
+Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
+Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false

eng/common/post-build/darc-gather-drop.ps1

@@ -19,7 +19,17 @@ try {
     ExitWithExitCode $exitCode
   }
 
+  # For now, only use a dry run.
+  # Ideally we would change darc to enable a quick request that
+  # would check whether the file exists that you can download it,
+  # and that it won't conflict with other files.
+  # https://github.com/dotnet/arcade/issues/3674
+  # Right now we can't remove continue-on-error because we ocassionally will have
+  # dependencies that have no associated builds (e.g. an old dependency).
+  # We need to add an option to baseline specific dependencies away, or add them manually
+  # to the BAR.
   darc gather-drop --non-shipping `
+    --dry-run `
     --continue-on-error `
     --id $BarBuildId `
     --output-dir $DropLocation `

eng/common/post-build/sourcelink-validation.ps1

@@ -1,8 +1,8 @@
 param(
   [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where Symbols.NuGet packages to be checked are stored
   [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
-  [Parameter(Mandatory=$true)][string] $GHRepoName,             # GitHub name of the repo including the Org. E.g., dotnet/arcade
-  [Parameter(Mandatory=$true)][string] $GHCommit,               # GitHub commit SHA used to build the packages
+  [Parameter(Mandatory=$false)][string] $GHRepoName,            # GitHub name of the repo including the Org. E.g., dotnet/arcade
+  [Parameter(Mandatory=$false)][string] $GHCommit,              # GitHub commit SHA used to build the packages
   [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion    # Version of SourceLink CLI to use
 )
 
@@ -13,6 +13,12 @@ param(
 # all files present in the repo at a specific commit point.
 $global:RepoFiles = @{}
 
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 6
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
 $ValidatePackage = {
   param( 
     [string] $PackagePath                                 # Full path to a Symbols.NuGet package
@@ -22,8 +28,8 @@ $ValidatePackage = {
 
   # Ensure input file exist
   if (!(Test-Path $PackagePath)) {
-    Write-PipelineTaskError "Input file does not exist: $PackagePath"
-    ExitWithExitCode 1
+    Write-Host "Input file does not exist: $PackagePath"
+    return 1
   }
 
   # Extensions for which we'll look for SourceLink information
@@ -38,7 +44,7 @@ $ValidatePackage = {
 
   Add-Type -AssemblyName System.IO.Compression.FileSystem
 
-  [System.IO.Directory]::CreateDirectory($ExtractPath);
+  [System.IO.Directory]::CreateDirectory($ExtractPath)  | Out-Null
 
   try {
     $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
@@ -138,62 +144,86 @@ $ValidatePackage = {
 
   if ($FailedFiles -eq 0) {
     Write-Host "Passed."
+    return 0
   }
   else {
-    Write-PipelineTaskError "$PackagePath has broken SourceLink links."
+    Write-Host "$PackagePath has broken SourceLink links."
+    return 1
   }
 }
 
 function ValidateSourceLinkLinks {
-  if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
+  if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
     if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) {
-      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>"
+      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
       ExitWithExitCode 1
     }
     else {
       $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2';
     }
   }
 
-  if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
-    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string"
+  if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
+    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
     ExitWithExitCode 1
   }
 
-  $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
-  $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
+  if ($GHRepoName -ne "" -and $GHCommit -ne "") {
+    $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
+    $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
 
-  try {
-    # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
-    $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+    try {
+      # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+      $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
 
-    foreach ($file in $Data) {
-      $Extension = [System.IO.Path]::GetExtension($file.path)
+      foreach ($file in $Data) {
+        $Extension = [System.IO.Path]::GetExtension($file.path)
 
-      if ($CodeExtensions.Contains($Extension)) {
-        $RepoFiles[$file.path] = 1
+        if ($CodeExtensions.Contains($Extension)) {
+          $RepoFiles[$file.path] = 1
+        }
       }
     }
+    catch {
+      Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+    }
   }
-  catch {
-    Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
-    Write-Host $_
-    ExitWithExitCode 1
+  elseif ($GHRepoName -ne "" -or $GHCommit -ne "") {
+    Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed."
   }
 
   if (Test-Path $ExtractPath) {
     Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
   }
 
   # Process each NuGet package in parallel
-  $Jobs = @()
   Get-ChildItem "$InputPath\*.symbols.nupkg" |
     ForEach-Object {
-      $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
+      Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+      $NumJobs = @(Get-Job -State 'Running').Count
+
+      while ($NumJobs -ge $MaxParallelJobs) {
+        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+        sleep $SecondsBetweenLoadChecks
+        $NumJobs = @(Get-Job -State 'Running').Count
+      }
+
+      foreach ($Job in @(Get-Job -State 'Completed')) {
+        Receive-Job -Id $Job.Id
+        Remove-Job -Id $Job.Id
+      }
     }
 
+  $ValidationFailures = 0
   foreach ($Job in $Jobs) {
-    Wait-Job -Id $Job.Id | Receive-Job
+    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+    if ($jobResult -ne "0") {
+      $ValidationFailures++
+    }
+  }
+  if ($ValidationFailures -gt 0) {
+    Write-PipelineTaskError " $ValidationFailures package(s) failed validation."
+    ExitWithExitCode 1
   }
 }
 

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -86,8 +86,8 @@ if ($TsaPublish) {
     if (-not $TsaRepositoryName) {
       $TsaRepositoryName = "$($Repository)-$($BranchName)"
     }
-    Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel"
-    & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory  --logger-level $GuardianLoggerLevel
+    Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel"
+    & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory  --logger-level $GuardianLoggerLevel
     if ($LASTEXITCODE -ne 0) {
       Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE

eng/common/sdl/run-sdl.ps1

@@ -25,19 +25,19 @@ if ($ValidPath -eq $False)
   exit 1
 }
 
+$configParam = @("--config")
+
 foreach ($tool in $ToolsList) {
   $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
-  $config = $False
   Write-Host $tool
   # We have to manually configure tools that run on source to look at the source directory only
   if ($tool -eq "credscan") {
-    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
-    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
+    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" `" OutputType : pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
+    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " "OutputType : pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
     if ($LASTEXITCODE -ne 0) {
       Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE
     }
-    $config = $True
   }
   if ($tool -eq "policheck") {
     Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
@@ -46,22 +46,14 @@ foreach ($tool in $ToolsList) {
       Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE
     }
-    $config = $True
   }
 
-  Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config"
-  if ($config) {
-    & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile
-    if ($LASTEXITCODE -ne 0) {
-      Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE."
-      exit $LASTEXITCODE
-    }
-  } else {
-    & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel
-    if ($LASTEXITCODE -ne 0) {
-      Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE."
-      exit $LASTEXITCODE
-    }
-  }
+  $configParam+=$gdnConfigFile
 }
 
+Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
+& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
+if ($LASTEXITCODE -ne 0) {
+  Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
+  exit $LASTEXITCODE
+}