[master] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21152.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21203.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>f95dc73e2028d630d67b782c9dcccbe464cf9677</Sha>
+      <Sha>1bfe91238cb39b9620b878b8f1bf0c789324b4cd</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/generate-locproject.ps1

@@ -0,0 +1,101 @@
+Param(
+    [Parameter(Mandatory=$true)][string] $SourcesDirectory,     # Directory where source files live; if using a Localize directory it should live in here
+    [string] $LanguageSet = 'VS_Main_Languages',                # Language set to be used in the LocProject.json
+    [switch] $UseCheckedInLocProjectJson,                       # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
+    [switch] $CreateNeutralXlfs                                 # Creates neutral xlf files. Only set to false when running locally
+)
+
+# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
+# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
+
+Set-StrictMode -Version 2.0
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
+$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
+$exclusions = @{ Exclusions = @() }
+if (Test-Path -Path $exclusionsFilePath)
+{
+    $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
+}
+
+Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
+
+# Template files
+$jsonFiles = @()
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
+$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+
+$xlfFiles = @()
+
+$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
+$langXlfFiles = @()
+if ($allXlfFiles) {
+    $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
+    $firstLangCode = $Matches.1
+    $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
+}
+$langXlfFiles | ForEach-Object {
+    $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
+
+    $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
+    $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$locFiles = $jsonFiles + $xlfFiles
+
+$locJson = @{
+    Projects = @(
+        @{
+            LanguageSet = $LanguageSet
+            LocItems = @(
+                $locFiles | ForEach-Object {
+                    $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" 
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($outputPath.Contains($exclusion))
+                        {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
+                        Remove-Item -Path $sourceFile
+                    }
+                    if ($continue)
+                    {
+                        return @{
+                            SourceFile = $sourceFile
+                            CopyOption = "LangIDOnName"
+                            OutputPath = $outputPath
+                        }
+                    }
+                }
+            )
+        }
+    )
+}
+
+$json = ConvertTo-Json $locJson -Depth 5
+Write-Host "LocProject.json generated:`n`n$json`n`n"
+Pop-Location
+
+if (!$UseCheckedInLocProjectJson) {
+    New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
+}
+else {
+    New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json
+
+    if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
+        Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
+
+        exit 1
+    }
+    else {
+        Write-Host "Generated LocProject.json and current LocProject.json are identical."
+    }
+}

eng/common/native/install-tool.ps1

@@ -105,7 +105,7 @@ try {
     Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
     exit 1
   } elseif (@($ToolFilePath).Length -Lt 1) {
-    Write-Host "$ToolName was not found in $ToolFilePath."
+    Write-Host "$ToolName was not found in $ToolInstallDirectory."
     exit 1
   }
 

eng/common/performance/android_scenarios.proj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <Python>python3</Python>
+    <HelixPreCommands>$(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk</HelixPreCommands>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+      <PayloadDirectory>%(Identity)</PayloadDirectory>
+    </HelixCorrelationPayload>
+  </ItemGroup>
+
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
+    <ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixWorkItem Include="SOD - Android HelloWorld APK Size">
+        <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+        <PreCommands>cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;$(Python) pre.py</PreCommands>
+        <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+        <PostCommands>$(Python) post.py</PostCommands>
+    </HelixWorkItem>
+    <HelixWorkItem Include="SOD - Android HelloWorld Extracted Size">
+        <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+        <PreCommands>cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;$(Python) pre.py --unzip</PreCommands>
+        <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+        <PostCommands>$(Python) post.py</PostCommands>
+    </HelixWorkItem>
+  </ItemGroup>
+</Project>

eng/common/performance/performance-setup.ps1

@@ -18,7 +18,9 @@ Param(
     [switch] $Internal,
     [switch] $Compare,
     [string] $MonoDotnet="",
-    [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind"
+    [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind",
+    [string] $LogicalMachine="",
+    [switch] $AndroidMono
 )
 
 $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
@@ -33,26 +35,24 @@ $Creator = $env:BUILD_DEFINITIONNAME
 $PerfLabArguments = ""
 $HelixSourcePrefix = "pr"
 
-$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
-
-# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5.
-if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) {
-    $Queue = "Windows.10.Amd64.ClientRS5.Open"
-}
-
-if ($Compare) {
-    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
-    $PerfLabArguments = ""
-    $ExtraBenchmarkDotNetArguments = ""
-}
+$Queue = ""
 
 if ($Internal) {
-    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
+    switch ($LogicalMachine) {
+        "perftiger" { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"  }
+        "perfowl" { $Queue = "Windows.10.Amd64.20H2.Owl.Perf"  }
+        "perfsurf" { $Queue = "Windows.10.Arm64.Perf.Surf"  }
+        "perfpixel4a" { $Queue = "Windows.10.Amd64.Pixel.Perf" }
+        Default { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" }
+    }
     $PerfLabArguments = "--upload-to-perflab-container"
     $ExtraBenchmarkDotNetArguments = ""
     $Creator = ""
     $HelixSourcePrefix = "official"
 }
+else {
+    $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
+}
 
 if($MonoInterpreter)
 {
@@ -85,7 +85,7 @@ if ($RunFromPerformanceRepo) {
     robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git
 }
 else {
-    git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
+    git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
 }
 
 if($MonoDotnet -ne "")
@@ -104,6 +104,15 @@ if ($UseBaselineCoreRun) {
     Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
 }
 
+if ($AndroidMono) {
+    if(!(Test-Path $WorkItemDirectory))
+    {
+        mkdir $WorkItemDirectory
+    }
+    Copy-Item -path "$SourceDirectory\artifacts\bin\AndroidSampleApp\arm64\Release\android-arm64\publish\apk\bin\HelloAndroid.apk" $PayloadDirectory
+    $SetupArguments = $SetupArguments -replace $Architecture, 'arm64'
+}
+
 $DocsDir = (Join-Path $PerformanceDirectory "docs")
 robocopy $DocsDir $WorkItemDirectory
 

eng/common/performance/performance-setup.sh

@@ -17,6 +17,7 @@ kind="micro"
 llvm=false
 monointerpreter=false
 monoaot=false
+monoaot_path=
 run_categories="Libraries Runtime"
 csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
 configurations="CompliationMode=$compilation_mode RunKind=$kind"
@@ -27,6 +28,7 @@ using_mono=false
 wasm_runtime_loc=
 using_wasm=false
 use_latest_dotnet=false
+logical_machine=
 
 while (($# > 0)); do
   lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
@@ -55,6 +57,10 @@ while (($# > 0)); do
       compilation_mode=$2
       shift 2
       ;;
+    --logicalmachine)
+      logical_machine=$2
+      shift 2
+      ;;
     --repository)
       repository=$2
       shift 2
@@ -102,7 +108,8 @@ while (($# > 0)); do
       ;;
     --monoaot)
       monoaot=true
-      shift 1
+      monoaot_path=$2
+      shift 2
       ;;
     --monodotnet)
       mono_dotnet=$2
@@ -179,19 +186,6 @@ queue=Ubuntu.1804.Amd64.Open
 creator=$BUILD_DEFINITIONNAME
 helix_source_prefix="pr"
 
-if [[ "$compare" == true ]]; then
-  extra_benchmark_dotnet_arguments=
-  perflab_arguments=
-
-  # No open queues for arm64
-  if [[ "$architecture" = "arm64" ]]; then
-    echo "Compare not available for arm64"
-    exit 1
-  fi
-
-  queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
-fi
-
 if [[ "$internal" == true ]]; then
     perflab_arguments="--upload-to-perflab-container"
     helix_source_prefix="official"
@@ -201,7 +195,11 @@ if [[ "$internal" == true ]]; then
     if [[ "$architecture" = "arm64" ]]; then
         queue=Ubuntu.1804.Arm64.Perf
     else
-        queue=Ubuntu.1804.Amd64.Tiger.Perf
+        if [[ "$logical_machine" = "perfowl" ]]; then
+            queue=Ubuntu.1804.Amd64.Owl.Perf
+        else
+            queue=Ubuntu.1804.Amd64.Tiger.Perf
+        fi
     fi
 
     if [[ "$alpine" = "true" ]]; then
@@ -234,6 +232,11 @@ if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
 fi
 
+if [[ "$monoaot" == "true" ]]; then
+    configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
+    extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoAOT"
+fi
+
 common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
 setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
 
@@ -243,7 +246,7 @@ if [[ "$run_from_perf_repo" = true ]]; then
     performance_directory=$workitem_directory
     setup_arguments="--perf-hash $commit_sha $common_setup_arguments"
 else
-    git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory
+    git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $performance_directory
 
     docs_directory=$performance_directory/docs
     mv $docs_directory $workitem_directory
@@ -256,12 +259,18 @@ if [[ "$wasm_runtime_loc" != "" ]]; then
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm"
 fi
 
-if [[ "$mono_dotnet" != "" ]]; then
+if [[ "$mono_dotnet" != "" ]] && [[ "$monoaot" == "false" ]]; then
     using_mono=true
     mono_dotnet_path=$payload_directory/dotnet-mono
     mv $mono_dotnet $mono_dotnet_path
 fi
 
+if [[ "$monoaot" == "true" ]]; then
+    monoaot_dotnet_path=$payload_directory/monoaot
+    mv $monoaot_path $monoaot_dotnet_path
+    extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --runtimes monoaotllvm --aotcompilerpath \$HELIX_CORRELATION_PAYLOAD/monoaot/sgen/mini/mono-sgen --customruntimepack \$HELIX_CORRELATION_PAYLOAD/monoaot/pack --aotcompilermode llvm"
+fi
+
 if [[ "$use_core_run" = true ]]; then
     new_core_root=$payload_directory/Core_Root
     mv $core_root_directory $new_core_root

eng/common/post-build/publish-using-darc.ps1

@@ -60,7 +60,7 @@ try {
   --id $buildId `
   --publishing-infra-version $PublishingInfraVersion `
   --default-channels `
-  --source-branch master `
+  --source-branch main `
   --azdev-pat $AzdoToken `
   --bar-uri $MaestroApiEndPoint `
   --password $MaestroToken `

eng/common/templates/job/execute-sdl.yml

@@ -45,6 +45,7 @@ jobs:
             buildId: $(AzDOBuildId)
             artifactName: ${{ artifactName }}
             downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+            checkDownloadedFiles: true
     - ${{ if eq(parameters.artifactNames, '') }}:
       - task: DownloadBuildArtifacts@0
         displayName: Download Build Artifacts
@@ -57,6 +58,7 @@ jobs:
           downloadType: specific files
           itemPattern: "**"
           downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+          checkDownloadedFiles: true
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
       -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
       -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts