Skip to content

Commit

Permalink
[master] Update dependencies from dotnet/arcade (microsoft#902)
Browse files Browse the repository at this point in the history
* Update dependencies from https://github.com/dotnet/arcade build 20210310.7

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21152.1 -> To Version 6.0.0-beta.21160.7

* Update dependencies from https://github.com/dotnet/arcade build 20210317.3

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21152.1 -> To Version 6.0.0-beta.21167.3

* Update dependencies from https://github.com/dotnet/arcade build 20210326.2

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21152.1 -> To Version 6.0.0-beta.21176.2

* Update dependencies from https://github.com/dotnet/arcade build 20210403.1

Microsoft.DotNet.Arcade.Sdk
 From Version 6.0.0-beta.21152.1 -> To Version 6.0.0-beta.21203.1

Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
  • Loading branch information
dotnet-maestro[bot] and dotnet-maestro[bot] committed Apr 7, 2021
1 parent 2fe9741 commit c7cd5b0
Show file tree
Hide file tree
Showing 18 changed files with 283 additions and 40 deletions.
4 changes: 2 additions & 2 deletions eng/Version.Details.xml
Expand Up @@ -3,9 +3,9 @@
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21152.1">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21203.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>f95dc73e2028d630d67b782c9dcccbe464cf9677</Sha>
<Sha>1bfe91238cb39b9620b878b8f1bf0c789324b4cd</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>
101 changes: 101 additions & 0 deletions eng/common/generate-locproject.ps1
@@ -0,0 +1,101 @@
Param(
[Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here
[string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json
[switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
[switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally
)

# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task

Set-StrictMode -Version 2.0
$ErrorActionPreference = "Stop"
. $PSScriptRoot\tools.ps1

Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')

$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
$exclusions = @{ Exclusions = @() }
if (Test-Path -Path $exclusionsFilePath)
{
$exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
}

Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work

# Template files
$jsonFiles = @()
$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern

$xlfFiles = @()

$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
$langXlfFiles = @()
if ($allXlfFiles) {
$null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
$firstLangCode = $Matches.1
$langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
}
$langXlfFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf

$destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
$xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
}

$locFiles = $jsonFiles + $xlfFiles

$locJson = @{
Projects = @(
@{
LanguageSet = $LanguageSet
LocItems = @(
$locFiles | ForEach-Object {
$outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
if ($outputPath.Contains($exclusion))
{
$continue = $false
}
}
$sourceFile = ($_.FullName | Resolve-Path -Relative)
if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
Remove-Item -Path $sourceFile
}
if ($continue)
{
return @{
SourceFile = $sourceFile
CopyOption = "LangIDOnName"
OutputPath = $outputPath
}
}
}
)
}
)
}

$json = ConvertTo-Json $locJson -Depth 5
Write-Host "LocProject.json generated:`n`n$json`n`n"
Pop-Location

if (!$UseCheckedInLocProjectJson) {
New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
}
else {
New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json

if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."

exit 1
}
else {
Write-Host "Generated LocProject.json and current LocProject.json are identical."
}
}
2 changes: 1 addition & 1 deletion eng/common/native/install-tool.ps1
Expand Up @@ -105,7 +105,7 @@ try {
Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
exit 1
} elseif (@($ToolFilePath).Length -Lt 1) {
Write-Host "$ToolName was not found in $ToolFilePath."
Write-Host "$ToolName was not found in $ToolInstallDirectory."
exit 1
}

Expand Down
34 changes: 34 additions & 0 deletions eng/common/performance/android_scenarios.proj
@@ -0,0 +1,34 @@
<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<Python>python3</Python>
<HelixPreCommands>$(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk</HelixPreCommands>
</PropertyGroup>

<ItemGroup>
<HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
<PayloadDirectory>%(Identity)</PayloadDirectory>
</HelixCorrelationPayload>
</ItemGroup>

<PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
</PropertyGroup>

<ItemGroup>
<HelixWorkItem Include="SOD - Android HelloWorld APK Size">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<PreCommands>cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;$(Python) pre.py</PreCommands>
<Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
<PostCommands>$(Python) post.py</PostCommands>
</HelixWorkItem>
<HelixWorkItem Include="SOD - Android HelloWorld Extracted Size">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<PreCommands>cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;$(Python) pre.py --unzip</PreCommands>
<Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
<PostCommands>$(Python) post.py</PostCommands>
</HelixWorkItem>
</ItemGroup>
</Project>
39 changes: 24 additions & 15 deletions eng/common/performance/performance-setup.ps1
Expand Up @@ -18,7 +18,9 @@ Param(
[switch] $Internal,
[switch] $Compare,
[string] $MonoDotnet="",
[string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind"
[string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind",
[string] $LogicalMachine="",
[switch] $AndroidMono
)

$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
Expand All @@ -33,26 +35,24 @@ $Creator = $env:BUILD_DEFINITIONNAME
$PerfLabArguments = ""
$HelixSourcePrefix = "pr"

$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"

# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5.
if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) {
$Queue = "Windows.10.Amd64.ClientRS5.Open"
}

if ($Compare) {
$Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
$PerfLabArguments = ""
$ExtraBenchmarkDotNetArguments = ""
}
$Queue = ""

if ($Internal) {
$Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
switch ($LogicalMachine) {
"perftiger" { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" }
"perfowl" { $Queue = "Windows.10.Amd64.20H2.Owl.Perf" }
"perfsurf" { $Queue = "Windows.10.Arm64.Perf.Surf" }
"perfpixel4a" { $Queue = "Windows.10.Amd64.Pixel.Perf" }
Default { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" }
}
$PerfLabArguments = "--upload-to-perflab-container"
$ExtraBenchmarkDotNetArguments = ""
$Creator = ""
$HelixSourcePrefix = "official"
}
else {
$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
}

if($MonoInterpreter)
{
Expand Down Expand Up @@ -85,7 +85,7 @@ if ($RunFromPerformanceRepo) {
robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git
}
else {
git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
}

if($MonoDotnet -ne "")
Expand All @@ -104,6 +104,15 @@ if ($UseBaselineCoreRun) {
Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
}

if ($AndroidMono) {
if(!(Test-Path $WorkItemDirectory))
{
mkdir $WorkItemDirectory
}
Copy-Item -path "$SourceDirectory\artifacts\bin\AndroidSampleApp\arm64\Release\android-arm64\publish\apk\bin\HelloAndroid.apk" $PayloadDirectory
$SetupArguments = $SetupArguments -replace $Architecture, 'arm64'
}

$DocsDir = (Join-Path $PerformanceDirectory "docs")
robocopy $DocsDir $WorkItemDirectory

Expand Down
43 changes: 26 additions & 17 deletions eng/common/performance/performance-setup.sh
Expand Up @@ -17,6 +17,7 @@ kind="micro"
llvm=false
monointerpreter=false
monoaot=false
monoaot_path=
run_categories="Libraries Runtime"
csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
configurations="CompliationMode=$compilation_mode RunKind=$kind"
Expand All @@ -27,6 +28,7 @@ using_mono=false
wasm_runtime_loc=
using_wasm=false
use_latest_dotnet=false
logical_machine=

while (($# > 0)); do
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
Expand Down Expand Up @@ -55,6 +57,10 @@ while (($# > 0)); do
compilation_mode=$2
shift 2
;;
--logicalmachine)
logical_machine=$2
shift 2
;;
--repository)
repository=$2
shift 2
Expand Down Expand Up @@ -102,7 +108,8 @@ while (($# > 0)); do
;;
--monoaot)
monoaot=true
shift 1
monoaot_path=$2
shift 2
;;
--monodotnet)
mono_dotnet=$2
Expand Down Expand Up @@ -179,19 +186,6 @@ queue=Ubuntu.1804.Amd64.Open
creator=$BUILD_DEFINITIONNAME
helix_source_prefix="pr"

if [[ "$compare" == true ]]; then
extra_benchmark_dotnet_arguments=
perflab_arguments=

# No open queues for arm64
if [[ "$architecture" = "arm64" ]]; then
echo "Compare not available for arm64"
exit 1
fi

queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
fi

if [[ "$internal" == true ]]; then
perflab_arguments="--upload-to-perflab-container"
helix_source_prefix="official"
Expand All @@ -201,7 +195,11 @@ if [[ "$internal" == true ]]; then
if [[ "$architecture" = "arm64" ]]; then
queue=Ubuntu.1804.Arm64.Perf
else
queue=Ubuntu.1804.Amd64.Tiger.Perf
if [[ "$logical_machine" = "perfowl" ]]; then
queue=Ubuntu.1804.Amd64.Owl.Perf
else
queue=Ubuntu.1804.Amd64.Tiger.Perf
fi
fi

if [[ "$alpine" = "true" ]]; then
Expand Down Expand Up @@ -234,6 +232,11 @@ if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
fi

if [[ "$monoaot" == "true" ]]; then
configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoAOT"
fi

common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"

Expand All @@ -243,7 +246,7 @@ if [[ "$run_from_perf_repo" = true ]]; then
performance_directory=$workitem_directory
setup_arguments="--perf-hash $commit_sha $common_setup_arguments"
else
git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory
git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $performance_directory

docs_directory=$performance_directory/docs
mv $docs_directory $workitem_directory
Expand All @@ -256,12 +259,18 @@ if [[ "$wasm_runtime_loc" != "" ]]; then
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm"
fi

if [[ "$mono_dotnet" != "" ]]; then
if [[ "$mono_dotnet" != "" ]] && [[ "$monoaot" == "false" ]]; then
using_mono=true
mono_dotnet_path=$payload_directory/dotnet-mono
mv $mono_dotnet $mono_dotnet_path
fi

if [[ "$monoaot" == "true" ]]; then
monoaot_dotnet_path=$payload_directory/monoaot
mv $monoaot_path $monoaot_dotnet_path
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --runtimes monoaotllvm --aotcompilerpath \$HELIX_CORRELATION_PAYLOAD/monoaot/sgen/mini/mono-sgen --customruntimepack \$HELIX_CORRELATION_PAYLOAD/monoaot/pack --aotcompilermode llvm"
fi

if [[ "$use_core_run" = true ]]; then
new_core_root=$payload_directory/Core_Root
mv $core_root_directory $new_core_root
Expand Down
2 changes: 1 addition & 1 deletion eng/common/post-build/publish-using-darc.ps1
Expand Up @@ -60,7 +60,7 @@ try {
--id $buildId `
--publishing-infra-version $PublishingInfraVersion `
--default-channels `
--source-branch master `
--source-branch main `
--azdev-pat $AzdoToken `
--bar-uri $MaestroApiEndPoint `
--password $MaestroToken `
Expand Down
2 changes: 2 additions & 0 deletions eng/common/templates/job/execute-sdl.yml
Expand Up @@ -45,6 +45,7 @@ jobs:
buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- ${{ if eq(parameters.artifactNames, '') }}:
- task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts
Expand All @@ -57,6 +58,7 @@ jobs:
downloadType: specific files
itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
Expand Down

0 comments on commit c7cd5b0

Please sign in to comment.