diff --git a/NuGet.config b/NuGet.config index 23206d2001..15e1ddd568 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,19 +1,19 @@ - + - - + + - + - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4cf005dce8..740e33cc0f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,25 +3,25 @@ - + https://github.com/dotnet/arcade - 98cc270f5e08096729fc7d54ffafa9e3c6576316 + 9b71be0663493cd0e111b55536a2e1eeb272f54c - + https://github.com/dotnet/arcade - 98cc270f5e08096729fc7d54ffafa9e3c6576316 + 9b71be0663493cd0e111b55536a2e1eeb272f54c - + https://github.com/dotnet/arcade - 98cc270f5e08096729fc7d54ffafa9e3c6576316 + 9b71be0663493cd0e111b55536a2e1eeb272f54c - + https://github.com/dotnet/arcade - 98cc270f5e08096729fc7d54ffafa9e3c6576316 + 9b71be0663493cd0e111b55536a2e1eeb272f54c - + https://github.com/dotnet/arcade - 98cc270f5e08096729fc7d54ffafa9e3c6576316 + 9b71be0663493cd0e111b55536a2e1eeb272f54c https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index a02fa656b3..842c083ffd 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -60,8 +60,8 @@ 2.4.1 2.0.3 2.4.1 - 5.0.0-beta.20052.1 - 5.0.0-beta.20052.1 + 2.2.0-beta.20302.3 + 1.0.0-beta.20302.3 1.22.0 1.1.2 2.0.0 @@ -73,7 +73,7 @@ 1.7.0 1.1.0-beta.19556.4 1.0.0-beta2-19554-01 - 5.0.0-beta.20052.1 + 1.0.0-beta.20302.3 1.0.0-beta.20055.1 diff --git a/eng/common/CheckSymbols.ps1 b/eng/common/CheckSymbols.ps1 index 5442eff386..b8d84607b8 100644 --- a/eng/common/CheckSymbols.ps1 +++ b/eng/common/CheckSymbols.ps1 @@ -5,12 +5,11 @@ param( ) Add-Type -AssemblyName System.IO.Compression.FileSystem -. $PSScriptRoot\pipeline-logging-functions.ps1 function FirstMatchingSymbolDescriptionOrDefault { param( [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParameter, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols [string] $SymbolsPath ) @@ -22,36 +21,36 @@ function FirstMatchingSymbolDescriptionOrDefault { # checking and which type of file was uploaded. # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName + $SymbolPath = $SymbolsPath + "\" + $FileName # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParameter $FullPath -o $SymbolsPath | Out-Null + .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null if (Test-Path $PdbPath) { - return 'PDB' + return "PDB" } elseif (Test-Path $NGenPdb) { - return 'NGen PDB' + return "NGen PDB" } elseif (Test-Path $SODbg) { - return 'DBG for SO' + return "DBG for SO" } elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' + return "Dwarf for Dylib" } elseif (Test-Path $SymbolPath) { - return 'Module' + return "Module" } else { return $null @@ -69,7 +68,7 @@ function CountMissingSymbols { } # Extensions for which we'll look for symbols - $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib') + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") # How many files are missing symbol information $MissingSymbols = 0 @@ -77,7 +76,7 @@ function CountMissingSymbols { $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageGuid = New-Guid $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) @@ -87,31 +86,31 @@ function CountMissingSymbols { Get-ChildItem -Recurse $ExtractPath | Where-Object {$RelevantExtensions -contains $_.Extension} | ForEach-Object { - if ($_.FullName -Match '\\ref\\') { + if ($_.FullName -Match "\\ref\\") { Write-Host "`t Ignoring reference assembly file" $_.FullName return } - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--microsoft-symbol-server' -SymbolsPath $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--internal-server' -SymbolsPath $SymbolsPath + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath Write-Host -NoNewLine "`t Checking file" $_.FullName "... " if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL (${$SymbolsOnMSDL}) and SymWeb (${$SymbolsOnSymWeb})" + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" } else { $MissingSymbols++ if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' + Write-Host "No symbols found on MSDL or SymWeb!" } else { if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' + Write-Host "No symbols found on MSDL!" } else { - Write-Host 'No symbols found on SymWeb!' + Write-Host "No symbols found on SymWeb!" } } } @@ -130,26 +129,26 @@ function CheckSymbolsAvailable { Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name - + # These packages from Arcade-Services include some native libraries that # our current symbol uploader can't handle. Below is a workaround until # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') { + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { Write-Host "Ignoring Arcade-services file: $FileName" Write-Host return } - elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') { + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { Write-Host "Ignoring Arcade-services file: $FileName" Write-Host return } - + Write-Host "Validating $FileName " $Status = CountMissingSymbols "$InputPath\$FileName" if ($Status -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" + Write-Error "Missing symbols for $Status modules in the package $FileName" } Write-Host diff --git a/eng/common/PublishToSymbolServers.proj b/eng/common/PublishToSymbolServers.proj index 311e2bbe0f..5d55e312b0 100644 --- a/eng/common/PublishToSymbolServers.proj +++ b/eng/common/PublishToSymbolServers.proj @@ -37,8 +37,6 @@ 3650 true - true - true false @@ -58,7 +56,7 @@ DryRun="false" ConvertPortablePdbsToWindowsPdbs="false" PdbConversionTreatAsWarning="" - Condition="$(PublishToSymbolServer) and $(PublishToMSDL)"/> + Condition="$(PublishToSymbolServer)"/> diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index c640123000..b37fd3d5e9 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -1,6 +1,6 @@ [CmdletBinding(PositionalBinding=$false)] Param( - [string] $verbosity = 'minimal', + [string] $verbosity = "minimal", [bool] $warnAsError = $true, [bool] $nodeReuse = $true, [switch] $ci, @@ -18,8 +18,9 @@ try { MSBuild @extraArgs } catch { + Write-Host $_ + Write-Host $_.Exception Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Build' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index d7d1a65109..41416862d9 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -145,12 +145,9 @@ function Get-File { New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null } - $TempPath = "$Path.tmp" if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" - Copy-Item -Path $Uri -Destination $TempPath - Write-Verbose "Moving temporary file to '$Path'" - Move-Item -Path $TempPath -Destination $Path + Write-Verbose "'$Uri' is a file path, copying file to '$Path'" + Copy-Item -Path $Uri -Destination $Path return $? } else { @@ -160,10 +157,8 @@ function Get-File { while($Attempt -Lt $DownloadRetries) { try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath - Write-Verbose "Downloaded to temporary location '$TempPath'" - Move-Item -Path $TempPath -Destination $Path - Write-Verbose "Moved temporary file to '$Path'" + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path + Write-Verbose "Downloaded to '$Path'" return $True } catch { @@ -364,21 +359,16 @@ function Expand-Zip { return $False } } - - $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" - if (Test-Path $TempOutputDirectory) { - Remove-Item $TempOutputDirectory -Force -Recurse + if (-Not (Test-Path $OutputDirectory)) { + New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null } - New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") if ($? -Eq $False) { Write-Error "Unable to extract '$ZipPath'" return $False } - - Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory } catch { Write-Host $_ diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh index bf272dcf55..271bddfac5 100644 --- a/eng/common/native/common-library.sh +++ b/eng/common/native/common-library.sh @@ -34,7 +34,7 @@ function ExpandZip { echo "'Force flag enabled, but '$output_directory' exists. Removing directory" rm -rf $output_directory if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'" + echo Unable to remove '$output_directory'>&2 return 1 fi fi @@ -45,7 +45,7 @@ function ExpandZip { echo "Extracting archive" tar -xf $zip_path -C $output_directory if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'" + echo "Unable to extract '$zip_path'" >&2 return 1 fi @@ -117,7 +117,7 @@ function DownloadAndExtract { # Download file GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'." + echo "Failed to download '$uri' to '$temp_tool_path'." >&2 return 1 fi @@ -125,7 +125,7 @@ function DownloadAndExtract { echo "extracting from $temp_tool_path to $installDir" ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'." + echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2 return 1 fi @@ -148,7 +148,7 @@ function NewScriptShim { fi if [[ ! -f $tool_file_path ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" + echo "Specified tool file path:'$tool_file_path' does not exist" >&2 return 1 fi diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh deleted file mode 100644 index aed19d07d5..0000000000 --- a/eng/common/native/find-native-compiler.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# -# This file locates the native compiler with the given name and version and sets the environment variables to locate it. -# - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -if [ $# -lt 0 ] -then - echo "Usage..." - echo "find-native-compiler.sh " - echo "Specify the name of compiler (clang or gcc)." - echo "Specify the major version of compiler." - echo "Specify the minor version of compiler." - exit 1 -fi - -. $scriptroot/../pipeline-logging-functions.sh - -compiler="$1" -cxxCompiler="$compiler++" -majorVersion="$2" -minorVersion="$3" - -if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi - -check_version_exists() { - desired_version=-1 - - # Set up the environment to be used for building with the desired compiler. - if command -v "$compiler-$1.$2" > /dev/null; then - desired_version="-$1.$2" - elif command -v "$compiler$1$2" > /dev/null; then - desired_version="$1$2" - elif command -v "$compiler-$1$2" > /dev/null; then - desired_version="-$1$2" - fi - - echo "$desired_version" -} - -if [ -z "$CLR_CC" ]; then - - # Set default versions - if [ -z "$majorVersion" ]; then - # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) - elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi - - for version in "${versions[@]}"; do - parts=(${version//./ }) - desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" - if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi - done - - if [ -z "$majorVersion" ]; then - if command -v "$compiler" > /dev/null; then - if [ "$(uname)" != "Darwin" ]; then - Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." - fi - export CC="$(command -v "$compiler")" - export CXX="$(command -v "$cxxCompiler")" - else - Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." - exit 1 - fi - else - if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then - if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then - if command -v "$compiler" > /dev/null; then - Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - export CC="$(command -v "$compiler")" - export CXX="$(command -v "$cxxCompiler")" - else - Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." - exit 1 - fi - fi - fi - fi - else - desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" - if [ "$desired_version" = "-1" ]; then - Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." - exit 1 - fi - fi - - if [ -z "$CC" ]; then - export CC="$(command -v "$compiler$desired_version")" - export CXX="$(command -v "$cxxCompiler$desired_version")" - if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi - fi -else - if [ ! -f "$CLR_CC" ]; then - Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" - exit 1 - fi - export CC="$CLR_CC" - export CXX="$CLR_CXX" -fi - -if [ -z "$CC" ]; then - Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." - exit 1 -fi - -export CCC_CC="$CC" -export CCC_CXX="$CXX" -export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh index 12339a4076..53ddf4e686 100644 --- a/eng/common/native/install-cmake-test.sh +++ b/eng/common/native/install-cmake-test.sh @@ -101,7 +101,7 @@ fi DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' + echo "Installation failed" >&2 exit 1 fi @@ -110,7 +110,7 @@ fi NewScriptShim $shim_path $tool_file_path true if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' + echo "Shim generation failed" >&2 exit 1 fi diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh index 18041be876..5f1a182fa9 100644 --- a/eng/common/native/install-cmake.sh +++ b/eng/common/native/install-cmake.sh @@ -101,7 +101,7 @@ fi DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' + echo "Installation failed" >&2 exit 1 fi @@ -110,7 +110,7 @@ fi NewScriptShim $shim_path $tool_file_path true if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' + echo "Shim generation failed" >&2 exit 1 fi diff --git a/eng/common/native/install-tool.ps1 b/eng/common/native/install-tool.ps1 index f397e1c75d..635ab3fd41 100644 --- a/eng/common/native/install-tool.ps1 +++ b/eng/common/native/install-tool.ps1 @@ -46,8 +46,6 @@ Param ( [int] $RetryWaitTimeInSeconds = 30 ) -. $PSScriptRoot\..\pipeline-logging-functions.ps1 - # Import common library modules Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") @@ -95,7 +93,7 @@ try { -Verbose:$Verbose if ($InstallStatus -Eq $False) { - Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping" + Write-Error "Installation failed" exit 1 } } @@ -105,7 +103,7 @@ try { Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" exit 1 } elseif (@($ToolFilePath).Length -Lt 1) { - Write-Host "$ToolName was not found in $ToolFilePath." + Write-Error "$ToolName was not found in $ToolFilePath." exit 1 } @@ -119,14 +117,14 @@ try { -Verbose:$Verbose if ($GenerateShimStatus -Eq $False) { - Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping" + Write-Error "Generate shim failed" return 1 } exit 0 } catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_ + Write-Host $_ + Write-Host $_.Exception exit 1 } diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 1db5e8a84d..e5826b5323 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,8 +6,7 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj @@ -41,13 +40,6 @@ $HELIX_WORKITEM_ROOT/testResults.xml - - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe - - - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun - - --corerun $(CoreRun) @@ -107,23 +99,4 @@ 4:00 - - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - \ No newline at end of file diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 31a99e4901..ec41965fc8 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -3,22 +3,18 @@ Param( [string] $CoreRootDirectory, [string] $BaselineCoreRootDirectory, [string] $Architecture="x64", - [string] $Framework="net5.0", + [string] $Framework="netcoreapp5.0", [string] $CompilationMode="Tiered", [string] $Repository=$env:BUILD_REPOSITORY_NAME, [string] $Branch=$env:BUILD_SOURCEBRANCH, [string] $CommitSha=$env:BUILD_SOURCEVERSION, [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="Libraries Runtime", + [string] $RunCategories="coreclr corefx", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", - [switch] $LLVM, - [switch] $MonoInterpreter, - [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, - [string] $MonoDotnet="", - [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" + [string] $Configurations="CompilationMode=$CompilationMode" ) $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") @@ -35,8 +31,7 @@ $HelixSourcePrefix = "pr" $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5. -if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) { +if ($Framework.StartsWith("netcoreapp")) { $Queue = "Windows.10.Amd64.ClientRS5.Open" } @@ -54,32 +49,9 @@ if ($Internal) { $HelixSourcePrefix = "official" } -if($MonoDotnet -ne "") -{ - $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" - if($ExtraBenchmarkDotNetArguments -eq "") - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } - else - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } -} - -# FIX ME: This is a workaround until we get this from the actual pipeline -$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" +$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" - -#This grabs the LKG version number of dotnet and passes it to our scripts -$VersionJSON = Get-Content global.json | ConvertFrom-Json -$DotNetVersion = $VersionJSON.tools.dotnet -$SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments" - - if ($RunFromPerformanceRepo) { $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" @@ -89,13 +61,6 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } -if($MonoDotnet -ne "") -{ - $UsingMono = "true" - $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") - Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath -} - if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot @@ -131,7 +96,6 @@ Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVar Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 9409e4d85e..2f2092166e 100644 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -4,7 +4,7 @@ source_directory=$BUILD_SOURCESDIRECTORY core_root_directory= baseline_core_root_directory= architecture=x64 -framework=net5.0 +framework=netcoreapp5.0 compilation_mode=tiered repository=$BUILD_REPOSITORY_NAME branch=$BUILD_SOURCEBRANCH @@ -12,18 +12,13 @@ commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false compare=false -mono_dotnet= kind="micro" -llvm=false -monointerpreter=false -monoaot=false -run_categories="Libraries Runtime" +run_categories="coreclr corefx" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations="CompliationMode=$compilation_mode RunKind=$kind" +configurations= run_from_perf_repo=false use_core_run=true use_baseline_core_run=true -using_mono=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -70,7 +65,6 @@ while (($# > 0)); do ;; --kind) kind=$2 - configurations="CompliationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -85,22 +79,6 @@ while (($# > 0)); do internal=true shift 1 ;; - --llvm) - llvm=true - shift 1 - ;; - --monointerpreter) - monointerpreter=true - shift 1 - ;; - --monoaot) - monoaot=true - shift 1 - ;; - --monodotnet) - mono_dotnet=$2 - shift 2 - ;; --compare) compare=true shift 1 @@ -129,7 +107,6 @@ while (($# > 0)); do echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." - echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo "" exit 0 ;; @@ -187,20 +164,9 @@ if [[ "$internal" == true ]]; then fi fi -if [[ "$mono_dotnet" != "" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" -fi - -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" +common_setup_arguments="--frameworks $framework --queue $queue --build-number $build_number --build-configs $configurations" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" - -# Get the tools section from the global.json. -# This grabs the LKG version number of dotnet and passes it to our scripts -dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` -setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" - - if [[ "$run_from_perf_repo" = true ]]; then payload_directory= workitem_directory=$source_directory @@ -213,12 +179,6 @@ else mv $docs_directory $workitem_directory fi -if [[ "$mono_dotnet" != "" ]]; then - using_mono=true - mono_dotnet_path=$payload_directory/dotnet-mono - mv $mono_dotnet $mono_dotnet_path -fi - if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root @@ -254,4 +214,3 @@ Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false -Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 8484451f3a..af5f48aace 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -12,7 +12,6 @@ $script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%" # TODO: BUG: Escape % ??? # TODO: Add test to verify don't need to escape "=". -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set function Write-PipelineTelemetryError { [CmdletBinding()] param( @@ -26,55 +25,49 @@ function Write-PipelineTelemetryError { [string]$SourcePath, [string]$LineNumber, [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force) + [switch]$AsOutput) - $PSBoundParameters.Remove('Category') | Out-Null + $PSBoundParameters.Remove("Category") | Out-Null + + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + $PSBoundParameters.Remove("Message") | Out-Null + $PSBoundParameters.Add("Message", $Message) - if($Force -Or ((Test-Path variable:ci) -And $ci)) { - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - } - $PSBoundParameters.Remove('Message') | Out-Null - $PSBoundParameters.Add('Message', $Message) Write-PipelineTaskError @PSBoundParameters } -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set function Write-PipelineTaskError { [CmdletBinding()] param( - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force - ) - - if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + if(!$ci) { if($Type -eq 'error') { - Write-Host $Message -ForegroundColor Red - return + Write-Host $Message -ForegroundColor Red + return } elseif ($Type -eq 'warning') { - Write-Host $Message -ForegroundColor Yellow - return + Write-Host $Message -ForegroundColor Yellow + return } - } - - if(($Type -ne 'error') -and ($Type -ne 'warning')) { + } + + if(($Type -ne 'error') -and ($Type -ne 'warning')) { Write-Host $Message return - } - $PSBoundParameters.Remove('Force') | Out-Null - if(-not $PSBoundParameters.ContainsKey('Type')) { + } + if(-not $PSBoundParameters.ContainsKey('Type')) { $PSBoundParameters.Add('Type', 'error') - } - Write-LogIssue @PSBoundParameters + } + Write-LogIssue @PSBoundParameters } function Write-PipelineSetVariable { @@ -87,7 +80,7 @@ function Write-PipelineTaskError { [switch]$AsOutput, [bool]$IsMultiJobVariable=$true) - if((Test-Path variable:ci) -And $ci) { + if($ci) { Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ 'variable' = $Name 'isSecret' = $Secret @@ -102,8 +95,7 @@ function Write-PipelineTaskError { [Parameter(Mandatory=$true)] [string]$Path, [switch]$AsOutput) - - if((Test-Path variable:ci) -And $ci) { + if($ci) { Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput } } @@ -239,4 +231,4 @@ function Write-LogIssue { } Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor -} +} \ No newline at end of file diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index 33c3f0d807..1c560a5061 100644 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -2,7 +2,6 @@ function Write-PipelineTelemetryError { local telemetry_category='' - local force=false local function_args=() local message='' while [[ $# -gt 0 ]]; do @@ -12,9 +11,6 @@ function Write-PipelineTelemetryError { telemetry_category=$2 shift ;; - -force|-f) - force=true - ;; -*) function_args+=("$1 $2") shift @@ -26,22 +22,19 @@ function Write-PipelineTelemetryError { shift done - if [[ $force != true ]] && [[ "$ci" != true ]]; then + if [[ "$ci" != true ]]; then echo "$message" >&2 return fi message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" function_args+=("$message") - if [[ $force == true ]]; then - function_args+=("-force") - fi Write-PipelineTaskError $function_args } function Write-PipelineTaskError { - if [[ $force != true ]] && [[ "$ci" != true ]]; then + if [[ "$ci" != true ]]; then echo "$@" >&2 return fi diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1 deleted file mode 100644 index de2d957922..0000000000 --- a/eng/common/post-build/add-build-to-channel.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -param( - [Parameter(Mandatory=$true)][int] $BuildId, - [Parameter(Mandatory=$true)][int] $ChannelId, - [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', - [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - # Check that the channel we are going to promote the build to exist - $channelInfo = Get-MaestroChannel -ChannelId $ChannelId - - if (!$channelInfo) { - Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!" - ExitWithExitCode 1 - } - - # Get info about which channel(s) the build has already been promoted to - $buildInfo = Get-MaestroBuild -BuildId $BuildId - - if (!$buildInfo) { - Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!" - ExitWithExitCode 1 - } - - # Find whether the build is already assigned to the channel or not - if ($buildInfo.channels) { - foreach ($channel in $buildInfo.channels) { - if ($channel.Id -eq $ChannelId) { - Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!" - ExitWithExitCode 0 - } - } - } - - Write-Host "Promoting build '$BuildId' to channel '$ChannelId'." - - Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId - - Write-Host 'done.' -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'" - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 deleted file mode 100644 index 38abc5392d..0000000000 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ /dev/null @@ -1,30 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to - [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - if ($PromoteToChannels -eq "") { - Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." - ExitWithExitCode 0 - } - - # Check that every channel that Maestro told to promote the build to - # is available in YAML - $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } - - foreach ($id in $PromoteToChannelsIds) { - if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { - Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." - } - } - - Write-Host 'done.' -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration." - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/darc-gather-drop.ps1 b/eng/common/post-build/darc-gather-drop.ps1 index 81ff2a4377..89854d3c1c 100644 --- a/eng/common/post-build/darc-gather-drop.ps1 +++ b/eng/common/post-build/darc-gather-drop.ps1 @@ -2,20 +2,20 @@ param( [Parameter(Mandatory=$true)][int] $BarBuildId, # ID of the build which assets should be downloaded [Parameter(Mandatory=$true)][string] $DropLocation, # Where the assets should be downloaded to [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, # Token used to access Maestro API - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', # Maestro API URL - [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' # Version of Maestro API to use + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", # Maestro API URL + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" # Version of Maestro API to use ) -try { - . $PSScriptRoot\post-build-utils.ps1 +. $PSScriptRoot\post-build-utils.ps1 - Write-Host 'Installing DARC ...' +try { + Write-Host "Installing DARC ..." . $PSScriptRoot\..\darc-init.ps1 $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category "Darc" -Message "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..." + Write-PipelineTaskError "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..." ExitWithExitCode $exitCode } @@ -38,7 +38,8 @@ try { --latest-location } catch { + Write-Host $_ + Write-Host $_.Exception Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "Darc" -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 index dab3534ab5..78ed0d540f 100644 --- a/eng/common/post-build/nuget-validation.ps1 +++ b/eng/common/post-build/nuget-validation.ps1 @@ -6,19 +6,20 @@ param( [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to ) -try { - . $PSScriptRoot\post-build-utils.ps1 +. $PSScriptRoot\post-build-utils.ps1 - $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1' +try { + $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" - New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force + New-Item -ItemType "directory" -Path ${ToolDestinationPath} -Force Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg } catch { + Write-PipelineTaskError "NuGet package validation failed. Please check error logs." + Write-Host $_ Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1 index 7d49744795..551ae113f8 100644 --- a/eng/common/post-build/post-build-utils.ps1 +++ b/eng/common/post-build/post-build-utils.ps1 @@ -1,17 +1,16 @@ # Most of the functions in this file require the variables `MaestroApiEndPoint`, # `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available. -$ErrorActionPreference = 'Stop' +$ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 # `tools.ps1` checks $ci to perform some actions. Since the post-build # scripts don't necessarily execute in the same agent that run the # build.ps1/sh script this variable isn't automatically set. $ci = $true -$disableConfigureToolsetImport = $true . $PSScriptRoot\..\tools.ps1 -function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') { +function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") { Validate-MaestroVars $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' @@ -51,20 +50,20 @@ function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) { return $result } -function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) { +function Trigger-Subscription([string]$SubscriptionId) { Validate-MaestroVars $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken - $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion" - Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null + $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion" + Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null } -function Trigger-Subscription([string]$SubscriptionId) { +function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) { Validate-MaestroVars $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken - $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion" - Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null + $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion" + Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null } function Validate-MaestroVars { @@ -73,18 +72,18 @@ function Validate-MaestroVars { Get-Variable MaestroApiVersion -Scope Global | Out-Null Get-Variable MaestroApiAccessToken -Scope Global | Out-Null - if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) { - Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" + if (!($MaestroApiEndPoint -Match "^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$")) { + Write-PipelineTaskError "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" ExitWithExitCode 1 } - if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) { - Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'" + if (!($MaestroApiVersion -Match "^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) { + Write-PipelineTaskError "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'" ExitWithExitCode 1 } } catch { - Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.' + Write-PipelineTaskError "Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script." Write-Host $_ ExitWithExitCode 1 } diff --git a/eng/common/post-build/promote-build.ps1 b/eng/common/post-build/promote-build.ps1 index ce45635fbd..e5ae85f251 100644 --- a/eng/common/post-build/promote-build.ps1 +++ b/eng/common/post-build/promote-build.ps1 @@ -2,18 +2,18 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', - [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" ) -try { - . $PSScriptRoot\post-build-utils.ps1 +. $PSScriptRoot\post-build-utils.ps1 +try { # Check that the channel we are going to promote the build to exist $channelInfo = Get-MaestroChannel -ChannelId $ChannelId if (!$channelInfo) { - Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!" + Write-Host "Channel with BAR ID $ChannelId was not found in BAR!" ExitWithExitCode 1 } @@ -21,7 +21,7 @@ try { $buildInfo = Get-MaestroBuild -BuildId $BuildId if (!$buildInfo) { - Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!" + Write-Host "Build with BAR ID $BuildId was not found in BAR!" ExitWithExitCode 1 } @@ -39,10 +39,10 @@ try { Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId - Write-Host 'done.' + Write-Host "done." } catch { + Write-Host "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'" Write-Host $_ - Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'" - ExitWithExitCode 1 + Write-Host $_.ScriptStackTrace } diff --git a/eng/common/post-build/setup-maestro-vars.ps1 b/eng/common/post-build/setup-maestro-vars.ps1 new file mode 100644 index 0000000000..d7f64dc63c --- /dev/null +++ b/eng/common/post-build/setup-maestro-vars.ps1 @@ -0,0 +1,26 @@ +param( + [Parameter(Mandatory=$true)][string] $ReleaseConfigsPath # Full path to ReleaseConfigs.txt asset +) + +. $PSScriptRoot\post-build-utils.ps1 + +try { + $Content = Get-Content $ReleaseConfigsPath + + $BarId = $Content | Select -Index 0 + + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 + + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index cc9d059d04..bbfdacca13 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -34,9 +34,9 @@ $ValidatePackage = { # Extensions for which we'll look for SourceLink information # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @('.dll', '.exe', '.pdb') + $RelevantExtensions = @(".dll", ".exe", ".pdb") - Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...' + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId @@ -58,7 +58,7 @@ $ValidatePackage = { $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName # We ignore resource DLLs - if ($FileName.EndsWith('.resources.dll')) { + if ($FileName.EndsWith(".resources.dll")) { return } @@ -96,7 +96,7 @@ $ValidatePackage = { $Uri = $Link -as [System.URI] # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode } else { @@ -143,19 +143,19 @@ $ValidatePackage = { } if ($FailedFiles -eq 0) { - Write-Host 'Passed.' + Write-Host "Passed." return 0 } else { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." + Write-Host "$PackagePath has broken SourceLink links." return 1 } } function ValidateSourceLinkLinks { - if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) { - if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'" + if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { + Write-PipelineTaskError "GHRepoName should be in the format / or -. '$GHRepoName'" ExitWithExitCode 1 } else { @@ -163,14 +163,14 @@ function ValidateSourceLinkLinks { } } - if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" + if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" ExitWithExitCode 1 } - if ($GHRepoName -ne '' -and $GHCommit -ne '') { - $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1') - $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript') + if ($GHRepoName -ne "" -and $GHCommit -ne "") { + $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") try { # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash @@ -188,8 +188,8 @@ function ValidateSourceLinkLinks { Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." } } - elseif ($GHRepoName -ne '' -or $GHCommit -ne '') { - Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.' + elseif ($GHRepoName -ne "" -or $GHCommit -ne "") { + Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed." } if (Test-Path $ExtractPath) { @@ -217,18 +217,18 @@ function ValidateSourceLinkLinks { $ValidationFailures = 0 foreach ($Job in @(Get-Job)) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job - if ($jobResult -ne '0') { + if ($jobResult -ne "0") { $ValidationFailures++ } } if ($ValidationFailures -gt 0) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." + Write-PipelineTaskError " $ValidationFailures package(s) failed validation." ExitWithExitCode 1 } } function InstallSourcelinkCli { - $sourcelinkCliPackageName = 'sourcelink' + $sourcelinkCliPackageName = "sourcelink" $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" @@ -239,7 +239,7 @@ function InstallSourcelinkCli { } else { Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global } } @@ -250,8 +250,8 @@ try { ValidateSourceLinkLinks } catch { + Write-Host $_ Write-Host $_.Exception Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 8e9527113c..096ac321d1 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,11 +1,13 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use - [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error - [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use ) +. $PSScriptRoot\post-build-utils.ps1 + +Add-Type -AssemblyName System.IO.Compression.FileSystem + function FirstMatchingSymbolDescriptionOrDefault { param( [string] $FullPath, # Full path to the module that has to be checked @@ -21,19 +23,19 @@ function FirstMatchingSymbolDescriptionOrDefault { # checking and which type of file was uploaded. # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName + $SymbolPath = $SymbolsPath + "\" + $FileName # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" @@ -41,19 +43,19 @@ function FirstMatchingSymbolDescriptionOrDefault { & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null if (Test-Path $PdbPath) { - return 'PDB' + return "PDB" } elseif (Test-Path $NGenPdb) { - return 'NGen PDB' + return "NGen PDB" } elseif (Test-Path $SODbg) { - return 'DBG for SO' + return "DBG for SO" } elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' + return "Dwarf for Dylib" } elseif (Test-Path $SymbolPath) { - return 'Module' + return "Module" } else { return $null @@ -72,7 +74,7 @@ function CountMissingSymbols { } # Extensions for which we'll look for symbols - $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib') + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") # How many files are missing symbol information $MissingSymbols = 0 @@ -80,54 +82,43 @@ function CountMissingSymbols { $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageGuid = New-Guid $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" - try { - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) - } - catch { - Write-Host "Something went wrong extracting $PackagePath" - Write-Host $_ - return -1 - } + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) Get-ChildItem -Recurse $ExtractPath | Where-Object {$RelevantExtensions -contains $_.Extension} | ForEach-Object { - if ($_.FullName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $_.FullName + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName return } - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--microsoft-symbol-server' $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--internal-server' $SymbolsPath + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath - Write-Host -NoNewLine "`t Checking file " $_.FullName "... " + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" } else { $MissingSymbols++ if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' + Write-Host "No symbols found on MSDL or SymWeb!" } else { if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' + Write-Host "No symbols found on MSDL!" } else { - Write-Host 'No symbols found on SymWeb!' + Write-Host "No symbols found on SymWeb!" } } } } - if ($Clean) { - Remove-Item $ExtractPath -Recurse -Force - } - Pop-Location return $MissingSymbols @@ -138,51 +129,38 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } - $TotalFailures = 0 - Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name - + # These packages from Arcade-Services include some native libraries that # our current symbol uploader can't handle. Below is a workaround until # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') { + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { Write-Host "Ignoring Arcade-services file: $FileName" Write-Host return } - elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') { + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { Write-Host "Ignoring Arcade-services file: $FileName" Write-Host return } - + Write-Host "Validating $FileName " $Status = CountMissingSymbols "$InputPath\$FileName" - + if ($Status -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - - if ($ContinueOnError) { - $TotalFailures++ - } - else { - ExitWithExitCode 1 - } + Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" + ExitWithExitCode $exitCode } Write-Host } - - if ($TotalFailures -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" - ExitWithExitCode 1 - } } function InstallDotnetSymbol { - $dotnetSymbolPackageName = 'dotnet-symbol' + $dotnetSymbolPackageName = "dotnet-symbol" $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" @@ -193,22 +171,19 @@ function InstallDotnetSymbol { } else { Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global } } try { - . $PSScriptRoot\post-build-utils.ps1 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - InstallDotnetSymbol CheckSymbolsAvailable } catch { + Write-Host $_ + Write-Host $_.Exception Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 index 55dea518ac..926d5b4551 100644 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -2,63 +2,56 @@ param( [Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', - [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" ) -try { - . $PSScriptRoot\post-build-utils.ps1 +. $PSScriptRoot\post-build-utils.ps1 - # Get all the $SourceRepo subscriptions - $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '') - $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId +# Get all the $SourceRepo subscriptions +$normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '') +$subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId - if (!$subscriptions) { - Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" - ExitWithExitCode 0 - } +if (!$subscriptions) { + Write-Host "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" + ExitWithExitCode 0 +} - $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] - $failedTriggeredSubscription = $false +$subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] +$failedTriggeredSubscription = $false - # Get all enabled subscriptions that need dependency flow on 'everyBuild' - foreach ($subscription in $subscriptions) { - if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { - Write-Host "Should trigger this subscription: ${$subscription.id}" - [void]$subscriptionsToTrigger.Add($subscription.id) - } +# Get all enabled subscriptions that need dependency flow on 'everyBuild' +foreach ($subscription in $subscriptions) { + if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { + Write-Host "Should trigger this subscription: $subscription.id" + [void]$subscriptionsToTrigger.Add($subscription.id) } +} - foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { - try { - Write-Host "Triggering subscription '$subscriptionToTrigger'." - - Trigger-Subscription -SubscriptionId $subscriptionToTrigger - - Write-Host 'done.' - } - catch - { - Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'" - Write-Host $_ - Write-Host $_.ScriptStackTrace - $failedTriggeredSubscription = $true - } +foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { + try { + Write-Host "Triggering subscription '$subscriptionToTrigger'." + + Trigger-Subscription -SubscriptionId $subscriptionToTrigger + + Write-Host "done." + } + catch + { + Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'" + Write-Host $_ + Write-Host $_.ScriptStackTrace + $failedTriggeredSubscription = $true } +} - if ($subscriptionsToTrigger.Count -eq 0) { - Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'." - } - elseif ($failedTriggeredSubscription) { - Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...' - ExitWithExitCode 1 - } - else { - Write-Host 'All subscriptions were triggered successfully!' - } +if ($subscriptionsToTrigger.Count -eq 0) { + Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'." } -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_ +elseif ($failedTriggeredSubscription) { + Write-Host "At least one subscription failed to be triggered..." ExitWithExitCode 1 } +else { + Write-Host "All subscriptions were triggered successfully!" +} diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index f997be4331..d0eec5163e 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -1,8 +1,8 @@ [CmdletBinding(PositionalBinding=$false)] Param( - [string] $configuration = 'Debug', + [string] $configuration = "Debug", [string] $task, - [string] $verbosity = 'minimal', + [string] $verbosity = "minimal", [string] $msbuildEngine = $null, [switch] $restore, [switch] $prepareMachine, @@ -32,7 +32,7 @@ function Print-Usage() { } function Build([string]$target) { - $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" } + $logSuffix = if ($target -eq "Execute") { "" } else { ".$target" } $log = Join-Path $LogDir "$task$logSuffix.binlog" $outputPath = Join-Path $ToolsetDir "$task\\" @@ -46,45 +46,33 @@ function Build([string]$target) { } try { - if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { + if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { Print-Usage exit 0 } if ($task -eq "") { - Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task '" -ForegroundColor Red + Write-Host "Missing required parameter '-task '" -ForegroundColor Red Print-Usage ExitWithExitCode 1 } - if( $msbuildEngine -eq "vs") { - # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty - } - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty - } - - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true - $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" - } - $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red + Write-Host "Unknown task: $task" -ForegroundColor Red ExitWithExitCode 1 } if ($restore) { - Build 'Restore' + Build "Restore" } - Build 'Execute' + Build "Execute" } catch { + Write-Host $_ + Write-Host $_.Exception Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Build' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index b7f61f9a2f..01799d63ff 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -1,114 +1,100 @@ Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located - [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error - [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located + [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error + [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") + [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") ) -try { - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - $disableConfigureToolsetImport = $true - $LASTEXITCODE = 0 +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - #Replace repo names to the format of org/repo - if (!($Repository.contains('/'))) { - $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; - } - else{ - $RepoName = $Repository; - } +#Replace repo names to the format of org/repo +if (!($Repository.contains('/'))) { + $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; +} +else{ + $RepoName = $Repository; +} - if ($GuardianPackageName) { - $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd')) - } else { - $guardianCliLocation = $GuardianCliLocation - } +if ($GuardianPackageName) { + $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) +} else { + $guardianCliLocation = $GuardianCliLocation +} - $workingDirectory = (Split-Path $SourceDirectory -Parent) - $ValidPath = Test-Path $guardianCliLocation +$workingDirectory = (Split-Path $SourceDirectory -Parent) +$ValidPath = Test-Path $guardianCliLocation - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.' - ExitWithExitCode 1 - } +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel - $gdnFolder = Join-Path $workingDirectory '.gdn' +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $workingDirectory ".gdn" - if ($TsaOnboard) { - if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.' - ExitWithExitCode 1 +if ($TsaOnboard) { + if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE } + } else { + Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." + exit 1 } +} - if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams - } - if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams - } +if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams +} +if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams +} - if ($UpdateBaseline) { - & (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline' - } +if ($UpdateBaseline) { + & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" +} - if ($TsaPublish) { - if ($TsaBranchName -and $BuildNumber) { - if (-not $TsaRepositoryName) { - $TsaRepositoryName = "$($Repository)-$($BranchName)" - } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.' - ExitWithExitCode 1 +if ($TsaPublish) { + if ($TsaBranchName -and $BuildNumber) { + if (-not $TsaRepositoryName) { + $TsaRepositoryName = "$($Repository)-$($BranchName)" + } + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE } + } else { + Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." + exit 1 } } -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - exit 1 -} diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 7f28d9c59e..6e6825013b 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -3,12 +3,54 @@ param( [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted ) -$ErrorActionPreference = 'Stop' +$ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +. $PSScriptRoot\..\tools.ps1 -function ExtractArtifacts { +$ExtractPackage = { + param( + [string] $PackagePath # Full path to a NuGet package + ) + + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + $RelevantExtensions = @(".dll", ".exe", ".pdb") + Write-Host -NoNewLine "Extracting" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + } + } + catch { + + } + finally { + $zip.Dispose() + } + } + function ExtractArtifacts { if (!(Test-Path $InputPath)) { Write-Host "Input Path does not exist: $InputPath" ExitWithExitCode 0 @@ -25,56 +67,11 @@ function ExtractArtifacts { } try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } Measure-Command { ExtractArtifacts } } catch { Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace ExitWithExitCode 1 } diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index a68bf0b88e..c737eb0e71 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -1,37 +1,31 @@ Param( [string] $GuardianCliLocation, [string] $Repository, - [string] $BranchName='master', + [string] $BranchName="master", [string] $WorkingDirectory, [string] $AzureDevOpsAccessToken, - [string] $GuardianLoggerLevel='Standard' + [string] $GuardianLoggerLevel="Standard" ) -$ErrorActionPreference = 'Stop' +$ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true $LASTEXITCODE = 0 -# `tools.ps1` checks $ci to perform some actions. Since the SDL -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -. $PSScriptRoot\..\tools.ps1 - # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" $zipFile = "$WorkingDirectory/gdn.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory '.gdn') -try { +$gdnFolder = (Join-Path $WorkingDirectory ".gdn") +Try +{ # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead - Write-Host 'Downloading gdn folder from internal config repostiory...' + Write-Host "Downloading gdn folder from internal config repostiory..." Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile if (Test-Path $gdnFolder) { # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) @@ -39,29 +33,19 @@ try { } [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) Write-Host $gdnFolder - ExitWithExitCode 0 -} catch [System.Net.WebException] { } # Catch and ignore webexception -try { +} Catch [System.Net.WebException] { # if the folder does not exist, we'll do a guardian init and push it to the remote repository - Write-Host 'Initializing Guardian...' + Write-Host "Initializing Guardian..." Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE + Write-Error "Guardian init failed with exit code $LASTEXITCODE." } # We create the mainbaseline so it can be edited later Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE + Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." } - & $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder' - ExitWithExitCode 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} + & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" +} \ No newline at end of file diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 968b39bef5..256ffbfb93 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 index d8fd2d82a6..79c707d6d8 100644 --- a/eng/common/sdl/push-gdn.ps1 +++ b/eng/common/sdl/push-gdn.ps1 @@ -1,69 +1,51 @@ Param( [string] $Repository, - [string] $BranchName='master', + [string] $BranchName="master", [string] $GdnFolder, [string] $AzureDevOpsAccessToken, [string] $PushReason ) -$ErrorActionPreference = 'Stop' +$ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true $LASTEXITCODE = 0 -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # We create the temp directory where we'll store the sdl-config repository - $sdlDir = Join-Path $env:TEMP 'sdl' - if (Test-Path $sdlDir) { - Remove-Item -Force -Recurse $sdlDir - } - - Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" - git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git clone failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - # We copy the .gdn folder from our local run into the git repository so it can be committed - $sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) '.gdn' - if (Get-Command Robocopy) { - Robocopy /S $GdnFolder $sdlRepositoryFolder - } else { - rsync -r $GdnFolder $sdlRepositoryFolder - } - # cd to the sdl-config directory so we can run git there - Push-Location $sdlDir - # git add . --> git commit --> git push - Write-Host 'git add .' - git add . - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git add failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" - git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git commit failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - Write-Host 'git push' - git push - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git push failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } +# We create the temp directory where we'll store the sdl-config repository +$sdlDir = Join-Path $env:TEMP "sdl" +if (Test-Path $sdlDir) { + Remove-Item -Force -Recurse $sdlDir +} - # Return to the original directory - Pop-Location +Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" +git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir +if ($LASTEXITCODE -ne 0) { + Write-Error "Git clone failed with exit code $LASTEXITCODE." +} +# We copy the .gdn folder from our local run into the git repository so it can be committed +$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" +if (Get-Command Robocopy) { + Robocopy /S $GdnFolder $sdlRepositoryFolder +} else { + rsync -r $GdnFolder $sdlRepositoryFolder } -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ - ExitWithExitCode 1 +# cd to the sdl-config directory so we can run git there +Push-Location $sdlDir +# git add . --> git commit --> git push +Write-Host "git add ." +git add . +if ($LASTEXITCODE -ne 0) { + Write-Error "Git add failed with exit code $LASTEXITCODE." } +Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" +git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" +if ($LASTEXITCODE -ne 0) { + Write-Error "Git commit failed with exit code $LASTEXITCODE." +} +Write-Host "git push" +git push +if ($LASTEXITCODE -ne 0) { + Write-Error "Git push failed with exit code $LASTEXITCODE." +} + +# Return to the original directory +Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index fe95ab35aa..9bc25314ae 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -5,69 +5,55 @@ Param( [string] $GdnFolder, [string[]] $ToolsList, [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard', + [string] $GuardianLoggerLevel="Standard", [string[]] $CrScanAdditionalRunConfigParams, [string[]] $PoliCheckAdditionalRunConfigParams ) -$ErrorActionPreference = 'Stop' +$ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true $LASTEXITCODE = 0 -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 +# We store config files in the r directory of .gdn +Write-Host $ToolsList +$gdnConfigPath = Join-Path $GdnFolder "r" +$ValidPath = Test-Path $GuardianCliLocation - # We store config files in the r directory of .gdn - Write-Host $ToolsList - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - $configParam = @('--config') +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} - foreach ($tool in $ToolsList) { - $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" - Write-Host $tool - # We have to manually configure tools that run on source to look at the source directory only - if ($tool -eq 'credscan') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } +$configParam = @("--config") + +foreach ($tool in $ToolsList) { + $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" + Write-Host $tool + # We have to manually configure tools that run on source to look at the source directory only + if ($tool -eq "credscan") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE } - if ($tool -eq 'policheck') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } + } + if ($tool -eq "policheck") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE } - - $configParam+=$gdnConfigFile } - Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } + $configParam+=$gdnConfigFile } -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 + +Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" +& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam +if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE } diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index c64c4f5686..52e2ff021d 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,12 +1,10 @@ parameters: - enable: 'false' # Whether the SDL validation job should execute or not overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named # 'continueOnError', the parameter value is not correctly picked up. # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; - downloadArtifacts: true # optional: determines if the artifacts should be dowloaded dependsOn: '' # Optional: dependencies of the job artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts # Usage: @@ -18,53 +16,37 @@ jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool - condition: eq( ${{ parameters.enable }}, 'true') variables: - group: DotNet-VSTS-Bot - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: name: Hosted VS2017 steps: - checkout: self clean: true - - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - - ${{ if eq(parameters.artifactNames, '') }}: + - ${{ if ne(parameters.artifactNames, '') }}: + - ${{ each artifactName in parameters.artifactNames }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: specific files - itemPattern: "**" + buildType: current + artifactName: ${{ artifactName }} downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + - ${{ if eq(parameters.artifactNames, '') }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: current + downloadType: specific files + itemPattern: "**" + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts + -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts + -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts + -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts + -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - task: NuGetToolInstaller@1 @@ -83,7 +65,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index fc39647f4b..ffda80a197 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,36 +1,67 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - parameters: # Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job cancelTimeoutInMinutes: '' + condition: '' - container: '' + continueOnError: false + + container: '' + dependsOn: '' + displayName: '' - pool: '' + steps: [] + + pool: '' + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' + # Job base template specific parameters + # Optional: Enable installing Microbuild plugin + # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix + # _TeamName - the name of your team + # _SignType - 'test' or 'real' enableMicrobuild: false + + # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false + + # Optional: Enable publishing to the build asset registry enablePublishBuildAssets: false - enablePublishTestResults: false + + # Optional: Prevent gather/push manifest from executing when using publishing pipelines enablePublishUsingPipelines: false - useBuildManifest: false - mergeTestResults: false - testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit + + # Optional: Include PublishTestResults task + enablePublishTestResults: false + + # Optional: enable sending telemetry + enableTelemetry: false + + # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') + helixRepo: '' + + # Optional: define the helix type for telemetry (example: 'build/product/') + helixType: '' + + # Required: name of the job name: '' - preSteps: [] + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + jobs: - job: ${{ parameters.name }} @@ -62,7 +93,7 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - ${{ if eq(parameters.enableTelemetry, 'true') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' - ${{ each variable in parameters.variables }}: @@ -94,12 +125,21 @@ jobs: workspace: ${{ parameters.workspace }} steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions + - task: sendStartTelemetry@0 + displayName: 'Send Helix Start Telemetry' + inputs: + helixRepo: ${{ parameters.helixRepo }} + ${{ if ne(parameters.helixType, '') }}: + helixType: ${{ parameters.helixType }} + buildConfig: $(_BuildConfig) + runAsPublic: ${{ parameters.runAsPublic }} + continueOnError: ${{ parameters.continueOnError }} + condition: always() - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildSigningPlugin@2 displayName: Install MicroBuild plugin inputs: @@ -111,16 +151,9 @@ jobs: continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: NuGetAuthenticate@0 - - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - ${{ each step in parameters.steps }}: - ${{ step }} @@ -133,60 +166,20 @@ jobs: env: TeamName: $(_TeamName) - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs - continueOnError: true - condition: always() - - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Gather Asset Manifests - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' - TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests' - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - - task: PublishBuildArtifacts@1 - displayName: Push Asset Manifests - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests' - PublishLocation: Container - ArtifactName: AssetManifests - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - ${{ if eq(parameters.enableTelemetry, 'true') }}: + # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions + - task: sendEndTelemetry@0 + displayName: 'Send Helix End Telemetry' + continueOnError: ${{ parameters.continueOnError }} + condition: always() + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + ArtifactName: $(Agent.Os)_$(Agent.JobName) continueOnError: true condition: always() @@ -197,8 +190,6 @@ jobs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ parameters.testRunTitle }} - mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() @@ -219,12 +210,3 @@ jobs: ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - - ${{ if eq(parameters.useBuildManifest, true) }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Build Manifest - inputs: - PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props' - PublishLocation: Container - ArtifactName: BuildManifests - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 055304ad89..b722975f9c 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -37,12 +37,6 @@ jobs: - name: _BuildConfig value: ${{ parameters.configuration }} - group: Publish-Build-Assets - # Skip component governance and codesign validation for SDL. These jobs - # create no content. - - name: skipComponentGovernanceDetection - value: true - - name: runCodesignValidationInjection - value: false steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index c08225a9a9..6a2f98c036 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,10 +1,19 @@ parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + # Optional: 'true' if failures in job.yml job should not fail the job continueOnError: false + # Optional: Enable installing Microbuild plugin + # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix + # _TeamName - the name of your team + # _SignType - 'test' or 'real' + enableMicrobuild: false + # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false + # Optional: Enable publishing to the build asset registry + enablePublishBuildAssets: false + # Optional: Enable publishing using release pipelines enablePublishUsingPipelines: false @@ -14,9 +23,19 @@ parameters: # Optional: Include toolset dependencies in the generated graph files includeToolset: false + # Optional: Include PublishTestResults task + enablePublishTestResults: false + + # Optional: enable sending telemetry + # if enabled then the 'helixRepo' parameter should also be specified + enableTelemetry: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job jobs: [] + # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') + helixRepo: '' + # Optional: Override automatically derived dependsOn value for "publish build assets" job publishBuildAssetsDependsOn: '' @@ -43,30 +62,29 @@ jobs: name: ${{ job.job }} -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - pool: - vmImage: vs2017-win2016 - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - - - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}: - - template: ../job/generate-graph-files.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} - dependsOn: - - Asset_Registry_Publish - pool: - vmImage: vs2017-win2016 +- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + pool: + vmImage: vs2017-win2016 + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + +- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: ../job/generate-graph-files.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} + dependsOn: + - Asset_Registry_Publish + pool: + vmImage: vs2017-win2016 diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 258ba4b770..ad9375f5e5 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -1,7 +1,4 @@ parameters: - artifactsPublishingAdditionalParameters: '' - dependsOn: - - Validate publishInstallersAndChecksums: false symbolPublishingAdditionalParameters: '' stageName: '' @@ -13,54 +10,37 @@ parameters: stages: - stage: ${{ parameters.stageName }} - dependsOn: ${{ parameters.dependsOn }} + dependsOn: validate variables: - template: ../common-variables.yml displayName: ${{ parameters.channelName }} Publishing jobs: - template: ../setup-maestro-vars.yml - - job: publish_symbols + - job: displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets - continueOnError: true - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - PdbArtifacts/** - BlobArtifacts/** - downloadPath: '$(Build.ArtifactStagingDirectory)' - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - task: NuGetAuthenticate@0 displayName: 'Authenticate to AzDO Feeds' - - task: PowerShell@2 - displayName: Enable cross-org publishing + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true - task: PowerShell@2 displayName: Publish @@ -73,48 +53,39 @@ stages: /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' /p:Configuration=Release - /p:PublishToMSDL=false ${{ parameters.symbolPublishingAdditionalParameters }} - - template: ../../steps/publish-logs.yml - parameters: - StageLabel: '${{ parameters.stageName }}' - JobLabel: 'SymbolPublishing' - - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars - timeoutInMinutes: 120 variables: + - group: DotNet-Blob-Feed + - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) pool: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets - continueOnError: true + displayName: Download Package Artifacts inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - PackageArtifacts/** - BlobArtifacts/** - AssetManifests/** - downloadPath: '$(Build.ArtifactStagingDirectory)' + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -153,6 +124,7 @@ stages: /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey) /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey) + /p:PublishToAzureDevOpsNuGetFeeds=true /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}' /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}' @@ -162,11 +134,6 @@ stages: /p:PublishToMSDL=false ${{ parameters.artifactsPublishingAdditionalParameters }} - - template: ../../steps/publish-logs.yml - parameters: - StageLabel: '${{ parameters.stageName }}' - JobLabel: 'AssetsPublishing' - - - template: ../../steps/add-build-to-channel.yml + - template: ../../steps/promote-build.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index bf98d990e8..c4bc1897d8 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -1,7 +1,5 @@ parameters: artifactsPublishingAdditionalParameters: '' - dependsOn: - - Validate publishInstallersAndChecksums: false symbolPublishingAdditionalParameters: '' stageName: '' @@ -10,47 +8,36 @@ parameters: transportFeed: '' shippingFeed: '' symbolsFeed: '' - # If the channel name is empty, no links will be generated - akaMSChannelName: '' stages: - stage: ${{ parameters.stageName }} - dependsOn: ${{ parameters.dependsOn }} + dependsOn: validate variables: - template: ../common-variables.yml displayName: ${{ parameters.channelName }} Publishing jobs: - template: ../setup-maestro-vars.yml - - job: publish_symbols + - job: displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - PdbArtifacts/** - BlobArtifacts/** - downloadPath: '$(Build.ArtifactStagingDirectory)' + artifactName: 'PDBArtifacts' + continueOnError: true # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -77,47 +64,37 @@ stages: /p:Configuration=Release ${{ parameters.symbolPublishingAdditionalParameters }} - - template: ../../steps/publish-logs.yml - parameters: - StageLabel: '${{ parameters.stageName }}' - JobLabel: 'SymbolPublishing' - - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars - timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - - name: ArtifactsCategory - value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }} - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) pool: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download Build Assets + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: 'specific' - itemPattern: | - PackageArtifacts/** - BlobArtifacts/** - AssetManifests/** - downloadPath: '$(Build.ArtifactStagingDirectory)' + buildType: current + artifactName: BlobArtifacts + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -137,7 +114,7 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ArtifactsCategory=$(ArtifactsCategory) + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) @@ -157,22 +134,15 @@ stages: /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:PublishToAzureDevOpsNuGetFeeds=true /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}' /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}' /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}' /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' - /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}' - /p:AkaMSClientId=$(akams-client-id) - /p:AkaMSClientSecret=$(akams-client-secret) ${{ parameters.artifactsPublishingAdditionalParameters }} - - template: ../../steps/publish-logs.yml - parameters: - StageLabel: '${{ parameters.stageName }}' - JobLabel: 'AssetsPublishing' - - - template: ../../steps/add-build-to-channel.yml + - template: ../../steps/promote-build.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index c99fd75037..1883d2b170 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -4,13 +4,13 @@ variables: - group: DotNet-DotNetCli-Storage - group: DotNet-MSRC-Storage - group: Publish-Build-Assets - + # .NET Core 3.1 Dev - name: PublicDevRelease_31_Channel_Id value: 128 - # .NET 5 Dev - - name: Net_5_Dev_Channel_Id + # .NET Core 5 Dev + - name: NetCore_5_Dev_Channel_Id value: 131 # .NET Eng - Validation @@ -29,14 +29,6 @@ variables: - name: NetCore_3_Tools_Channel_Id value: 344 - # .NET Core 3.0 Internal Servicing - - name: InternalServicing_30_Channel_Id - value: 184 - - # .NET Core 3.0 Release - - name: PublicRelease_30_Channel_Id - value: 19 - # .NET Core 3.1 Release - name: PublicRelease_31_Channel_Id value: 129 @@ -49,10 +41,6 @@ variables: - name: NetCore_31_Blazor_Features_Channel_Id value: 531 - # .NET Core Experimental - - name: NetCore_Experimental_Channel_Id - value: 562 - # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} @@ -63,7 +51,7 @@ variables: - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion - value: "2020-02-20" + value: "2019-01-16" - name: SourceLinkCLIVersion value: 3.0.0 @@ -90,10 +78,3 @@ variables: value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json - name: InternalInstallersBlobFeedKey value: $(dotnetclimsrc-access-key) - - # Skip component governance and codesign validation for SDL. These jobs - # create no content. - - name: skipComponentGovernanceDetection - value: true - - name: runCodesignValidationInjection - value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index b51bc5375e..e1cff6e702 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -9,272 +9,156 @@ parameters: continueOnError: false params: '' artifactNames: '' - downloadArtifacts: true # These parameters let the user customize the call to sdk-task.ps1 for publishing # symbols & general artifacts as well as for signing validation symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' - useBuildManifest: false # Which stages should finish execution before post-build stages start - validateDependsOn: - - build - publishDependsOn: - - Validate + dependsOn: [build] - # Channel ID's instantiated in this file. - # When adding a new channel implementation the call to `check-channel-consistency.ps1` - # needs to be updated with the new channel ID - NetEngLatestChannelId: 2 - NetEngValidationChannelId: 9 - NetDev5ChannelId: 131 - GeneralTestingChannelId: 529 - NETCoreToolingDevChannelId: 548 - NETCoreToolingReleaseChannelId: 549 - NETInternalToolingChannelId: 551 - NETCoreExperimentalChannelId: 562 - NetEngServicesIntChannelId: 678 - NetEngServicesProdChannelId: 679 - Net5Preview3ChannelId: 739 - Net5Preview4ChannelId: 856 - Net5Preview5ChannelId: 857 - NetCoreSDK313xxChannelId: 759 - NetCoreSDK313xxInternalChannelId: 760 - NetCoreSDK314xxChannelId: 921 - NetCoreSDK314xxInternalChannelId: 922 - stages: -- stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} +- stage: validate + dependsOn: ${{ parameters.dependsOn }} displayName: Validate - variables: - - template: common-variables.yml jobs: - - template: setup-maestro-vars.yml - - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} - - - job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableNugetValidation }}, 'true') - pool: - vmImage: 'windows-2019' - variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - ${{ if eq(parameters.useBuildManifest, true) }}: + - ${{ if eq(parameters.enableNugetValidation, 'true') }}: + - job: + displayName: NuGet Validation + pool: + vmImage: 'windows-2019' + steps: - task: DownloadBuildArtifacts@0 - displayName: Download build manifest + displayName: Download Package Artifacts inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BuildManifests - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts + buildType: current + artifactName: PackageArtifacts - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if eq(parameters.enableSigningValidation, 'true') }}: + - job: + displayName: Signing Validation + pool: + vmImage: 'windows-2019' + steps: + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.signingValidationAdditionalParameters }} - - job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: SourceLink Validation + variables: + - template: common-variables.yml + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} - dependsOn: setupMaestroVars - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NetCore_Dev5_Publish' - channelName: '.NET 5 Dev' - akaMSChannelName: 'net5/dev' - channelId: ${{ parameters.NetDev5ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + stageName: 'NetCore_Dev31_Publish' + channelName: '.NET Core 3.1 Dev' + channelId: 128 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview3_Publish' - channelName: '.NET 5 Preview 3' - akaMSChannelName: 'net5/preview3' - channelId: ${{ parameters.Net5Preview3ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + stageName: 'Net_Eng_Latest_Publish' + channelName: '.NET Eng - Latest' + channelId: 2 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview4_Publish' - channelName: '.NET 5 Preview 4' - akaMSChannelName: 'net5/preview4' - channelId: ${{ parameters.Net5Preview4ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + stageName: 'Net_Eng_Validation_Publish' + channelName: '.NET Eng - Validation' + channelId: 9 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview5_Publish' - channelName: '.NET 5 Preview 5' - akaMSChannelName: 'net5/preview5' - channelId: ${{ parameters.Net5Preview5ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + stageName: 'NetCore_3_Tools_Validation_Publish' + channelName: '.NET 3 Tools - Validation' + channelId: 390 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Latest_Publish' - channelName: '.NET Eng - Latest' - akaMSChannelName: 'eng/daily' - channelId: ${{ parameters.NetEngLatestChannelId }} + stageName: 'NetCore_3_Tools_Publish' + channelName: '.NET 3 Tools' + channelId: 344 transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' @@ -282,27 +166,47 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Validation_Publish' - channelName: '.NET Eng - Validation' - akaMSChannelName: 'eng/validation' - channelId: ${{ parameters.NetEngValidationChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + stageName: 'NetCore_Release31_Publish' + channelName: '.NET Core 3.1 Release' + channelId: 129 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NetCore_Blazor31_Features_Publish' + channelName: '.NET Core 3.1 Blazor Features' + channelId: 531 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-blazor/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-blazor/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-blazor-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NetCore_31_Internal_Servicing_Publishing' + channelName: '.NET Core 3.1 Internal Servicing' + channelId: 550 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'General_Testing_Publish' channelName: 'General Testing' - akaMSChannelName: 'generaltesting' - channelId: ${{ parameters.GeneralTestingChannelId }} + channelId: 529 transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' @@ -310,12 +214,11 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Tooling_Dev_Publishing' channelName: '.NET Core Tooling Dev' - channelId: ${{ parameters.NETCoreToolingDevChannelId }} + channelId: 548 transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' @@ -323,77 +226,95 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Tooling_Release_Publishing' channelName: '.NET Core Tooling Release' - channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} + channelId: 549 transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_311xx_Publishing' + channelName: '.NET Core SDK 3.1.1xx' + channelId: 560 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NET_Internal_Tooling_Publishing' - channelName: '.NET Internal Tooling' - channelId: ${{ parameters.NETInternalToolingChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' + stageName: 'NETCore_SDK_311xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.1xx Internal' + channelId: 559 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Experimental_Publishing' - channelName: '.NET Core Experimental' - channelId: ${{ parameters.NETCoreExperimentalChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' + stageName: 'NETCore_SDK_312xx_Publishing' + channelName: '.NET Core SDK 3.1.2xx' + channelId: 558 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml +- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Int_Publish' - channelName: '.NET Eng Services - Int' - channelId: ${{ parameters.NetEngServicesIntChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + stageName: 'NETCore_SDK_312xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.2xx Internal' + channelId: 557 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Prod_Publish' - channelName: '.NET Eng Services - Prod' - channelId: ${{ parameters.NetEngServicesProdChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + stageName: 'NETCore_SDK_313xx_Publishing' + channelName: '.NET Core SDK 3.1.3xx' + channelId: 759 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' +- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.3xx Internal' + channelId: 760 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_SDK_314xx_Publishing' channelName: '.NET Core SDK 3.1.4xx' - channelId: ${{ parameters.NetCoreSDK314xxChannelId }} + channelId: 921 transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' @@ -401,38 +322,47 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_SDK_314xx_Internal_Publishing' channelName: '.NET Core SDK 3.1.4xx Internal' - channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} + channelId: 922 transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_313xx_Publishing' - channelName: '.NET Core SDK 3.1.3xx' - channelId: ${{ parameters.NetCoreSDK313xxChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + stageName: 'VS16_6_Publishing' + channelName: 'VS 16.6' + channelId: 1010 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' -- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_313xx_Internal_Publishing' - channelName: '.NET Core SDK 3.1.3xx Internal' - channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + stageName: 'VS16_7_Publishing' + channelName: 'VS 16.7' + channelId: 1011 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_Master_Publishing' + channelName: 'VS Master' + channelId: 1012 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' \ No newline at end of file diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index b3d29d4498..56242b068e 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,22 +1,11 @@ jobs: - job: setupMaestroVars displayName: Setup Maestro Vars - variables: - - template: common-variables.yml - - name: BuildId - value: $[ coalesce(variables.BARBuildId, 0) ] - - name: PromoteToMaestroChannels - value: $[ coalesce(variables.PromoteToChannelIds, 0) ] - - name: PromoteToMaestroChannel - value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' steps: - - checkout: none - - task: DownloadBuildArtifacts@0 displayName: Download Release Configs - condition: and(eq(variables.PromoteToMaestroChannels, 0), eq(variables.PromoteToMaestroChannel, 0)) inputs: buildType: current artifactName: ReleaseConfigs @@ -25,53 +14,5 @@ jobs: name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - script: | - try { - if ($Env:PromoteToMaestroChannels -eq 0 -and $Env:PromoteToMaestroChannel -eq 0) { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels][$Env:PromoteToMaestroChannel]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" - - Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + filePath: $(Build.SourcesDirectory)/eng/common/post-build/setup-maestro-vars.ps1 + arguments: -ReleaseConfigsPath '$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt' diff --git a/eng/common/templates/steps/add-build-to-channel.yml b/eng/common/templates/steps/add-build-to-channel.yml deleted file mode 100644 index f67a210d62..0000000000 --- a/eng/common/templates/steps/add-build-to-channel.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - ChannelId: 0 - -steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml deleted file mode 100644 index f91751fe78..0000000000 --- a/eng/common/templates/steps/publish-logs.yml +++ /dev/null @@ -1,23 +0,0 @@ -parameters: - StageLabel: '' - JobLabel: '' - -steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 5eceb48725..05df886f55 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -10,7 +10,7 @@ parameters: HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects @@ -18,12 +18,11 @@ parameters: XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int) Creator: '' # optional -- if the build is external, use this to specify who is sending the job DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() @@ -56,7 +55,6 @@ steps: DotNetCliVersion: ${{ parameters.DotNetCliVersion }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) @@ -87,7 +85,6 @@ steps: DotNetCliVersion: ${{ parameters.DotNetCliVersion }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d8dfc5e004..bc228dfdf9 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -5,13 +5,11 @@ [bool]$ci = if (Test-Path variable:ci) { $ci } else { $false } # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. -[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' } - -# Set to true to opt out of outputting binary log while running in CI -[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false } +[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog } +# Binary log must be enabled on CI. +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md @@ -26,7 +24,7 @@ [bool]$restore = if (Test-Path variable:restore) { $restore } else { $true } # Adjusts msbuild verbosity level. -[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { 'minimal' } +[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } # Set to true to reuse msbuild nodes. Recommended to not reuse on CI. [bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci } @@ -43,22 +41,22 @@ # Enable repos to use a particular version of the on-line dotnet-install scripts. # default URL: https://dot.net/v1/dotnet-install.ps1 -[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' } +[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { "v1" } # True to use global NuGet cache instead of restoring packages to repository-local directory. [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } # An array of names of processes to stop on script exit if prepareMachine is true. -$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') } - -$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null } +$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") } set-strictmode -version 2.0 -$ErrorActionPreference = 'Stop' +$ErrorActionPreference = "Stop" [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -function Create-Directory ([string[]] $path) { - New-Item -Path $path -Force -ItemType 'Directory' | Out-Null +function Create-Directory([string[]] $path) { + if (!(Test-Path $path)) { + New-Item -path $path -force -itemType "Directory" | Out-Null + } } function Unzip([string]$zipfile, [string]$outpath) { @@ -98,10 +96,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } -# createSdkLocationFile parameter enables a file being generated under the toolset directory -# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations -# as dot sourcing isn't possible. -function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { +function InitializeDotNetCli([bool]$install) { if (Test-Path variable:global:_DotNetInstallDir) { return $global:_DotNetInstallDir } @@ -124,7 +119,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { - $dotnetCmd = Get-Command 'dotnet.exe' -ErrorAction SilentlyContinue + $dotnetExecutable = GetExecutableFileName 'dotnet' + $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue + if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent } @@ -137,13 +134,13 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { - $dotnetRoot = Join-Path $RepoRoot '.dotnet' + $dotnetRoot = Join-Path $RepoRoot ".dotnet" if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" ExitWithExitCode 1 } } @@ -151,24 +148,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_INSTALL_DIR = $dotnetRoot } - # Creates a temporary file under the toolset dir. - # The following code block is protecting against concurrent access so that this function can - # be called in parallel. - if ($createSdkLocationFile) { - do { - $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) - } - until (!(Test-Path $sdkCacheFileTemp)) - Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot - - try { - Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' - } catch { - # Somebody beat us - Remove-Item -Path $sdkCacheFileTemp - } - } - # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom # build steps from using anything other than what we've downloaded. # It also ensures that VS msbuild will use the downloaded sdk targets. @@ -177,6 +156,15 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build Write-PipelinePrependPath -Path $dotnetRoot + # Work around issues with Azure Artifacts credential provider + # https://github.com/dotnet/arcade/issues/3932 + if ($ci) { + $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 + $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20 + Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20' + Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' + } + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' @@ -184,53 +172,27 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { } function GetDotNetInstallScript([string] $dotnetRoot) { - $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' + $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - - $maxRetries = 5 - $retries = 1 - - $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" - - while($true) { - try { - Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript - break - } - catch { - Write-Host "Failed to download '$uri'" - Write-Error $_.Exception.Message -ErrorAction Continue - } - - if (++$retries -le $maxRetries) { - $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff - Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." - Start-Sleep -Seconds $delayInSeconds - } - else { - throw "Unable to download file in $maxRetries attempts." - } - - } + Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript } return $installScript } -function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') { +function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { InstallDotNet $dotnetRoot $version $architecture } -function InstallDotNet([string] $dotnetRoot, - [string] $version, - [string] $architecture = '', - [string] $runtime = '', - [bool] $skipNonVersionedFiles = $false, - [string] $runtimeSourceFeed = '', - [string] $runtimeSourceFeedKey = '') { +function InstallDotNet([string] $dotnetRoot, + [string] $version, + [string] $architecture = "", + [string] $runtime = "", + [bool] $skipNonVersionedFiles = $false, + [string] $runtimeSourceFeed = "", + [string] $runtimeSourceFeedKey = "") { $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ @@ -246,7 +208,7 @@ function InstallDotNet([string] $dotnetRoot, & $installScript @installParameters } catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from public location." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from public location." # Only the runtime can be installed from a custom [private] location. if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) { @@ -262,7 +224,7 @@ function InstallDotNet([string] $dotnetRoot, & $installScript @installParameters } catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'." ExitWithExitCode 1 } } else { @@ -283,21 +245,25 @@ function InstallDotNet([string] $dotnetRoot, # Throws on failure. # function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { + if (-not (IsWindowsPlatform)) { + throw "Cannot initialize Visual Studio on non-Windows" + } + if (Test-Path variable:global:_MSBuildExe) { return $global:_MSBuildExe } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { '15.9' } + $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. if ($env:VSINSTALLDIR -ne $null) { - $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue + $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { # Workaround for https://github.com/dotnet/roslyn/issues/35793 # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ - $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0]) + $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) if ($msbuildVersion -ge $vsMinVersion) { return $global:_MSBuildExe = $msbuildCmd.Path @@ -317,7 +283,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion } else { - if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') { + if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") { $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] } else { @@ -325,12 +291,9 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" } - $vsInstallDir = $null - if ($xcopyMSBuildVersion.Trim() -ine "none") { - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install - } + $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install if ($vsInstallDir -eq $null) { - throw 'Unable to find Visual Studio that has required version and components installed' + throw "Unable to find Visual Studio that has required version and components installed" } } @@ -354,7 +317,7 @@ function InstallXCopyMSBuild([string]$packageVersion) { } function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { - $packageName = 'RoslynTools.MSBuild' + $packageName = "RoslynTools.MSBuild" $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" @@ -370,7 +333,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Unzip $packagePath $packageDir } - return Join-Path $packageDir 'tools' + return Join-Path $packageDir "tools" } # @@ -387,37 +350,36 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ + if (-not (IsWindowsPlatform)) { + throw "Cannot run vswhere on non-Windows platforms." + } + if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { - $vswhereVersion = '2.5.2' + $vswhereVersion = "2.5.2" } $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" - $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe' + $vsWhereExe = Join-Path $vsWhereDir "vswhere.exe" if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir - Write-Host 'Downloading vswhere' - try { - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe - } - catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - } + Write-Host "Downloading vswhere" + Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') + $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") - if (Get-Member -InputObject $vsRequirements -Name 'version') { - $args += '-version' + if (Get-Member -InputObject $vsRequirements -Name "version") { + $args += "-version" $args += $vsRequirements.version } - if (Get-Member -InputObject $vsRequirements -Name 'components') { + if (Get-Member -InputObject $vsRequirements -Name "components") { foreach ($component in $vsRequirements.components) { - $args += '-requires' + $args += "-requires" $args += $component } } @@ -443,27 +405,28 @@ function InitializeBuildTool() { # Initialize dotnet cli if listed in 'tools' $dotnetRoot = $null - if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') { + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { $dotnetRoot = InitializeDotNetCli -install:$restore } - if ($msbuildEngine -eq 'dotnet') { + if ($msbuildEngine -eq "dotnet") { if (!$dotnetRoot) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } - $buildTool = @{ Path = Join-Path $dotnetRoot 'dotnet.exe'; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore } catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 } @@ -472,15 +435,15 @@ function InitializeBuildTool() { function GetDefaultMSBuildEngine() { # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. - if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { - return 'vs' + if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { + return "vs" } - if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') { - return 'dotnet' + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { + return "dotnet" } - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." ExitWithExitCode 1 } @@ -488,11 +451,10 @@ function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { # Use local cache on CI to ensure deterministic build, # use global cache in dev builds to avoid cost of downloading packages. - # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { - $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' + $env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages" } else { - $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' + $env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages" } } @@ -505,7 +467,7 @@ function GetSdkTaskProject([string]$taskName) { } function InitializeNativeTools() { - if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) { + if (Get-Member -InputObject $GlobalJson -Name "native-tools") { $nativeArgs= @{} if ($ci) { $nativeArgs = @{ @@ -534,20 +496,20 @@ function InitializeToolset() { } if (-not $restore) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." ExitWithExitCode 1 } $buildTool = InitializeBuildTool - $proj = Join-Path $ToolsetDir 'restore.proj' - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' } + $proj = Join-Path $ToolsetDir "restore.proj" + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } '' | Set-Content $proj MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile - $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 + $path = Get-Content $toolsetLocationFile -TotalCount 1 if (!(Test-Path $path)) { throw "Invalid toolset path: $path" } @@ -563,7 +525,7 @@ function ExitWithExitCode([int] $exitCode) { } function Stop-Processes() { - Write-Host 'Killing running build processes...' + Write-Host "Killing running build processes..." foreach ($processName in $processesToStopOnExit) { Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process } @@ -580,18 +542,13 @@ function MSBuild() { # Work around issues with Azure Artifacts credential provider # https://github.com/dotnet/arcade/issues/3932 - if ($ci -and $buildTool.Tool -eq 'dotnet') { + if ($ci -and $buildTool.Tool -eq "dotnet") { dotnet nuget locals http-cache -c - - $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 - $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20 - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20' - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' } $toolsetBuildProject = InitializeToolset $path = Split-Path -parent $toolsetBuildProject - $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll') + $path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") $args += "/logger:$path" } @@ -605,13 +562,13 @@ function MSBuild() { # function MSBuild-Core() { if ($ci) { - if (!$binaryLog -and !$excludeCIBinarylog) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' + if (!$binaryLog) { + Write-PipelineTaskError -Message "Binary log must be enabled in CI build." ExitWithExitCode 1 } if ($nodeReuse) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.' + Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." ExitWithExitCode 1 } } @@ -621,10 +578,10 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" if ($warnAsError) { - $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true' + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" } else { - $cmdArgs += ' /p:TreatWarningsAsErrors=false' + $cmdArgs += " /p:TreatWarningsAsErrors=false" } foreach ($arg in $args) { @@ -633,12 +590,10 @@ function MSBuild-Core() { } } - $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" - $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.' + Write-PipelineTaskError -Message "Build failed." $buildLog = GetMSBuildBinaryLogCommandLineArgument $args if ($buildLog -ne $null) { @@ -653,12 +608,12 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { foreach ($argument in $arguments) { if ($argument -ne $null) { $arg = $argument.Trim() - if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) { - return $arg.Substring('/bl:'.Length) + if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { + return $arg.Substring("/bl:".Length) } - if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) { - return $arg.Substring('/binaryLogger:'.Length) + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { + return $arg.Substring("/binaryLogger:".Length) } } } @@ -666,16 +621,29 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } +function GetExecutableFileName($baseName) { + if (IsWindowsPlatform) { + return "$baseName.exe" + } + else { + return $baseName + } +} + +function IsWindowsPlatform() { + return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT +} + . $PSScriptRoot\pipeline-logging-functions.ps1 -$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') -$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..') -$ArtifactsDir = Join-Path $RepoRoot 'artifacts' -$ToolsetDir = Join-Path $ArtifactsDir 'toolset' -$ToolsDir = Join-Path $RepoRoot '.tools' -$LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration -$TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration -$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") +$EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$ArtifactsDir = Join-Path $RepoRoot "artifacts" +$ToolsetDir = Join-Path $ArtifactsDir "toolset" +$ToolsDir = Join-Path $RepoRoot ".tools" +$LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration +$TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration +$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json # true if global.json contains a "runtimes" section $globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } @@ -688,18 +656,3 @@ Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir Write-PipelineSetVariable -Name 'TMP' -Value $TempDir - -# Import custom tools configuration, if present in the repo. -# Note: Import in global scope so that the script set top-level variables without qualification. -if (!$disableConfigureToolsetImport) { - $configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1' - if (Test-Path $configureToolsetScript) { - . $configureToolsetScript - if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) { - if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) { - Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code' - ExitWithExitCode $LastExitCode - } - } - } -} diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e94fce22ec..acbb0c5b3f 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -18,17 +18,9 @@ fi # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} -# Set to true to opt out of outputting binary log while running in CI -exclude_ci_binary_log=${exclude_ci_binary_log:-false} - -if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then - binary_log_default=true -else - binary_log_default=false -fi - # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -binary_log=${binary_log:-$binary_log_default} +# Binary log must be enabled on CI. +binary_log=${binary_log:-$ci} # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). prepare_machine=${prepare_machine:-false} @@ -49,7 +41,7 @@ fi # Configures warning treatment in msbuild. warn_as_error=${warn_as_error:-true} -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} @@ -85,11 +77,11 @@ function ResolvePath { function ReadGlobalVersion { local key=$1 - local line=$(awk "/$key/ {print; exit}" "$global_json_file") + local line=`grep -m 1 "$key" "$global_json_file"` local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file" + Write-PipelineTelemetryError -category 'InitializeToolset' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -160,6 +152,15 @@ function InitializeDotNetCli { # build steps from using anything other than what we've downloaded. Write-PipelinePrependPath -path "$dotnet_root" + # Work around issues with Azure Artifacts credential provider + # https://github.com/dotnet/arcade/issues/3932 + if [[ "$ci" == true ]]; then + export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 + export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 + Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" + Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" + fi + Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" @@ -180,7 +181,7 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 - + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -233,28 +234,6 @@ function InstallDotNet { } } -function with_retries { - local maxRetries=5 - local retries=1 - echo "Trying to run '$@' for maximum of $maxRetries attempts." - while [[ $((retries++)) -le $maxRetries ]]; do - "$@" - - if [[ $? == 0 ]]; then - echo "Ran '$@' successfully." - return 0 - fi - - timeout=$((2**$retries-1)) - echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2 - sleep $timeout - done - - echo "Failed to execute '$@' for $maxRetries times." 1>&2 - - return 1 -} - function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" @@ -267,13 +246,13 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code } - else - with_retries wget -v -O "$install_script" "$install_script_url" || { + else + wget -q -O "$install_script" "$install_script_url" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code @@ -288,11 +267,11 @@ function InitializeBuildTool { if [[ -n "${_InitializeBuildTool:-}" ]]; then return fi - + InitializeDotNetCli $restore # return values - _InitializeBuildTool="$_InitializeDotNetCli/dotnet" + _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" _InitializeBuildToolFramework="netcoreapp2.1" } @@ -311,9 +290,6 @@ function GetNuGetPackageCachePath { } function InitializeNativeTools() { - if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then - return - fi if grep -Fq "native-tools" $global_json_file then local nativeArgs="" @@ -356,14 +332,14 @@ function InitializeToolset { if [[ "$binary_log" == true ]]; then bl="/bl:$log_dir/ToolsetRestore.binlog" fi - + echo '' > "$proj" MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then - Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj" + Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" ExitWithExitCode 3 fi @@ -394,12 +370,7 @@ function MSBuild { # Work around issues with Azure Artifacts credential provider # https://github.com/dotnet/arcade/issues/3932 if [[ "$ci" == true ]]; then - "$_InitializeBuildTool" nuget locals http-cache -c - - export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 - export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 - Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" - Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" + dotnet nuget locals http-cache -c fi local toolset_dir="${_InitializeToolset%/*}" @@ -412,13 +383,13 @@ function MSBuild { function MSBuild-Core { if [[ "$ci" == true ]]; then - if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then - Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." + if [[ "$binary_log" != true ]]; then + Write-PipelineTaskError "Binary log must be enabled in CI build." ExitWithExitCode 1 fi if [[ "$node_reuse" == true ]]; then - Write-PipelineTelemetryError -category 'Build' "Node reuse must be disabled in CI build." + Write-PipelineTaskError "Node reuse must be disabled in CI build." ExitWithExitCode 1 fi fi @@ -430,17 +401,11 @@ function MSBuild-Core { warnaserror_switch="/warnaserror" fi - function RunBuildTool { - export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@" - - "$_InitializeBuildTool" "$@" || { - local exit_code=$? - Write-PipelineTaskError "Build failed (exit code '$exit_code')." - ExitWithExitCode $exit_code - } + "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { + local exit_code=$? + Write-PipelineTaskError "Build failed (exit code '$exit_code')." + ExitWithExitCode $exit_code } - - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } ResolvePath "${BASH_SOURCE[0]}" @@ -459,7 +424,7 @@ temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false -dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true +dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true if [[ -n "$dotnetlocal_key" ]]; then global_json_has_runtimes=true fi @@ -479,18 +444,3 @@ Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" Write-PipelineSetVariable -name "Temp" -value "$temp_dir" Write-PipelineSetVariable -name "TMP" -value "$temp_dir" - -# Import custom tools configuration, if present in the repo. -if [ -z "${disable_configure_toolset_import:-}" ]; then - configure_toolset_script="$eng_root/configure-toolset.sh" - if [[ -a "$configure_toolset_script" ]]; then - . "$configure_toolset_script" - fi -fi - -# TODO: https://github.com/dotnet/arcade/issues/1468 -# Temporary workaround to avoid breaking change. -# Remove once repos are updated. -if [[ -n "${useInstalledDotNetCli:-}" ]]; then - use_installed_dotnet_cli="$useInstalledDotNetCli" -fi diff --git a/global.json b/global.json index d2637af033..301cd03aa1 100644 --- a/global.json +++ b/global.json @@ -9,7 +9,7 @@ "dotnet": "3.1.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20052.1", - "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20052.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.20302.3", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.20302.3" } -} \ No newline at end of file +}