From 2932098d03b10dc10484bde2cdbfb261bc4db84d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 7 Mar 2019 23:28:07 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20190307.4 (#3012) This change updates the following dependencies - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19157.4 - Microsoft.DotNet.SignTool - 1.0.0-beta.19157.4 --- eng/Version.Details.xml | 8 +-- eng/Versions.props | 2 +- eng/common/build.ps1 | 13 +++-- eng/common/build.sh | 61 +++++++++++--------- eng/common/darc-init.ps1 | 2 +- eng/common/darc-init.sh | 23 +++++++- eng/common/generate-graph-files.ps1 | 61 ++++++++++++++++++++ eng/common/templates/job/job.yml | 4 ++ eng/common/templates/steps/send-to-helix.yml | 2 + eng/common/tools.ps1 | 10 ++-- eng/common/tools.sh | 10 +++- global.json | 2 +- 12 files changed, 146 insertions(+), 52 deletions(-) create mode 100644 eng/common/generate-graph-files.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4b0d0f524db8..34d9e06047ff 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,13 +2,13 @@ - + https://github.com/dotnet/arcade - 9a5da1c8a3232c6304955c7a6ffe6109a1b13d9f + 951b378f6b173d58c6d93ef33ce7ca83c2ce5ec5 - + https://github.com/dotnet/arcade - 9a5da1c8a3232c6304955c7a6ffe6109a1b13d9f + 951b378f6b173d58c6d93ef33ce7ca83c2ce5ec5 diff --git a/eng/Versions.props b/eng/Versions.props index db9f5828bd2a..f798a1d71655 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -22,7 +22,7 @@ 2.0.0 4.0.0 1.5.0 - 1.0.0-beta.19112.3 + 1.0.0-beta.19157.4 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 8279dc713391..d0147db4bd52 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -11,7 +11,7 @@ Param( [switch][Alias('b')]$build, [switch] $rebuild, [switch] $deploy, - [switch] $test, + [switch][Alias('t')]$test, [switch] $integrationTest, [switch] $performanceTest, [switch] $sign, @@ -40,10 +40,10 @@ function Print-Usage() { Write-Host " -rebuild Rebuild solution" Write-Host " -deploy Deploy built VSIXes" Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" - Write-Host " -test Run all unit tests in the solution" - Write-Host " -pack Package build outputs into NuGet packages and Willow components" + Write-Host " -test Run all unit tests in the solution (short: -t)" Write-Host " -integrationTest Run all integration tests in the solution" Write-Host " -performanceTest Run all performance tests in the solution" + Write-Host " -pack Package build outputs into NuGet packages and Willow components" Write-Host " -sign Sign build outputs" Write-Host " -publish Publish artifacts (e.g. symbols)" Write-Host "" @@ -51,9 +51,11 @@ function Print-Usage() { Write-Host "Advanced settings:" Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" Write-Host " -ci Set when running on CI server" - Write-Host " -prepareMachine Prepare machine for CI run" + Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" + Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host "" + Write-Host "Command line arguments not listed above are passed thru to msbuild." Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." } @@ -99,12 +101,11 @@ function Build { /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` /p:Publish=$publish ` - /p:ContinuousIntegrationBuild=$ci ` @properties } try { - if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { + if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { Print-Usage exit 0 } diff --git a/eng/common/build.sh b/eng/common/build.sh index 4fe8b41ed706..0227c6e1e554 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -10,20 +10,24 @@ set -e usage() { echo "Common settings:" - echo " --configuration Build configuration: 'Debug' or 'Release' (short: --c)" + echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)" echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" echo " --binaryLog Create MSBuild binary log (short: -bl)" + echo " --help Print help and exit (short: -h)" echo "" + echo "Actions:" echo " --restore Restore dependencies (short: -r)" - echo " --build Build all projects (short: -b)" - echo " --rebuild Rebuild all projects" - echo " --test Run all unit tests (short: -t)" + echo " --build Build solution (short: -b)" + echo " --rebuild Rebuild solution" + echo " --test Run all unit tests in the solution (short: -t)" + echo " --integrationTest Run all integration tests in the solution" + echo " --performanceTest Run all performance tests in the solution" + echo " --pack Package build outputs into NuGet packages and Willow components" echo " --sign Sign build outputs" echo " --publish Publish artifacts (e.g. symbols)" - echo " --pack Package build outputs into NuGet packages and Willow components" - echo " --help Print help and exit (short: -h)" echo "" + echo "Advanced settings:" echo " --projects Project or solution file(s) to build" echo " --ci Set when running on CI server" @@ -32,6 +36,7 @@ usage() echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" echo "" echo "Command line arguments starting with '/p:' are passed through to MSBuild." + echo "Arguments can also be passed in with a single hyphen." } source="${BASH_SOURCE[0]}" @@ -50,10 +55,10 @@ restore=false build=false rebuild=false test=false -pack=false -publish=false integration_test=false performance_test=false +pack=false +publish=false sign=false public=false ci=false @@ -66,68 +71,69 @@ projects='' configuration='Debug' prepare_machine=false verbosity='minimal' + properties='' while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" case "$opt" in - --help|-h) + -help|-h) usage exit 0 ;; - --configuration|-c) + -configuration|-c) configuration=$2 shift ;; - --verbosity|-v) + -verbosity|-v) verbosity=$2 shift ;; - --binarylog|-bl) + -binarylog|-bl) binary_log=true ;; - --restore|-r) + -restore|-r) restore=true ;; - --build|-b) + -build|-b) build=true ;; - --rebuild) + -rebuild) rebuild=true ;; - --pack) + -pack) pack=true ;; - --test|-t) + -test|-t) test=true ;; - --integrationtest) + -integrationtest) integration_test=true ;; - --performancetest) + -performancetest) performance_test=true ;; - --sign) + -sign) sign=true ;; - --publish) + -publish) publish=true ;; - --preparemachine) + -preparemachine) prepare_machine=true ;; - --projects) + -projects) projects=$2 shift ;; - --ci) + -ci) ci=true ;; - --warnaserror) + -warnaserror) warn_as_error=$2 shift ;; - --nodereuse) + -nodereuse) node_reuse=$2 shift ;; @@ -191,7 +197,6 @@ function Build { /p:PerformanceTest=$performance_test \ /p:Sign=$sign \ /p:Publish=$publish \ - /p:ContinuousIntegrationBuild=$ci \ $properties ExitWithExitCode 0 diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 57db185df194..2467ebdd4225 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -19,7 +19,7 @@ function InstallDarcCli ($darcVersion) { # Until we can anonymously query the BAR API for the latest arcade-services # build applied to the PROD channel, this is hardcoded. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19081.1' + $darcVersion = '1.1.0-beta.19151.3' } $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json' diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index bad07c3ae618..8d63dd711b2a 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,6 +1,24 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" +darcVersion="1.1.0-beta.19151.3" + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --darcversion) + darcVersion=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done # resolve $source until the file is no longer a symlink while [[ -h "$source" ]]; do @@ -27,12 +45,11 @@ function InstallDarcCli { echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) fi - ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" - local toolset_version=$_ReadGlobalVersion + local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json" echo "Installing Darc CLI version $toolset_version..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) } InstallDarcCli diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 new file mode 100644 index 000000000000..c04c80e4f61d --- /dev/null +++ b/eng/common/generate-graph-files.ps1 @@ -0,0 +1,61 @@ +Param( + [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens + [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) + [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) + [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created + [string] $darcVersion = '1.1.0-beta.19156.4', # darc's version + [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about + # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies +) + +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +function CheckExitCode ([string]$stage) +{ + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Push-Location $PSScriptRoot + + Write-Host "Installing darc..." + . .\darc-init.ps1 -darcVersion $darcVersion + CheckExitCode "Running darc-init" + + $darcExe = "$env:USERPROFILE\.dotnet\tools" + $darcExe = Resolve-Path "$darcExe\darc.exe" + + Create-Directory $outputFolder + + $graphVizFilePath = "$outputFolder\graphviz.txt" + $graphFilePath = "$outputFolder\graph.txt" + $options = "get-dependency-graph --graphviz '$graphVizFilePath' --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken --output-file $graphFilePath" + + if ($includeToolset) { + Write-Host "Toolsets will be included in the graph..." + $options += " --include-toolset" + } + + Write-Host "Generating dependency graph..." + $darc = Invoke-Expression "& `"$darcExe`" $options" + CheckExitCode "Generating dependency graph" + + $graph = Get-Content $graphVizFilePath + Set-Content $graphVizFilePath -Value "Paste the following digraph object in http://www.webgraphviz.com `r`n", $graph + Write-Host "'$graphVizFilePath' and '$graphFilePath' created!" +} +catch { + if (!$includeToolset) { + Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow + Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow + } + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 98ca94c326cf..cd4e5731a624 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -108,6 +108,10 @@ jobs: - name: ${{ pair.key }} value: ${{ pair.value }} + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + ${{ if ne(parameters.workspace, '') }}: workspace: ${{ parameters.workspace }} diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 1fbf8b889732..3a449210f5b0 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -50,6 +50,7 @@ steps: EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog @@ -77,5 +78,6 @@ steps: EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d1ed296dd6b5..de7523cae581 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -417,15 +417,15 @@ function InitializeToolset() { $proj = Join-Path $ToolsetDir "restore.proj" $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } - + '' | Set-Content $proj - MSBuild $proj $bl /t:__WriteToolsetLocation /noconsolelogger /p:__ToolsetLocationOutputFile=$toolsetLocationFile - + MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile + $path = Get-Content $toolsetLocationFile -TotalCount 1 if (!(Test-Path $path)) { throw "Invalid toolset path: $path" } - + return $global:_ToolsetBuildProj = $path } @@ -461,7 +461,7 @@ function MSBuild() { $buildTool = InitializeBuildTool - $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse" + $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" if ($warnAsError) { $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 115a56a827d7..ecdece1f8552 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -238,11 +238,15 @@ function InitializeToolset { ExitWithExitCode 2 fi - local toolset_restore_log="$log_dir/ToolsetRestore.binlog" local proj="$toolset_dir/restore.proj" + local bl="" + if [[ "$binary_log" == true ]]; then + bl="/bl:$log_dir/ToolsetRestore.binlog" + fi + echo '' > "$proj" - MSBuild "$proj" /t:__WriteToolsetLocation /noconsolelogger /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file" + MSBuild "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" local toolset_build_proj=`cat "$toolset_location_file"` @@ -289,7 +293,7 @@ function MSBuild { warnaserror_switch="/warnaserror" fi - "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error "$@" || { + "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { local exit_code=$? echo "Build failed (exit code '$exit_code')." >&2 ExitWithExitCode $exit_code diff --git a/global.json b/global.json index f25efd31ee80..d66eb8e798ac 100644 --- a/global.json +++ b/global.json @@ -6,6 +6,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19112.3" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19157.4" } }