Update dependencies from https://github.com/dotnet/arcade build 20190…

…307.4 (#3012)

This change updates the following dependencies
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19157.4
- Microsoft.DotNet.SignTool - 1.0.0-beta.19157.4

eng/Version.Details.xml

@@ -2,13 +2,13 @@
 <Dependencies>
   <ProductDependencies></ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19112.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19157.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9a5da1c8a3232c6304955c7a6ffe6109a1b13d9f</Sha>
+      <Sha>951b378f6b173d58c6d93ef33ce7ca83c2ce5ec5</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="1.0.0-beta.19112.3">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="1.0.0-beta.19157.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>9a5da1c8a3232c6304955c7a6ffe6109a1b13d9f</Sha>
+      <Sha>951b378f6b173d58c6d93ef33ce7ca83c2ce5ec5</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -22,7 +22,7 @@
     <PlatformAbstractionsVersion>2.0.0</PlatformAbstractionsVersion>
     <SystemDiagnosticsFileVersionInfoVersion>4.0.0</SystemDiagnosticsFileVersionInfoVersion>
     <SystemReflectionMetadataVersion>1.5.0</SystemReflectionMetadataVersion>
-    <MicrosoftDotNetSignToolVersion>1.0.0-beta.19112.3</MicrosoftDotNetSignToolVersion>
+    <MicrosoftDotNetSignToolVersion>1.0.0-beta.19157.4</MicrosoftDotNetSignToolVersion>
   </PropertyGroup>
   <!-- Get .NET Framework reference assemblies from NuGet packages -->
   <PropertyGroup>

eng/common/build.ps1

@@ -11,7 +11,7 @@ Param(
   [switch][Alias('b')]$build,
   [switch] $rebuild,
   [switch] $deploy,
-  [switch] $test,
+  [switch][Alias('t')]$test,
   [switch] $integrationTest,
   [switch] $performanceTest,
   [switch] $sign,
@@ -40,20 +40,22 @@ function Print-Usage() {
     Write-Host "  -rebuild                Rebuild solution"
     Write-Host "  -deploy                 Deploy built VSIXes"
     Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
-    Write-Host "  -test                   Run all unit tests in the solution"
-    Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
+    Write-Host "  -test                   Run all unit tests in the solution (short: -t)"
     Write-Host "  -integrationTest        Run all integration tests in the solution"
     Write-Host "  -performanceTest        Run all performance tests in the solution"
+    Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
     Write-Host "  -sign                   Sign build outputs"
     Write-Host "  -publish                Publish artifacts (e.g. symbols)"
     Write-Host ""
 
     Write-Host "Advanced settings:"
     Write-Host "  -projects <value>       Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
     Write-Host "  -ci                     Set when running on CI server"
-    Write-Host "  -prepareMachine         Prepare machine for CI run"
+    Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
+    Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
     Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
     Write-Host ""
+
     Write-Host "Command line arguments not listed above are passed thru to msbuild."
     Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
 }
@@ -99,12 +101,11 @@ function Build {
     /p:PerformanceTest=$performanceTest `
     /p:Sign=$sign `
     /p:Publish=$publish `
-    /p:ContinuousIntegrationBuild=$ci `
     @properties
 }
 
 try {
-  if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
+  if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
     Print-Usage
     exit 0
   }

eng/common/build.sh

@@ -10,20 +10,24 @@ set -e
 usage()
 {
   echo "Common settings:"
-  echo "  --configuration <value>    Build configuration: 'Debug' or 'Release' (short: --c)"
+  echo "  --configuration <value>    Build configuration: 'Debug' or 'Release' (short: -c)"
   echo "  --verbosity <value>        Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
   echo "  --binaryLog                Create MSBuild binary log (short: -bl)"
+  echo "  --help                     Print help and exit (short: -h)"
   echo ""
+
   echo "Actions:"
   echo "  --restore                  Restore dependencies (short: -r)"
-  echo "  --build                    Build all projects (short: -b)"
-  echo "  --rebuild                  Rebuild all projects"
-  echo "  --test                     Run all unit tests (short: -t)"
+  echo "  --build                    Build solution (short: -b)"
+  echo "  --rebuild                  Rebuild solution"
+  echo "  --test                     Run all unit tests in the solution (short: -t)"
+  echo "  --integrationTest          Run all integration tests in the solution"
+  echo "  --performanceTest          Run all performance tests in the solution"
+  echo "  --pack                     Package build outputs into NuGet packages and Willow components"
   echo "  --sign                     Sign build outputs"
   echo "  --publish                  Publish artifacts (e.g. symbols)"
-  echo "  --pack                     Package build outputs into NuGet packages and Willow components"
-  echo "  --help                     Print help and exit (short: -h)"
   echo ""
+
   echo "Advanced settings:"
   echo "  --projects <value>       Project or solution file(s) to build"
   echo "  --ci                     Set when running on CI server"
@@ -32,6 +36,7 @@ usage()
   echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   echo ""
   echo "Command line arguments starting with '/p:' are passed through to MSBuild."
+  echo "Arguments can also be passed in with a single hyphen."
 }
 
 source="${BASH_SOURCE[0]}"
@@ -50,10 +55,10 @@ restore=false
 build=false
 rebuild=false
 test=false
-pack=false
-publish=false
 integration_test=false
 performance_test=false
+pack=false
+publish=false
 sign=false
 public=false
 ci=false
@@ -66,68 +71,69 @@ projects=''
 configuration='Debug'
 prepare_machine=false
 verbosity='minimal'
+
 properties=''
 
 while [[ $# > 0 ]]; do
-  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
   case "$opt" in
-    --help|-h)
+    -help|-h)
       usage
       exit 0
       ;;
-    --configuration|-c)
+    -configuration|-c)
       configuration=$2
       shift
       ;;
-    --verbosity|-v)
+    -verbosity|-v)
       verbosity=$2
       shift
       ;;
-    --binarylog|-bl)
+    -binarylog|-bl)
       binary_log=true
       ;;
-    --restore|-r)
+    -restore|-r)
       restore=true
       ;;
-    --build|-b)
+    -build|-b)
       build=true
       ;;
-    --rebuild)
+    -rebuild)
       rebuild=true
       ;;
-    --pack)
+    -pack)
       pack=true
       ;;
-    --test|-t)
+    -test|-t)
       test=true
       ;;
-    --integrationtest)
+    -integrationtest)
       integration_test=true
       ;;
-    --performancetest)
+    -performancetest)
       performance_test=true
       ;;
-    --sign)
+    -sign)
       sign=true
       ;;
-    --publish)
+    -publish)
       publish=true
       ;;
-    --preparemachine)
+    -preparemachine)
       prepare_machine=true
       ;;
-    --projects)
+    -projects)
       projects=$2
       shift
       ;;
-    --ci)
+    -ci)
       ci=true
       ;;
-    --warnaserror)
+    -warnaserror)
       warn_as_error=$2
       shift
       ;;
-    --nodereuse)
+    -nodereuse)
       node_reuse=$2
       shift
       ;;
@@ -191,7 +197,6 @@ function Build {
     /p:PerformanceTest=$performance_test \
     /p:Sign=$sign \
     /p:Publish=$publish \
-    /p:ContinuousIntegrationBuild=$ci \
     $properties
 
   ExitWithExitCode 0

eng/common/darc-init.ps1

@@ -19,7 +19,7 @@ function InstallDarcCli ($darcVersion) {
   # Until we can anonymously query the BAR API for the latest arcade-services
   # build applied to the PROD channel, this is hardcoded.
   if (-not $darcVersion) {
-    $darcVersion = '1.1.0-beta.19081.1'
+    $darcVersion = '1.1.0-beta.19151.3'
   }
 
   $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json'

eng/common/darc-init.sh

@@ -1,6 +1,24 @@
 #!/usr/bin/env bash
 
 source="${BASH_SOURCE[0]}"
+darcVersion="1.1.0-beta.19151.3"
+
+while [[ $# > 0 ]]; do
+  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  case "$opt" in
+    --darcversion)
+      darcVersion=$2
+      shift
+      ;;
+    *)
+      echo "Invalid argument: $1"
+      usage
+      exit 1
+      ;;
+  esac
+
+  shift
+done
 
 # resolve $source until the file is no longer a symlink
 while [[ -h "$source" ]]; do
@@ -27,12 +45,11 @@ function InstallDarcCli {
     echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
   fi
 
-  ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
-  local toolset_version=$_ReadGlobalVersion
+  local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-arcade/index.json"
 
   echo "Installing Darc CLI version $toolset_version..."
   echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
-  echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g)
+  echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
 }
 
 InstallDarcCli

eng/common/generate-graph-files.ps1

@@ -0,0 +1,61 @@
+Param(
+  [Parameter(Mandatory=$true)][string] $barToken,       # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens
+  [Parameter(Mandatory=$true)][string] $gitHubPat,      # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
+  [Parameter(Mandatory=$true)][string] $azdoPat,        # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
+  [Parameter(Mandatory=$true)][string] $outputFolder,   # Where the graphviz.txt file will be created
+  [string] $darcVersion = '1.1.0-beta.19156.4',         # darc's version
+  [switch] $includeToolset                              # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
+                                                        # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
+)
+
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\tools.ps1
+
+function CheckExitCode ([string]$stage)
+{
+  $exitCode = $LASTEXITCODE
+  if ($exitCode  -ne 0) {
+    Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
+    ExitWithExitCode $exitCode
+  }
+}
+
+try {
+  Push-Location $PSScriptRoot
+
+  Write-Host "Installing darc..."
+  . .\darc-init.ps1 -darcVersion $darcVersion
+  CheckExitCode "Running darc-init"
+
+  $darcExe = "$env:USERPROFILE\.dotnet\tools"
+  $darcExe = Resolve-Path "$darcExe\darc.exe"
+
+  Create-Directory $outputFolder
+
+  $graphVizFilePath = "$outputFolder\graphviz.txt"
+  $graphFilePath = "$outputFolder\graph.txt"
+  $options = "get-dependency-graph --graphviz '$graphVizFilePath' --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken --output-file $graphFilePath"
+
+  if ($includeToolset) {
+    Write-Host "Toolsets will be included in the graph..."
+    $options += " --include-toolset"
+  }
+
+  Write-Host "Generating dependency graph..."
+  $darc = Invoke-Expression "& `"$darcExe`" $options"
+  CheckExitCode "Generating dependency graph"
+
+  $graph = Get-Content $graphVizFilePath
+  Set-Content $graphVizFilePath -Value "Paste the following digraph object in http://www.webgraphviz.com `r`n", $graph
+  Write-Host "'$graphVizFilePath' and '$graphFilePath' created!"
+}
+catch {
+  if (!$includeToolset) {
+    Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow
+    Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow
+  }
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}

eng/common/templates/job/job.yml

@@ -108,6 +108,10 @@ jobs:
         - name: ${{ pair.key }}
           value: ${{ pair.value }}
 
+  # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds
+  - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - group: DotNet-HelixApi-Access
+
   ${{ if ne(parameters.workspace, '') }}:
     workspace: ${{ parameters.workspace }}
 

eng/common/templates/steps/send-to-helix.yml

@@ -50,6 +50,7 @@ steps:
       EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
       WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
       Creator: ${{ parameters.Creator }}
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
     continueOnError: ${{ parameters.continueOnError }}
   - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
@@ -77,5 +78,6 @@ steps:
       EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
       WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
       Creator: ${{ parameters.Creator }}
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
     continueOnError: ${{ parameters.continueOnError }}

eng/common/tools.ps1

@@ -417,15 +417,15 @@ function InitializeToolset() {
 
   $proj = Join-Path $ToolsetDir "restore.proj"
   $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" }
-  
+
   '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
-  MSBuild $proj $bl /t:__WriteToolsetLocation /noconsolelogger /p:__ToolsetLocationOutputFile=$toolsetLocationFile
-  
+  MSBuild $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
+
   $path = Get-Content $toolsetLocationFile -TotalCount 1
   if (!(Test-Path $path)) {
     throw "Invalid toolset path: $path"
   }
-  
+
   return $global:_ToolsetBuildProj = $path
 }
 
@@ -461,7 +461,7 @@ function MSBuild() {
 
   $buildTool = InitializeBuildTool
 
-  $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse"
+  $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
 
   if ($warnAsError) { 
     $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true"