Update dependency files (#83)

* Update dependencies from https://github.com/dotnet/arcade build 598 This change updates the following dependencies - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.18604.15 * Updating global.json to include a specific VS dependency, and updating Arcade SDK version. * Removing vswhere from global.json. This will default to the version specified in eng\common\tools.ps1 (2.5.2)
dotnet · Dec 5, 2018 · e11966c · e11966c
1 parent 07a7a47
commit e11966c
Show file tree

Hide file tree

Showing 5 changed files with 198 additions and 43 deletions.
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -2,9 +2,9 @@
 <Dependencies>
   <ProductDependencies></ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.18577.9">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.18604.15">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>085cb30bcceacc3dd2fdb563800b4b02bc71d02d</Sha>
+      <Sha>f14da17c737c4f2b5408c4c0801d7e913a95766f</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
@@ -43,8 +43,8 @@ parameters:
   # Optional: enable sending telemetry
   #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
   #             _HelixBuildConfig - differentiate between Debug, Release, other
-  #             _HelixSource - Example: build/product
-  #             _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
+  #             _HelixType - Example: build/product/
+  #             _HelixSource - Example: official/dotnet/arcade/$(Build.SourceBranch)
   enableTelemetry: false
 
   # Optional: If specified, then automatically derive "_HelixSource" variable for telemetry

diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml
@@ -0,0 +1,71 @@
+parameters:
+  HelixSource: 'pr/default'          # required
+  HelixType: ̓'tests/default/'        # required
+  HelixBuild: $(Build.BuildNumber)   # required
+  HelixTargetQueues: ''              # required
+  HelixAccessToken: ''               # required
+  HelixPreCommands: ''
+  HelixPostCommands: ''
+  WorkItemDirectory: ''
+  WorkItemCommand: ''
+  CorrelationPayloadDirectory: ''
+  XUnitProjects: ''
+  XUnitTargetFramework: ''
+  XUnitRunnerVersion: ''
+  IncludeDotNetCli: false
+  DotNetCliPackageType: ''
+  DotNetCliVersion: ''
+  EnableXUnitReporter: false
+  WaitForWorkItemCompletion: true
+  condition: succeeded()
+  continueOnError: false
+
+steps:
+  - script: '%BUILD_SOURCESDIRECTORY%\eng\common\msbuild.ps1 %BUILD_SOURCESDIRECTORY%\eng\common\helixpublish.proj /bl:%BUILD_SOURCESDIRECTORY%\artifacts\log\%BuildConfig%\SendToHelix.binlog'
+    displayName: Send job to Helix (Windows)
+    env:
+      BuildConfig: $(_BuildConfig)
+      HelixSource: ${{ parameters.HelixSource }}
+      HelixType: ${{ parameters.HelixType }}
+      HelixBuild: ${{ parameters.HelixBuild }}
+      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+      HelixAccessToken: ${{ parameters.HelixAccessToken }}
+      HelixPreCommands: ${{ parameters.HelixPreCommands }}
+      HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+      WorkItemCommand: ${{ parameters.WorkItemCommand }}
+      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+      XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitTargetFramework: ${{ parameters.XUnitTargetFramework }}
+      XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+      EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+    condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
+    continueOnError: ${{ parameters.continueOnError }}
+  - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
+    displayName: Send job to Helix (Unix)
+    env:
+      BuildConfig: $(_BuildConfig)
+      HelixSource: ${{ parameters.HelixSource }}
+      HelixType: ${{ parameters.HelixType }}
+      HelixBuild: ${{ parameters.HelixBuild }}
+      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+      HelixAccessToken: ${{ parameters.HelixAccessToken }}
+      HelixPreCommands: ${{ parameters.HelixPreCommands }}
+      HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+      WorkItemCommand: ${{ parameters.WorkItemCommand }}
+      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+      XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitTargetFramework: ${{ parameters.XUnitTargetFramework }}
+      XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+      EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+    condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
+    continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
@@ -20,6 +20,11 @@ function Create-Directory([string[]] $path) {
   }
 }
 
+function Unzip([string]$zipfile, [string]$outpath) {
+  Add-Type -AssemblyName System.IO.Compression.FileSystem
+  [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
+}
+
 function InitializeDotNetCli([bool]$install) {
   # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
   $env:DOTNET_MULTILEVEL_LOOKUP=0
@@ -79,39 +84,104 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version) {
   }
 }
 
-function InitializeVisualStudioBuild {
-  $vsToolsPath = $env:VS150COMNTOOLS
-  if ($vsToolsPath -eq $null) { 
-    $vsToolsPath = $env:VS160COMNTOOLS
+#
+# Locates Visual Studio MSBuild installation. 
+# The preference order for MSBuild to use is as follows:
+#
+#   1. MSBuild from an active VS command prompt
+#   2. MSBuild from a compatible VS installation
+#   3. MSBuild from the xcopy tool package
+#
+# Returns full path to msbuild.exe.
+# Throws on failure.
+#
+function InitializeVisualStudioMSBuild {
+  $vsMinVersionStr = if (!$GlobalJson.tools.vs.version) { $GlobalJson.tools.vs.version } else { "15.9" }
+  $vsMinVersion = [Version]::new($vsMinVersionStr) 
+
+  # Try msbuild command available in the environment.
+  if ($env:VSINSTALLDIR -ne $null) {
+    $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue
+    if ($msbuildCmd -ne $null) {
+      if ($msbuildCmd.Version -ge $vsMinVersion) {
+        return $msbuildCmd.Path
+      }
+
+      # Report error - the developer environment is initialized with incompatible VS version.
+      throw "Developer Command Prompt for VS $($env:VisualStudioVersion) is not recent enough. Please upgrade to $vsMinVersionStr or build from a plain CMD window"
+    }
   }
 
-  if (($vsToolsPath -ne $null) -and (Test-Path $vsToolsPath)) {
-    $vsInstallDir = [System.IO.Path]::GetFullPath((Join-Path $vsToolsPath "..\.."))
+  # Locate Visual Studio installation or download x-copy msbuild.
+  $vsInfo = LocateVisualStudio  
+  if ($vsInfo -ne $null) {
+    $vsInstallDir = $vsInfo.installationPath
+    $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
+
+    InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
   } else {
-    $vsInfo = LocateVisualStudio
 
-    $vsInstallDir = $vsInfo.installationPath
-    $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
-    $vsVersion = $vsInfo.installationVersion.Split('.')[0] + "0"
+    if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") {
+      $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
+      $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
+    } else {
+      $vsMajorVersion = $vsMinVersion.Major
+      $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha"
+    }
+
+    $vsInstallDir = InstallXCopyMSBuild $xcopyMSBuildVersion
+  }
+
+  $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
+  return Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe"
+}
 
-    Set-Item "env:VS$($vsVersion)COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")    
-    Set-Item "env:VSSDK$($vsVersion)Install" $vsSdkInstallDir
+function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
+  $env:VSINSTALLDIR = $vsInstallDir
+  Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")
+
+  $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
+  if (Test-Path $vsSdkInstallDir) {
+    Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir
     $env:VSSDKInstall = $vsSdkInstallDir
   }
+}
+
+function InstallXCopyMSBuild([string] $packageVersion) {
+  $packageName = "RoslynTools.MSBuild"
+  $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
+  $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
+
+  if (!(Test-Path $packageDir)) {
+    Create-Directory $packageDir
+    Write-Host "Downloading $packageName $packageVersion"
+    Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath
+    Unzip $packagePath $packageDir
+  }
 
-  return $vsInstallDir
+  return Join-Path $packageDir "tools"
 }
 
+#
+# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json.
+#
+# The following properties of tools.vs are recognized:
+#   "version": "{major}.{minor}"    
+#       Two part minimal VS version, e.g. "15.9", "16.0", etc.
+#   "components": ["componentId1", "componentId2", ...] 
+#       Array of ids of workload components that must be available in the VS instance.
+#       See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017
+#
+# Returns JSON describing the located VS instance (same format as returned by vswhere), 
+# or $null if no instance meeting the requirements is found on the machine.
+#
 function LocateVisualStudio {
-  $vswhereVersion = $GlobalJson.tools.vswhere
-
-  if (!$vsWhereVersion) {
-    Write-Host "vswhere version must be specified in /global.json." -ForegroundColor Red
-    ExitWithExitCode 1
+  $vswhereVersion = Get-Member -InputObject $GlobalJson.tools -Name "vswhere"
+  if ($vsWhereVersion -eq $null) {
+    $vswhereVersion = "2.5.2"
   }
 
-  $toolsRoot = Join-Path $RepoRoot ".tools"
-  $vsWhereDir = Join-Path $toolsRoot "vswhere\$vswhereVersion"
+  $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
   $vsWhereExe = Join-Path $vsWhereDir "vswhere.exe"
 
   if (!(Test-Path $vsWhereExe)) {
@@ -120,17 +190,25 @@ function LocateVisualStudio {
     Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
   }
 
-  $vsInfo = & $vsWhereExe `
-    -latest `
-    -prerelease `
-    -format json `
-    -requires Microsoft.Component.MSBuild `
-    -requires Microsoft.VisualStudio.Component.VSSDK `
-    -requires Microsoft.VisualStudio.Component.Roslyn.Compiler | ConvertFrom-Json
+  $vs = $GlobalJson.tools.vs
+  $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild")
+
+  if (Get-Member -InputObject $vs -Name "version") { 
+    $args += "-version"
+    $args += $vs.version
+  }
+
+  if (Get-Member -InputObject $vs -Name "components") { 
+    foreach ($component in $vs.components) {
+      $args += "-requires"
+      $args += $component
+    }    
+  }
+
+  $vsInfo =& $vsWhereExe $args | ConvertFrom-Json
 
   if ($lastExitCode -ne 0) {
-    Write-Host "Failed to locate Visual Studio (exit code '$lastExitCode')." -ForegroundColor Red
-    ExitWithExitCode $lastExitCode
+    return $null
   }
 
   # use first matching instance
@@ -153,18 +231,18 @@ function InitializeTools() {
 
   # Initialize dotnet cli if listed in 'tools'
   $dotnetRoot = $null
-  if ((Get-Member -InputObject $tools -Name "dotnet") -ne $null) {
+  if (Get-Member -InputObject $tools -Name "dotnet") {
     $dotnetRoot = InitializeDotNetCli -install:$restore
   }
 
   if (-not $msbuildEngine) {
-    # Presence of vswhere.version indicates the repo needs to build using VS msbuild.
-    if ((Get-Member -InputObject $tools -Name "vswhere") -ne $null) {
+    # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
+    if (Get-Member -InputObject $tools -Name "vs") {
       $msbuildEngine = "vs"
     } elseif ($dotnetRoot -ne $null) {
       $msbuildEngine = "dotnet"
     } else {
-      Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vswhere'." -ForegroundColor Red
+      Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red
       ExitWithExitCode 1
     }
   }
@@ -178,10 +256,13 @@ function InitializeTools() {
     $script:buildDriver = Join-Path $dotnetRoot "dotnet.exe"
     $script:buildArgs = "msbuild"
   } elseif ($msbuildEngine -eq "vs") {
-    $vsInstallDir = InitializeVisualStudioBuild
-
-    $script:buildDriver = Join-Path $vsInstallDir "MSBuild\15.0\Bin\msbuild.exe"
-    $script:buildArgs = ""
+    try {
+      $script:buildDriver = InitializeVisualStudioMSBuild
+      $script:buildArgs = ""
+    } catch { 
+      Write-Host $_ -ForegroundColor Red
+      ExitWithExitCode 1
+    }
   } else {
     Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red
     ExitWithExitCode 1
@@ -262,6 +343,7 @@ $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
 $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
 $ArtifactsDir = Join-Path $RepoRoot "artifacts"
 $ToolsetDir = Join-Path $ArtifactsDir "toolset"
+$ToolsDir = Join-Path $RepoRoot ".tools"
 $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration
 $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration
 $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json

diff --git a/global.json b/global.json
@@ -1,9 +1,11 @@
 {
   "tools": {
     "dotnet": "3.0.100-preview-009764",
-    "vswhere": "2.5.2"
+    "vs": {
+      "version": "15.9"
+    }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18577.9"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18605.3"
   }
 }