From 4414abdd3d3993998dbfa9f8b6e03b203f456394 Mon Sep 17 00:00:00 2001 From: Chris Sienkiewicz Date: Thu, 12 Dec 2019 17:31:38 -0800 Subject: [PATCH 1/3] Rework build definitions to accept VS optprof drop --- azure-pipelines-official.yml | 6 +- eng/InternalTools.props | 16 ++++ eng/Tools.props | 13 +++ eng/Versions.props | 1 + ...dio.AcquireOptimizationData.Roslyn.targets | 92 +++++++++++++++++++ eng/build.ps1 | 54 +++++------ eng/config/PublishData.json | 15 +-- 7 files changed, 157 insertions(+), 40 deletions(-) create mode 100644 eng/InternalTools.props create mode 100644 eng/Tools.props create mode 100644 eng/VisualStudio.AcquireOptimizationData.Roslyn.targets diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml index ea3320369d887..89a1d6b051119 100644 --- a/azure-pipelines-official.yml +++ b/azure-pipelines-official.yml @@ -8,8 +8,7 @@ resources: # SignType: real # SkipTests: false # SkipApplyOptimizationData: false -# IbcSourceBranchName: 'default' -# IbcDropId: 'default' +# IbcDrop: 'default' # The variables `_DotNetArtifactsCategory` and `_DotNetValidationArtifactsCategory` are required for proper publishing of build artifacts. See https://github.com/dotnet/roslyn/pull/38259 variables: @@ -76,8 +75,7 @@ stages: -officialSkipTests $(SkipTests) -officialSkipApplyOptimizationData $(SkipApplyOptimizationData) -officialSourceBranchName $(SourceBranchName) - -officialIbcSourceBranchName $(IbcSourceBranchName) - -officialIbcDropId $(IbcDropId) + -officialIbcDrop $(IbcDrop) -skipAnalyzers /p:RepositoryName=$(Build.Repository.Name) /p:VisualStudioDropAccessToken=$(System.AccessToken) diff --git a/eng/InternalTools.props b/eng/InternalTools.props new file mode 100644 index 0000000000000..5a9f48f83e7d6 --- /dev/null +++ b/eng/InternalTools.props @@ -0,0 +1,16 @@ + + + + + + $(RestoreSources); + https://devdiv.pkgs.visualstudio.com/_packaging/Engineering/nuget/v3/index.json; + https://api.nuget.org/v3/index.json; + + + + + + + + \ No newline at end of file diff --git a/eng/Tools.props b/eng/Tools.props new file mode 100644 index 0000000000000..eb5fc1d6e5260 --- /dev/null +++ b/eng/Tools.props @@ -0,0 +1,13 @@ + + + + + + + false + + + + + + \ No newline at end of file diff --git a/eng/Versions.props b/eng/Versions.props index 8b512f711fff4..f1c632c8c0257 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -66,6 +66,7 @@ $(RoslynDiagnosticsNugetPackageVersion) 1.0.31 4.3.0 + 1.0.210 0.8.31-beta 1.0.35 1.3.0 diff --git a/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets b/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets new file mode 100644 index 0000000000000..15158fa020726 --- /dev/null +++ b/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets @@ -0,0 +1,92 @@ + + + + + + + + + <_VisualStudioBuildTasksAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.visualstudio\$(MicrosoftDotNetBuildTasksVisualStudioVersion)\tools\net472\Microsoft.DotNet.Build.Tasks.VisualStudio.dll + + + + + + + true + + $(EnablePartialNgenOptimization) + + + + + + + + + <_DropToolPath>$(NuGetPackageRoot)drop.app\$(DropAppVersion)\lib\net45\drop.exe + <_DropToolExists>false + <_DropToolExists Condition="Exists('$(_DropToolPath)')">true + + + + + + + + + + + + + + + + <_DropServiceUrl>https://devdiv.artifacts.visualstudio.com + <_PatAuthArg Condition="'$(VisualStudioDropAccessToken)' != ''">--patAuth "$(VisualStudioDropAccessToken)" + <_DestArg>$(IbcOptimizationDataDir.TrimEnd('\')) + <_DropsJsonPath>$(ArtifactsLogDir)OptimizationDataDrops.json + <_DropsLogPath>$(ArtifactsLogDir)OptimizationDataAcquisition.log + <_DropNamePrefix>OptimizationData/$(RepositoryName)/$(VisualStudioIbcSourceBranchName) + <_DropName>$(_DropNamePrefix)/$(VisualStudioIbcDropId) + + + <_DropName Condition="'$(VisualStudioIbcDrop)' != ''">$(VisualStudioIbcDrop) + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/eng/build.ps1 b/eng/build.ps1 index e85b12d5793a8..b97a66d81bb33 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -49,8 +49,7 @@ param ( [string]$officialSkipApplyOptimizationData = "", [string]$officialSkipTests = "", [string]$officialSourceBranchName = "", - [string]$officialIbcSourceBranchName = "", - [string]$officialIbcDropId = "", + [string]$officialIbcDrop = "", # Test actions [switch]$test32, @@ -108,10 +107,8 @@ function Print-Usage() { Write-Host " -officialSkipTests Pass 'true' to not run tests" Write-Host " -officialSkipApplyOptimizationData Pass 'true' to not apply optimization data" Write-Host " -officialSourceBranchName The source branch name" - Write-Host " -officialIbcDropId IBC data drop to use (e.g. '20190210.1/935479/1')." + Write-Host " -officialIbcDrop IBC data drop to use (e.g. 'ProfilingOutputs/DevDiv/VS/..')." Write-Host " 'default' for the most recent available for the branch." - Write-Host " -officialIbcSourceBranchName IBC source branch (e.g. 'master-vs-deps')" - Write-Host " 'default' to select branch based on eng/config/PublishData.json." Write-Host "" Write-Host "Command line arguments starting with '/p:' are passed through to MSBuild." } @@ -146,8 +143,6 @@ function Process-Arguments() { OfficialBuildOnly "officialSkipTests" OfficialBuildOnly "officialSkipApplyOptimizationData" OfficialBuildOnly "officialSourceBranchName" - OfficialBuildOnly "officialIbcDropId" - OfficialBuildOnly "officialIbcSourceBranchName" if ($officialBuildId) { $script:useGlobalNuGetCache = $false @@ -220,8 +215,7 @@ function BuildSolution() { $testTargetFrameworks = if ($testCoreClr) { "netcoreapp3.0%3Bnetcoreapp2.1" } else { "" } - $ibcSourceBranchName = GetIbcSourceBranchName - $ibcDropId = if ($officialIbcDropId -ne "default") { $officialIbcDropId } else { "" } + $ibcDropName = GetIbcDropName # Do not set this property to true explicitly, since that would override values set in projects. $suppressExtensionDeployment = if (!$deployExtensions) { "/p:DeployExtension=false" } else { "" } @@ -257,10 +251,9 @@ function BuildSolution() { /p:BootstrapBuildPath=$bootstrapDir ` /p:TestTargetFrameworks=$testTargetFrameworks ` /p:TreatWarningsAsErrors=true ` - /p:VisualStudioIbcSourceBranchName=$ibcSourceBranchName ` - /p:VisualStudioIbcDropId=$ibcDropId ` /p:EnableNgenOptimization=$applyOptimizationData ` /p:IbcOptimizationDataDir=$ibcDir ` + /p:VisualStudioIbcDrop=$ibcDropName ` $suppressExtensionDeployment ` $msbuildWarnAsError ` $buildFromSource ` @@ -280,33 +273,42 @@ function GetIbcSourceBranchName() { } function calculate { - $fallback = "master-vs-deps" - - if (!$officialIbcSourceBranchName) { - return $fallback - } - - if ($officialIbcSourceBranchName -ne "default") { - return $officialIbcSourceBranchName - } + $fallback = "master" $branchData = GetBranchPublishData $officialSourceBranchName if ($branchData -eq $null) { Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using IBC data from '$fallback'." -ForegroundColor Yellow - Write-Host "Override by setting IbcSourceBranchName build variable." -ForegroundColor Yellow + Write-Host "Override by setting IbcDrop build variable." -ForegroundColor Yellow return $fallback } - if (Get-Member -InputObject $branchData -Name "ibcSourceBranch") { - return $branchData.ibcSourceBranch - } - - return $officialSourceBranchName + return $branchData.vsBranch } return $global:_IbcSourceBranchName = calculate } +function GetIbcDropName() { + + if ($officialIbcDrop -and $officialIbcDrop -ne "default"){ + return $officialIbcDrop + } + + # Don't try and get the ibc drop if we're not in an official build as it won't be used anyway + if (!$officialBuildId) { + return "" + } + + # Bring in the ibc tools + $packagePath = Join-Path (Get-PackageDir "Microsoft.DevDiv.Optimization.Data.PowerShell") "lib\net461" + Import-Module (Join-Path $packagePath "Optimization.Data.PowerShell.dll") + + # Find the matching drop + $branch = GetIbcSourceBranchName + $drop = Find-OptimizationInputsStoreForBranch -ProjectName "DevDiv" -RepositoryName "VS" -BranchName $branch + return $drop.Name +} + # Set VSO variables used by MicroBuildBuildVSBootstrapper pipeline task function SetVisualStudioBootstrapperBuildArgs() { $fallbackBranch = "master-vs-deps" diff --git a/eng/config/PublishData.json b/eng/config/PublishData.json index 6b45dbbf0095e..31b63952e0b09 100644 --- a/eng/config/PublishData.json +++ b/eng/config/PublishData.json @@ -79,8 +79,7 @@ "vsix": [ "https://dotnet.myget.org/F/roslyn-nonnull/vsix/upload" ], "channels": [ "nonnull" ], "vsBranch": "lab/d16.1stg", - "vsMajorVersion": 16, - "ibcSourceBranch": "master-vs-deps" + "vsMajorVersion": 16 }, "features/dataflow": { "nugetKind": "PerBuildPreRelease", @@ -89,8 +88,7 @@ "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ], "channels": [ "dataflow" ], "vsBranch": "lab/d16.1stg", - "vsMajorVersion": 16, - "ibcSourceBranch": "master-vs-deps" + "vsMajorVersion": 16 }, "features/razorSupport2": { "nugetKind": [ "Shipping", "NonShipping" ], @@ -99,8 +97,7 @@ "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ], "channels": [ "razorSupport2" ], "vsBranch": "lab/d16.1stg", - "vsMajorVersion": 16, - "ibcSourceBranch": "master-vs-deps" + "vsMajorVersion": 16 }, "features/compilerNext": { "nugetKind": [ "Shipping", "NonShipping" ], @@ -109,8 +106,7 @@ "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ], "channels": [ "compilerNext" ], "vsBranch": "master", - "vsMajorVersion": 16, - "ibcSourceBranch": "master-vs-deps" + "vsMajorVersion": 16 }, "features/dotnetFormat": { "nugetKind": [ "Shipping", "NonShipping" ], @@ -119,8 +115,7 @@ "vsix": [ "https://dotnet.myget.org/F/roslyn/vsix/upload" ], "channels": [ "dotnetFormat" ], "vsBranch": "master", - "vsMajorVersion": 16, - "ibcSourceBranch": "master-vs-deps" + "vsMajorVersion": 16 } }, "releases": { From fd50a7ee71f33e8e5636326d381cf0afaf0043c6 Mon Sep 17 00:00:00 2001 From: Chris Sienkiewicz Date: Fri, 13 Dec 2019 11:32:51 -0800 Subject: [PATCH 2/3] Fix target error conditions --- eng/VisualStudio.AcquireOptimizationData.Roslyn.targets | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets b/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets index 15158fa020726..c6e3bf31ae1d0 100644 --- a/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets +++ b/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets @@ -53,11 +53,11 @@ - - + - + + <_DropServiceUrl>https://devdiv.artifacts.visualstudio.com From 717e15c8e701feffbd313363150fa314b90a628f Mon Sep 17 00:00:00 2001 From: Chris Sienkiewicz Date: Sat, 14 Dec 2019 12:44:24 -0800 Subject: [PATCH 3/3] Remove temp workarounds --- eng/Tools.props | 13 --- ...dio.AcquireOptimizationData.Roslyn.targets | 92 ------------------- 2 files changed, 105 deletions(-) delete mode 100644 eng/Tools.props delete mode 100644 eng/VisualStudio.AcquireOptimizationData.Roslyn.targets diff --git a/eng/Tools.props b/eng/Tools.props deleted file mode 100644 index eb5fc1d6e5260..0000000000000 --- a/eng/Tools.props +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - false - - - - - - \ No newline at end of file diff --git a/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets b/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets deleted file mode 100644 index c6e3bf31ae1d0..0000000000000 --- a/eng/VisualStudio.AcquireOptimizationData.Roslyn.targets +++ /dev/null @@ -1,92 +0,0 @@ - - - - - - - - - <_VisualStudioBuildTasksAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.visualstudio\$(MicrosoftDotNetBuildTasksVisualStudioVersion)\tools\net472\Microsoft.DotNet.Build.Tasks.VisualStudio.dll - - - - - - - true - - $(EnablePartialNgenOptimization) - - - - - - - - - <_DropToolPath>$(NuGetPackageRoot)drop.app\$(DropAppVersion)\lib\net45\drop.exe - <_DropToolExists>false - <_DropToolExists Condition="Exists('$(_DropToolPath)')">true - - - - - - - - - - - - - - - - <_DropServiceUrl>https://devdiv.artifacts.visualstudio.com - <_PatAuthArg Condition="'$(VisualStudioDropAccessToken)' != ''">--patAuth "$(VisualStudioDropAccessToken)" - <_DestArg>$(IbcOptimizationDataDir.TrimEnd('\')) - <_DropsJsonPath>$(ArtifactsLogDir)OptimizationDataDrops.json - <_DropsLogPath>$(ArtifactsLogDir)OptimizationDataAcquisition.log - <_DropNamePrefix>OptimizationData/$(RepositoryName)/$(VisualStudioIbcSourceBranchName) - <_DropName>$(_DropNamePrefix)/$(VisualStudioIbcDropId) - - - <_DropName Condition="'$(VisualStudioIbcDrop)' != ''">$(VisualStudioIbcDrop) - - - - - - - - - - - - - - - - - - - \ No newline at end of file