Add support for source-build (#1740)

dotnet · May 23, 2019 · ad4067b · ad4067b
1 parent 030b50f
commit ad4067b
Show file tree

Hide file tree

Showing 20 changed files with 176 additions and 26 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -15,6 +15,8 @@
     <IsReferenceAssemblyProject>false</IsReferenceAssemblyProject>
     <IsReferenceAssemblyProject Condition="$(MSBuildProjectDirectory.EndsWith('ref'))">true</IsReferenceAssemblyProject>
     <OutDirName Condition="'$(IsReferenceAssemblyProject)' == 'true'">$(MSBuildProjectName)-ref</OutDirName>
+
+    <ExcludeFromSourceBuild Condition="'$(IsUnitTestProject)' == 'true' or '$(IsBenchmarkProject)' == 'true' or '$(IsTestAssetProject)' == 'true' or '$(IsSampleProject)' == 'true'">true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <Import Project="eng\FlakyTests.BeforeArcade.props" />
@@ -63,7 +65,7 @@
     </Content>
   </ItemDefinitionGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(DotNetBuildFromSource)' != 'true'">
     <Reference Include="Microsoft.DotNet.GenAPI" Version="$(MicrosoftDotNetGenApiPackageVersion)" PrivateAssets="All" />
   </ItemGroup>
 

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -204,3 +204,47 @@ jobs:
           searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Flaky'
         continueOnError: true
         condition: always()
+- job: Source_Build
+  displayName: 'Test: Linux Source Build'
+  container: centos:7
+  pool:
+    vmImage: 'ubuntu-16.04'
+  variables:
+    DotNetCoreSdkDir: $(Agent.ToolsDirectory)/dotnet
+    DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: true
+  steps:
+  - script: |
+      source eng/common/native/common-library.sh
+      mkdir -p $HOME/bin
+      GetFile https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 $HOME/bin/jq
+      chmod +x $HOME/bin/jq
+      echo "##vso[task.prependpath]$HOME/bin"
+    displayName: Install jq
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk'
+    inputs:
+      packageType: sdk
+      # The SDK version selected here is intentionally supposed to use the latest release
+      # For the purpose of building Linux distros, we can't depend on features of the SDK
+      # which may not exist in pre-built versions of the SDK
+      version: 3.0.x
+      installationPath: $(DotNetCoreSdkDir)
+      includePreviewVersions: true
+  - script: ./eng/scripts/ci-source-build.sh --ci --configuration Release
+    displayName: Run ci-source-build.sh
+  - task: PublishBuildArtifacts@1
+    displayName: Upload logs
+    continueOnError: true
+    inputs:
+      pathtoPublish: artifacts/log/
+      artifactName: Source_Build_Logs
+      artifactType: Container
+      parallel: true
+  - task: PublishBuildArtifacts@1
+    displayName: Upload package artifacts
+    condition: succeeded()
+    inputs:
+      pathtoPublish: artifacts/packages/
+      artifactName: Source_Build_Packages
+      artifactType: Container
+      parallel: true
diff --git a/eng/Dependencies.props b/eng/Dependencies.props
@@ -17,7 +17,7 @@ and are generated based on the last package release.
     </LatestPackageReference>
   </ItemDefinitionGroup>
 
-  <ItemGroup Label=".NET Core dependencies">
+  <ItemGroup Label=".NET team dependencies">
     <LatestPackageReference Include="Microsoft.Win32.Registry" Version="$(MicrosoftWin32RegistryPackageVersion)" />
     <LatestPackageReference Include="System.ComponentModel.Annotations" Version="$(SystemComponentModelAnnotationsPackageVersion)" />
     <LatestPackageReference Include="System.Data.SqlClient" Version="$(SystemDataSqlClientPackageVersion)" />
@@ -36,12 +36,16 @@ and are generated based on the last package release.
     <LatestPackageReference Include="System.Text.Json" Version="$(SystemTextJsonPackageVersion)" />
     <LatestPackageReference Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsPackageVersion)" />
     <LatestPackageReference Include="System.ValueTuple" Version="$(SystemValueTuplePackageVersion)" />
+    <LatestPackageReference Include="Microsoft.CodeAnalysis.Common" Version="$(MicrosoftCodeAnalysisCommonPackageVersion)" />
+    <LatestPackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="$(MicrosoftCodeAnalysisCSharpPackageVersion)" />
+    <LatestPackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="$(MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion)" />
+    <LatestPackageReference Include="Microsoft.Extensions.DependencyModel" Version="$(MicrosoftExtensionsDependencyModelPackageVersion)" />
+    <LatestPackageReference Include="Microsoft.DotNet.GenAPI" Version="$(MicrosoftDotNetGenApiPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' != '.NETFramework' " Label="MSBuild">
-    <LatestPackageReference Include="Microsoft.Build.Framework" Version="15.8.166" />
-    <LatestPackageReference Include="Microsoft.Build.Tasks.Core" Version="15.8.166" />
-    <LatestPackageReference Include="Microsoft.Build.Utilities.Core" Version="15.8.166" />
+    <LatestPackageReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildFrameworkPackageVersion)" />
+    <LatestPackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildUtilitiesCorePackageVersion)" />
   </ItemGroup>
 
   <!-- These versions must not be renamed because they override the implicit package references from Arcade. -->
@@ -51,17 +55,12 @@ and are generated based on the last package release.
     <XUnitRunnerConsoleVersion>$(XUnitVersion)</XUnitRunnerConsoleVersion>
   </PropertyGroup>
 
-  <ItemGroup Label="External dependencies">
+  <ItemGroup Label="External dependencies" Condition="'$(DotNetBuildFromSource)' != 'true'">
     <LatestPackageReference Include="BenchmarkDotNet" Version="0.10.13" />
     <LatestPackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.6" />
     <LatestPackageReference Include="Microsoft.Azure.KeyVault" Version="2.3.2" />
     <LatestPackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.0.3" />
-    <LatestPackageReference Include="Microsoft.CodeAnalysis.Common" Version="2.8.0" />
-    <LatestPackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="2.8.0" />
-    <LatestPackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="2.8.0" />
-    <LatestPackageReference Include="Microsoft.Extensions.DependencyModel" Version="2.1.0" />
     <LatestPackageReference Include="Microsoft.TypeScript.MSBuild" Version="2.9.2" />
-    <LatestPackageReference Include="Microsoft.DotNet.GenAPI" Version="$(MicrosoftDotNetGenApiPackageVersion)" />
     <LatestPackageReference Include="Moq" Version="4.10.0" />
     <LatestPackageReference Include="Newtonsoft.Json" Version="11.0.2" />
     <LatestPackageReference Include="Polly.Extensions.Http" Version="3.0.0" />

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -52,6 +52,14 @@
     Versions below this comment are not managed by automation and can be changed as needed.
   -->
   <PropertyGroup Label="Manual">
+    <!-- Stable MSBuild task dependencies -->
+    <MicrosoftBuildFrameworkPackageVersion>15.8.166</MicrosoftBuildFrameworkPackageVersion>
+    <MicrosoftBuildUtilitiesCorePackageVersion>15.8.166</MicrosoftBuildUtilitiesCorePackageVersion>
+    <!-- Used for building and testing analyzers -->
+    <MicrosoftCodeAnalysisCommonPackageVersion>2.8.0</MicrosoftCodeAnalysisCommonPackageVersion>
+    <MicrosoftCodeAnalysisCSharpPackageVersion>2.8.0</MicrosoftCodeAnalysisCSharpPackageVersion>
+    <MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>2.8.0</MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>
+    <MicrosoftExtensionsDependencyModelPackageVersion>2.1.0</MicrosoftExtensionsDependencyModelPackageVersion>
     <!-- Stable dotnet/corefx packages no longer updated for .NET Core 3 -->
     <SystemMemoryPackageVersion>4.5.2</SystemMemoryPackageVersion>
     <SystemNetHttpPackageVersion>4.3.2</SystemNetHttpPackageVersion>

diff --git a/eng/Workarounds.AfterArcade.targets b/eng/Workarounds.AfterArcade.targets
@@ -32,4 +32,9 @@
     <PackageReference Remove="xunit.runner.visualstudio" />
     <PackageReference Remove="xunit.runner.console" />
   </ItemGroup>
+
+  <!-- Workaround https://github.com/dotnet/arcade/pull/2867 -->
+  <PropertyGroup Condition="'$(DotNetBuildFromSource)' == 'true'">
+    <BeforePack>$(BeforePack.Replace(';_AddSourcePackageSourceLinkFile', ''))</BeforePack>
+  </PropertyGroup>
 </Project>
diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh
diff --git a/eng/configure-toolset.sh b/eng/configure-toolset.sh
@@ -2,4 +2,6 @@
 # We could potentially try to find an existing installation that has all the required runtimes,
 # but it's unlikely one will be available.
 
-use_installed_dotnet_cli="false"
+if [ -z "${DotNetBuildFromSource:-}" ] || [ "${DotNetBuildFromSource:-}" = false ]; then
+    use_installed_dotnet_cli="false"
+fi
diff --git a/eng/scripts/ci-source-build.sh b/eng/scripts/ci-source-build.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+#
+# This script is meant for testing source build by imitating some of the input parameters and conditions.
+#
+
+set -euo pipefail
+
+scriptroot="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+reporoot="$(dirname "$(dirname "$scriptroot")")"
+
+# For local development, make a backup copy of this file first
+if [ ! -f "$reporoot/global.bak.json" ]; then
+    mv "$reporoot/global.json" "$reporoot/global.bak.json"
+fi
+
+# Detect the current version of .NET Core installed
+export SDK_VERSION=$(dotnet --version)
+echo "The ambient version of .NET Core SDK version = $SDK_VERSION"
+
+# Update the global.json file to match the current .NET environment
+cat "$reporoot/global.bak.json" | \
+    jq '.sdk.version=env.SDK_VERSION' | \
+    jq '.tools.dotnet=env.SDK_VERSION' | \
+    jq 'del(.tools.runtimes)' \
+    > "$reporoot/global.json"
+
+# Restore the original global.json file
+trap "{
+    mv "$reporoot/global.bak.json" "$reporoot/global.json"
+}" EXIT
+
+export DotNetBuildFromSource='true'
+"$reporoot/eng/common/build.sh" --restore --build --pack "$@"
diff --git a/eng/targets/ResolveReferences.targets b/eng/targets/ResolveReferences.targets
@@ -1,6 +1,7 @@
 <Project>
 
   <PropertyGroup>
+    <EnableCustomReferenceResolution Condition="'$(EnableCustomReferenceResolution)' == '' AND ('$(DotNetBuildFromSource)' != 'true' OR '$(ExcludeFromSourceBuild)' != 'true')">true</EnableCustomReferenceResolution>
     <ResolveReferencesDependsOn>
       ResolveCustomReferences;
       $(ResolveReferencesDependsOn);
@@ -45,7 +46,7 @@
     </Reference>
   </ItemDefinitionGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(EnableCustomReferenceResolution)' == 'true'">
     <Reference Update="@(Reference)">
       <IsSharedSource Condition="'%(IsSharedSource)' == '' AND $([System.String]::new('%(Identity)').EndsWith('.Sources'))">true</IsSharedSource>
     </Reference>
@@ -71,22 +72,10 @@
     </ProjectReference>
 
     <Reference Remove="@(_ProjectReferenceByAssemblyName)" />
-
-    <!-- Use _ReferenceTemp to workaround issues in Visual Studio which causes a conflict between Reference, packages, and projects. -->
-    <_ReferenceTemp Include="@(Reference)" />
-    <Reference Remove="@(Reference)" />
   </ItemGroup>
 
-  <!-- Ensure package output paths are available. -->
-  <Target Name="CreatePackageOutputPath" BeforeTargets="BeforeBuild">
-    <MakeDir Directories="$(PackageOutputPath)" />
-  </Target>
-
-  <Target Name="ResolveCustomReferences" BeforeTargets="CollectPackageReferences;ResolveAssemblyReferencesDesignTime;ResolveAssemblyReferences" Condition=" '$(TargetFramework)' != '' ">
+  <Target Name="ResolveCustomReferences" BeforeTargets="CollectPackageReferences;ResolveAssemblyReferencesDesignTime;ResolveAssemblyReferences" Condition=" '$(TargetFramework)' != '' AND '$(EnableCustomReferenceResolution)' == 'true'">
     <ItemGroup>
-      <Reference Include="@(_ReferenceTemp)" />
-      <_ReferenceTemp Remove="@(_ReferenceTemp)" />
-
       <!-- Ensure only content asset and msbuild props/targets are consumed from .Sources packages -->
       <Reference>
         <IncludeAssets Condition="'%(IsSharedSource)' == 'true'">ContentFiles;Build</IncludeAssets>

diff --git a/src/Caching/SqlServer/Directory.Build.props b/src/Caching/SqlServer/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- These projects depend on code which is not open source. -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>
diff --git a/src/Caching/StackExchangeRedis/Directory.Build.props b/src/Caching/StackExchangeRedis/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- These projects depend on a 3rd party source -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>
diff --git a/src/Configuration/Config.AzureKeyVault/Directory.Build.props b/src/Configuration/Config.AzureKeyVault/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- These projects depend on a 3rd party source. -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>
diff --git a/src/Configuration/Config.NewtonsoftJson/Directory.Build.props b/src/Configuration/Config.NewtonsoftJson/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- These projects depend on a 3rd party source. -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>
diff --git a/...ecification.Tests/src/Microsoft.Extensions.DependencyInjection.Specification.Tests.csproj b/...ecification.Tests/src/Microsoft.Extensions.DependencyInjection.Specification.Tests.csproj
@@ -11,6 +11,7 @@
     <IsUnitTestProject>false</IsUnitTestProject>
     <IsShipping>true</IsShipping>
     <HasReferenceAssembly>false</HasReferenceAssembly>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/src/Hosting/IntegrationTesting/src/Microsoft.Extensions.Hosting.IntegrationTesting.csproj b/src/Hosting/IntegrationTesting/src/Microsoft.Extensions.Hosting.IntegrationTesting.csproj
@@ -11,6 +11,7 @@
     <!-- This is used as a package by aspnet benchmarking infrastructure. It is meant for internal-use only. -->
     <IsShippingPackage>false</IsShippingPackage>
     <IsPackable>true</IsPackable>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/src/Hosting/WindowsServices/Directory.Build.props b/src/Hosting/WindowsServices/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- Source build does not support Windows. -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>
diff --git a/src/HttpClientFactory/Polly/Directory.Build.props b/src/HttpClientFactory/Polly/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- These projects depend on a 3rd party source -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>
diff --git a/src/Logging/Logging.AzureAppServices/Directory.Build.props b/src/Logging/Logging.AzureAppServices/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- These projects depend on a 3rd party source -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>
diff --git a/src/Logging/Logging.Testing/src/Microsoft.Extensions.Logging.Testing.csproj b/src/Logging/Logging.Testing/src/Microsoft.Extensions.Logging.Testing.csproj
@@ -10,6 +10,7 @@
     <!-- This is an internal package for testing purposes only. -->
     <IsPackable>true</IsPackable>
     <IsShipping>false</IsShipping>
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/src/TestingUtils/Directory.Build.props b/src/TestingUtils/Directory.Build.props
@@ -0,0 +1,8 @@
+<Project>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)..\, Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <!-- These projects depend on a 3rd party source -->
+    <ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
+  </PropertyGroup>
+</Project>