Update src source build content

eng/pipelines/templates/jobs/vmr-build.yml

@@ -371,7 +371,7 @@ jobs:
           set -ex
 
           dockerVolumeArgs="-v $(sourcesPath):/vmr"
-          sourceOnlyArgs=''
+          customBuildArgs=''
           extraBuildProperties=''
 
           if [[ ! -z '${{ parameters.targetOS }}' ]]; then
@@ -382,11 +382,15 @@ jobs:
             extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}"
           fi
 
+          if [[ '${{ parameters.useDevVersions }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --dev"
+          fi
+
           if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
             if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
-              sourceOnlyArgs='--poison'
+              customBuildArgs="$customBuildArgs --poison"
             fi
-            sourceOnlyArgs="$sourceOnlyArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
+            customBuildArgs="$customBuildArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
           fi
 
           if [[ -n "${{ parameters.extraProperties }}" ]]; then
@@ -395,10 +399,10 @@ jobs:
 
           # Only use Docker when a container is specified
           if [[ -n "${{ parameters.container }}" ]]; then
-            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs)
+            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
           else
             cd $(sourcesPath)
-            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs)
+            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
           fi
 
         displayName: Run Tests
@@ -425,6 +429,10 @@ jobs:
         CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
 
+        if (Test-Path "artifacts/scenario-tests/") {
+            CopyWithRelativeFolders "artifacts/scenario-tests/" $targetFolder "*.binlog"
+        }
+
         if (Test-Path "artifacts/TestResults/*") {
             CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.binlog"
             CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.diff"
@@ -452,13 +460,11 @@ jobs:
         cd "$(sourcesPath)"
         find artifacts/log/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/log/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
-
-        if [ -d "artifacts/TestResults/" ]; then
-          find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
-          find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
-          find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
-          find artifacts/TestResults/ -type f -name "*.trx" -exec rsync -R {} -t ${targetFolder} \;
-        fi
+        [ -d "artifacts/scenario-tests/" ] && find artifacts/scenario-tests/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.trx" -exec rsync -R {} -t ${targetFolder} \;
 
         if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
           find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
@@ -490,13 +496,25 @@ jobs:
       condition: succeededOrFailed()
       continueOnError: true
       inputs:
-        testRunner: vSTest
+        testRunner: VSTest
         testResultsFiles: 'artifacts/TestResults/Release/*.trx'
         searchFolder: $(sourcesPath)
         mergeTestResults: true
         publishRunAttachments: true
         testRunTitle: Tests_$(Agent.JobName)
 
+    - task: PublishTestResults@2
+      displayName: Publish Scenario Test Results
+      condition: succeededOrFailed()
+      continueOnError: true
+      inputs:
+        testRunner: xUnit
+        testResultsFiles: 'artifacts/TestResults/**/scenario-tests/*.xml'
+        searchFolder: $(sourcesPath)
+        mergeTestResults: true
+        publishRunAttachments: true
+        testRunTitle: ScenarioTests_$(Agent.JobName)
+
   - task: CopyFiles@2
     inputs:
       SourceFolder: $(sourcesPath)/artifacts

src/Installer/redist-installer/redist-installer.csproj

@@ -16,11 +16,12 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\Layout\redist\redist.csproj" ReferenceOutputAssembly="false" />
+    <ProjectReference Include="$(RepoRoot)template_feed\*\*.csproj" ReferenceOutputAssembly="false" />
     <!--
       These two projects reference assets from multiple architectures, so they can't be built until we have a join point job that can pull assets from multiple legs.
       https://github.com/dotnet/source-build/issues/4336
     -->
-    <ProjectReference Include="..\..\Layout\redist\redist.csproj" ReferenceOutputAssembly="false" />
     <ProjectReference Include="projects\SdkResolver.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuild)' != 'true'" />
     <ProjectReference Include="projects\VSTemplateLocator.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuild)' != 'true'" />
   </ItemGroup>

src/SourceBuild/content/Directory.Build.props

@@ -160,6 +160,7 @@
     <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
     <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'AssetManifests'))</AssetManifestsIntermediateDir>
     <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
+    <DotNetSdkExtractDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk'))</DotNetSdkExtractDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
     <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>

src/SourceBuild/content/build.sh

@@ -88,7 +88,7 @@ exclude_ci_binary_log=false
 prepare_machine=false
 use_dev_versioning=false
 
-properties=''
+properties=()
 while [[ $# > 0 ]]; do
   opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
@@ -120,13 +120,13 @@ while [[ $# > 0 ]]; do
     # Source-only settings
     -source-only|-source-build|-so|-sb)
       sourceOnly=true
-      properties="$properties /p:DotNetBuildSourceOnly=true"
+      properties+=( "/p:DotNetBuildSourceOnly=true" )
       ;;
     -online)
-      properties="$properties /p:DotNetBuildWithOnlineFeeds=true"
+      properties+=( "/p:DotNetBuildWithOnlineFeeds=true" )
       ;;
     -poison)
-      properties="$properties /p:EnablePoison=true"
+      properties+=( "/p:EnablePoison=true" )
       ;;
     -release-manifest)
       releaseManifest="$2"
@@ -162,14 +162,14 @@ while [[ $# > 0 ]]; do
       ;;
 
     # Advanced settings
-    -build-tests)
-      properties="$properties /p:DotNetBuildTests=true"
+    -build-repo-tests)
+      properties+=( "/p:DotNetBuildTests=true" )
       ;;
     -ci)
       ci=true
       ;;
     -clean-while-building|-cwb)
-      properties="$properties /p:CleanWhileBuilding=true"
+      properties+=( "/p:CleanWhileBuilding=true" )
       ;;
     -excludecibinarylog|-nobl)
       exclude_ci_binary_log=true
@@ -178,13 +178,13 @@ while [[ $# > 0 ]]; do
       prepare_machine=true
       ;;
     -use-mono-runtime)
-      properties="$properties /p:SourceBuildUseMonoRuntime=true"
+      properties+=( "/p:SourceBuildUseMonoRuntime=true" )
       ;;
     -dev)
       use_dev_versioning=true
       ;;
     *)
-      properties="$properties $1"
+      properties+=( "$1" )
       ;;
   esac
 
@@ -198,7 +198,7 @@ if [[ "$ci" == true ]]; then
 fi
 
 if [[ "$use_dev_versioning" == true && "$sourceOnly" != true ]]; then
-  properties="$properties /p:UseOfficialBuildVersioning=false"
+  properties+=( "/p:UseOfficialBuildVersioning=false" )
 fi
 
 # Never use the global nuget cache folder
@@ -234,14 +234,14 @@ function Build {
       $targets \
       $bl \
       /p:Configuration=$configuration \
-      $properties
+      "${properties[@]}"
 
     ExitWithExitCode 0
 
   else
 
     if [ "$ci" == "true" ]; then
-      properties="$properties /p:ContinuousIntegrationBuild=true"
+      properties+=( "/p:ContinuousIntegrationBuild=true" )
     fi
 
     if [ "$test" != "true" ]; then
@@ -251,7 +251,7 @@ function Build {
       fi
 
       "$CLI_ROOT/dotnet" build-server shutdown
-      "$CLI_ROOT/dotnet" msbuild "$scriptroot/eng/init-source-only.proj" $initSourceOnlyBinaryLog $properties
+      "$CLI_ROOT/dotnet" msbuild "$scriptroot/eng/init-source-only.proj" $initSourceOnlyBinaryLog "${properties[@]}"
       # kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
       "$CLI_ROOT/dotnet" build-server shutdown
     fi
@@ -264,7 +264,7 @@ function Build {
       bl="/bl:\"$log_dir/Build.binlog\""
     fi
 
-    "$CLI_ROOT/dotnet" msbuild --restore "$project" $bl $targets $properties
+    "$CLI_ROOT/dotnet" msbuild --restore "$project" $bl $targets "${properties[@]}"
   fi
 }
 
@@ -331,9 +331,9 @@ if [[ "$sourceOnly" == "true" ]]; then
   # Support custom source built package locations
   if [ "$CUSTOM_PACKAGES_DIR" != "" ]; then
     if [ "$test" == "true" ]; then
-      properties="$properties /p:CustomSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR"
+      properties+=( "/p:CustomSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR" )
     else
-      properties="$properties /p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR"
+      properties+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR" )
     fi
   fi
 
@@ -365,7 +365,7 @@ if [[ "$sourceOnly" == "true" ]]; then
     packageVersionsPath="$CUSTOM_PACKAGES_DIR/PackageVersions.props"
   elif [ -d "$packagesArchiveDir" ]; then
     sourceBuiltArchive=$(find "$packagesArchiveDir" -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz')
-    if [ -f "${packagesPreviouslySourceBuiltDir}}PackageVersions.props" ]; then
+    if [ -f "${packagesPreviouslySourceBuiltDir}PackageVersions.props" ]; then
       packageVersionsPath=${packagesPreviouslySourceBuiltDir}PackageVersions.props
     elif [ -f "$sourceBuiltArchive" ]; then
       tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props
@@ -375,8 +375,9 @@ if [[ "$sourceOnly" == "true" ]]; then
 
   if [ ! -f "$packageVersionsPath" ]; then
     echo "Cannot find PackagesVersions.props.  Debugging info:"
-    echo "  Attempted archive path: $packagesArchiveDir"
     echo "  Attempted custom PVP path: $CUSTOM_PACKAGES_DIR/PackageVersions.props"
+    echo "  Attempted previously-source-built path: ${packagesPreviouslySourceBuiltDir}PackageVersions.props"
+    echo "  Attempted archive path: $packagesArchiveDir"
     exit 1
   fi
 

src/SourceBuild/content/eng/Version.Details.xml

@@ -2,9 +2,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24223.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24253.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>91ce5946fec9f81f15f81d20e1aab63527302410</Sha>
+      <Sha>020255bcf7d0b8beed7de05338d97396982ae527</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

src/SourceBuild/content/eng/Versions.props

@@ -23,8 +23,9 @@
       of a .NET major or minor release, prebuilts may be needed. When the release is mature, prebuilts
       are not necessary, and this property is removed from the file.
     -->
-    <PrivateSourceBuiltSdkVersion>9.0.100-preview.4.24223.1</PrivateSourceBuiltSdkVersion>
-    <PrivateSourceBuiltArtifactsVersion>9.0.100-preview.4.24223.1</PrivateSourceBuiltArtifactsVersion>
+    <PrivateSourceBuiltSdkVersion>9.0.100-preview.5.24256.1</PrivateSourceBuiltSdkVersion>
+    <PrivateSourceBuiltArtifactsVersion>9.0.100-preview.5.24256.1</PrivateSourceBuiltArtifactsVersion>
+    <PrivateSourceBuiltPrebuiltsVersion>9.0.100-preview.5.24256.1</PrivateSourceBuiltPrebuiltsVersion>
     <!-- command-line-api dependencies -->
     <SystemCommandLineVersion>2.0.0-beta4.24126.1</SystemCommandLineVersion>
     <!-- msbuild dependencies -->

src/SourceBuild/content/eng/allowed-vmr-binaries.txt

@@ -38,6 +38,10 @@ src/cecil/symbols/**/Test/Resources/assemblies/*.pdb
 src/cecil/symbols/**/Test/Resources/assemblies/*.dll
 src/cecil/symbols/**/Test/Resources/assemblies/*.mdb
 
+# diagnostics
+src/diagnostics/src/tests/Microsoft.FileFormats.UnitTests/TestBinaries/**/*
+src/diagnostics/src/tests/Microsoft.SymbolStore.UnitTests/TestBinaries/**/*
+
 # efcore
 src/efcore/test/EFCore.Sqlite.FunctionalTests/northwind.db # https://github.com/dotnet/source-build/issues/4326
 src/efcore/benchmark/EFCore.Sqlite.Benchmarks/AdventureWorks2014.db # https://github.com/dotnet/source-build/issues/4326

src/SourceBuild/content/eng/extract-sdk-archive.proj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+  </PropertyGroup>
+
+  <Target Name="ExtractSdkArchive"
+          BeforeTargets="AfterBuild"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion"
+          Inputs="$(SdkTarballPath)"
+          Outputs="$(DotNetSdkExtractDir)">
+    <MakeDir Directories="$(DotNetSdkExtractDir)" />
+    <Exec Condition="'$(ArchiveExtension)' == '.tar.gz'"
+          Command="tar -xzf $(SdkTarballPath) -C $(DotNetSdkExtractDir)" />
+    <Unzip Condition="'$(ArchiveExtension)' == '.zip'"
+           SourceFiles="$(SdkTarballPath)"
+           DestinationFolder="$(DotNetSdkExtractDir)" />
+  </Target>
+
+</Project>

src/SourceBuild/content/eng/finish-source-only.proj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj" />
+    <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
     <ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
   </ItemGroup>
 
@@ -59,13 +60,9 @@
           Outputs="$(SdkSymbolsTarball)">
     <PropertyGroup>
       <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
-      <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
     </PropertyGroup>
 
-    <MakeDir Directories="$(IntermediateSdkLayout)" />
-    <Exec Command="tar -xzf $(SdkTarballPath) -C $(IntermediateSdkLayout)" />
-
-    <CreateSdkSymbolsLayout SdkLayoutPath="$(IntermediateSdkLayout)"
+    <CreateSdkSymbolsLayout SdkLayoutPath="$(DotNetSdkExtractDir)"
                             AllSymbolsPath="$(IntermediateSymbolsRootDir)"
                             SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
                             FailOnMissingPDBs="false" />
@@ -76,7 +73,6 @@
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
     <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
-    <RemoveDir Directories="$(IntermediateSdkLayout)" />
   </Target>
 
   <!--

src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml

@@ -64,14 +64,14 @@ jobs:
     parameters:
       buildName: ${{ parameters.buildName }}
       architecture: ${{ parameters.architecture }}
-      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-${{ parameters.targetRid }}.tar.gz' 
+      patterns: '**/assets/Release/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-${{ parameters.targetRid }}.tar.gz' 
       displayName: Download Source Build SDK
 
   - template: ../steps/download-vmr-artifact.yml
     parameters:
       buildName: ${{ parameters.buildName }}
       architecture: ${{ parameters.architecture }}
-      patterns: '**/Private.SourceBuilt.Artifacts.+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*).${{ parameters.targetRid }}.tar.gz' 
+      patterns: '**/assets/Release/Private.SourceBuilt.Artifacts.+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*).${{ parameters.targetRid }}.tar.gz' 
       displayName: Download Source Built Artifacts
 
   - script: |

src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml

@@ -32,7 +32,6 @@ jobs:
   steps:
   - script: |
       vmrSrcDir="$(Build.SourcesDirectory)/src"
-
       # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path,
       # allowing a job to be run for each src repo.
       matrix=""

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/CreateSdkSymbolsLayout.cs

@@ -89,8 +89,9 @@ private IList<string> GenerateSymbolsLayout(Hashtable allPdbGuids)
                     {
                         // Copy matching pdb to symbols path, preserving sdk binary's hierarchy
                         string sourcePath = (string)allPdbGuids[debugId]!;
+                        string fileRelativePath = file.Substring(SdkLayoutPath.Length);
                         string destinationPath =
-                            file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
+                            Path.Combine(SdkSymbolsLayoutPath, fileRelativePath)
                                 .Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
 
                         Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);

src/SourceBuild/content/global.json

@@ -1,10 +1,10 @@
 {
   "tools": {
-    "dotnet": "9.0.100-preview.4.24223.7"
+    "dotnet": "9.0.100-preview.5.24253.16"
   },
   "msbuild-sdks": {
     "Microsoft.Build.NoTargets": "3.7.0",
     "Microsoft.Build.Traversal": "3.4.0",
-    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24223.3"
+    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24253.1"
   }
 }