Add new Pipeline for Running Libs Tests with TestReadyToRun (#91229)

eng/pipelines/common/platform-matrix.yml

@@ -25,6 +25,7 @@ parameters:
 jobs:
 
 # Linux arm
+
 - ${{ if or(containsValue(parameters.platforms, 'linux_arm'), in(parameters.platformGroup, 'all', 'gcstress')) }}:
   - template: xplat-setup.yml
     parameters:
@@ -45,6 +46,7 @@ jobs:
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Linux armv6
+
 - ${{ if containsValue(parameters.platforms, 'linux_armv6') }}:
   - template: xplat-setup.yml
     parameters:

eng/pipelines/coreclr/crossgen2.yml

@@ -60,6 +60,29 @@ extends:
             displayNameArgs: R2R_CG2
             liveLibrariesBuildConfig: Release
 
+      - template: /eng/pipelines/common/platform-matrix.yml
+        parameters:
+          jobTemplate: /eng/pipelines/common/global-build-job.yml
+          helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
+          buildConfig: Release
+          platforms:
+            - linux_x64
+            - osx_x64
+            - windows_x64
+          jobParameters:
+            testGroup: innerloop
+            buildArgs: -s clr+libs+libs.tests
+                       -c $(_BuildConfig)
+                       /p:TestReadyToRun=true
+                       /p:ArchiveTests=true
+            nameSuffix: TestReadyToRun_Libraries
+            timeoutInMinutes: 360
+            postBuildSteps:
+              - template: /eng/pipelines/libraries/helix.yml
+                parameters:
+                  creator: dotnet-bot
+                  testRunNamePrefixSuffix: TestReadyToRun_$(_BuildConfig)
+
       # Run pri0 tests with hot/cold splitting enabled (only supported on x64 at the moment)
       # TODO: test on arm64 once supported
       - template: /eng/pipelines/common/platform-matrix.yml

eng/testing/tests.singlefile.targets

@@ -70,6 +70,10 @@
     <DefineConstants>$(DefineConstants);SINGLE_FILE_TEST_RUNNER</DefineConstants>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <DefineConstants>$(DefineConstants);TEST_READY_TO_RUN_COMPILED</DefineConstants>
+  </PropertyGroup>
+
   <Import Project="$(CoreCLRBuildIntegrationDir)Microsoft.DotNet.ILCompiler.SingleEntry.targets" Condition="'$(TestNativeAot)' == 'true'" />
 
   <ItemGroup Condition="'$(TestNativeAot)' == 'true'">
@@ -106,6 +110,114 @@
     </ItemGroup>
   </Target>
 
+  <!--
+      For TestReadyToRun, we need the whole framework to be R2R-compiled besides
+      the actual test assembly. However, this is a very lengthy process and it's
+      unnecessary in this case because we already have an R2R-compiled framework.
+      So, we have to tell the build that we already have these binaries so that it
+      doesn't build them again for each test.
+  -->
+  <Target Name="ExcludeExistingR2RBinaries"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          BeforeTargets="_PrepareForReadyToRunCompilation">
+    <PropertyGroup>
+      <ArtifactsNetCoreAppBundlePath>$(ArtifactsObjDir)Microsoft.NETCore.App.Bundle/</ArtifactsNetCoreAppBundlePath>
+      <ArtifactsNetCoreAppBundlePath>$(ArtifactsNetCoreAppBundlePath)$(Configuration)/$(NetCoreAppCurrent)/$(OutputRID)/output/</ArtifactsNetCoreAppBundlePath>
+      <ArtifactsNetCoreAppBundlePath>$(ArtifactsNetCoreAppBundlePath)shared/$(MicrosoftNetCoreAppFrameworkName)/$(PackageVersion)/</ArtifactsNetCoreAppBundlePath>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_BundleAssembliesToCopy Include="$(ArtifactsNetCoreAppBundlePath)*.dll" />
+      <ResolvedFileToPublish Remove="@(_BundleAssembliesToCopy)" MatchOnMetadata="Filename" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      For TestReadyToRun, each crossgen'd assembly needs to reference the whole
+      framework. For this, it looks at the contents of the same list that contains
+      all the assemblies we're going to R2R-compile. However, since we removed those
+      belonging to the framework we have ready to use in the previous target, then
+      the references list generated in _PrepareForReadyToRunCompilation is incomplete.
+      So, we add those missing assemblies only to the references list in this target.
+  -->
+  <Target Name="AddExistingR2RBinariesReferencesForCrossgen2"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          AfterTargets="_PrepareForReadyToRunCompilation">
+    <ItemGroup>
+      <_ReadyToRunAssembliesToReference Include="@(_BundleAssembliesToCopy)" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      For TestReadyToRun, debugging binaries bloat the test sizes way too much and
+      makes the Helix machines run out of disk. Since we don't need them for the
+      TestReadyToRun test runs, we remove them from the list that is later on copied
+      to the final location.
+  -->
+  <Target Name="RemoveDbgBinsFromTestR2ROutput"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          BeforeTargets="_CopyFilesMarkedCopyLocal">
+    <ItemGroup>
+      <ReferenceCopyLocalPaths
+          Remove="@(ReferenceCopyLocalPaths->WithMetadataValue('Extension', '.dbg'))" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      Very similarly to the previous target, we need to get rid of the debugging
+      binaries from the publishing directory as well.
+  -->
+  <Target Name="RemoveDbgBinsFromTestR2RPublish"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          BeforeTargets="_CopyResolvedFilesToPublishPreserveNewest">
+    <ItemGroup>
+      <_ResolvedFileToPublishPreserveNewest
+          Remove="@(_ResolvedFileToPublishPreserveNewest->WithMetadataValue('Extension', '.dbg'))" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+      As explained in Target 'ExcludeExistingR2RBinaries' up above, for TestReadyToRun
+      we need the fully R2R-compiled framework, but we already have it elsewhere. So,
+      once the test's specific stuff is constructed, we copy the R2R-compiled framework
+      to the test's self-contained directory so the test can use it when called.
+  -->
+  <Target Name="CopyExistingR2RBinaries"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          AfterTargets="_CopyResolvedFilesToPublishAlways">
+
+    <Copy SourceFiles="@(_BundleAssembliesToCopy)"
+          DestinationFolder="$(PublishDir)"
+          OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+          Retries="$(CopyRetryCount)"
+          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+          UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+          UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)" />
+
+  </Target>
+
+  <!--
+      There are a few tests that need a 'live-ref-pack', which is missing from the
+      publish directory in TestReadyToRun builds. This target copies it there.
+  -->
+  <Target Name="CopyLiveRefPackIfPresent"
+          Condition="'$(TestReadyToRun)' == 'true'"
+          AfterTargets="CopyExistingR2RBinaries">
+
+    <ItemGroup>
+      <OutDirLiveRefPackFiles Include="$(OutDir)live-ref-pack/*" />
+    </ItemGroup>
+
+    <Copy SourceFiles="@(OutDirLiveRefPackFiles)"
+          DestinationFolder="$(PublishDir)live-ref-pack"
+          OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+          Retries="$(CopyRetryCount)"
+          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+          UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+          UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)" />
+
+  </Target>
+
   <Target Name="__UpdateExcludedAssembliesFromSingleFile"
           Inputs="ExcludeFromSingleFile"
           Outputs="ResolvedFileToPublish"

src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj

@@ -25,8 +25,8 @@
 
   <ItemGroup>
     <BundleComponentReference Include="../Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj" />
-    <BundleComponentReference Include="../installers/dotnet-hostfxr.proj" />
-    <BundleComponentReference Include="../installers/dotnet-host.proj" />
+    <BundleComponentReference Include="../installers/dotnet-hostfxr.proj" Condition="'$(TestReadyToRun)' != 'true'" />
+    <BundleComponentReference Include="../installers/dotnet-host.proj" Condition="'$(TestReadyToRun)' != 'true'" />
   </ItemGroup>
 
   <Target Name="PublishToDisk">

src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs

@@ -30,6 +30,17 @@ public static int Main(string[] args)
         // The current RemoteExecutor implementation is not compatible with the SingleFileTestRunner.
         Environment.SetEnvironmentVariable("DOTNET_REMOTEEXECUTOR_SUPPORTED", "0");
 
+        // To detect ReadyToRun testing mode, we set a constant in
+        // eng/testing/tests.singlefile.targets, which we use in the following
+        // preprocessor directive. In the case that it is defined, we set an
+        // environment variable that we consume later to implement
+        // PlatformDetection.IsReadyToRunCompiled. This last value is used for the
+        // [ActiveIssue] annotations designed to exclude tests from running.
+
+#if TEST_READY_TO_RUN_COMPILED
+        Environment.SetEnvironmentVariable("TEST_READY_TO_RUN_MODE" ,"1");
+#endif
+
         var diagnosticSink = new ConsoleDiagnosticMessageSink();
         var testsFinished = new TaskCompletionSource();
         var testSink = new TestMessageSink();

src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs

@@ -211,6 +211,8 @@ private static bool GetLinqExpressionsBuiltWithIsInterpretingOnly()
         public static bool HasHostExecutable => HasAssemblyFiles; // single-file don't have a host
         public static bool IsSingleFile => !HasAssemblyFiles;
 
+        public static bool IsReadyToRunCompiled => Environment.GetEnvironmentVariable("TEST_READY_TO_RUN_MODE") == "1";
+
         private static volatile Tuple<bool> s_lazyNonZeroLowerBoundArraySupported;
         public static bool IsNonZeroLowerBoundArraySupported
         {

src/libraries/Directory.Build.props

@@ -85,6 +85,10 @@
     <EnableCoverageSupport Condition="'$(ContinuousIntegrationBuild)' != 'true'">true</EnableCoverageSupport>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <UseLocalAppHostPack>true</UseLocalAppHostPack>
+  </PropertyGroup>
+
   <!-- To enable the interpreter for mono desktop, we need to pass an env switch -->
   <PropertyGroup>
     <MonoEnvOptions Condition="'$(MonoEnvOptions)' == '' and '$(TargetsMobile)' != 'true' and '$(MonoForceInterpreter)' == 'true'">--interpreter</MonoEnvOptions>

src/libraries/System.Runtime/tests/System.Runtime.Tests/System/EnumTests.cs

@@ -1599,6 +1599,7 @@ public void GetValues_EnumTypeNotEnum_ThrowsArgumentException(Type enumType)
         private class ClassWithEnumConstraint<T> where T : Enum { }
 
         [Fact]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/94189", typeof(PlatformDetection), nameof(PlatformDetection.IsReadyToRunCompiled))]
         public void EnumConstraint_ThrowsArgumentException()
         {
             Type genericArgumentWithEnumConstraint = typeof(ClassWithEnumConstraint<>).GetGenericArguments()[0];

src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/JitInfoTests.cs

@@ -100,6 +100,7 @@ public void JitInfoIsNotPopulated()
 
         [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsReflectionEmitSupported))]
         [SkipOnMono("Mono does not track thread specific JIT information")]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/94189", typeof(PlatformDetection), nameof(PlatformDetection.IsReadyToRunCompiled))]
         public void JitInfoCurrentThreadIsPopulated()
         {
             TimeSpan t1_beforeCompilationTime = TimeSpan.Zero;

src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Type/TypeTests.Get.cs

@@ -55,6 +55,7 @@ public static IEnumerable<object[]> GetInterface_TestData()
 
         [Theory]
         [MemberData(nameof(GetInterface_TestData))]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/94189", typeof(PlatformDetection), nameof(PlatformDetection.IsReadyToRunCompiled))]
         public void GetInterface_Invoke_ReturnsExpected(Type type, string name, bool ignoreCase, Type expected)
         {
             if (!ignoreCase)

src/libraries/pretest.proj

@@ -26,7 +26,14 @@
     <ProjectReference Include="$(MonoProjectRoot)wasm\symbolicator\WasmSymbolicator.csproj" Condition="'$(TargetOS)' == 'browser'" />
 
     <!-- needed to test workloads for wasm -->
-    <ProjectReference Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Runtime.sfxproj" Pack="true" Condition="'$(TargetOS)' == 'browser' or '$(TargetOS)' == 'wasi'" />
+    <ProjectReference Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Runtime.sfxproj"
+                      Pack="true"
+                      Condition="'$(TargetOS)' == 'browser' or '$(TargetOS)' == 'wasi'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <ProjectReference Include="$(InstallerProjectRoot)pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj" />
+    <ProjectReference Include="$(InstallerProjectRoot)pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj" />
   </ItemGroup>
 
   <Import Project="$(RepositoryEngineeringDir)testing\wasm-provisioning.targets"

src/libraries/tests.proj

@@ -611,6 +611,31 @@
     -->
   </ItemGroup>
 
+  <!--
+      There is a decent number of hidden tests that fail with TestReadyToRun, and
+      it's proven to be a neverending task to flush all of them at once. So, we will
+      start by only enabling the tests we have confirmed work correctly, and we will
+      gradually enable the rest in subsequent PR's.
+
+      Tracking Issue for this work item: https://github.com/dotnet/runtime/issues/95928
+  -->
+  <ItemGroup Condition="'$(TestReadyToRun)' == 'true'">
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.Extensions.DependencyInjection/tests/DI.External.Tests/Microsoft.Extensions.DependencyInjection.ExternalContainers.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.ComponentModel.Composition/tests/System.ComponentModel.Composition.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Composition/tests/System.Composition.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Console/tests/System.Console.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime/tests/System.Dynamic.Runtime.Tests/System.Dynamic.Runtime.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.IO.FileSystem.Watcher/tests/System.IO.FileSystem.Watcher.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime/tests/System.Reflection.Tests/System.Reflection.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Reflection.TypeExtensions/tests/System.Reflection.TypeExtensions.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime/tests/System.Runtime.Extensions.Tests/System.Runtime.Extensions.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime.Loader/tests/DefaultContext/System.Runtime.Loader.DefaultContext.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Runtime.Loader/tests/System.Runtime.Loader.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Text.RegularExpressions/tests/FunctionalTests/System.Text.RegularExpressions.Tests.csproj" />
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Threading.Thread/tests/System.Threading.Thread.Tests.csproj" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Condition="'$(RunSmokeTestsOnly)' == 'true'"
                       Include="@(SmokeTestProject)" />
@@ -621,7 +646,11 @@
                       Include="@(GrpcTestProject)" />
   </ItemGroup>
 
-  <ItemGroup Condition="'$(RunSmokeTestsOnly)' != 'true' and '$(RunGrpcTestsOnly)' != 'true' and '$(RunHighAOTResourceRequiringTestsOnly)' != 'true' and '$(RunLimitedSetOfTests)' != 'true'">
+  <ItemGroup Condition="'$(RunSmokeTestsOnly)' != 'true'
+                        and '$(RunGrpcTestsOnly)' != 'true'
+                        and '$(RunHighAOTResourceRequiringTestsOnly)' != 'true'
+                        and '$(RunLimitedSetOfTests)' != 'true'">
+
     <ProjectReference Include="$(MSBuildThisFileDirectory)*\tests\**\*.Tests.csproj"
                       Exclude="@(ProjectExclusions)"
                       Condition="'$(TestAssemblies)' == 'true'"

src/tasks/Crossgen2Tasks/ShimFilesSimulatingLogicInSdkRepo/Microsoft.NET.CrossGen.props

@@ -1,6 +1,6 @@
 <!--
 ***********************************************************************************************
-Microsoft.NET.CrossGen.targets
+Microsoft.NET.CrossGen.props
 
 WARNING:  DO NOT MODIFY this file unless you are knowledgeable about MSBuild and have
           created a backup copy.  Incorrect changes to this file will make it
@@ -17,4 +17,4 @@ Copyright (c) .NET Foundation. All rights reserved.
   <UsingTask TaskName="PrepareForReadyToRunCompilation" AssemblyFile="$(MSBuildThisFileDirectory)Crossgen2Tasks.dll" />
   <UsingTask TaskName="ResolveReadyToRunCompilers" AssemblyFile="$(MSBuildThisFileDirectory)Crossgen2Tasks.dll" />
   <UsingTask TaskName="RunReadyToRunCompiler" AssemblyFile="$(MSBuildThisFileDirectory)Crossgen2Tasks.dll" />
-</Project>
+</Project>