Add mono interpreter test leg to CI (#35568)

This change enables the interpreter on CI as well as providing an option for the local test run.

eng/pipelines/libraries/base-job.yml

@@ -17,6 +17,7 @@ parameters:
   variables: {}
   name: ''
   displayName: ''
+  testDisplayName: ''
   testScope: ''
   pool: ''
 
@@ -93,7 +94,10 @@ jobs:
           - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}:
             - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
             - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)'
-          - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }}
+          - ${{ if eq(parameters.testDisplayName, '') }}:
+            - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }}
+          - ${{ if ne(parameters.testDisplayName, '') }}:
+            - _testRunNamePrefixSuffix: ${{ parameters.testDisplayName }}
 
         # Windows variables
         - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:

eng/pipelines/libraries/helix-queues-setup.yml

@@ -50,26 +50,30 @@ jobs:
 
     # Linux x64
     - ${{ if eq(parameters.platform, 'Linux_x64') }}:
-      - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
-        - Centos.7.Amd64.Open
-        - RedHat.7.Amd64.Open
-        - Debian.9.Amd64.Open
-        - Ubuntu.1604.Amd64.Open
-        - Ubuntu.1804.Amd64.Open
-        - SLES.12.Amd64.Open
-        - SLES.15.Amd64.Open
-        - (Fedora.30.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-helix-4f8cef7-20200121150022
-        - (Fedora.32.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-helix-20200512010618-efb9f14
-        - (Ubuntu.1910.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64-cfcfd50-20191030180623
-        - (Debian.10.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-bfcd90a-20200121150006
-      - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}:
-        - Centos.7.Amd64.Open
-        - RedHat.7.Amd64.Open
+      - ${{ if eq(parameters.jobParameters.interpreter, '') }}:
+        - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
+          - Centos.7.Amd64.Open
+          - RedHat.7.Amd64.Open
+          - Debian.9.Amd64.Open
+          - Ubuntu.1604.Amd64.Open
+          - Ubuntu.1804.Amd64.Open
+          - SLES.12.Amd64.Open
+          - SLES.15.Amd64.Open
+          - (Fedora.30.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-helix-4f8cef7-20200121150022
+          - (Fedora.32.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-helix-20200512010618-efb9f14
+          - (Ubuntu.1910.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64-cfcfd50-20191030180623
+          - (Debian.10.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64-bfcd90a-20200121150006
+        - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}:
+          - Centos.7.Amd64.Open
+          - RedHat.7.Amd64.Open
+          - Debian.9.Amd64.Open
+          - Ubuntu.1604.Amd64.Open
+          - Ubuntu.1804.Amd64.Open
+          - SLES.15.Amd64.Open
+          - (Fedora.30.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-helix-4f8cef7-20200121150022
+      - ${{ if eq(parameters.jobParameters.interpreter, 'true') }}:
+        # Limiting interp runs as we don't need as much coverage.
         - Debian.9.Amd64.Open
-        - Ubuntu.1604.Amd64.Open
-        - Ubuntu.1804.Amd64.Open
-        - SLES.15.Amd64.Open
-        - (Fedora.30.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-helix-4f8cef7-20200121150022
 
     # OSX x64
     - ${{ if eq(parameters.platform, 'OSX_x64') }}:

eng/pipelines/libraries/helix.yml

@@ -9,6 +9,7 @@ parameters:
   targetOS: ''
   testRunNamePrefixSuffix: ''
   testScope: 'innerloop' # innerloop | outerloop | all
+  interpreter: ''
   condition: always()
   extraHelixArguments: ''
 
@@ -20,6 +21,7 @@ steps:
             /p:TargetArchitecture=${{ parameters.archGroup }}
             /p:Configuration=${{ parameters.configuration }}
             /p:TargetOS=${{ parameters.targetOS }}
+            /p:MonoEnableInterpreter=${{ parameters.interpreter }}
             /p:TestScope=${{ parameters.testScope }}
             /p:TestRunNamePrefixSuffix=${{ parameters.testRunNamePrefixSuffix }}
             /p:HelixBuild=$(Build.BuildNumber)

eng/pipelines/libraries/run-test-job.yml

@@ -7,6 +7,8 @@ parameters:
   isOfficialBuild: false
   liveRuntimeBuildConfig: ''
   runtimeFlavor: 'coreclr'
+  runtimeDisplayName: 'coreclr'
+  interpreter: ''
   timeoutInMinutes: 150
   pool: ''
   runtimeVariant: ''
@@ -35,11 +37,13 @@ jobs:
       pool: ${{ parameters.pool }}
       testScope: ${{ parameters.testScope }}
       ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
-        displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }}
-        name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeFlavor) }}
+        displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
+        name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }}
       ${{ if eq(parameters.liveRuntimeBuildConfig, '') }}:
         displayName: 'Test Run'
         name: test_run
+      ${{ if eq(parameters.interpreter, 'true') }}:
+        testDisplayName: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }}
 
       dependsOn:
       - ${{ if notIn(parameters.framework, 'allConfigurations', 'net472') }}:
@@ -91,6 +95,7 @@ jobs:
             msbuildScript: $(_msbuildCommand)
             framework: ${{ parameters.framework }}
             testScope: ${{ parameters.testScope }}
+            interpreter: ${{ parameters.interpreter }}
             creator: dotnet-bot
             helixToken: ''
             testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix)

eng/pipelines/runtime.yml

@@ -826,6 +826,37 @@ jobs:
     jobParameters:
       isOfficialBuild: false
       isFullMatrix: ${{ variables.isFullMatrix }}
+      runtimeDisplayName: mono
+      testScope: innerloop
+      liveRuntimeBuildConfig: release
+      dependsOnTestBuildConfiguration: ${{ variables.debugOnPrReleaseOnRolling }}
+      dependsOnTestArchitecture: x64
+      condition: >-
+        or(
+          eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true),
+          eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true),
+          eq(variables['isFullMatrix'], true))
+
+#
+# Libraries Release Test Execution against a release mono interpreter runtime.
+# Only when libraries or mono changed
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/libraries/run-test-job.yml
+    runtimeFlavor: mono
+    buildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
+    platforms:
+    # - Windows_NT_x64
+    #- OSX_x64
+    #- Linux_arm64
+    - Linux_x64
+    helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
+    jobParameters:
+      isOfficialBuild: false
+      isFullMatrix: ${{ variables.isFullMatrix }}
+      interpreter: true
+      runtimeDisplayName: mono_interpreter
       testScope: innerloop
       liveRuntimeBuildConfig: release
       dependsOnTestBuildConfiguration: ${{ variables.debugOnPrReleaseOnRolling }}

eng/testing/tests.props

@@ -33,6 +33,12 @@
     <PlatformManifestFile />
   </PropertyGroup>
 
+  <!-- Provide runtime options to Mono (interpreter, aot, debugging, etc) -->
+  <ItemGroup Condition="'$(MonoEnvOptions)' != ''">
+    <RunScriptCommands Condition="'$(TargetsWindows)' == 'true'" Include="set MONO_ENV_OPTIONS='$(MonoEnvOptions)'" />
+    <RunScriptCommands Condition="'$(TargetsWindows)' != 'true'" Include="export MONO_ENV_OPTIONS='$(MonoEnvOptions)'" />
+  </ItemGroup>
+
   <!--
     Unit/Functional/Integration test support.
     Supported runners: xunit.

src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs

@@ -22,6 +22,7 @@ public static partial class PlatformDetection
         public static bool IsNetCore => Environment.Version.Major >= 5 || RuntimeInformation.FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase);
         public static bool IsMonoRuntime => Type.GetType("Mono.RuntimeStructs") != null;
         public static bool IsMonoInterpreter => GetIsRunningOnMonoInterpreter();
+        public static bool IsNotMonoInterpreter => !IsMonoInterpreter;
         public static bool IsFreeBSD => RuntimeInformation.IsOSPlatform(OSPlatform.Create("FREEBSD"));
         public static bool IsNetBSD => RuntimeInformation.IsOSPlatform(OSPlatform.Create("NETBSD"));
         public static bool IsiOS => RuntimeInformation.IsOSPlatform(OSPlatform.Create("IOS"));

src/libraries/Directory.Build.props

@@ -330,6 +330,11 @@
     <EnableCoverageSupport Condition="'$(ContinuousIntegrationBuild)' != 'true'">true</EnableCoverageSupport>
   </PropertyGroup>
 
+  <!-- To enable the interpreter for mono, we need to pass an env switch -->
+  <PropertyGroup>
+    <MonoEnvOptions Condition="'$(MonoEnvOptions)' == '' and '$(MonoEnableInterpreter)' == 'true'">--interpreter</MonoEnvOptions>
+  </PropertyGroup>
+
   <Import Project="$(RepositoryEngineeringDir)testing\tests.props" Condition="'$(EnableTestSupport)' == 'true'" />
 
   <!-- Use msbuild path functions as that property is used in bash scripts. -->

src/libraries/System.Drawing.Common/tests/AssemblyInfo.cs

@@ -0,0 +1,8 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Xunit;
+
+[assembly: ActiveIssue("https://github.com/dotnet/runtime/issues/35917", typeof(PlatformDetection), nameof(PlatformDetection.IsNotMonoInterpreter))]

src/libraries/System.Drawing.Common/tests/StringFormatTests.cs

@@ -84,7 +84,7 @@ public void Ctor_Format()
             }
         }
 
-        [Fact]
+        [ConditionalFact(Helpers.IsDrawingSupported)]
         public void Ctor_NullFormat_ThrowsArgumentNullException()
         {
             AssertExtensions.Throws<ArgumentNullException>("format", () => new StringFormat(null));

src/libraries/System.Drawing.Common/tests/System.Drawing.Common.Tests.csproj

@@ -5,6 +5,7 @@
     <TargetFrameworks>$(NetCoreAppCurrent);$(NetFrameworkCurrent)</TargetFrameworks>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="BitmapTests.cs" />
     <Compile Include="BrushTests.cs" />
     <Compile Include="BrushesTests.cs" />

src/libraries/System.Net.ServicePoint/tests/ServicePointManagerTest.cs

@@ -284,6 +284,7 @@ public static void FindServicePoint_ReturnsCachedServicePoint()
         }
 
         [Fact]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/36217", typeof(PlatformDetection), nameof(PlatformDetection.IsNotMonoInterpreter))]
         public static void FindServicePoint_Collectible()
         {
             RemoteExecutor.Invoke(() =>

src/libraries/System.Runtime.InteropServices/tests/System/Runtime/InteropServices/Marshal/ReadWrite/ByteTests.cs

@@ -132,6 +132,7 @@ public void ReadByte_StructWithReferenceTypes_ReturnsExpected()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void ReadByte_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.ReadByte(IntPtr.Zero));
@@ -159,6 +160,7 @@ public void ReadByte_NotReadable_ThrowsArgumentException()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void WriteByte_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.WriteByte(IntPtr.Zero, 0));

src/libraries/System.Runtime.InteropServices/tests/System/Runtime/InteropServices/Marshal/ReadWrite/Int16Tests.cs

@@ -127,6 +127,7 @@ public void ReadInt16_StructWithReferenceTypes_ReturnsExpected()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void ReadInt16_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.ReadInt16(IntPtr.Zero));
@@ -154,6 +155,7 @@ public void ReadInt16_NotReadable_ThrowsArgumentException()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void WriteInt16_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.WriteInt16(IntPtr.Zero, 0));

src/libraries/System.Runtime.InteropServices/tests/System/Runtime/InteropServices/Marshal/ReadWrite/Int32Tests.cs

@@ -129,6 +129,7 @@ public void ReadInt32_StructWithReferenceTypes_ReturnsExpected()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void ReadInt32_ZeroPoint_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.ReadInt32(IntPtr.Zero));
@@ -156,6 +157,7 @@ public void ReadInt32_NotReadable_ThrowsArgumentException()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void WriteInt32_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.WriteInt32(IntPtr.Zero, 0));

src/libraries/System.Runtime.InteropServices/tests/System/Runtime/InteropServices/Marshal/ReadWrite/Int64Tests.cs

@@ -141,6 +141,7 @@ public void ReadInt64_StructWithReferenceTypes_ReturnsExpected()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void ReadInt64_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.ReadInt64(IntPtr.Zero));
@@ -168,6 +169,7 @@ public void ReadInt64_NotReadable_ThrowsArgumentException()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void WriteInt64_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.WriteInt64(IntPtr.Zero, 0));

src/libraries/System.Runtime.InteropServices/tests/System/Runtime/InteropServices/Marshal/ReadWrite/IntPtrTests.cs

@@ -136,6 +136,7 @@ public void ReadIntPtr_StructWithReferenceTypes_ReturnsExpected()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void ReadIntPtr_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.ReadIntPtr(IntPtr.Zero));
@@ -163,6 +164,7 @@ public void ReadIntPtr_NotReadable_ThrowsArgumentException()
         }
 
         [Fact]
+        [SkipOnMono("Marshal.ReadByte will not be implemented in Mono, see https://github.com/mono/mono/issues/15085.")]
         public void WriteIntPtr_ZeroPointer_ThrowsException()
         {
             AssertExtensions.ThrowsAny<AccessViolationException, NullReferenceException>(() => Marshal.WriteIntPtr(IntPtr.Zero, (IntPtr)0));

src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs

@@ -24,6 +24,7 @@ public partial class BinaryFormatterTests : FileCleanupTestBase
         // On 32-bit we can't test these high inputs as they cause OutOfMemoryExceptions.
         [ConditionalTheory(typeof(Environment), nameof(Environment.Is64BitProcess))]
         [SkipOnCoreClr("Long running tests: https://github.com/dotnet/runtime/issues/11191", RuntimeConfiguration.Checked)]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/35915", typeof(PlatformDetection), nameof(PlatformDetection.IsNotMonoInterpreter))]
         [InlineData(2 * 6_584_983 - 2)] // previous limit
         [InlineData(2 * 7_199_369 - 2)] // last pre-computed prime number
         public void SerializeHugeObjectGraphs(int limit)

src/libraries/System.Text.Json/tests/Serialization/Stream.Collections.cs

@@ -18,6 +18,7 @@ namespace System.Text.Json.Serialization.Tests
     public static partial class StreamTests
     {
         [Fact]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/35927", typeof(PlatformDetection), nameof(PlatformDetection.IsNotMonoInterpreter))]
         public static async Task HandleCollectionsAsync()
         {
             await RunTest<string>();

src/libraries/System.Threading.Tasks/tests/AssemblyInfo.cs

@@ -0,0 +1,8 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Xunit;
+
+[assembly: ActiveIssue("https://github.com/dotnet/runtime/issues/35916", typeof(PlatformDetection), nameof(PlatformDetection.IsNotMonoInterpreter))]

src/libraries/System.Threading.Tasks/tests/System.Threading.Tasks.Tests.csproj

@@ -5,6 +5,7 @@
     <TargetFrameworks>$(NetCoreAppCurrent)</TargetFrameworks>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="AssemblyInfo.cs" />
     <Compile Include="Helpers.cs" />
     <Compile Include="XunitAssemblyAttributes.cs" />
     <Compile Include="UnwrapTests.cs" />

src/libraries/sendtohelix.proj

@@ -3,7 +3,6 @@
     <!-- Set helix build to build number if available -->
     <HelixBuild Condition="'$(HelixBuild)' == ''">$(BUILD_BUILDNUMBER)</HelixBuild>
     <HelixBuild Condition="'$(HelixBuild)' == ''">default</HelixBuild>
-
     <!-- For arm/arm64 we set a 30 min timeout temporarily until we split up slow test assemblies. -->
     <TimeoutInSeconds Condition="'$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm'">1800</TimeoutInSeconds>
     <TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">900</TimeoutInSeconds>
@@ -59,6 +58,9 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(HelixCommand)' == ''">
+    <HelixPreCommands Condition="'$(MonoEnvOptions)' != '' and '$(OSGroup)' == 'Windows_NT'">set MONO_ENV_OPTIONS='$(MonoEnvOptions)'</HelixPreCommands>
+    <HelixPreCommands Condition="'$(MonoEnvOptions)' != '' and '$(OSGroup)' != 'Windows_NT'">export MONO_ENV_OPTIONS='$(MonoEnvOptions)'</HelixPreCommands>
+
     <!--
       For windows we need to use call, since the command is going to be called from a bat script created by Helix
       and we exit /b at the end of RunTests.cmd, Helix runs some other commands after ours within the bat script,

src/mono/mono.proj

@@ -3,6 +3,7 @@
 
   <PropertyGroup>
     <MonoCrossDir Condition="'$(MonoCrossDir)' == '' and '$(ROOTFS_DIR)' != ''">$(ROOTFS_DIR)</MonoCrossDir>
+    <MonoEnableInterpreter Condition="'$(MonoEnableInterpreter)' == ''">false</MonoEnableInterpreter>
     <DotNetExec Condition="'$(OS)' == 'Windows_NT'">dotnet.exe</DotNetExec>
     <DotNetExec Condition="'$(DotNetExec)' == ''">dotnet</DotNetExec>
     <LocalDotnetDir>$(RepoRoot).dotnet</LocalDotnetDir>