Merge branch 'dev/infrastructure' into dev/infra-merge1

docs/workflow/testing/coreclr/test-configuration.md

@@ -37,15 +37,27 @@ Test cases are categorized by priority level. The most important subset should b
         // The .NET Foundation licenses this file to you under the MIT license.
         // See the LICENSE file in the project root for more information.
     ```
-* Disable building of a test by conditionally setting the `<DisableProjectBuild>` property.
-    * e.g. `<DisableProjectBuild Condition=" '$(Platform)' == 'arm64' ">true</DisableProjectBuild>`
+* The managed portion of all tests should be able to build on any platform.
+In fact in CI the managed portion of all tests will be built on OSX.
+Each target will run a subset of the tests built on OSX.
+Therefore the managed portion of each test **must not contain**:
+  * Target platform dependent conditionally compiled code.
+  * Target platform dependent conditionally included files
+  * Target platform dependent conditional `<DefineConstants/>`
+* Disable building and running a test on selected Targets by conditionally setting the `<CLRTestTargetUnsupported>` property.
+    * e.g. `<CLRTestTargetUnsupported Condition="'$(TargetArchitecture)' == 'arm64'">true</CLRTestTargetUnsupported>`
+* Disable building of a test by unconditionally setting the `<DisableProjectBuild>` property.
+    * e.g. `<DisableProjectBuild>true</DisableProjectBuild>`
 * Exclude test from GCStress runs by adding the following to the csproj:
     * `<GCStressIncompatible>true</GCStressIncompatible>`
 * Exclude test from JIT stress runs runs by adding the following to the csproj:
     * `<JitOptimizationSensitive>true</JitOptimizationSensitive>`
 * Add NuGet references by updating the following [test project](https://github.com/dotnet/runtime/blob/master/src/coreclr/tests/src/Common/test_dependencies/test_dependencies.csproj).
 * Get access to System.Private.CoreLib types and methods that are not exposed via public surface by adding the following to the csproj:
     * `<ReferenceSystemPrivateCoreLib>true</ReferenceSystemPrivateCoreLib>`
+* Any System.Private.CoreLib types and methods used by tests must be available for building on all platforms.
+This means there must be enough implementation for the C# compiler to find the referenced types and methods. Unsupported target platforms
+should simply `throw new PlatformNotSupportedException()` in its dummy method implementations.
 * Update exclusion list at [tests/issues.targets](https://github.com/dotnet/runtime/blob/master/src/coreclr/tests/issues.targets) if the test fails due to active bug.
 
 ### Creating a C# test project

docs/workflow/testing/coreclr/unix-test-instructions.md

@@ -55,21 +55,26 @@ Please use the following command for help.
 
 ### Unsupported and temporarily disabled tests
 
-Unsupported tests outside of Windows have two annotations in their csproj to
-ignore them when run.
+To support building all tests for all targets on single target, we use
+the conditional property
 
 ```xml
-<TestUnsupportedOutsideWindows>true</TestUnsupportedOutsideWindows>
+<CLRTestTargetUnsupported Condition="...">true</CLRTestTargetUnsupported>
 ```
 
-This will write in the bash target to skip the test by returning a passing value if run outside Windows.
+This property disables building of a test in a default build. It also
+disables running a test in the bash/batch wrapper scripts. It allows the
+test to be built on any target in CI when the `allTargets` option is
+passed to the `build-test.*` scripts.
+
+Tests which never should be built or run are marked
 
-In addition:
 ```xml
-<DisableProjectBuild Condition="'$(TargetsUnix)' == 'true'">true</DisableProjectBuild>
+<DisableProjectBuild>true</DisableProjectBuild>
 ```
 
-Is used to disable the build, that way if building on Unix cycles are saved building/running.
+This propoerty should not be conditioned on Target properties to allow
+all tests to be built for `allTargets`.
 
 PAL tests
 ---------

eng/pipelines/common/platform-matrix.yml

@@ -19,7 +19,6 @@ parameters:
   helixQueuesTemplate: ''
   stagedBuild: false
   # When set to false, suppresses reuse of OSX managed build artifacts (for pipelines without an OSX obj)
-  managedOsxBuild: true
   # When set to true, passes the 'platforms' value as a job parameter also named 'platforms'.
   # Handled as an opt-in parameter to avoid excessive yaml.
   passPlatforms: false
@@ -47,7 +46,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Linux
         crossrootfsDir: '/crossrootfs/arm'
         ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -71,7 +69,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Linux
         crossrootfsDir: '/crossrootfs/arm64'
         ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -100,11 +97,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        ${{ if ne(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: Linux
-          managedTestBuildOsSubgroup: _musl
-        ${{ if eq(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Linux musl arm64
@@ -128,7 +120,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Linux
         crossrootfsDir: '/crossrootfs/arm64'
         ${{ insert }}: ${{ parameters.jobParameters }}
 
@@ -152,10 +143,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        ${{ if ne(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: Linux
-        ${{ if eq(parameters.managedOsxBuild, true) }}:
-          managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # WebAssembly
@@ -305,7 +292,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # tvOS arm64
@@ -325,7 +311,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # iOS x64
@@ -345,7 +330,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # iOS x86
@@ -385,7 +369,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # iOS arm64
@@ -405,12 +388,11 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # macOS x64
 
-- ${{ if or(containsValue(parameters.platforms, 'OSX_x64'), eq(parameters.platformGroup, 'all')) }}:
+- ${{ if or(containsValue(parameters.platforms, 'OSX_x64'), containsValue(parameters.platforms, 'CoreClrTestBuildHost'), eq(parameters.platformGroup, 'all')) }}:
   - template: xplat-setup.yml
     parameters:
       jobTemplate: ${{ parameters.jobTemplate }}
@@ -425,7 +407,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: OSX
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows x64
@@ -445,7 +426,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows x86
@@ -465,7 +445,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows arm
@@ -484,7 +463,6 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows arm64
@@ -504,5 +482,4 @@ jobs:
         ${{ if eq(parameters.passPlatforms, true) }}:
           platforms: ${{ parameters.platforms }}
         helixQueueGroup: ${{ parameters.helixQueueGroup }}
-        managedTestBuildOsGroup: Windows_NT
         ${{ insert }}: ${{ parameters.jobParameters }}

eng/pipelines/common/templates/runtimes/build-test-job.yml

@@ -39,8 +39,6 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
-    managedTestBuildOsGroup: ${{ parameters.osGroup }}
-    managedTestBuildOsSubgroup: ${{ parameters.osSubgroup }}
     container: ${{ parameters.container }}
     runtimeVariant: ${{ parameters.runtimeVariant }}
     testGroup: ${{ parameters.testGroup }}
@@ -53,13 +51,13 @@ jobs:
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:
       continueOnError: true
 
+    # Compute job name from template parameters
     ${{ if eq(parameters.testGroup, 'innerloop') }}:
-      name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
-
+      name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
     ${{ if ne(parameters.testGroup, 'innerloop') }}:
-      name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
 
     # Since the condition is being altered, merge the default with the additional conditions.
     # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions
@@ -118,7 +116,7 @@ jobs:
         displayName: Disk Usage before Build
     
     # Build managed test components
-    - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) allTargets skipstressdependencies skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
       displayName: Build managed test components
 
     - ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}: 
@@ -132,11 +130,11 @@ jobs:
       parameters:
         rootFolder: $(managedTestArtifactRootFolderPath)
         includeRootFolder: false
-        archiveType: $(archiveType)
-        tarCompression: $(tarCompression)
-        archiveExtension: $(archiveExtension)
-        artifactName: $(managedTestArtifactName)
-        displayName: 'managed test components'
+        archiveExtension: '.tar.gz'
+        archiveType: tar
+        tarCompression: gz
+        artifactName: $(managedGenericTestArtifactName)
+        displayName: 'managed test components (generic)'
 
 
     # Publish .packages/microsoft.net.sdk.il needed for traversing
@@ -145,9 +143,9 @@ jobs:
       parameters:
         rootFolder: $(microsoftNetSdkIlFolderPath)
         includeRootFolder: false
-        archiveType: $(archiveType)
-        tarCompression: $(tarCompression)
-        archiveExtension: $(archiveExtension)
+        archiveType: tar
+        tarCompression: gz
+        archiveExtension: '.tar.gz'
         artifactName: $(microsoftNetSdkIlArtifactName)
         displayName: 'Microsoft.NET.Sdk.IL package'
 
@@ -157,6 +155,6 @@ jobs:
       displayName: Publish Logs
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}'
+        artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_AnyOS_AnyCPU_$(buildConfig)_Lib${{ parameters.liveLibrariesBuildConfig }}_${{ parameters.testGroup }}'
       continueOnError: true
       condition: always()

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -3,8 +3,6 @@ parameters:
   archType: ''
   osGroup: ''
   osSubgroup: ''
-  managedTestBuildOsGroup: ''
-  managedTestBuildOsSubgroup: ''
   container: ''
   testGroup: ''
   crossrootfsDir: ''
@@ -37,8 +35,6 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
-    managedTestBuildOsGroup: ${{ parameters.managedTestBuildOsGroup }}
-    managedTestBuildOsSubgroup: ${{ parameters.managedTestBuildOsSubgroup }}
     container: ${{ parameters.container }}
     testGroup: ${{ parameters.testGroup }}
     crossrootfsDir: ${{ parameters.crossrootfsDir }}
@@ -57,9 +53,9 @@ jobs:
     dependsOn:
     - ${{ if ne(parameters.corefxTests, true) }}:
       - ${{ if eq(parameters.testGroup, 'innerloop') }}:
-        - '${{ parameters.runtimeFlavor }}_common_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+        - '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{parameters.buildConfig }}'
       - ${{ if ne(parameters.testGroup, 'innerloop') }}:
-        - '${{ parameters.runtimeFlavor }}_common_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+        - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}'
     - ${{ if ne(parameters.stagedBuild, true) }}:
       - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
@@ -195,9 +191,9 @@ jobs:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
           unpackFolder: '$(managedTestArtifactRootFolderPath)'
-          artifactFileName: '$(managedTestArtifactName)$(archiveExtension)'
-          artifactName: '$(managedTestArtifactName)'
-          displayName: 'managed test artifacts (built on ${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }})'
+          artifactFileName: '$(managedGenericTestArtifactName).tar.gz'
+          artifactName: '$(managedGenericTestArtifactName)'
+          displayName: 'generic managed test artifacts'
 
 
     # Download product binaries directory
@@ -218,13 +214,14 @@ jobs:
           artifactName: '$(coreClrProductArtifactName)'
           displayName: 'CoreCLR product download for Mono'
 
+
     # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing
     # ilproj test projects during copynativeonly.
     - ${{ if ne(parameters.corefxTests, true) }}:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
           unpackFolder: '$(microsoftNetSdkIlFolderPath)'
-          artifactFileName: '$(microsoftNetSdkIlArtifactName)$(archiveExtension)'
+          artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz'
           artifactName: '$(microsoftNetSdkIlArtifactName)'
           displayName: 'Microsoft.NET.Sdk.IL package'
 
@@ -245,7 +242,7 @@ jobs:
     # the native artifacts to the final test folders is dependent on availability of the
     # managed test artifacts.
     - ${{ if ne(parameters.corefxTests, true) }}:
-      - script: $(coreClrRepoRootDir)build-test$(scriptExt) copynativeonly $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) $(librariesOverrideArg)
+      - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipstressdependencies copynativeonly $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) $(librariesOverrideArg)
         displayName: Copy native test components to test output folder
 
 
@@ -260,9 +257,15 @@ jobs:
       displayName: Generate test wrappers
 
 
+    # Compose the Core_Root folder containing all artifacts needed for running
+    # CoreCLR tests.
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) generatelayoutonly $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
+      displayName: Generate CORE_ROOT
+
+
     # Crossgen framework assemblies prior to triggering readyToRun execution runs.
     - ${{ if eq(parameters.readyToRun, true) }}:
-      - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
+      - script: $(coreClrRepoRootDir)build-test$(scriptExt) crossgenframeworkonly $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
         displayName: Crossgen framework assemblies
 
     # Overwrite coreclr runtime binaries with mono ones

eng/pipelines/coreclr/ci.yml

@@ -98,13 +98,7 @@ jobs:
     jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     buildConfig: checked
     platforms:
-    - Linux_arm
-    - Linux_arm64
-    - OSX_x64
-    - Windows_NT_arm
-    - Windows_NT_arm64
-    - Windows_NT_x64
-    - Windows_NT_x86
+    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     testGroup: outerloop
     jobParameters:
       liveLibrariesBuildConfig: Release

eng/pipelines/coreclr/crossgen2-composite.yml

@@ -24,6 +24,7 @@ jobs:
     - Linux_x64
     - OSX_x64
     - Windows_NT_x64
+    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     jobParameters:
       testGroup: innerloop
 
@@ -32,8 +33,7 @@ jobs:
     jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     buildConfig: checked
     platforms:
-    - OSX_x64
-    - Windows_NT_x64
+    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     jobParameters:
       testGroup: innerloop
       liveLibrariesBuildConfig: Release