Skip to content

Enable use of RuntimeVariant to exclude runtime tests #51522

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
naricc merged 19 commits into from
May 25, 2021
Merged

Enable use of RuntimeVariant to exclude runtime tests #51522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
8b2db81
Allow filtering of runtime tests based on runtimevariant.
Apr 19, 2021
5923757
Changed artifact path.
Apr 26, 2021
a1fe67e
Fixed quotes.
Apr 27, 2021
9706235
Removed extra spaces.
Apr 27, 2021
5915dfb
Fixed spacing in issues.targets, and removed AltJit.
Apr 27, 2021
bdd4232
Made wasm use interpreter
Apr 27, 2021
e483d70
Fixed runtime variant arg spelling.
Apr 27, 2021
0649e99
Seperate exclusions for llvmaout.
Apr 28, 2021
ea6452b
Added more failing tests.
Apr 28, 2021
3fdb7d4
Fixed spacing.
Apr 28, 2021
b70f193
Added another test to issues.targets.
Apr 28, 2021
e673a48
Added runtimeVariant parameter in aot compile step.
Apr 29, 2021
df60230
Added more failures to issues.targets.
Apr 29, 2021
5d49775
Fixed issues.targets.
Apr 29, 2021
291ff57
Fixed issues.targets.
Apr 29, 2021
03ac47a
Removed extra space.
Apr 29, 2021
3a67811
Added priority arg.
Apr 29, 2021
31bd379
Added missing quote.
Apr 29, 2021
08e89d1
Added another exclusion.
May 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions eng/pipelines/common/templates/runtimes/run-test-job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,11 @@ jobs:
- ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
- '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}'
- ${{ if ne(parameters.stagedBuild, true) }}:
- ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
- ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }}:
# minijit and mono interpreter runtimevariants do not require any special build of the runtime
- ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
- ${{ if not(or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'))) }}:
- ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
- ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
- ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}

Expand All @@ -84,6 +88,13 @@ jobs:
- name: runtimeFlavorArgs
value: '-excludemonofailures'

- name: runtimeVariantArg
value: ''

- ${{ if not(eq(parameters.runtimeVariant, '')) }}:
- name: runtimeVariantArg
value: '/p:RuntimeVariant=${{ parameters.runtimeVariant }}'

- name: crossgenArg
value: ''
- name: LogNamePrefix
Expand Down Expand Up @@ -255,7 +266,7 @@ jobs:


# Generate test wrappers. This is the step that examines issues.targets to exclude tests.
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) buildtestwrappersonly $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) buildtestwrappersonly $(runtimeFlavorArgs) $(runtimeVariantArg) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
displayName: Generate test wrappers


Expand Down Expand Up @@ -291,10 +302,10 @@ jobs:

- ${{ if and(eq(parameters.runtimeFlavor, 'mono'), eq(parameters.runtimeVariant, 'llvmaot')) }}:
- ${{ if eq(parameters.archType, 'x64') }}:
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) mono_aot $(buildConfig) $(archType)
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) mono_aot $(buildConfig) $(archType) $(runtimeVariantArg)
displayName: "LLVM AOT compile CoreCLR tests"
- ${{ if eq(parameters.archType, 'arm64') }}:
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) mono_aot $(buildConfig) $(archType) cross
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) mono_aot $(buildConfig) $(archType) cross $(runtimeVariantArg)
displayName: "LLVM AOT cross-compile CoreCLR tests"
env:
__MonoToolPrefix: aarch64-linux-gnu-
Expand Down Expand Up @@ -354,8 +365,7 @@ jobs:
- normal
- ${{ if eq(parameters.runtimeFlavor, 'coreclr') }}:
- no_tiered_compilation
- ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
- interpreter

${{ if in(parameters.testGroup, 'jitstress') }}:
scenarios:
- jitminopts
Expand Down
2 changes: 1 addition & 1 deletion eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ parameters:


steps:
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -excludemonofailures os Browser wasm $(buildConfigUpper)
- script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=monointerpreter /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -excludemonofailures os Browser wasm $(buildConfigUpper)
displayName: Build Tests

# Send tests to Helix
Expand Down
8 changes: 8 additions & 0 deletions eng/pipelines/mono/templates/xplat-pipeline-job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ jobs:
- name: buildProductArtifactName
value: 'MonoProduct__${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

# minijit and monointerpreter do not use seperate product builds.
- ${{ if or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }}:
- name : buildProductArtifactName
value : 'MonoProduct___$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'

- name: binTestsPath
value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr'

Expand Down Expand Up @@ -92,6 +97,9 @@ jobs:
value: 'Checked'
${{ if eq(parameters.liveRuntimeBuildConfig, 'debug') }}:
value: 'Debug'

- name: priorityArg
value: ''

- librariesBuildArtifactName: ''
- librariesOverrideArg: ''
Expand Down
27 changes: 26 additions & 1 deletion eng/pipelines/runtime.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -853,7 +853,7 @@ jobs:
eq(variables['isFullMatrix'], true))

#
# Mono CoreCLR runtime Test executions using live libraries
# Mono CoreCLR runtime Test executions using live libraries in jit mode
# Only when Mono is changed
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
Expand All @@ -869,6 +869,31 @@ jobs:
testGroup: innerloop
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
liveRuntimeBuildConfig: release
runtimeVariant: minijit
condition: >-
or(
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true),
eq(variables['isFullMatrix'], true))

#
# Mono CoreCLR runtime Test executions using live libraries in interpreter mode
# Only when Mono is changed
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
buildConfig: release
runtimeFlavor: mono
platforms:
- OSX_x64
- Linux_arm64
helixQueueGroup: pr
helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
jobParameters:
testGroup: innerloop
liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }}
liveRuntimeBuildConfig: release
runtimeVariant: monointerpreter
condition: >-
or(
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
Expand Down
42 changes: 36 additions & 6 deletions src/tests/issues.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1039,7 +1039,7 @@
</ExcludeList>
</ItemGroup>

<!-- Known failures for mono runtime on *all* architectures/operating systems -->
<!-- Known failures for mono runtime on *all* architectures/operating systems in *all* runtime modes -->
<ItemGroup Condition="'$(RuntimeFlavor)' == 'mono'" >
<ExcludeList Include="$(XunitTestBinBase)/JIT/Directed/Convert/out_of_range_fp_to_int_conversions/*">
<Issue>Mono does not define out of range fp to int conversions</Issue>
Expand Down Expand Up @@ -1937,10 +1937,11 @@
<ExcludeList Include = "$(XunitTestBinBase)tracing/eventsource/eventsourcetrace/eventsourcetrace/**">
<Issue>needs triage</Issue>
</ExcludeList>
</ItemGroup>

<!-- The following only fail in the mono interpreter, but we don't have a way to exclude based on scenario yet -->

<ExcludeList Include = "$(XunitTestBinBase)/JIT/jit64/opt/cse/HugeArray1/**">
<!-- Known failures for mono runtime on *all* architectures/operating systems in interpreter runtime mode -->

@lambdageek lambdageek Apr 21, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎉

<ItemGroup Condition="'$(RuntimeFlavor)' == 'mono' and '$(RuntimeVariant)' == 'monointerpreter' ">
<ExcludeList Include = "$(XunitTestBinBase)/JIT/jit64/opt/cse/HugeArray1/**">
<Issue>https://github.com/dotnet/runtime/issues/46622</Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/JIT/Regression/JitBlue/GitHub_21990/**">
Expand Down Expand Up @@ -2516,7 +2517,34 @@
<!-- End interpreter issues -->
</ItemGroup>

<ItemGroup Condition=" '$(RuntimeFlavor)' == 'mono' and ('$(TargetArchitecture)' == 'arm64' or '$(AltJitArch)' == 'arm64') and '$(TargetsWindows)' != 'true' " >
<ItemGroup Condition="'$(RuntimeFlavor)' == 'mono' and '$(RuntimeVariant)' == 'llvmaot' ">
<ExcludeList Include = "$(XunitTestBinBase)JIT/Methodical/ELEMENT_TYPE_IU/_il_relu_conv/**">
<Issue>needs triage</Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_r/**">
<Issue>needs triage</Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_d/**">
<Issue>needs triage</Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/JIT/Methodical/eh/deadcode/deadoponerrorinfunclet_d/**">
<Issue>needs triage</Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/JIT/Directed/nullabletypes/isinstvaluetype_do/**">
<Issue> needs triage </Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/JIT/Generics/Instantiation/Structs/struct01/**">
<Issue> needs triage </Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/JIT/jit64/opt/cse/HugeArray1/**">
<Issue> needs triage </Issue>
</ExcludeList>
<ExcludeList Include = "$(XunitTestBinBase)/GC/Scenarios/Dynamo/dynamo/**">
<Issue> needs triage </Issue>
</ExcludeList>
</ItemGroup>

<ItemGroup Condition=" '$(RuntimeFlavor)' == 'mono' and '$(TargetArchitecture)' == 'arm64' and '$(TargetsWindows)' != 'true' " >
<ExcludeList Include="$(XunitTestBinBase)/baseservices/typeequivalence/simple/Simple/**">
<Issue>Doesn't compile with LLVM AOT.</Issue>
</ExcludeList>
Expand Down Expand Up @@ -2668,12 +2696,14 @@
<ExcludeList Include = "$(XunitTestBinBase)/JIT/Regression/VS-ia64-JIT/M00/b108366/b108366/**">
<Issue>needs triage</Issue>
</ExcludeList>
</ItemGroup>

<!-- These are known failures on mono-arm64 in the interpreter runtime variant -->
<ItemGroup Condition=" '$(RuntimeFlavor)' == 'mono' and ('$(TargetArchitecture)' == 'arm64') and '$(TargetsWindows)' != 'true' and '$(RuntimeVariant)' == 'monointerpreter' " >
<!-- The following issues only effect the Mono interpreter; there is currently no way to exclude based on that scenario -->
<ExcludeList Include = "$(XunitTestBinBase)JIT/IL_Conformance/Old/Base/ckfinite/**">
<Issue>needs triage</Issue>
</ExcludeList>
<!-- End interpreter issues -->
</ItemGroup>


Expand Down