Skip to content

Commit

Permalink
Rework environment-specific custom xUnit attributes
Browse files Browse the repository at this point in the history
  • Loading branch information
AndreyAkinshin committed Jul 21, 2023
1 parent ab17946 commit 28809a1
Show file tree
Hide file tree
Showing 39 changed files with 143 additions and 200 deletions.
Expand Up @@ -234,7 +234,7 @@ public class AllSetupAndCleanupAttributeBenchmarksAsyncValueTaskSetup
public void Benchmark() => Console.WriteLine(BenchmarkCalled);
}

[FactNotGitHubActionsWindows]
[FactEnvSpecific(EnvRequirement.NonWindows)]
public void AllSetupAndCleanupMethodRunsAsyncGenericValueTaskSetupTest()
{
var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob");
Expand Down
5 changes: 3 additions & 2 deletions tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs
Expand Up @@ -318,8 +318,9 @@ public void AcceptsSpan(Span<int> span)
}
}

[TheoryNetCoreOnly("the implicit cast operator is available only in .NET Core 2.1+ (See https://github.com/dotnet/corefx/issues/30121 for more)"),
MemberData(nameof(GetToolchains))]
[TheoryEnvSpecific("The implicit cast operator is available only in .NET Core 2.1+ (See https://github.com/dotnet/corefx/issues/30121 for more)",
EnvRequirement.DotNetCoreOnly)]
[MemberData(nameof(GetToolchains))]
public void StringCanBePassedToBenchmarkAsReadOnlySpan(IToolchain toolchain) => CanExecute<WithStringToReadOnlySpan>(toolchain);

public class WithStringToReadOnlySpan
Expand Down
Expand Up @@ -25,7 +25,9 @@ public class BenchmarkSwitcherTest

public BenchmarkSwitcherTest(ITestOutputHelper output) => Output = output;

[FactDotNetCoreOnly("When CommandLineParser wants to display help, it tries to get the Title of the Entry Assembly which is an xunit runner, which has no Title and fails..")]
[FactEnvSpecific(
"When CommandLineParser wants to display help, it tries to get the Title of the Entry Assembly which is an xunit runner, which has no Title and fails..",
EnvRequirement.DotNetCoreOnly)]
public void WhenInvalidCommandLineArgumentIsPassedAnErrorMessageIsDisplayedAndNoBenchmarksAreExecuted()
{
var logger = new OutputLogger(Output);
Expand Down Expand Up @@ -332,7 +334,7 @@ public void WhenUserCreatesStaticBenchmarkMethodWeDisplayAnError_FromAssembly()
Assert.Contains("static", logger.GetLog());
}

[FactDotNetCoreOnly("For some reason this test is flaky on Full Framework")]
[FactEnvSpecific("For some reason this test is flaky on Full Framework", EnvRequirement.DotNetCoreOnly)]
public void WhenUserAddTheResumeAttributeAndRunTheBenchmarks()
{
var logger = new OutputLogger(Output);
Expand Down
2 changes: 1 addition & 1 deletion tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs
Expand Up @@ -42,7 +42,7 @@ public void ExporterWritesToFile()
}
}

[FactWindowsOnly("On Unix, it's possible to write to an opened file")]
[FactEnvSpecific("On Unix, it's possible to write to an opened file", EnvRequirement.WindowsOnly)]
public void ExporterWorksWhenFileIsLocked()
{
string resultsDirectoryPath = Path.GetTempPath();
Expand Down
Expand Up @@ -11,7 +11,7 @@ public ExtraAttributesForEntryMethodTests(ITestOutputHelper output) : base(outpu
{
}

[FactClassicDotNetOnly("STAThread attribute is not respected in netcoreapp https://github.com/dotnet/coreclr/issues/13688")]
[FactEnvSpecific("STAThread attribute is not respected in netcoreapp https://github.com/dotnet/coreclr/issues/13688", EnvRequirement.FullFrameworkOnly)]
public void UserCanMarkBenchmarkAsRequiringSTA() => CanExecute<RequiresSTA>();

public class RequiresSTA
Expand Down
Expand Up @@ -104,7 +104,7 @@ public void InProcessBenchmarkSimpleCasesReflectionEmitSupported()
}
}

[TheoryFullFrameworkOnly("We can't use Roslyn toolchain for .NET Core because we don't know which assemblies to reference and .NET Core does not support dynamic assembly saving")]
[TheoryEnvSpecific("We can't use Roslyn toolchain for .NET Core because we don't know which assemblies to reference and .NET Core does not support dynamic assembly saving", EnvRequirement.FullFrameworkOnly)]
[InlineData(typeof(SampleBenchmark))]
[InlineData(typeof(RunnableVoidCaseBenchmark))]
[InlineData(typeof(RunnableRefStructCaseBenchmark))]
Expand Down
Expand Up @@ -20,7 +20,7 @@ public class JitRuntimeValidationTest : BenchmarkTestExecutor
private const string RyuJitNotAvailable = "// ERROR: RyuJIT is requested but it is not available in current environment";
private const string ToolchainSupportsOnlyRyuJit = "Currently dotnet cli toolchain supports only RyuJit";

[TheoryWindowsOnly("CLR is a valid job only on Windows")]
[TheoryEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)]
[InlineData(Jit.LegacyJit, Platform.X86, null)]
[InlineData(Jit.LegacyJit, Platform.X64, null)]
[InlineData(Jit.RyuJit, Platform.X86, RyuJitNotAvailable)]
Expand Down
Expand Up @@ -19,7 +19,7 @@ public class LargeAddressAwareTest

public LargeAddressAwareTest(ITestOutputHelper outputHelper) => output = outputHelper;

[FactWindowsOnly("CLR is a valid job only on Windows")]
[FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)]
public void BenchmarkCanAllocateMoreThan2Gb()
{
var summary = BenchmarkRunner
Expand Down
Expand Up @@ -70,7 +70,7 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain)
});
}

[FactDotNetCoreOnly("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 7.0)")]
[FactEnvSpecific("We don't want to test NativeAOT twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)]
public void MemoryDiagnoserSupportsNativeAOT()
{
if (RuntimeInformation.IsMacOS())
Expand All @@ -82,7 +82,7 @@ public void MemoryDiagnoserSupportsNativeAOT()
.ToToolchain());
}

[FactDotNetCoreOnly("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 7.0)")]
[FactEnvSpecific("We don't want to test MonoVM twice (for .NET Framework 4.6.2 and .NET 7.0)", EnvRequirement.DotNetCoreOnly)]
public void MemoryDiagnoserSupportsModernMono()
{
MemoryDiagnoserIsAccurate(MonoToolchain.Mono70);
Expand Down Expand Up @@ -263,7 +263,9 @@ public void Allocate()
}
}

[TheoryNetCore30(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work"), MemberData(nameof(GetToolchains))]
[TheoryEnvSpecific(".NET Core 3.0 preview6+ exposes a GC.GetTotalAllocatedBytes method which makes it possible to work",
EnvRequirement.DotNetCore30Only)]
[MemberData(nameof(GetToolchains))]
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolchain)
{
Expand Down
2 changes: 1 addition & 1 deletion tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs
Expand Up @@ -10,7 +10,7 @@ namespace BenchmarkDotNet.IntegrationTests
{
public class MonoTests : BenchmarkTestExecutor
{
[FactDotNetCoreOnly("UseMonoRuntime option is available in .NET Core only starting from .NET 6")]
[FactEnvSpecific("UseMonoRuntime option is available in .NET Core only starting from .NET 6", EnvRequirement.DotNetCoreOnly)]
public void Mono70IsSupported()
{
var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono70));
Expand Down
Expand Up @@ -22,7 +22,7 @@ public class MultipleRuntimesTest

public MultipleRuntimesTest(ITestOutputHelper outputHelper) => output = outputHelper;

[FactWindowsOnly("CLR is a valid job only on Windows")]
[FactEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)]
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
public void SingleBenchmarkCanBeExecutedForMultipleRuntimes()
{
Expand Down
2 changes: 1 addition & 1 deletion tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs
Expand Up @@ -15,7 +15,7 @@ public class NativeAotTests : BenchmarkTestExecutor
{
public NativeAotTests(ITestOutputHelper outputHelper) : base(outputHelper) { }

[FactDotNetCoreOnly("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process")]
[FactEnvSpecific("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET Core or NativeAOT process", EnvRequirement.DotNetCoreOnly)]
public void LatestNativeAotVersionIsSupported()
{
if (!RuntimeInformation.Is64BitPlatform()) // NativeAOT does not support 32bit yet
Expand Down
4 changes: 2 additions & 2 deletions tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs
Expand Up @@ -18,7 +18,7 @@ public NuGetReferenceTests(ITestOutputHelper output) : base(output)
{
}

[FactNotLinux("For some reason this test is unstable on Ubuntu for both AzureDevOps and Travis CI")]
[FactEnvSpecific("For some reason this test is unstable on Ubuntu for both AzureDevOps and Travis CI", EnvRequirement.NonLinux)]
public void UserCanSpecifyCustomNuGetPackageDependency()
{
var toolchain = RuntimeInformation.GetCurrentRuntime().GetToolchain(preferMsBuildToolchains: true);
Expand All @@ -29,7 +29,7 @@ public void UserCanSpecifyCustomNuGetPackageDependency()
CanExecute<WithCallToNewtonsoft>(config);
}

[FactClassicDotNetOnly("Roslyn toolchain does not support .NET Core")]
[FactEnvSpecific("Roslyn toolchain does not support .NET Core", EnvRequirement.FullFrameworkOnly)]
public void RoslynToolchainDoesNotSupportNuGetPackageDependency()
{
var toolchain = RoslynToolchain.Instance;
Expand Down
2 changes: 1 addition & 1 deletion tests/BenchmarkDotNet.IntegrationTests/PathTooLongTests.cs
Expand Up @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.IntegrationTests
{
public class PathTooLongTests : BenchmarkTestExecutor
{
[FactWindowsOnly("Testing Windows long path limitation")]
[FactEnvSpecific("Testing Windows long path limitation", EnvRequirement.WindowsOnly)]
public void PathTooLongTest() =>
CanExecute<
VeryLongName012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789>();
Expand Down
Expand Up @@ -14,7 +14,7 @@ public class PowerManagementApplierTests : BenchmarkTestExecutor

public PowerManagementApplierTests(ITestOutputHelper output) : base(output) { }

[FactWindowsOnly("Setting high-performance plan is suitable only on Windows")]
[FactEnvSpecific("Setting high-performance plan is suitable only on Windows", EnvRequirement.WindowsOnly)]
public void TestSettingAndRevertingBackGuid()
{
var userPlan = PowerManagementHelper.CurrentPlan;
Expand All @@ -29,7 +29,7 @@ public void TestSettingAndRevertingBackGuid()
Assert.Equal(userPlan, PowerManagementHelper.CurrentPlan);
}

[FactWindowsOnly("Setting high-performance plan is suitable only on Windows")]
[FactEnvSpecific("Setting high-performance plan is suitable only on Windows", EnvRequirement.WindowsOnly)]
public void TestPowerPlanShouldNotChange()
{
var userPlan = PowerManagementHelper.CurrentPlan;
Expand Down
Expand Up @@ -17,13 +17,13 @@ public ProcessPropertiesTests(ITestOutputHelper output)
{
}

[FactWindowsOnly("Process.set_PriorityClass requires root on Unix")]
[FactEnvSpecific("Process.set_PriorityClass requires root on Unix", EnvRequirement.WindowsOnly)]
public void HighPriorityIsSet()
{
CanExecute<HighPriority>();
}

[FactWindowsOnly("Process.set_ProcessorAffinity requires root on Unix")]
[FactEnvSpecific("Process.set_ProcessorAffinity requires root on Unix", EnvRequirement.WindowsOnly)]
public void CustomAffinityCanBeSet()
{
var config = ManualConfig.CreateEmpty()
Expand Down
Expand Up @@ -3,6 +3,7 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Tests.Loggers;
using BenchmarkDotNet.Tests.XUnit;
using BenchmarkDotNet.Toolchains.Roslyn;
using Xunit;
using Xunit.Abstractions;
Expand All @@ -14,7 +15,7 @@ public class RoslynToolchainTest : BenchmarkTestExecutor
public RoslynToolchainTest(ITestOutputHelper output) : base(output) { }

/// <summary>Prooftest for #1039.</summary>
[Tests.XUnit.TheoryFullFrameworkOnly("Roslyn toolchain does not support .NET Core")]
[TheoryEnvSpecific("Roslyn toolchain does not support .NET Core", EnvRequirement.FullFrameworkOnly)]
[InlineData("en-US")]
[InlineData("fr-FR")]
[InlineData("ru-RU")]
Expand Down
Expand Up @@ -52,7 +52,7 @@ public class NonTailCallBenchmarks
public long Factorial() => FactorialWithoutTailing(7);
}

[TheoryWindowsOnly(WindowsOnly)]
[TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)]
[MemberData(nameof(GetJits))]
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime)
Expand All @@ -62,7 +62,7 @@ public void TailCallDiagnoserCatchesTailCallEvents(Jit jit, Platform platform, R
Assert.Contains(output.CapturedOutput, x => x.Text.Contains(TAIL_CALL_MARK));
}

[TheoryWindowsOnly(WindowsOnly)]
[TheoryEnvSpecific(WindowsOnly, EnvRequirement.WindowsOnly)]
[MemberData(nameof(GetJits))]
[Trait(Constants.Category, Constants.BackwardCompatibilityCategory)]
public void TailCallDiagnoserNotCatchesTailCallEvents(Jit jit, Platform platform, Runtime runtime)
Expand Down
2 changes: 1 addition & 1 deletion tests/BenchmarkDotNet.Tests/AppConfigGeneratorTests.cs
Expand Up @@ -142,7 +142,7 @@ public void GeneratesRightJitSettings(Jit jit, string expectedRuntimeNode)
}
}

[FactWindowsOnly("Full Framework is supported only on Windows")]
[FactEnvSpecific("Full Framework is supported only on Windows", EnvRequirement.WindowsOnly)]
public void RemovesStartupSettingsForPrivateBuildsOfClr()
{
const string input =
Expand Down
2 changes: 1 addition & 1 deletion tests/BenchmarkDotNet.Tests/ArtifactFileNameHelperTests.cs
Expand Up @@ -13,7 +13,7 @@ namespace BenchmarkDotNet.Tests
{
public class ArtifactFileNameHelperTests
{
[FactWindowsOnly(nonWindowsSkipReason: "ETW Sessions can be created only on Windows")]
[FactEnvSpecific("ETW Sessions can be created only on Windows", EnvRequirement.WindowsOnly)]
public void OnWindowsWeMustAlwaysUseOldLongPathsLimitForSessionFiles()
{
var config = DefaultConfig.Instance
Expand Down
18 changes: 10 additions & 8 deletions tests/BenchmarkDotNet.Tests/ConfigParserTests.cs
Expand Up @@ -110,7 +110,9 @@ public void UserCanChooseStrategy()
Assert.Equal(RunStrategy.ColdStart, job.Run.RunStrategy);
}

[FactDotNetCoreOnly("When CommandLineParser wants to display help, it tries to get the Title of the Entry Assembly which is an xunit runner, which has no Title and fails..")]
[FactEnvSpecific(
"When CommandLineParser wants to display help, it tries to get the Title of the Entry Assembly which is an xunit runner, which has no Title and fails..",
EnvRequirement.DotNetCoreOnly)]
public void UnknownConfigMeansFailure()
{
Assert.False(ConfigParser.Parse(new[] { "--unknown" }, new OutputLogger(Output)).isSuccess);
Expand All @@ -133,7 +135,7 @@ public void NonExistingPathMeansFailure()
Assert.False(ConfigParser.Parse(new[] { "--coreRun", nonExistingFile }, new OutputLogger(Output)).isSuccess);
}

[FactDotNetCoreOnly("Detecting current version of .NET Core works only for .NET Core processes")]
[FactEnvSpecific("Detecting current version of .NET Core works only for .NET Core processes", EnvRequirement.DotNetCoreOnly)]
public void CoreRunConfigParsedCorrectlyWhenRuntimeNotSpecified()
{
var fakeDotnetCliPath = typeof(object).Assembly.Location;
Expand All @@ -150,7 +152,7 @@ public void CoreRunConfigParsedCorrectlyWhenRuntimeNotSpecified()
Assert.Equal(fakeRestorePackages, toolchain.RestorePath.FullName);
}

[FactClassicDotNetOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
[FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.FullFrameworkOnly)]
public void SpecifyingCoreRunWithFullFrameworkTargetsMostRecentTfm()
{
var fakePath = typeof(object).Assembly.Location;
Expand All @@ -163,7 +165,7 @@ public void SpecifyingCoreRunWithFullFrameworkTargetsMostRecentTfm()
Assert.Equal("net8.0", generator.TargetFrameworkMoniker);
}

[FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
[FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)]
public void SpecifyingCoreRunAndRuntimeCreatesTwoJobs()
{
const string runtime = "net7.0";
Expand Down Expand Up @@ -191,7 +193,7 @@ public void SpecifyingCoreRunAndRuntimeCreatesTwoJobs()
Assert.Equal(fakeRestorePackages, generator.PackagesPath);
}

[FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
[FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)]
public void FirstJobIsBaseline_RuntimesCoreRun()
{
const string runtime1 = "net5.0";
Expand All @@ -205,7 +207,7 @@ public void FirstJobIsBaseline_RuntimesCoreRun()
Assert.Equal(runtime1, ((DotNetCliGenerator)baselineJob.GetToolchain().Generator).TargetFrameworkMoniker);
}

[FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
[FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)]
public void FirstJobIsBaseline_CoreRunsRuntimes()
{
const string runtime1 = "net5.0";
Expand All @@ -219,7 +221,7 @@ public void FirstJobIsBaseline_CoreRunsRuntimes()
Assert.Equal(fakePath1, ((CoreRunToolchain)baselineJob.GetToolchain()).SourceCoreRun.FullName);
}

[FactDotNetCoreOnly("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process")]
[FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)]
public void UserCanSpecifyMultipleCoreRunPaths()
{
var fakeCoreRunPath_1 = typeof(object).Assembly.Location;
Expand All @@ -244,7 +246,7 @@ public void MonoPathParsedCorrectly()
Assert.Single(config.GetJobs().Where(job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath));
}

[FactWindowsOnly("Testing local builds of Full .NET Framework is supported only on Windows")]
[FactEnvSpecific("Testing local builds of Full .NET Framework is supported only on Windows", EnvRequirement.WindowsOnly)]
public void ClrVersionParsedCorrectly()
{
const string clrVersion = "secret";
Expand Down

0 comments on commit 28809a1

Please sign in to comment.