diff --git a/docs/benchmarkdotnet.md b/docs/benchmarkdotnet.md index f8c9e222849..be429f57ad7 100644 --- a/docs/benchmarkdotnet.md +++ b/docs/benchmarkdotnet.md @@ -289,7 +289,7 @@ M00_L00: The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**. -Available options are: Mono, wasmnet70, CoreRT, net462, net47, net471, net472, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. +Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0. Example: run the benchmarks for .NET 7.0 and 8.0: @@ -361,18 +361,6 @@ dotnet run -c Release -f net9.0 --cli "C:\Projects\performance\.dotnet\dotnet.ex This is very useful when you want to compare different builds of .NET. -### Private CLR Build - -It's possible to benchmark a private build of .NET Runtime. You just need to pass the value of `COMPLUS_Version` to BenchmarkDotNet. You can do that by either using `--clrVersion $theVersion` as an argument or `Job.ShortRun.With(new ClrRuntime(version: "$theVersion"))` in the code. - -So if you made a change in CLR and want to measure the difference, you can run the benchmarks with: - -```cmd -dotnet run -c Release -f net48 -- --clrVersion $theVersion -``` - -More info can be found in [BenchmarkDotNet issue #706](https://github.com/dotnet/BenchmarkDotNet/issues/706). - ### Private CoreRT Build To run benchmarks with private CoreRT build you need to provide the `IlcPath`. Example: diff --git a/eng/Versions.props b/eng/Versions.props index a045425ef6c..07ade91729c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 11.0.0-preview.5.26261.101 11.0.0-preview.5.26261.101 11.0.0-preview.5.26261.101 - 0.16.0-nightly.20260320.467 + 0.16.0-nightly.20260518.1249 11.0.0-preview.5.26261.101 11.0.0-prerelease.26204.1 diff --git a/scripts/ci_setup.py b/scripts/ci_setup.py index e622a9fe69b..07c0a95733f 100644 --- a/scripts/ci_setup.py +++ b/scripts/ci_setup.py @@ -424,6 +424,11 @@ def main(args: CiSetupArgs): if args.experiment_name == "jitoptrepeat": experiment_config = variable_format % ('DOTNET_JitOptRepeat', '*') + if args.experiment_name == "runtimeasync": + # Surfaced to MSBuild as the $(EnableRuntimeAsync) property; gates the + # runtime-async Features flag in src/Directory.Build.targets. + experiment_config = variable_format % ('EnableRuntimeAsync', 'true') + output = '' with push_dir(get_repo_root_path()): diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 96647e856eb..b168fcecc65 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -5,27 +5,27 @@ false - + latest - + False - + $(NoWarn);NU1507 $(NoWarn);NETSDK1138 $(NoWarn);CS9057 True 4 - + True false false - + false diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets new file mode 100644 index 00000000000..63d3b91f501 --- /dev/null +++ b/src/Directory.Build.targets @@ -0,0 +1,11 @@ + + + + + + $(Features);runtime-async=on + + \ No newline at end of file diff --git a/src/benchmarks/micro/MicroBenchmarks.csproj b/src/benchmarks/micro/MicroBenchmarks.csproj index 179557c61d4..d7b14d9336f 100644 --- a/src/benchmarks/micro/MicroBenchmarks.csproj +++ b/src/benchmarks/micro/MicroBenchmarks.csproj @@ -122,14 +122,6 @@ - - - - - - - - diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 121d9dd9b98..8d2a590cae9 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Threading.Tasks; using BenchmarkDotNet.Running; using System.IO; using BenchmarkDotNet.Extensions; @@ -14,7 +15,7 @@ namespace MicroBenchmarks { class Program { - static int Main(string[] args) + static async Task Main(string[] args) { var argsList = new List(args); int? partitionCount; @@ -40,9 +41,14 @@ static int Main(string[] args) return 1; } - return BenchmarkSwitcher + // Use RunAsync (not Run) so BDN does not install its single-threaded + // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync + // entrypoint installs that context before benchmark discovery, which + // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource] + // callbacks (e.g. SslStreamTests.GetTls13Support). + var summaries = await BenchmarkSwitcher .FromAssembly(typeof(Program).Assembly) - .Run(argsList.ToArray(), + .RunAsync(argsList.ToArray(), RecommendedConfig.Create( artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")), mandatoryCategories: ImmutableHashSet.Create([Categories.Libraries, Categories.Runtime, Categories.ThirdParty, Categories.Sve]), @@ -52,7 +58,9 @@ static int Main(string[] args) categoryExclusionFilterValue: categoryExclusionFilterValue, getDiffableDisasm: getDiffableDisasm) .AddValidator(new NoWasmValidator(Categories.NoWASM))) - .ToExitCode(); + .ConfigureAwait(false); + + return summaries.ToExitCode(); } } } \ No newline at end of file diff --git a/src/benchmarks/micro/README.md b/src/benchmarks/micro/README.md index 590edec71af..d19ed650f5f 100644 --- a/src/benchmarks/micro/README.md +++ b/src/benchmarks/micro/README.md @@ -12,7 +12,7 @@ To learn more about designing benchmarks, please read [Microbenchmark Design Gui ## Quick Start -The first thing that you need to choose is the Target Framework. Available options are: `netcoreapp3.1|net6.0|net7.0|net8.0|net9.0|net10.0|net11.0|net472`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework. +The first thing that you need to choose is the Target Framework. Available options are: `net8.0|net9.0|net10.0|net11.0`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework. The following commands are run from the `src/benchmarks/micro` directory. diff --git a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index ef8782db2aa..a54dc819171 100644 --- a/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -1,7 +1,11 @@  Library - netstandard2.0 + + net8.0 enable true @@ -13,7 +17,7 @@ - + diff --git a/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs b/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs index 44f81dd29e9..3124e82c546 100644 --- a/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs @@ -61,26 +61,26 @@ internal static string BuildDisassemblyString(DisassemblyResult disassemblyResul private static Func GetElementGetter(string name) { - var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier"); + var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!; - type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic); + type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic)!; - var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic); + var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic)!; - var method = property.GetGetMethod(nonPublic: true); + var method = property.GetGetMethod(nonPublic: true)!; var generic = typeof(Func<,>).MakeGenericType(type, typeof(T)); var @delegate = method.CreateDelegate(generic); - return (obj) => (T)@delegate.DynamicInvoke(obj); // cast to (Func) throws + return (obj) => (T)@delegate.DynamicInvoke(obj)!; // cast to (Func) throws } private static Func> GetPrettifyMethod() { - var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier"); + var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!; - var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic); + var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic)!; var @delegate = method.CreateDelegate(typeof(Func>)); diff --git a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs index 7b3b2d38f3b..280e43653d9 100644 --- a/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs @@ -21,7 +21,7 @@ public class MandatoryCategoryValidator : IValidator public MandatoryCategoryValidator(ImmutableHashSet categories) => _mandatoryCategories = categories; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => !benchmark.Descriptor.Categories.Any(category => _mandatoryCategories.Contains(category))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) @@ -30,6 +30,7 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: TreatsWarningsAsErrors, $"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]") - ); + ) + .ToAsyncEnumerable(); } } \ No newline at end of file diff --git a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs index 5b2cfec63ca..68fe73a4645 100644 --- a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs @@ -23,7 +23,7 @@ public class NoWasmValidator : IValidator public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal))) .Select(benchmark => benchmark.Descriptor.GetFilterName()) @@ -32,7 +32,8 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: TreatsWarningsAsErrors, $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]") - ); + ) + .ToAsyncEnumerable(); private bool IsAsyncMethod(MethodInfo workloadMethod) { diff --git a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs index c1ed4268e0f..d308afb27a8 100644 --- a/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs +++ b/src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs @@ -4,24 +4,67 @@ using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using Reporting; using System; +using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace BenchmarkDotNet.Extensions { - public class PerfLabExporter : ExporterBase + // Implements IExporter directly (not ExporterBase) because PerfLabExporter writes + // a file with a custom name pattern ("{type}-perf-lab-report.json") via + // File.WriteAllTextAsync and manages the file lifecycle itself, rather than having + // ExporterBase open and hand us a writer for a default-named file. + public class PerfLabExporter : IExporter { - protected override string FileExtension => "json"; - protected override string FileCaption => "perf-lab-report"; + private const string FileExtension = "json"; + private const string FileCaption = "perf-lab-report"; - public PerfLabExporter() + public string Name => nameof(PerfLabExporter); + + public async ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken) + { + string? jsonOutput = BuildJson(summary); + if (jsonOutput is null) + return; + + string filePath = GetArtifactFullName(summary); + if (File.Exists(filePath)) + { + try + { + File.Delete(filePath); + } + catch (IOException) + { + string uniqueString = DateTime.Now.ToString("yyyyMMdd-HHmmss"); + string altPath = $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}-{uniqueString}.{FileExtension}"; + logger.WriteLineError($"Could not overwrite file {filePath}. Exporting to {altPath}"); + filePath = altPath; + } + } + + await File.WriteAllTextAsync(filePath, jsonOutput, cancellationToken).ConfigureAwait(false); + logger.WriteLineInfo($" {filePath}"); + } + + private string GetArtifactFullName(Summary summary) + => $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}.{FileExtension}"; + + private static string GetFileName(Summary summary) { + var targets = summary.BenchmarksCases.Select(b => b.Descriptor.Type).Distinct().ToArray(); + if (targets.Length == 1) + return FolderNameHelper.ToFolderName(targets.Single()); + return summary.Title; } - public override void ExportToLog(Summary summary, ILogger logger) + private static string? BuildJson(Summary summary) { var reporter = new Reporter(); @@ -49,7 +92,7 @@ public override void ExportToLog(Summary summary, ILogger logger) var test = new Test(); test.Name = FullNameProvider.GetBenchmarkName(report.BenchmarkCase); test.Categories = report.BenchmarkCase.Descriptor.Categories; - + if (hasCriticalErrors) { test.AdditionalData["criticalErrors"] = "true"; @@ -58,7 +101,7 @@ public override void ExportToLog(Summary summary, ILogger logger) var results = from result in report.AllMeasurements where result.IterationMode == Engines.IterationMode.Workload && result.IterationStage == Engines.IterationStage.Result orderby result.LaunchIndex, result.IterationIndex - select new { result.Nanoseconds, result.Operations}; + select new { result.Nanoseconds, result.Operations }; var overheadResults = from result in report.AllMeasurements where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jitting @@ -104,7 +147,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit HigherIsBetter = true, MetricName = "Count", Results = (from result in results - select (double)result.Operations).ToList() + select (double)result.Operations).ToList() }); foreach (var metric in report.Metrics.Keys) @@ -130,9 +173,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit reporter.AddTest(test); } - var jsonOutput = reporter.GetJson(); - if (jsonOutput is not null) - logger.WriteLine(jsonOutput); + return reporter.GetJson(); } } } diff --git a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs index aaf8a62317a..9df1b3d99cb 100644 --- a/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/TooManyTestCasesValidator.cs @@ -19,7 +19,7 @@ public class TooManyTestCasesValidator : IValidator public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) { var byDescriptor = validationParameters.Benchmarks .Where(benchmark => !SkipValidation(benchmark.Descriptor.WorkloadMethod)) @@ -29,10 +29,11 @@ public IEnumerable Validate(ValidationParameters validationPara new ValidationError( isCritical: true, message: $"{group.Key.Descriptor.Type.Name}.{group.Key.Descriptor.WorkloadMethod.Name} has {group.Count()} test cases. It MUST NOT have more than {Limit} test cases. We don't have inifinite amount of time to run all the benchmarks!!", - benchmarkCase: group.First())); + benchmarkCase: group.First())) + .ToAsyncEnumerable(); } - private static bool SkipValidation(MemberInfo member) + private static bool SkipValidation(MemberInfo? member) { while (member != null) { diff --git a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs index 532e9b003f5..e3903769a69 100644 --- a/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/UniqueArgumentsValidator.cs @@ -14,7 +14,9 @@ public class UniqueArgumentsValidator : IValidator { public bool TreatsWarningsAsErrors => true; - public IEnumerable Validate(ValidationParameters validationParameters) + // Use ToAsyncEnumerable() (not async + yield return) to avoid the AsyncIteratorMethodBuilder + // state machine deadlocking with BDN's BenchmarkSynchronizationContext (matches BDN's own validators). + public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) => validationParameters.Benchmarks .Where(benchmark => benchmark.HasArguments || benchmark.HasParameters) .GroupBy(benchmark => (benchmark.Descriptor.Type, benchmark.Descriptor.WorkloadMethod, benchmark.Job)) @@ -25,12 +27,16 @@ public IEnumerable Validate(ValidationParameters validationPara return numberOfTestCases != numberOfUniqueTestCases; }) - .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())); + .Select(duplicate => new ValidationError(true, $"Benchmark Arguments should be unique, {duplicate.Key.Type}.{duplicate.Key.WorkloadMethod} has duplicate arguments.", duplicate.First())) + .ToAsyncEnumerable(); private class BenchmarkArgumentsComparer : IEqualityComparer { - public bool Equals(BenchmarkCase x, BenchmarkCase y) + public bool Equals(BenchmarkCase? x, BenchmarkCase? y) { + if (x is null || y is null) + return ReferenceEquals(x, y); + if (FullNameProvider.GetBenchmarkName(x).Equals(FullNameProvider.GetBenchmarkName(y), System.StringComparison.Ordinal)) return true; diff --git a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs index 107c298e93f..e36efdf9747 100644 --- a/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs +++ b/src/harness/BenchmarkDotNet.Extensions/ValuesGenerator.cs @@ -130,7 +130,7 @@ public static byte[] ArrayBase64EncodingBytes(int count) /// the stored values are randomly generated. /// GenerateValue is used to generate a random value in the appropriate range for both the key and value /// - public static Dictionary Dictionary(int count) + public static Dictionary Dictionary(int count) where TKey : notnull { if (count > 2 && typeof(TKey) == typeof(bool)) throw new ArgumentOutOfRangeException("count", "Cannot exceed 2 for Dictionary"); diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs index fdabaa78db0..5715ffcf676 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueArgumentsValidatorTests.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Threading.Tasks; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; @@ -26,7 +27,7 @@ public void DuplicatedArgumentsAreDetected(Type typeWithBenchmarks, bool shouldR var benchmarksForType = BenchmarkConverter.TypeToBenchmarks(typeWithBenchmarks); var validationParameters = new ValidationParameters(benchmarksForType.BenchmarksCases, benchmarksForType.Config); - var validationErrors = new UniqueArgumentsValidator().Validate(validationParameters); + var validationErrors = new UniqueArgumentsValidator().ValidateAsync(validationParameters).ToBlockingEnumerable(); if (shouldReportError) Assert.NotEmpty(validationErrors); diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs index ff3b8f0f6e0..58a6124938f 100644 --- a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/UniqueValuesGeneratorTests.cs @@ -151,14 +151,14 @@ private static void SupportsNonDefaultValue() Assert.NotEqual(default, value); } - private static void SupportsDictionary(int count) + private static void SupportsDictionary(int count) where TKey : notnull { var dictionary = ValuesGenerator.Dictionary(count); Assert.NotNull(dictionary); Assert.Equal(count, dictionary.Count); } - private static void Supports(int count = 10) + private static void Supports(int count = 10) where T : notnull { SupportsArray(count); SupportsNonDefaultValue();