Update bdn 0.16 binarydata and async

docs/benchmarkdotnet.md

@@ -27,7 +27,7 @@
     - [Running In Process](#running-in-process)
     - [CoreRun](#corerun)
     - [dotnet cli](#dotnet-cli)
     - [Private CLR Build](#private-clr-build)
     - [Private CoreRT Build](#private-corert-build)
 
 ## Main Concepts
@@ -289,7 +289,7 @@
 
 The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**.
 
-Available options are: Mono, wasmnet70, CoreRT, net462, net47, net471, net472, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0.
+Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0.
 
 Example: run the benchmarks for .NET 7.0 and 8.0:
 
@@ -361,18 +361,6 @@
 
 This is very useful when you want to compare different builds of .NET.
 
-### Private CLR Build
-
-It's possible to benchmark a private build of .NET Runtime. You just need to pass the value of `COMPLUS_Version` to BenchmarkDotNet. You can do that by either using `--clrVersion $theVersion` as an argument or `Job.ShortRun.With(new ClrRuntime(version: "$theVersion"))` in the code.
-
-So if you made a change in CLR and want to measure the difference, you can run the benchmarks with:
-
-```cmd
-dotnet run -c Release -f net48 -- --clrVersion $theVersion
-```
-
-More info can be found in [BenchmarkDotNet issue #706](https://github.com/dotnet/BenchmarkDotNet/issues/706).
-
 ### Private CoreRT Build
 
 To run benchmarks with private CoreRT build you need to provide the `IlcPath`. Example:

eng/Versions.props

@@ -11,7 +11,7 @@
     <MicrosoftNETILLinkPackageVersion>11.0.0-preview.5.26261.101</MicrosoftNETILLinkPackageVersion>
     <SystemThreadingChannelsPackageVersion>11.0.0-preview.5.26261.101</SystemThreadingChannelsPackageVersion>
     <MicrosoftExtensionsLoggingPackageVersion>11.0.0-preview.5.26261.101</MicrosoftExtensionsLoggingPackageVersion>
-    <BenchmarkDotNetVersion>0.16.0-nightly.20260320.467</BenchmarkDotNetVersion>
+    <BenchmarkDotNetVersion>0.16.0-nightly.20260518.1249</BenchmarkDotNetVersion>
     <MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>11.0.0-preview.5.26261.101</MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>
     <MicrosoftDotNetXHarnessCLIVersion>11.0.0-prerelease.26204.1</MicrosoftDotNetXHarnessCLIVersion>
   </PropertyGroup>

scripts/ci_setup.py

@@ -424,6 +424,11 @@ def main(args: CiSetupArgs):
     if args.experiment_name == "jitoptrepeat":
         experiment_config = variable_format % ('DOTNET_JitOptRepeat', '*')
 
+    if args.experiment_name == "runtimeasync":
+        # Surfaced to MSBuild as the $(EnableRuntimeAsync) property; gates the
+        # runtime-async Features flag in src/Directory.Build.targets.
+        experiment_config = variable_format % ('EnableRuntimeAsync', 'true')
+
     output = ''
 
     with push_dir(get_repo_root_path()):

src/Directory.Build.props

@@ -5,27 +5,27 @@
     <!-- Avoid spawning any-long living compiler processes to avoid BenchmarkDotNet issues with "file in use",
          and automation failing to clean up the folders once the runs are over -->
     <UseSharedCompilation>false</UseSharedCompilation>
-    
+
     <!-- Use the latest C# compiler features -->
     <LangVersion>latest</LangVersion>
-    
+
     <!-- This repo does not produce any libraries, therefore generating docs is disabled -->
     <GenerateDocumentationFile>False</GenerateDocumentationFile>
-    
+
     <!-- every warning is important, we want to enforce best practices here to keep high quality of the code and avoid common mistakes -->
     <NoWarn>$(NoWarn);NU1507</NoWarn> <!-- Darc does not seem to support auto updating of packageSourceMapping causing tool publishes to fail without manual updating of the NuGet.config with package mappings, disable this check so we don't have to manually update the source mappings. -->
     <NoWarn>$(NoWarn);NETSDK1138</NoWarn> <!-- Disable warning about EOL target frameworks as we target them to test them -->
     <NoWarn>$(NoWarn);CS9057</NoWarn> <!-- Suppress analyzer version mismatch when BDN analyzers target a newer Roslyn than the SDK provides -->
     <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
     <WarningLevel>4</WarningLevel>
-    
+
     <!-- we are testing latest bits and we must use preview SDK version -->
     <SuppressNETCoreSdkPreviewMessage>True</SuppressNETCoreSdkPreviewMessage>
 
     <!-- Disable SourceLink -->
     <EnableSourceLink>false</EnableSourceLink>
     <EnableSourceControlManagerQueries>false</EnableSourceControlManagerQueries>
-    
+
     <!-- Explicit disable signing the built assemblies here to stop Arcade attempting to sign them -->
     <SignAssembly>false</SignAssembly>
   </PropertyGroup>

src/Directory.Build.targets

@@ -0,0 +1,11 @@
+<Project>
+    <Import Project="../Directory.Build.targets" />
+
+    <PropertyGroup>
+      <!-- Runtime-async is gated behind the 'runtimeasync' experiment so it only runs
+           in the dedicated experiment lane. ci_setup.py sets EnableRuntimeAsync=true as
+           an env var (which MSBuild surfaces as a property) when the experiment name
+           is 'runtimeasync'. -->
+      <Features Condition="'$(EnableRuntimeAsync)' == 'true' and $([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net11.0'))">$(Features);runtime-async=on</Features>
+    </PropertyGroup>
+</Project>

src/benchmarks/micro/MicroBenchmarks.csproj

@@ -122,14 +122,6 @@
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(ExtensionsVersion)" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="$(ExtensionsVersion)" />
   </ItemGroup>
-  <!-- These package versions are only for the opt-in net472 path (for example, PERFLAB_TARGET_FRAMEWORKS=net472). -->
-  <ItemGroup Condition="'$(TargetFramework)' != '' and '$(TargetFrameworkIdentifier)' == '.NETFramework'">
-    <PackageReference Include="System.Memory" Version="4.6.3" />
-    <PackageReference Include="System.Numerics.Vectors" Version="4.6.1" />
-    <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.6.3" />
-    <PackageReference Include="System.IO.Pipelines" Version="10.0.3" />
-    <PackageReference Include="System.Threading.Channels" Version="$(SystemVersion)" />
-  </ItemGroup>
   <ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net8.0')) and !$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net10.0'))">
     <!-- Newer System.Numerics.Tensors packages require net10+, so keep net8.0/net9.0 on the last compatible package line. -->
     <PackageReference Include="System.Numerics.Tensors" Version="10.0.*-*" />

src/benchmarks/micro/Program.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Threading.Tasks;
 using BenchmarkDotNet.Running;
 using System.IO;
 using BenchmarkDotNet.Extensions;
@@ -14,7 +15,7 @@ namespace MicroBenchmarks
 {
     class Program
     {
-        static int Main(string[] args)
+        static async Task<int> Main(string[] args)
         {
             var argsList = new List<string>(args);
             int? partitionCount;
@@ -40,9 +41,14 @@ static int Main(string[] args)
                 return 1;
             }
 
-            return BenchmarkSwitcher
+            // Use RunAsync (not Run) so BDN does not install its single-threaded
+            // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync
+            // entrypoint installs that context before benchmark discovery, which
+            // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource]
+            // callbacks (e.g. SslStreamTests.GetTls13Support).
+            var summaries = await BenchmarkSwitcher
                 .FromAssembly(typeof(Program).Assembly)
-                .Run(argsList.ToArray(), 
+                .RunAsync(argsList.ToArray(),
                     RecommendedConfig.Create(
                         artifactsPath: new DirectoryInfo(Path.Combine(AppContext.BaseDirectory, "BenchmarkDotNet.Artifacts")), 
                         mandatoryCategories: ImmutableHashSet.Create([Categories.Libraries, Categories.Runtime, Categories.ThirdParty, Categories.Sve]),
@@ -52,7 +58,9 @@ static int Main(string[] args)
                         categoryExclusionFilterValue: categoryExclusionFilterValue,
                         getDiffableDisasm: getDiffableDisasm)
                     .AddValidator(new NoWasmValidator(Categories.NoWASM)))
-                .ToExitCode();
+                .ConfigureAwait(false);
+
+            return summaries.ToExitCode();
         }
     }
 }

src/benchmarks/micro/README.md

@@ -12,7 +12,7 @@ To learn more about designing benchmarks, please read [Microbenchmark Design Gui
 
 ## Quick Start
 
-The first thing that you need to choose is the Target Framework. Available options are: `netcoreapp3.1|net6.0|net7.0|net8.0|net9.0|net10.0|net11.0|net472`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework.
+The first thing that you need to choose is the Target Framework. Available options are: `net8.0|net9.0|net10.0|net11.0`. You can specify the target framework using `-f|--framework` argument. For the sake of simplicity, all examples below use `net11.0` as the target framework.
 
 The following commands are run from the `src/benchmarks/micro` directory.
 

src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj

@@ -1,7 +1,11 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Library</OutputType>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <!-- net8.0 is the lowest TFM exercised by the perf pipelines (see scripts/channel_map.py).
+         net472 (and other netfx) consumers are no longer supported; the harness uses BDN APIs
+         (e.g. IValidator.ValidateAsync returning IAsyncEnumerable<T>) that would otherwise require
+         polyfill packages on netstandard2.0. -->
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ProduceReferenceAssembly>true</ProduceReferenceAssembly>
   </PropertyGroup>
@@ -13,7 +17,7 @@
     <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="$(BenchmarkDotNetVersion)" />
   </ItemGroup>
   <ItemGroup Condition="'$(BenchmarkDotNetSources)' != ''">
-    <ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet\BenchmarkDotNet.csproj" SetTargetFramework="TargetFramework=netstandard2.0" />
+    <ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet\BenchmarkDotNet.csproj" SetTargetFramework="TargetFramework=net8.0" />
     <ProjectReference Include="$(_BenchmarkDotNetSourcesN)src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" SetTargetFramework="TargetFramework=netstandard2.0" />
   </ItemGroup>
   <ItemGroup>

src/harness/BenchmarkDotNet.Extensions/DiffableDisassemblyExporter.cs

@@ -61,26 +61,26 @@ internal static string BuildDisassemblyString(DisassemblyResult disassemblyResul
 
         private static Func<object, T> GetElementGetter<T>(string name)
         {
-            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier");
+            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!;
 
-            type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic);
+            type = type.GetNestedType("Element", BindingFlags.Instance | BindingFlags.NonPublic)!;
 
-            var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic);
+            var property = type.GetProperty(name, BindingFlags.Instance | BindingFlags.NonPublic)!;
 
-            var method = property.GetGetMethod(nonPublic: true);
+            var method = property.GetGetMethod(nonPublic: true)!;
 
             var generic = typeof(Func<,>).MakeGenericType(type, typeof(T));
 
             var @delegate = method.CreateDelegate(generic);
 
-            return (obj) => (T)@delegate.DynamicInvoke(obj); // cast to (Func<object, T>) throws
+            return (obj) => (T)@delegate.DynamicInvoke(obj)!; // cast to (Func<object, T>) throws
         }
 
         private static Func<DisassembledMethod, DisassemblyResult, DisassemblyDiagnoserConfig, string, IReadOnlyList<object>> GetPrettifyMethod()
         {
-            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier");
+            var type = typeof(DisassemblyDiagnoser).Assembly.GetType("BenchmarkDotNet.Disassemblers.Exporters.DisassemblyPrettifier")!;
 
-            var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic);
+            var method = type.GetMethod("Prettify", BindingFlags.Static | BindingFlags.NonPublic)!;
 
             var @delegate = method.CreateDelegate(typeof(Func<DisassembledMethod, DisassemblyResult, DisassemblyDiagnoserConfig, string, IReadOnlyList<object>>));
 

src/harness/BenchmarkDotNet.Extensions/MandatoryCategoryValidator.cs

@@ -21,7 +21,7 @@ public class MandatoryCategoryValidator : IValidator
 
         public MandatoryCategoryValidator(ImmutableHashSet<string> categories) => _mandatoryCategories = categories;
 
-        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+        public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
             => validationParameters.Benchmarks
                 .Where(benchmark => !benchmark.Descriptor.Categories.Any(category => _mandatoryCategories.Contains(category)))
                 .Select(benchmark => benchmark.Descriptor.GetFilterName())
@@ -30,6 +30,7 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
                     new ValidationError(
                         isCritical: TreatsWarningsAsErrors,
                         $"{benchmarkId} does not belong to one of the mandatory categories: {string.Join(", ", _mandatoryCategories)}. Use [BenchmarkCategory(Categories.$)]")
-                );
+                )
+                .ToAsyncEnumerable();
     }
 }

src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs

@@ -23,7 +23,7 @@ public class NoWasmValidator : IValidator
 
         public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory;
 
-        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
+        public IAsyncEnumerable<ValidationError> ValidateAsync(ValidationParameters validationParameters)
             => validationParameters.Benchmarks
                 .Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal)))
                 .Select(benchmark => benchmark.Descriptor.GetFilterName())
@@ -32,7 +32,8 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
                     new ValidationError(
                         isCritical: TreatsWarningsAsErrors,
                         $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]")
-                );
+                )
+                .ToAsyncEnumerable();
 
         private bool IsAsyncMethod(MethodInfo workloadMethod)
         {

src/harness/BenchmarkDotNet.Extensions/PerfLabExporter.cs

@@ -4,24 +4,67 @@
 
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Reports;
 using Reporting;
 using System;
+using System.IO;
 using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace BenchmarkDotNet.Extensions
 {
-    public class PerfLabExporter : ExporterBase
+    // Implements IExporter directly (not ExporterBase) because PerfLabExporter writes
+    // a file with a custom name pattern ("{type}-perf-lab-report.json") via
+    // File.WriteAllTextAsync and manages the file lifecycle itself, rather than having
+    // ExporterBase open and hand us a writer for a default-named file.
+    public class PerfLabExporter : IExporter
     {
-        protected override string FileExtension => "json";
-        protected override string FileCaption => "perf-lab-report";
+        private const string FileExtension = "json";
+        private const string FileCaption = "perf-lab-report";
 
-        public PerfLabExporter()
+        public string Name => nameof(PerfLabExporter);
+
+        public async ValueTask ExportAsync(Summary summary, ILogger logger, CancellationToken cancellationToken)
+        {
+            string? jsonOutput = BuildJson(summary);
+            if (jsonOutput is null)
+                return;
+
+            string filePath = GetArtifactFullName(summary);
+            if (File.Exists(filePath))
+            {
+                try
+                {
+                    File.Delete(filePath);
+                }
+                catch (IOException)
+                {
+                    string uniqueString = DateTime.Now.ToString("yyyyMMdd-HHmmss");
+                    string altPath = $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}-{uniqueString}.{FileExtension}";
+                    logger.WriteLineError($"Could not overwrite file {filePath}. Exporting to {altPath}");
+                    filePath = altPath;
+                }
+            }
+
+            await File.WriteAllTextAsync(filePath, jsonOutput, cancellationToken).ConfigureAwait(false);
+            logger.WriteLineInfo($"  {filePath}");
+        }
+
+        private string GetArtifactFullName(Summary summary)
+            => $"{Path.Combine(summary.ResultsDirectoryPath, GetFileName(summary))}-{FileCaption}.{FileExtension}";
+
+        private static string GetFileName(Summary summary)
         {
+            var targets = summary.BenchmarksCases.Select(b => b.Descriptor.Type).Distinct().ToArray();
+            if (targets.Length == 1)
+                return FolderNameHelper.ToFolderName(targets.Single());
+            return summary.Title;
         }
 
-        public override void ExportToLog(Summary summary, ILogger logger)
+        private static string? BuildJson(Summary summary)
         {
             var reporter = new Reporter();
 
@@ -49,7 +92,7 @@ public override void ExportToLog(Summary summary, ILogger logger)
                 var test = new Test();
                 test.Name = FullNameProvider.GetBenchmarkName(report.BenchmarkCase);
                 test.Categories = report.BenchmarkCase.Descriptor.Categories;
-                
+
                 if (hasCriticalErrors)
                 {
                     test.AdditionalData["criticalErrors"] = "true";
@@ -58,7 +101,7 @@ public override void ExportToLog(Summary summary, ILogger logger)
                 var results = from result in report.AllMeasurements
                               where result.IterationMode == Engines.IterationMode.Workload && result.IterationStage == Engines.IterationStage.Result
                               orderby result.LaunchIndex, result.IterationIndex
-                              select new { result.Nanoseconds, result.Operations};
+                              select new { result.Nanoseconds, result.Operations };
 
                 var overheadResults = from result in report.AllMeasurements
                                       where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jitting
@@ -104,7 +147,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit
                     HigherIsBetter = true,
                     MetricName = "Count",
                     Results = (from result in results
-                               select  (double)result.Operations).ToList()
+                               select (double)result.Operations).ToList()
                 });
 
                 foreach (var metric in report.Metrics.Keys)
@@ -130,9 +173,7 @@ where result.IsOverhead() && result.IterationStage != Engines.IterationStage.Jit
                 reporter.AddTest(test);
             }
 
-            var jsonOutput = reporter.GetJson();
-            if (jsonOutput is not null)
-                logger.WriteLine(jsonOutput);
+            return reporter.GetJson();
         }
     }
 }