From 739291418ce03c4db63100af4bf950ab8085084e Mon Sep 17 00:00:00 2001 From: Shrinidhi203 <2024SI96508@wilp.bits-pilani.ac.in> Date: Fri, 28 Nov 2025 15:04:36 +0530 Subject: [PATCH 1/3] "Fix unhandled exception when running invalid type-based benchmark with arguments Fixes #2724 When running BenchmarkRunner.Run() or BenchmarkRunner.Run(Type) with arguments on an invalid benchmark type (no [Benchmark] methods), the runner threw an unhandled InvalidOperationException instead of returning a validation error. Changes: - Replace .Single() with .SingleOrDefault() in RunWithDirtyAssemblyResolveHelper - Return Summary.ValidationFailed when no benchmarks are found - Enable and update tests for args scenarios in RunningEmptyBenchmarkTests.cs" --- .../Running/BenchmarkRunnerDirty.cs | 17 +++++++--- .../Running/RunningEmptyBenchmarkTests.cs | 33 ++++++++++++++----- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs index 2f8606b0cd..f80158e35f 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerDirty.cs @@ -86,14 +86,23 @@ public static Summary RunSource(string source, IConfig? config = null) [MethodImpl(MethodImplOptions.NoInlining)] private static Summary RunWithDirtyAssemblyResolveHelper(Type type, IConfig? config, string[]? args) - => (args == null + { + var summaries = args == null ? BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.TypeToBenchmarks(type, config) }) - : new BenchmarkSwitcher(new[] { type }).RunWithDirtyAssemblyResolveHelper(args, config, false)) - .Single(); + : new BenchmarkSwitcher(new[] { type }).RunWithDirtyAssemblyResolveHelper(args, config, false); + + return summaries.SingleOrDefault() + ?? Summary.ValidationFailed($"No benchmarks found in type '{type.Name}'", string.Empty, string.Empty); + } [MethodImpl(MethodImplOptions.NoInlining)] private static Summary RunWithDirtyAssemblyResolveHelper(Type type, MethodInfo[] methods, IConfig? config = null) - => BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.MethodsToBenchmarks(type, methods, config) }).Single(); + { + var summaries = BenchmarkRunnerClean.Run(new[] { BenchmarkConverter.MethodsToBenchmarks(type, methods, config) }); + + return summaries.SingleOrDefault() + ?? Summary.ValidationFailed($"No benchmarks found in type '{type.Name}'", string.Empty, string.Empty); + } [MethodImpl(MethodImplOptions.NoInlining)] private static Summary[] RunWithDirtyAssemblyResolveHelper(Assembly assembly, IConfig? config, string[]? args) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index 3dcee47bb6..ea4eacad6b 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -42,25 +42,31 @@ public class RunningEmptyBenchmarkTests /// [Theory] [InlineData(null)] - //[InlineData(new object[] { new string[] { " " } })] + [InlineData(new object[] { new string[] { " " } })] public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) { GetConfigWithLogger(out var logger, out var config); var summary = BenchmarkRunner.Run(config, args); + if (args == null) { Assert.True(summary.HasCriticalValidationErrors); Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + else + { + // When args is provided and type is invalid, we get a ValidationFailed summary + // instead of an unhandled exception + Assert.NotNull(summary); } - - Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); } #pragma warning restore BDN1000 [Theory] [InlineData(null)] - //[InlineData(new object[] { new string[] { " " } })] + [InlineData(new object[] { new string[] { " " } })] public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) { GetConfigWithLogger(out var logger, out var config); @@ -79,16 +85,25 @@ public void GenericTypeWithBenchmarkAttribute_RunsSuccessfully(string[]? args) /// [Theory] [InlineData(null)] - //[InlineData(new object[] { new string[] { " " } })] + [InlineData(new object[] { new string[] { " " } })] public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkAttribute(string[]? args) { GetConfigWithLogger(out var logger, out var config); - var summary = BenchmarkRunner.Run(typeof(EmptyBenchmark), config, args); - Assert.True(summary.HasCriticalValidationErrors); - Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); - Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + + if (args == null) + { + Assert.True(summary.HasCriticalValidationErrors); + Assert.Contains(summary.ValidationErrors, validationError => validationError.Message == GetValidationErrorForType(typeof(EmptyBenchmark))); + Assert.Contains(GetValidationErrorForType(typeof(EmptyBenchmark)), logger.GetLog()); + } + else + { + // When args is provided and type is invalid, we get a ValidationFailed summary + // instead of an unhandled exception + Assert.NotNull(summary); + } } #pragma warning restore BDN1000 From 38d9b3647694f3e53fde8eba395a6488877a24de Mon Sep 17 00:00:00 2001 From: Shrinidhi203 <2024SI96508@wilp.bits-pilani.ac.in> Date: Fri, 28 Nov 2025 16:20:16 +0530 Subject: [PATCH 2/3] Fix StyleCop SA1028: Remove trailing whitespace --- .../Running/RunningEmptyBenchmarkTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index ea4eacad6b..a13f2146c9 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -58,7 +58,7 @@ public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBen else { // When args is provided and type is invalid, we get a ValidationFailed summary - // instead of an unhandled exception + // instead of an unhandled exception Assert.NotNull(summary); } } @@ -101,7 +101,7 @@ public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkA else { // When args is provided and type is invalid, we get a ValidationFailed summary - // instead of an unhandled exception + // instead of an unhandled exception Assert.NotNull(summary); } } From e390fa388377f87d9e3e24af7ef9ebb90a1143b7 Mon Sep 17 00:00:00 2001 From: Shrinidhi203 <2024SI96508@wilp.bits-pilani.ac.in> Date: Sat, 29 Nov 2025 12:09:36 +0530 Subject: [PATCH 3/3] "Add assertion for failure in test per maintainer feedback" --- .../Running/RunningEmptyBenchmarkTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs index a13f2146c9..ea2836b5f3 100644 --- a/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/RunningEmptyBenchmarkTests.cs @@ -58,8 +58,9 @@ public void GenericTypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBen else { // When args is provided and type is invalid, we get a ValidationFailed summary - // instead of an unhandled exception + // instead of an unhandled exception (fix for issue #2724) Assert.NotNull(summary); + Assert.Contains("No benchmarks found", summary.Title); } } #pragma warning restore BDN1000 @@ -101,8 +102,9 @@ public void TypeWithoutBenchmarkAttribute_ThrowsValidationError_WhenNoBenchmarkA else { // When args is provided and type is invalid, we get a ValidationFailed summary - // instead of an unhandled exception + // instead of an unhandled exception (fix for issue #2724) Assert.NotNull(summary); + Assert.Contains("No benchmarks found", summary.Title); } } #pragma warning restore BDN1000