From 6a58dae292af6e3b05ed960068b6e2b1cdb12ea4 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Sun, 30 Nov 2025 19:27:46 -0500 Subject: [PATCH 01/11] Compare and group benchmark parameters by object value rather than string representation --- src/BenchmarkDotNet/Order/DefaultOrderer.cs | 4 ++- .../Parameters/ParameterComparer.cs | 34 ++++++++++++++----- .../Parameters/ParameterEqualityComparer.cs | 20 +++++++++++ 3 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index d0dc2d475b..6acfff7dc4 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -89,6 +89,8 @@ public string GetHighlightGroupKey(BenchmarkCase benchmarkCase) public string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, BenchmarkCase benchmarkCase) { + var paramSets = allBenchmarksCases.Select(benchmarkCase => benchmarkCase.Parameters).Distinct(ParameterEqualityComparer.Instance).ToArray(); + var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList(); var implicitRules = new List(); bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline); @@ -125,7 +127,7 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase keys.Add(benchmarkCase.Job.DisplayInfo); break; case BenchmarkLogicalGroupRule.ByParams: - keys.Add(benchmarkCase.Parameters.ValueInfo); + keys.Add($"Distinct Param Set {Array.FindIndex(paramSets, (paramSet) => ParameterEqualityComparer.Instance.Equals(paramSet, benchmarkCase.Parameters))}"); break; case BenchmarkLogicalGroupRule.ByCategory: keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories)); diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index 6cadf6a946..05a5e05766 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -1,5 +1,7 @@ using System; +using System.Collections; using System.Collections.Generic; +using System.Linq; namespace BenchmarkDotNet.Parameters { @@ -18,6 +20,7 @@ public int Compare(ParameterInstances x, ParameterInstances y) if (compareTo != 0) return compareTo; } + return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo); } @@ -25,18 +28,33 @@ private int CompareValues(object x, object y) { // Detect IComparable implementations. // This works for all primitive types in addition to user types that implement IComparable. - if (x != null && y != null && x.GetType() == y.GetType() && - x is IComparable xComparable) + if (x != null && y != null && x.GetType() == y.GetType()) { - try + if (x is IComparable xComparable) { - return xComparable.CompareTo(y); + try + { + return xComparable.CompareTo(y); + } + // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. + // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 + // For now, catch and ignore the exception, and fallback to string comparison below. + catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + { + } } - // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. - // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 - // For now, catch and ignore the exception, and fallback to string comparison below. - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // collection equality support { + if (x is Array xArr && y is Array yArr) // check rank here for arrays because their values will get compared when flattened + { + if (xArr.Rank != yArr.Rank) + return xArr.Rank.CompareTo(yArr.Rank); + } + + var xFlat = xEnumerable.OfType().ToArray(); + var yFlat = yEnumerable.OfType().ToArray(); + + return StructuralComparisons.StructuralComparer.Compare(xFlat, yFlat); } } diff --git a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs new file mode 100644 index 0000000000..f1da16f340 --- /dev/null +++ b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace BenchmarkDotNet.Parameters +{ + internal class ParameterEqualityComparer : IEqualityComparer + { + public static readonly ParameterEqualityComparer Instance = new ParameterEqualityComparer(); + + public bool Equals(ParameterInstances? x, ParameterInstances? y) + { + return ParameterComparer.Instance.Compare(x, y) == 0; + } + + public int GetHashCode([DisallowNull] ParameterInstances obj) + { + return obj.ValueInfo.GetHashCode(); + } + } +} \ No newline at end of file From a090785043390da4af8570f607da30a0d38e647f Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Sun, 30 Nov 2025 19:28:06 -0500 Subject: [PATCH 02/11] Add test for grouping benchmark parameters which are collections --- .../ArgumentsTests.cs | 113 +++++++++++++++++- 1 file changed, 109 insertions(+), 4 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs index 76edfe5d4e..e14001bf52 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; @@ -298,6 +299,60 @@ public IEnumerable Sources() public void Any(string name, IEnumerable source) => source.Any(); } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void IEnumerableArgumentsCanBeGrouped(IToolchain toolchain) + { + var summary = CanExecute(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping + + // There should be two logical groups, one for the first argument and one for the second argument + // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group + + var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor); + + Assert.True( + descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2) + ); + } + + public class EnumerableOnDifferentMethods + { + public IEnumerable EnumerableOne() + { + yield return 1; + yield return 2; + yield return 3; + } + + public IEnumerable EnumerableTwo() + { + yield return 1; + yield return 2; + yield return 3; + } + + public IEnumerable> GetEnumerables() + { + yield return EnumerableOne(); + yield return EnumerableTwo(); + } + + [Benchmark(Baseline = true)] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables(IEnumerable arr) + { + if (arr.Count() != 3) + throw new ArgumentException("Incorrect length"); + } + + [Benchmark] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables2(IEnumerable collection) + { + if (collection.Count() != 3) + throw new ArgumentException("Incorrect length"); + } + } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void JaggedArrayCanBeUsedAsArgument(IToolchain toolchain) => CanExecute(toolchain); @@ -311,9 +366,9 @@ public void Test(int[][] array) throw new ArgumentNullException(nameof(array)); for (int i = 0; i < 10; i++) - for (int j = 0; j < i; j++) - if (array[i][j] != i) - throw new ArgumentException("Invalid value"); + for (int j = 0; j < i; j++) + if (array[i][j] != i) + throw new ArgumentException("Invalid value"); } public IEnumerable CreateMatrix() @@ -543,6 +598,46 @@ public void AcceptsArrays(int[] even, int[] notEven) } } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void ArrayArgumentsCanBeGrouped(IToolchain toolchain) + { + var summary = CanExecute(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping + + // There should be two logical groups, one for the first argument and one for the second argument + // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group + + var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor); + + Assert.True( + descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2) + ); + } + + public class ArrayOnDifferentMethods + { + public IEnumerable GetArrays() + { + yield return new int[] { 1, 2, 3 }; + yield return new int[] { 2, 3, 4 }; + } + + [Benchmark(Baseline = true)] + [ArgumentsSource(nameof(GetArrays))] + public void AcceptsArrays(int[] arr) + { + if (arr.Length != 3) + throw new ArgumentException("Incorrect length"); + } + + [Benchmark] + [ArgumentsSource(nameof(GetArrays))] + public void AcceptsArrays2(int[] arr) + { + if (arr.Length != 3) + throw new ArgumentException("Incorrect length"); + } + } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] public void VeryBigIntegersAreSupported(IToolchain toolchain) => CanExecute(toolchain); @@ -1026,6 +1121,16 @@ public Disposable(int id) } } - private void CanExecute(IToolchain toolchain) => CanExecute(CreateSimpleConfig(job: Job.Dry.WithToolchain(toolchain))); + private Reports.Summary CanExecute(IToolchain toolchain, Func furtherConfigure = null) + { + var config = CreateSimpleConfig(job: Job.Dry.WithToolchain(toolchain)); + + if (furtherConfigure is not null) + { + config = furtherConfigure(config); + } + + return CanExecute(config); + } } } \ No newline at end of file From 5c265ae45e25c75b9240849a36d057e5677b87f5 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Mon, 1 Dec 2025 07:47:12 -0500 Subject: [PATCH 03/11] Remove whitespace from params logical group key --- src/BenchmarkDotNet/Order/DefaultOrderer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 6acfff7dc4..b30a841118 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -127,7 +127,7 @@ public string GetLogicalGroupKey(ImmutableArray allBenchmarksCase keys.Add(benchmarkCase.Job.DisplayInfo); break; case BenchmarkLogicalGroupRule.ByParams: - keys.Add($"Distinct Param Set {Array.FindIndex(paramSets, (paramSet) => ParameterEqualityComparer.Instance.Equals(paramSet, benchmarkCase.Parameters))}"); + keys.Add($"DistinctParamSet{Array.FindIndex(paramSets, (paramSet) => ParameterEqualityComparer.Instance.Equals(paramSet, benchmarkCase.Parameters))}"); break; case BenchmarkLogicalGroupRule.ByCategory: keys.Add(string.Join(",", benchmarkCase.Descriptor.Categories)); From 24086de7a7e346edbaa795d984f9103393815232 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Mon, 1 Dec 2025 08:17:09 -0500 Subject: [PATCH 04/11] Update Verify files to account for logical group key changes --- ...rTest_Invalid_TwoJobBaselines.verified.txt | 18 ++--- ...st_Invalid_TwoMethodBaselines.verified.txt | 10 +-- ...rTest_JobBaseline_MethodsJobs.verified.txt | 20 +++--- ...JobBaseline_MethodsParamsJobs.verified.txt | 38 +++++------ ...erTest_MethodBaseline_Methods.verified.txt | 10 +-- ...st_MethodBaseline_MethodsJobs.verified.txt | 18 ++--- ..._MethodBaseline_MethodsParams.verified.txt | 18 ++--- ...hodBaseline_MethodsParamsJobs.verified.txt | 34 +++++----- ...MethodJobBaseline_MethodsJobs.verified.txt | 12 ++-- ...JobBaseline_MethodsJobsParams.verified.txt | 22 +++---- ..._MethodsParamsJobs_GroupByAll.verified.txt | 66 +++++++++---------- ...odsParamsJobs_GroupByCategory.verified.txt | 50 +++++++------- ...thodsParamsJobs_GroupByParams.verified.txt | 30 ++++----- 13 files changed, 173 insertions(+), 173 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt index 314e1f16bd..e7fedb599a 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoJobBaselines.verified.txt @@ -8,14 +8,14 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- | - Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Invalid_TwoJobBaselines.Foo | Yes | - Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 2 | Invalid_TwoJobBaselines.Foo | Yes | - | | | | | | | | | | - Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | Invalid_TwoJobBaselines.Bar | Yes | - Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.94 | 0.09 | 2 | Invalid_TwoJobBaselines.Bar | Yes | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-Invalid_TwoJobBaselines.Foo | Yes | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 2 | DistinctParamSet0-Invalid_TwoJobBaselines.Foo | Yes | + | | | | | | | | | | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | DistinctParamSet0-Invalid_TwoJobBaselines.Bar | Yes | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.94 | 0.09 | 2 | DistinctParamSet0-Invalid_TwoJobBaselines.Bar | Yes | Errors: 2 -* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2 -* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Bar in class Invalid_TwoJobBaselines has 2 +* Only 1 job in a group can have "Baseline = true" applied to it, group DistinctParamSet0-Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2 +* Only 1 job in a group can have "Baseline = true" applied to it, group DistinctParamSet0-Invalid_TwoJobBaselines.Bar in class Invalid_TwoJobBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt index 11fbd70a0b..61eb0a37dc 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_Invalid_TwoMethodBaselines.verified.txt @@ -7,10 +7,10 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC DefaultJob : extra output line - Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | - Bar | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | Yes | + Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | + Foo | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-DefaultJob | Yes | + Bar | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-DefaultJob | Yes | Errors: 1 -* Only 1 benchmark method in a group can have "Baseline = true" applied to it, group DefaultJob in class Invalid_TwoMethodBaselines has 2 +* Only 1 benchmark method in a group can have "Baseline = true" applied to it, group DistinctParamSet0-DefaultJob in class Invalid_TwoMethodBaselines has 2 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt index 5c8cbaad01..2bb1e98434 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsJobs.verified.txt @@ -8,15 +8,15 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | - Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | JobBaseline_MethodsJobs.Base | Yes | - Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | JobBaseline_MethodsJobs.Base | No | - | | | | | | | | | | - Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | JobBaseline_MethodsJobs.Foo | Yes | - Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | JobBaseline_MethodsJobs.Foo | No | - | | | | | | | | | | - Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | JobBaseline_MethodsJobs.Bar | Yes | - Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | JobBaseline_MethodsJobs.Bar | No | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------------------------- |--------- | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-JobBaseline_MethodsJobs.Base | Yes | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | DistinctParamSet0-JobBaseline_MethodsJobs.Base | No | + | | | | | | | | | | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | DistinctParamSet0-JobBaseline_MethodsJobs.Foo | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | DistinctParamSet0-JobBaseline_MethodsJobs.Foo | No | + | | | | | | | | | | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | DistinctParamSet0-JobBaseline_MethodsJobs.Bar | Yes | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | DistinctParamSet0-JobBaseline_MethodsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt index 4f5f37b6c7..150f6b5d91 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_JobBaseline_MethodsParamsJobs.verified.txt @@ -8,24 +8,24 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- | - Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No | - | | | | | | | | | | | - Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No | - | | | | | | | | | | | - Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No | - | | | | | | | | | | | - Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^ - Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.42 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No | - | | | | | | | | | | | - Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes | - Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.37 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No | - | | | | | | | | | | | - Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes | - Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.33 | 0.02 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|----------------------------------------------------- |--------- | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 2 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Base | No | + | | | | | | | | | | | + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.06 | 1 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 2.40 | 0.11 | 2 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Foo | No | + | | | | | | | | | | | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.96 | 0.06 | 2 | DistinctParamSet0-JobBaseline_MethodsParamsJobs.Bar | No | + | | | | | | | | | | | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Base | Yes | ^ + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.42 | 0.02 | 2 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Base | No | + | | | | | | | | | | | + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Foo | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.37 | 0.02 | 2 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Foo | No | + | | | | | | | | | | | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Bar | Yes | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.33 | 0.02 | 2 | DistinctParamSet1-JobBaseline_MethodsParamsJobs.Bar | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt index aa22c9caab..9bbfa03f47 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_Methods.verified.txt @@ -7,10 +7,10 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC DefaultJob : extra output line - Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DefaultJob | Yes | - Foo | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DefaultJob | No | - Bar | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DefaultJob | No | + Method | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | + Base | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-DefaultJob | Yes | + Foo | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-DefaultJob | No | + Bar | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt index 297f782355..6741559eb2 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsJobs.verified.txt @@ -8,14 +8,14 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | Job1 | Yes | - Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | Job1 | No | - Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | Job1 | No | - | | | | | | | | | | - Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | Job2 | Yes | - Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | Job2 | No | - Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | Job2 | No | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------- |--------- | + Base | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-Job1 | Yes | + Foo | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-Job1 | No | + Bar | Job1 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-Job1 | No | + | | | | | | | | | | + Base | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | DistinctParamSet0-Job2 | Yes | + Foo | Job2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | DistinctParamSet0-Job2 | No | + Bar | Job2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | DistinctParamSet0-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt index e30ce8166a..eff7eb4cf2 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParams.verified.txt @@ -7,14 +7,14 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC DefaultJob : extra output line - Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |------ |---------:|--------:|--------:|------:|--------:|-----:|---------------------- |--------- | - Base | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-DefaultJob | Yes | ^ - Foo | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-DefaultJob | No | - Bar | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-DefaultJob | No | - | | | | | | | | | | - Base | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=10]-DefaultJob | Yes | ^ - Foo | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=10]-DefaultJob | No | - Bar | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=10]-DefaultJob | No | + Method | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |------ |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- | + Base | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-DefaultJob | Yes | ^ + Foo | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-DefaultJob | No | + Bar | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-DefaultJob | No | + | | | | | | | | | | + Base | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | DistinctParamSet1-DefaultJob | Yes | ^ + Foo | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | DistinctParamSet1-DefaultJob | No | + Bar | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | DistinctParamSet1-DefaultJob | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt index f85ce9e878..3b94cd3819 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodBaseline_MethodsParamsJobs.verified.txt @@ -8,22 +8,22 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- | - Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2]-Job1 | Yes | ^ - Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2]-Job1 | No | - Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2]-Job1 | No | - | | | | | | | | | | | - Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | [Param=2]-Job2 | Yes | - Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | [Param=2]-Job2 | No | - Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | [Param=2]-Job2 | No | - | | | | | | | | | | | - Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10]-Job1 | Yes | ^ - Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | [Param=10]-Job1 | No | - Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.28 | 0.02 | 3 | [Param=10]-Job1 | No | - | | | | | | | | | | | - Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | [Param=10]-Job2 | Yes | - Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.10 | 0.01 | 2 | [Param=10]-Job2 | No | - Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.20 | 0.01 | 3 | [Param=10]-Job2 | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|----------------------- |--------- | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0-Job1 | Yes | ^ + Foo | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0-Job1 | No | + Bar | Job1 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0-Job1 | No | + | | | | | | | | | | | + Base | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.03 | 1 | DistinctParamSet0-Job2 | Yes | + Foo | Job2 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 1.24 | 0.03 | 2 | DistinctParamSet0-Job2 | No | + Bar | Job2 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 1.48 | 0.04 | 3 | DistinctParamSet0-Job2 | No | + | | | | | | | | | | | + Base | Job1 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1-Job1 | Yes | ^ + Foo | Job1 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | DistinctParamSet1-Job1 | No | + Bar | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 1.28 | 0.02 | 3 | DistinctParamSet1-Job1 | No | + | | | | | | | | | | | + Base | Job2 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | DistinctParamSet1-Job2 | Yes | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.10 | 0.01 | 2 | DistinctParamSet1-Job2 | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.20 | 0.01 | 3 | DistinctParamSet1-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt index e71eeca6b6..1074de050f 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobs.verified.txt @@ -8,11 +8,11 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | * | Yes | - Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | * | No | - Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | * | No | - Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | * | No | + Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------------ |--------- | + Foo | Job1 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0 | Yes | + Bar | Job1 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0 | No | + Foo | Job2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0 | No | + Bar | Job2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | DistinctParamSet0 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt index 30d92f0b9e..173aff0467 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_MethodJobBaseline_MethodsJobsParams.verified.txt @@ -8,16 +8,16 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- | - Foo | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | [Param=2] | Yes | ^ - Bar | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | [Param=2] | No | - Foo | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | [Param=2] | No | - Bar | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | [Param=2] | No | - | | | | | | | | | | | - Foo | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | [Param=10] | Yes | ^ - Bar | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | [Param=10] | No | - Foo | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.39 | 0.03 | 3 | [Param=10] | No | - Bar | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.58 | 0.03 | 4 | [Param=10] | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |---------:|--------:|--------:|------:|--------:|-----:|------------------ |--------- | + Foo | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | DistinctParamSet0 | Yes | ^ + Bar | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | DistinctParamSet0 | No | + Foo | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 2.76 | 0.22 | 3 | DistinctParamSet0 | No | + Bar | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 3.64 | 0.29 | 4 | DistinctParamSet0 | No | + | | | | | | | | | | | + Foo | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | DistinctParamSet1 | Yes | ^ + Bar | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | DistinctParamSet1 | No | + Foo | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.39 | 0.03 | 3 | DistinctParamSet1 | No | + Bar | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.58 | 0.03 | 4 | DistinctParamSet1 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt index a7a7350881..9012bdff5e 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByAll.verified.txt @@ -8,38 +8,38 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------------------------- |--------- | - A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=2]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-[Param=10]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=2]-CatA | Yes | ^ - | | | | | | | | | | | - A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-[Param=10]-CatA | Yes | ^ - | | | | | | | | | | | - A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=2]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-[Param=10]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=2]-CatA | No | ^ - | | | | | | | | | | | - A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-[Param=10]-CatA | No | ^ - | | | | | | | | | | | - B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=2]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-[Param=10]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=2]-CatB | Yes | ^ - | | | | | | | | | | | - B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-[Param=10]-CatB | Yes | ^ - | | | | | | | | | | | - B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=2]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-[Param=10]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=2]-CatB | No | ^ - | | | | | | | | | | | - B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-[Param=10]-CatB | No | ^ + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|----------------------------------------------------------------------- |--------- | + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-DistinctParamSet0-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job1-DistinctParamSet1-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-DistinctParamSet0-CatA | Yes | ^ + | | | | | | | | | | | + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A1-Job2-DistinctParamSet1-CatA | Yes | ^ + | | | | | | | | | | | + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-DistinctParamSet0-CatA | No | ^ + | | | | | | | | | | | + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job1-DistinctParamSet1-CatA | No | ^ + | | | | | | | | | | | + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-DistinctParamSet0-CatA | No | ^ + | | | | | | | | | | | + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.A2-Job2-DistinctParamSet1-CatA | No | ^ + | | | | | | | | | | | + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-DistinctParamSet0-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job1-DistinctParamSet1-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-DistinctParamSet0-CatB | Yes | ^ + | | | | | | | | | | | + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B1-Job2-DistinctParamSet1-CatB | Yes | ^ + | | | | | | | | | | | + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-DistinctParamSet0-CatB | No | ^ + | | | | | | | | | | | + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job1-DistinctParamSet1-CatB | No | ^ + | | | | | | | | | | | + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-DistinctParamSet0-CatB | No | ^ + | | | | | | | | | | | + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | ? | ? | 1 | NoBaseline_MethodsParamsJobs_GroupByAll.B2-Job2-DistinctParamSet1-CatB | No | ^ Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt index bb98037dd3..af6dca36cd 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByCategory.verified.txt @@ -8,30 +8,30 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|--------------------- |--------- | - A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | CatA-[Param=2]-Job1 | Yes | ^ - A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | CatA-[Param=2]-Job1 | No | - | | | | | | | | | | | - A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | CatA-[Param=2]-Job2 | Yes | - A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.32 | 0.05 | 2 | CatA-[Param=2]-Job2 | No | - | | | | | | | | | | | - A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job1 | Yes | ^ - A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | CatA-[Param=10]-Job1 | No | - | | | | | | | | | | | - A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-[Param=10]-Job2 | Yes | - A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | CatA-[Param=10]-Job2 | No | - | | | | | | | | | | | - B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job1 | Yes | ^ - B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.11 | 0.01 | 2 | CatB-[Param=2]-Job1 | No | - | | | | | | | | | | | - B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=2]-Job2 | Yes | - B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.09 | 0.01 | 2 | CatB-[Param=2]-Job2 | No | - | | | | | | | | | | | - B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job1 | Yes | ^ - B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | 1.08 | 0.01 | 2 | CatB-[Param=10]-Job1 | No | - | | | | | | | | | | | - B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-[Param=10]-Job2 | Yes | - B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | 1.07 | 0.01 | 2 | CatB-[Param=10]-Job2 | No | + Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- | + A1 | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1.01 | 0.11 | 1 | CatA-DistinctParamSet0-Job1 | Yes | ^ + A2 | Job1 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 1.88 | 0.16 | 2 | CatA-DistinctParamSet0-Job1 | No | + | | | | | | | | | | | + A1 | Job2 | 2 | 314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.04 | 1 | CatA-DistinctParamSet0-Job2 | Yes | + A2 | Job2 | 2 | 414.5 ns | 5.88 ns | 8.80 ns | 1.32 | 0.05 | 2 | CatA-DistinctParamSet0-Job2 | No | + | | | | | | | | | | | + A1 | Job1 | 10 | 514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-DistinctParamSet1-Job1 | Yes | ^ + A2 | Job1 | 10 | 614.5 ns | 5.88 ns | 8.80 ns | 1.19 | 0.03 | 2 | CatA-DistinctParamSet1-Job1 | No | + | | | | | | | | | | | + A1 | Job2 | 10 | 714.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.02 | 1 | CatA-DistinctParamSet1-Job2 | Yes | + A2 | Job2 | 10 | 814.5 ns | 5.88 ns | 8.80 ns | 1.14 | 0.02 | 2 | CatA-DistinctParamSet1-Job2 | No | + | | | | | | | | | | | + B1 | Job1 | 2 | 914.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet0-Job1 | Yes | ^ + B2 | Job1 | 2 | 1,014.5 ns | 5.88 ns | 8.80 ns | 1.11 | 0.01 | 2 | CatB-DistinctParamSet0-Job1 | No | + | | | | | | | | | | | + B1 | Job2 | 2 | 1,114.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet0-Job2 | Yes | + B2 | Job2 | 2 | 1,214.5 ns | 5.88 ns | 8.80 ns | 1.09 | 0.01 | 2 | CatB-DistinctParamSet0-Job2 | No | + | | | | | | | | | | | + B1 | Job1 | 10 | 1,314.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet1-Job1 | Yes | ^ + B2 | Job1 | 10 | 1,414.5 ns | 5.88 ns | 8.80 ns | 1.08 | 0.01 | 2 | CatB-DistinctParamSet1-Job1 | No | + | | | | | | | | | | | + B1 | Job2 | 10 | 1,514.5 ns | 5.88 ns | 8.80 ns | 1.00 | 0.01 | 1 | CatB-DistinctParamSet1-Job2 | Yes | + B2 | Job2 | 10 | 1,614.5 ns | 5.88 ns | 8.80 ns | 1.07 | 0.01 | 2 | CatB-DistinctParamSet1-Job2 | No | Errors: 0 diff --git a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt index 6fa6a6534a..a481b0f6f9 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/VerifiedFiles/MarkdownExporterVerifyTests.GroupExporterTest_NoBaseline_MethodsParamsJobs_GroupByParams.verified.txt @@ -8,20 +8,20 @@ Frequency: 2531248 Hz, Resolution: 395.062 ns, Timer: TSC Job2 : extra output line - Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | -------- |----- |------ |-----------:|--------:|--------:|-----:|------------- |--------- | - Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=2] | No | ^ - Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=2] | No | - Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=2] | No | - Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=2] | No | - Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=2] | No | - Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=2] | No | - | | | | | | | | | - Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 1 | [Param=10] | No | ^ - Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | [Param=10] | No | - Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | [Param=10] | No | - Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 4 | [Param=10] | No | - Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | [Param=10] | No | - Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | [Param=10] | No | + Method | Job | Param | Mean | Error | StdDev | Rank | LogicalGroup | Baseline | +------- |----- |------ |-----------:|--------:|--------:|-----:|------------------ |--------- | + Base | Job1 | 2 | 114.5 ns | 5.88 ns | 8.80 ns | 1 | DistinctParamSet0 | No | ^ + Base | Job2 | 2 | 214.5 ns | 5.88 ns | 8.80 ns | 2 | DistinctParamSet0 | No | + Foo | Job1 | 2 | 514.5 ns | 5.88 ns | 8.80 ns | 3 | DistinctParamSet0 | No | + Bar | Job1 | 2 | 614.5 ns | 5.88 ns | 8.80 ns | 4 | DistinctParamSet0 | No | + Foo | Job2 | 2 | 714.5 ns | 5.88 ns | 8.80 ns | 5 | DistinctParamSet0 | No | + Bar | Job2 | 2 | 814.5 ns | 5.88 ns | 8.80 ns | 6 | DistinctParamSet0 | No | + | | | | | | | | | + Base | Job1 | 10 | 314.5 ns | 5.88 ns | 8.80 ns | 1 | DistinctParamSet1 | No | ^ + Base | Job2 | 10 | 414.5 ns | 5.88 ns | 8.80 ns | 2 | DistinctParamSet1 | No | + Foo | Job1 | 10 | 914.5 ns | 5.88 ns | 8.80 ns | 3 | DistinctParamSet1 | No | + Bar | Job1 | 10 | 1,014.5 ns | 5.88 ns | 8.80 ns | 4 | DistinctParamSet1 | No | + Foo | Job2 | 10 | 1,114.5 ns | 5.88 ns | 8.80 ns | 5 | DistinctParamSet1 | No | + Bar | Job2 | 10 | 1,214.5 ns | 5.88 ns | 8.80 ns | 6 | DistinctParamSet1 | No | Errors: 0 From ce085a1e99ca47b649414d67c09682c9c57981b9 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Mon, 1 Dec 2025 08:43:00 -0500 Subject: [PATCH 05/11] Implement param equality comparison directly --- .../Parameters/ParameterEqualityComparer.cs | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs index f1da16f340..10f3906f5b 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs @@ -1,5 +1,8 @@ -using System.Collections.Generic; +using System; +using System.Collections; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; namespace BenchmarkDotNet.Parameters { @@ -7,14 +10,57 @@ internal class ParameterEqualityComparer : IEqualityComparer { public static readonly ParameterEqualityComparer Instance = new ParameterEqualityComparer(); - public bool Equals(ParameterInstances? x, ParameterInstances? y) + public bool Equals(ParameterInstances x, ParameterInstances y) { - return ParameterComparer.Instance.Compare(x, y) == 0; + if (x == null && y == null) return true; + if (x != null && y == null) return false; + if (x == null) return false; + + for (int i = 0; i < Math.Min(x.Count, y.Count); i++) + { + var isEqual = ValuesEqual(x[i]?.Value, y[i]?.Value); + + if (!isEqual) return false; + } + + return true; + } + + private bool ValuesEqual(object x, object y) + { + if (x == null && y == null) return true; + + // Detect IComparable implementations. + // This works for all primitive types in addition to user types that implement IComparable. + if (x != null && y != null && x.GetType() == y.GetType()) + { + if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // collection equality support + { + if (x is Array xArr && y is Array yArr) // check rank here for arrays because their values will get compared when flattened + { + if (xArr.Rank != yArr.Rank) return false; + if (xArr.Length != yArr.Length) return false; + } + + var xFlat = xEnumerable.OfType().ToArray(); + var yFlat = yEnumerable.OfType().ToArray(); + + return StructuralComparisons.StructuralEqualityComparer.Equals(xFlat, yFlat); + } + else + { + // The user should define a value-based Equals check + return x.Equals(y); + } + } + + // The objects are of different types or one is null, they cannot be equal + return false; } - public int GetHashCode([DisallowNull] ParameterInstances obj) + public int GetHashCode(ParameterInstances obj) { - return obj.ValueInfo.GetHashCode(); + return obj?.ValueInfo.GetHashCode() ?? 0; } } } \ No newline at end of file From f9562a3ae091ea45ab7f54570813b511e2c75d26 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Mon, 1 Dec 2025 08:57:16 -0500 Subject: [PATCH 06/11] Ensure that params enumerable comparisons return false when collection lengths are different --- src/BenchmarkDotNet/Parameters/ParameterComparer.cs | 3 +++ src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index 05a5e05766..3bba1eb5c0 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -54,6 +54,9 @@ private int CompareValues(object x, object y) var xFlat = xEnumerable.OfType().ToArray(); var yFlat = yEnumerable.OfType().ToArray(); + if (xFlat.Length != yFlat.Length) + return xFlat.Length.CompareTo(yFlat.Length); + return StructuralComparisons.StructuralComparer.Compare(xFlat, yFlat); } } diff --git a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs index 10f3906f5b..f920abaf8f 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs @@ -39,12 +39,13 @@ private bool ValuesEqual(object x, object y) if (x is Array xArr && y is Array yArr) // check rank here for arrays because their values will get compared when flattened { if (xArr.Rank != yArr.Rank) return false; - if (xArr.Length != yArr.Length) return false; } var xFlat = xEnumerable.OfType().ToArray(); var yFlat = yEnumerable.OfType().ToArray(); + if (xFlat.Length != yFlat.Length) return false; + return StructuralComparisons.StructuralEqualityComparer.Equals(xFlat, yFlat); } else From 4921e1b064a0a2216acf8a416a0f6a574182ba40 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Mon, 1 Dec 2025 12:42:05 -0500 Subject: [PATCH 07/11] Remove false comment from ParameterEqualityComparer.cs --- src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs index f920abaf8f..41dcdf3a26 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs @@ -30,13 +30,11 @@ private bool ValuesEqual(object x, object y) { if (x == null && y == null) return true; - // Detect IComparable implementations. - // This works for all primitive types in addition to user types that implement IComparable. if (x != null && y != null && x.GetType() == y.GetType()) { - if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // collection equality support + if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // Collection equality support { - if (x is Array xArr && y is Array yArr) // check rank here for arrays because their values will get compared when flattened + if (x is Array xArr && y is Array yArr) // Check rank here for arrays because their values will get compared when flattened { if (xArr.Rank != yArr.Rank) return false; } From 7c651b7d489200751cfdfa9af321ebccbce6a813 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Mon, 1 Dec 2025 12:42:23 -0500 Subject: [PATCH 08/11] Add parameter comparison fallback if collection elements are not comparable --- .../Parameters/ParameterComparer.cs | 12 +++- .../ArgumentsTests.cs | 55 ++++++++++++++----- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index 3bba1eb5c0..bbefb55ee0 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -57,7 +57,17 @@ private int CompareValues(object x, object y) if (xFlat.Length != yFlat.Length) return xFlat.Length.CompareTo(yFlat.Length); - return StructuralComparisons.StructuralComparer.Compare(xFlat, yFlat); + try + { + return StructuralComparisons.StructuralComparer.Compare(xFlat, yFlat); + } + // Inner element type does not support comparison, hash elements to compare collections + catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + { + var xFlatHashed = xFlat.Select(elem => elem.GetHashCode()).ToArray(); + var yFlatHashed = yFlat.Select(elem => elem.GetHashCode()).ToArray(); + return StructuralComparisons.StructuralComparer.Compare(xFlatHashed, yFlatHashed); + } } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs index e14001bf52..778328bfda 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ArgumentsTests.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; @@ -316,39 +317,65 @@ public void IEnumerableArgumentsCanBeGrouped(IToolchain toolchain) public class EnumerableOnDifferentMethods { - public IEnumerable EnumerableOne() + public IEnumerable> GetEnumerables() { - yield return 1; - yield return 2; - yield return 3; + yield return Enumerable.Range(1, 10); + yield return Enumerable.Range(2, 10); } - public IEnumerable EnumerableTwo() + [Benchmark(Baseline = true)] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables(IEnumerable arg) { - yield return 1; - yield return 2; - yield return 3; + if (arg.IsEmpty()) + throw new ArgumentException("Incorrect length"); + } + + [Benchmark] + [ArgumentsSource(nameof(GetEnumerables))] + public void AcceptsEnumerables2(IEnumerable arg) + { + if (arg.IsEmpty()) + throw new ArgumentException("Incorrect length"); } + } + [Theory, MemberData(nameof(GetToolchains), DisableDiscoveryEnumeration = true)] + public void IEnumerableArgumentsOfDifferentLengthsCanBeGrouped(IToolchain toolchain) + { + var summary = CanExecute(toolchain, (config) => config.AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByParams)); // We must group by params to test array argument grouping + + // There should be two logical groups, one for the first argument and one for the second argument + // Thus there should be two pairs per descriptor, and each pair should be distinct because it belongs to a different group + + var descriptorGroupPairs = summary.BenchmarksCases.Select(benchmarkCase => (benchmarkCase.Descriptor, summary.GetLogicalGroupKey(benchmarkCase))).GroupBy(benchmarkCase => benchmarkCase.Descriptor); + + Assert.True( + descriptorGroupPairs.All(group => group.Select(pair => pair.Item2).Distinct().Count() == 2) + ); + } + + public class EnumerableOfDiffLengthOnDifferentMethods + { public IEnumerable> GetEnumerables() { - yield return EnumerableOne(); - yield return EnumerableTwo(); + yield return Enumerable.Range(1, 10); + yield return Enumerable.Range(1, 11); } [Benchmark(Baseline = true)] [ArgumentsSource(nameof(GetEnumerables))] - public void AcceptsEnumerables(IEnumerable arr) + public void AcceptsEnumerables(IEnumerable arg) { - if (arr.Count() != 3) + if (arg.IsEmpty()) throw new ArgumentException("Incorrect length"); } [Benchmark] [ArgumentsSource(nameof(GetEnumerables))] - public void AcceptsEnumerables2(IEnumerable collection) + public void AcceptsEnumerables2(IEnumerable arg) { - if (collection.Count() != 3) + if (arg.IsEmpty()) throw new ArgumentException("Incorrect length"); } } From 21003f6e8f5b7a67c0e63faf49718a3b2b4d84e5 Mon Sep 17 00:00:00 2001 From: Carl Furtado Date: Mon, 1 Dec 2025 18:53:31 -0500 Subject: [PATCH 09/11] Optimize parameter comparison for enumerables --- .../Parameters/ParameterComparer.cs | 134 +++++++++++++++--- .../Parameters/ParameterEqualityComparer.cs | 117 +++++++++++++-- 2 files changed, 223 insertions(+), 28 deletions(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index bbefb55ee0..ffce8be42c 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Reflection; namespace BenchmarkDotNet.Parameters { @@ -24,7 +25,7 @@ public int Compare(ParameterInstances x, ParameterInstances y) return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo); } - private int CompareValues(object x, object y) + private int CompareValues(T1 x, T2 y) { // Detect IComparable implementations. // This works for all primitive types in addition to user types that implement IComparable. @@ -43,36 +44,131 @@ private int CompareValues(object x, object y) { } } - else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // collection equality support + else if (x is IStructuralComparable) { - if (x is Array xArr && y is Array yArr) // check rank here for arrays because their values will get compared when flattened + if (x is Array xArr && y is Array yArr) { - if (xArr.Rank != yArr.Rank) - return xArr.Rank.CompareTo(yArr.Rank); - } + if (xArr.Rank != yArr.Rank) return xArr.Rank.CompareTo(yArr.Rank); - var xFlat = xEnumerable.OfType().ToArray(); - var yFlat = yEnumerable.OfType().ToArray(); + for (int dim = 0; dim < xArr.Rank; dim++) + { + if (xArr.GetLength(dim) != yArr.GetLength(dim)) + return xArr.GetLength(dim).CompareTo(yArr.GetLength(dim)); + } - if (xFlat.Length != yFlat.Length) - return xFlat.Length.CompareTo(yFlat.Length); + // 1D, 2D, and 3D array comparison is optimized with dedicated methods + if (xArr.Rank == 1) return StructuralComparisonWithFallback(xArr, yArr); - try + if (xArr.Rank == 2) + { + return (int) GetType() + .GetMethod(nameof(CompareTwoDimensionalArray), BindingFlags.NonPublic | BindingFlags.Instance) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(this, [xArr, yArr]); + } + + if (xArr.Rank == 3) + { + return (int) GetType() + .GetMethod(nameof(CompareThreeDimensionalArray), BindingFlags.NonPublic | BindingFlags.Instance) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(this, [xArr, yArr]); + } + + return CompareEnumerables(xArr, yArr); + } + else // Probably a user-defined IStructuralComparable, as tuples would be handled by the IComparable case { - return StructuralComparisons.StructuralComparer.Compare(xFlat, yFlat); + return StructuralComparisons.StructuralComparer.Compare(x, y); } - // Inner element type does not support comparison, hash elements to compare collections - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + } + else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection equality support + { + return CompareEnumerables(xEnumerable, yEnumerable); + } + } + + // Anything else to differentiate between objects. + var stringComp = string.CompareOrdinal(x?.ToString(), y?.ToString()); + + if (stringComp != 0) return stringComp; + + return x?.GetHashCode().CompareTo(y?.GetHashCode() ?? 0) ?? 0; + } + + private int StructuralComparisonWithFallback(Array x, Array y) + { + try + { + return StructuralComparisons.StructuralComparer.Compare(x, y); + } + // Inner element type does not support comparison, hash elements to compare collections + catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + { + var xFlatHashed = x.OfType().Select(elem => elem.GetHashCode()); + var yFlatHashed = y.OfType().Select(elem => elem.GetHashCode()); + return CompareEnumerables(xFlatHashed, yFlatHashed); + } + } + + private int CompareEnumerables(IEnumerable nonGenericX, IEnumerable nonGenericY) + { + // Use this instead of StructuralComparisons.StructuralComparer to avoid resolving the whole enumerable to object[] + + var x = nonGenericX.OfType(); + var y = nonGenericY.OfType(); + + foreach (var (xElem, yElem) in x.Zip(y, (x, y) => (x, y))) + { + int res = CompareValues(xElem, yElem); + + if (res != 0) return res; + } + + return 0; + } + + private int CompareTwoDimensionalArray(T1[,] arrOne, T2[,] arrTwo) + { + // Assume that arrOne & arrTwo are the same Length & Rank + + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + var x = arrOne[i, j]; + var y = arrTwo[i, j]; + + var res = CompareValues(x, y); + + if (res != 0) return res; + } + } + + return 0; + } + + private int CompareThreeDimensionalArray(T1[,,] arrOne, T2[,,] arrTwo) + { + // Assume that arrOne & arrTwo are the same Length & Rank + + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + for (int k = 0; k elem.GetHashCode()).ToArray(); - var yFlatHashed = yFlat.Select(elem => elem.GetHashCode()).ToArray(); - return StructuralComparisons.StructuralComparer.Compare(xFlatHashed, yFlatHashed); + var x = arrOne[i, j, k]; + var y = arrTwo[i, j, k]; + + var res = CompareValues(x, y); + + if (res != 0) return res; } } } - // Anything else. - return string.CompareOrdinal(x?.ToString(), y?.ToString()); + return 0; } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs index 41dcdf3a26..a9aa732fd5 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs @@ -1,8 +1,8 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.Linq; +using System.Reflection; namespace BenchmarkDotNet.Parameters { @@ -26,25 +26,51 @@ public bool Equals(ParameterInstances x, ParameterInstances y) return true; } - private bool ValuesEqual(object x, object y) + private bool ValuesEqual(T1 x, T2 y) { if (x == null && y == null) return true; if (x != null && y != null && x.GetType() == y.GetType()) { - if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // Collection equality support + if (x is IStructuralEquatable) { - if (x is Array xArr && y is Array yArr) // Check rank here for arrays because their values will get compared when flattened + if (x is Array xArr && y is Array yArr) { if (xArr.Rank != yArr.Rank) return false; - } - var xFlat = xEnumerable.OfType().ToArray(); - var yFlat = yEnumerable.OfType().ToArray(); + for (int dim = 0; dim < xArr.Rank; dim++) + { + if (xArr.GetLength(dim) != yArr.GetLength(dim)) return false; + } + + if (xArr.Rank == 1) return StructuralEqualityWithFallback(xArr, yArr); - if (xFlat.Length != yFlat.Length) return false; + if (xArr.Rank == 2) + { + return (bool) GetType() + .GetMethod(nameof(TwoDimensionalArrayEquals), BindingFlags.NonPublic | BindingFlags.Instance) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(this, [xArr, yArr]); + } - return StructuralComparisons.StructuralEqualityComparer.Equals(xFlat, yFlat); + if (xArr.Rank == 3) + { + return (bool) GetType() + .GetMethod(nameof(ThreeDimensionalArrayEquals), BindingFlags.NonPublic | BindingFlags.Instance) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(this, [xArr, yArr]); + } + + return EnumerablesEqual(xArr, yArr); + } + else // Probably a user-defined IStructuralComparable, as tuples would be handled by the IComparable case + { + return StructuralComparisons.StructuralEqualityComparer.Equals(x, y); + } + } + else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection equality support + { + return EnumerablesEqual(xEnumerable, yEnumerable); } else { @@ -57,9 +83,82 @@ private bool ValuesEqual(object x, object y) return false; } + private bool StructuralEqualityWithFallback(Array x, Array y) + { + try + { + return StructuralComparisons.StructuralEqualityComparer.Equals(x, y); + } + // Inner element type does not support comparison, hash elements to compare collections + catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + { + return EnumerablesEqual(x.OfType().Select(elem => elem.GetHashCode()), y.OfType().Select(elem => elem.GetHashCode())); + } + } + + private bool EnumerablesEqual(IEnumerable nonGenericX, IEnumerable nonGenericY) + { + // Use this instead of StructuralComparisons.StructuralEqualityComparer to avoid resolving the whole enumerable to object[] + + var x = nonGenericX.OfType(); + var y = nonGenericY.OfType(); + + foreach (var (xElem, yElem) in x.Zip(y, (x, y) => (x, y))) + { + bool res = ValuesEqual(xElem, yElem); + + if (!res) return false; + } + + return true; + } + public int GetHashCode(ParameterInstances obj) { return obj?.ValueInfo.GetHashCode() ?? 0; } + + private bool TwoDimensionalArrayEquals(T1[,] arrOne, T2[,] arrTwo) + { + // Assume that arrOne & arrTwo are the same Length & Rank + + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + var x = arrOne[i, j]; + var y = arrTwo[i, j]; + + var res = ValuesEqual(x, y); + + if (!res) return false; + } + } + + return true; + } + + private bool ThreeDimensionalArrayEquals(T1[,,] arrOne, T2[,,] arrTwo) + { + // Assume that arrOne & arrTwo are the same Length & Rank + + for (int i = 0; i < arrOne.GetLength(0); i++) + { + for (int j = 0; j < arrOne.GetLength(1); j++) + { + for (int k = 0; k Date: Mon, 1 Dec 2025 22:16:07 -0500 Subject: [PATCH 10/11] Implement feedback for parameter comparison --- .../Parameters/ParameterComparer.cs | 103 ++++++++++++------ .../Parameters/ParameterEqualityComparer.cs | 60 +++++----- 2 files changed, 101 insertions(+), 62 deletions(-) diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index ffce8be42c..1943497ac1 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Parameters { @@ -31,20 +32,7 @@ private int CompareValues(T1 x, T2 y) // This works for all primitive types in addition to user types that implement IComparable. if (x != null && y != null && x.GetType() == y.GetType()) { - if (x is IComparable xComparable) - { - try - { - return xComparable.CompareTo(y); - } - // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. - // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 - // For now, catch and ignore the exception, and fallback to string comparison below. - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) - { - } - } - else if (x is IStructuralComparable) + if (x is IStructuralComparable xStructuralComparable) { if (x is Array xArr && y is Array yArr) { @@ -59,7 +47,7 @@ private int CompareValues(T1 x, T2 y) // 1D, 2D, and 3D array comparison is optimized with dedicated methods if (xArr.Rank == 1) return StructuralComparisonWithFallback(xArr, yArr); - if (xArr.Rank == 2) + if (xArr.Rank == 2 && !RuntimeInformation.IsAot) { return (int) GetType() .GetMethod(nameof(CompareTwoDimensionalArray), BindingFlags.NonPublic | BindingFlags.Instance) @@ -67,7 +55,7 @@ private int CompareValues(T1 x, T2 y) .Invoke(this, [xArr, yArr]); } - if (xArr.Rank == 3) + if (xArr.Rank == 3 && !RuntimeInformation.IsAot) { return (int) GetType() .GetMethod(nameof(CompareThreeDimensionalArray), BindingFlags.NonPublic | BindingFlags.Instance) @@ -77,23 +65,70 @@ private int CompareValues(T1 x, T2 y) return CompareEnumerables(xArr, yArr); } - else // Probably a user-defined IStructuralComparable, as tuples would be handled by the IComparable case + else // Probably a user-defined IStructuralComparable or tuple { - return StructuralComparisons.StructuralComparer.Compare(x, y); + return StructuralComparisonWithFallback(xStructuralComparable, (IStructuralComparable) y); } } - else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection equality support + else if (x is IComparable xComparable) + { + try + { + return xComparable.CompareTo(y); + } + // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. + // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 + // For now, catch and ignore the exception, and fallback to string comparison below. + catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + { + } + } + else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection comparison support { return CompareEnumerables(xEnumerable, yEnumerable); } } // Anything else to differentiate between objects. - var stringComp = string.CompareOrdinal(x?.ToString(), y?.ToString()); + return string.CompareOrdinal(x?.ToString(), y?.ToString()); + } + + private int StructuralComparisonWithFallback(IStructuralComparable x, IStructuralComparable y) + { + try + { + return StructuralComparisons.StructuralComparer.Compare(x, y); + } + // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. + // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 + // For now, catch and ignore the exception, and fallback to string comparison below. + catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + { + var ITuple = Type.GetType("System.Runtime.CompilerServices.ITuple"); + + if (ITuple.IsAssignableFrom(x.GetType())) + { + var lengthProperty = ITuple.GetProperty("Length"); + + var xLength = (int) lengthProperty.GetValue(x); + var yLength = (int) lengthProperty.GetValue(y); + + if (xLength != yLength) return xLength.CompareTo(yLength); - if (stringComp != 0) return stringComp; + var indexerProperty = ITuple.GetProperty("Item"); - return x?.GetHashCode().CompareTo(y?.GetHashCode() ?? 0) ?? 0; + object ItemGetter(IStructuralComparable tup, int i) => indexerProperty.GetValue(tup, [i]); + + var xFlatTransformed = Enumerable.Range(0, xLength).Select(i => ItemGetter(x, i).ToString()); + var yFlatTransformed = Enumerable.Range(0, xLength).Select(i => ItemGetter(y, i).ToString()); + + return CompareEnumerables(xFlatTransformed, yFlatTransformed); + } + else + { + return string.CompareOrdinal(x?.ToString(), y?.ToString()); + } + } } private int StructuralComparisonWithFallback(Array x, Array y) @@ -102,29 +137,35 @@ private int StructuralComparisonWithFallback(Array x, Array y) { return StructuralComparisons.StructuralComparer.Compare(x, y); } - // Inner element type does not support comparison, hash elements to compare collections + // Inner element type does not support comparison, ToString (GetHashCode not preferred) elements to compare collections catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) { - var xFlatHashed = x.OfType().Select(elem => elem.GetHashCode()); - var yFlatHashed = y.OfType().Select(elem => elem.GetHashCode()); - return CompareEnumerables(xFlatHashed, yFlatHashed); + var xFlatTransformed = x.OfType().Select(elem => elem.ToString()); + var yFlatTransformed = y.OfType().Select(elem => elem.ToString()); + return CompareEnumerables(xFlatTransformed, yFlatTransformed); } } - private int CompareEnumerables(IEnumerable nonGenericX, IEnumerable nonGenericY) + private int CompareEnumerables(IEnumerable x, IEnumerable y) { // Use this instead of StructuralComparisons.StructuralComparer to avoid resolving the whole enumerable to object[] - var x = nonGenericX.OfType(); - var y = nonGenericY.OfType(); + var xEnumer = x.GetEnumerator(); + var yEnumer = y.GetEnumerator(); - foreach (var (xElem, yElem) in x.Zip(y, (x, y) => (x, y))) + bool xHasElement, yHasElement; + + // Use bitwise AND to avoid short-circuiting, which destroys this function's length checking logic + while ((xHasElement = xEnumer.MoveNext()) & (yHasElement = yEnumer.MoveNext())) { - int res = CompareValues(xElem, yElem); + int res = CompareValues(xEnumer.Current, yEnumer.Current); if (res != 0) return res; } + if (xHasElement) return 1; + if (yHasElement) return -1; + return 0; } diff --git a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs index a9aa732fd5..3682dc42bd 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterEqualityComparer.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Parameters { @@ -32,7 +33,7 @@ private bool ValuesEqual(T1 x, T2 y) if (x != null && y != null && x.GetType() == y.GetType()) { - if (x is IStructuralEquatable) + if (x is IStructuralEquatable xStructuralEquatable) { if (x is Array xArr && y is Array yArr) { @@ -43,9 +44,10 @@ private bool ValuesEqual(T1 x, T2 y) if (xArr.GetLength(dim) != yArr.GetLength(dim)) return false; } - if (xArr.Rank == 1) return StructuralEqualityWithFallback(xArr, yArr); + // 1D, 2D, and 3D array comparison is optimized with dedicated methods + if (xArr.Rank == 1) return StructuralEquals(xArr, yArr); - if (xArr.Rank == 2) + if (xArr.Rank == 2 && !RuntimeInformation.IsAot) { return (bool) GetType() .GetMethod(nameof(TwoDimensionalArrayEquals), BindingFlags.NonPublic | BindingFlags.Instance) @@ -53,7 +55,7 @@ private bool ValuesEqual(T1 x, T2 y) .Invoke(this, [xArr, yArr]); } - if (xArr.Rank == 3) + if (xArr.Rank == 3 && !RuntimeInformation.IsAot) { return (bool) GetType() .GetMethod(nameof(ThreeDimensionalArrayEquals), BindingFlags.NonPublic | BindingFlags.Instance) @@ -63,18 +65,17 @@ private bool ValuesEqual(T1 x, T2 y) return EnumerablesEqual(xArr, yArr); } - else // Probably a user-defined IStructuralComparable, as tuples would be handled by the IComparable case + else // Probably a user-defined IStructuralEquatable or tuple { - return StructuralComparisons.StructuralEqualityComparer.Equals(x, y); + return StructuralEquals(xStructuralEquatable, (IStructuralEquatable) y); } } - else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection equality support + else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection equality support { return EnumerablesEqual(xEnumerable, yEnumerable); } else { - // The user should define a value-based Equals check return x.Equals(y); } } @@ -83,39 +84,31 @@ private bool ValuesEqual(T1 x, T2 y) return false; } - private bool StructuralEqualityWithFallback(Array x, Array y) + private bool StructuralEquals(IStructuralEquatable x, IStructuralEquatable y) { - try - { - return StructuralComparisons.StructuralEqualityComparer.Equals(x, y); - } - // Inner element type does not support comparison, hash elements to compare collections - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) - { - return EnumerablesEqual(x.OfType().Select(elem => elem.GetHashCode()), y.OfType().Select(elem => elem.GetHashCode())); - } + return StructuralComparisons.StructuralEqualityComparer.Equals(x, y); } - private bool EnumerablesEqual(IEnumerable nonGenericX, IEnumerable nonGenericY) + private bool EnumerablesEqual(IEnumerable x, IEnumerable y) { - // Use this instead of StructuralComparisons.StructuralEqualityComparer to avoid resolving the whole enumerable to object[] + // Use this instead of StructuralComparisons.StructuralComparer to avoid resolving the whole enumerable to object[] - var x = nonGenericX.OfType(); - var y = nonGenericY.OfType(); + var xEnumer = x.GetEnumerator(); + var yEnumer = y.GetEnumerator(); - foreach (var (xElem, yElem) in x.Zip(y, (x, y) => (x, y))) + bool xHasElement, yHasElement; + + // Use bitwise AND to avoid short-circuiting, which destroys this function's length checking logic + while ((xHasElement = xEnumer.MoveNext()) & (yHasElement = yEnumer.MoveNext())) { - bool res = ValuesEqual(xElem, yElem); + bool res = ValuesEqual(xEnumer.Current, yEnumer.Current); if (!res) return false; } - return true; - } + if (xHasElement || yHasElement) return false; - public int GetHashCode(ParameterInstances obj) - { - return obj?.ValueInfo.GetHashCode() ?? 0; + return true; } private bool TwoDimensionalArrayEquals(T1[,] arrOne, T2[,] arrTwo) @@ -129,7 +122,7 @@ private bool TwoDimensionalArrayEquals(T1[,] arrOne, T2[,] arrTwo) var x = arrOne[i, j]; var y = arrTwo[i, j]; - var res = ValuesEqual(x, y); + bool res = ValuesEqual(x, y); if (!res) return false; } @@ -151,7 +144,7 @@ private bool ThreeDimensionalArrayEquals(T1[,,] arrOne, T2[,,] arrTwo) var x = arrOne[i, j, k]; var y = arrTwo[i, j, k]; - var res = ValuesEqual(x, y); + bool res = ValuesEqual(x, y); if (!res) return false; } @@ -160,5 +153,10 @@ private bool ThreeDimensionalArrayEquals(T1[,,] arrOne, T2[,,] arrTwo) return true; } + + public int GetHashCode(ParameterInstances obj) + { + return obj?.ValueInfo.GetHashCode() ?? 0; + } } } \ No newline at end of file From 1ab11e4e6cee45246f4a58bcda1807ef284251e1 Mon Sep 17 00:00:00 2001 From: Tim Cassell Date: Tue, 2 Dec 2025 06:59:11 -0500 Subject: [PATCH 11/11] Implement PR feedback. Added a TODO note for performance. --- src/BenchmarkDotNet/Order/DefaultOrderer.cs | 2 + .../Parameters/ParameterComparer.cs | 328 +++++++++++------- .../Parameters/ParameterEqualityComparer.cs | 290 +++++++++++----- 3 files changed, 411 insertions(+), 209 deletions(-) diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index b30a841118..51451a5602 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -89,6 +89,8 @@ public string GetHighlightGroupKey(BenchmarkCase benchmarkCase) public string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, BenchmarkCase benchmarkCase) { + // TODO: GetLogicalGroupKey is called for every benchmarkCase, so as the number of cases grows, this can get very expensive to recompute for each call. + // We should somehow amortize the cost by computing it only once per summary. var paramSets = allBenchmarksCases.Select(benchmarkCase => benchmarkCase.Parameters).Distinct(ParameterEqualityComparer.Instance).ToArray(); var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList(); diff --git a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs index 1943497ac1..4f39311b82 100644 --- a/src/BenchmarkDotNet/Parameters/ParameterComparer.cs +++ b/src/BenchmarkDotNet/Parameters/ParameterComparer.cs @@ -1,215 +1,287 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Linq; using System.Reflection; using BenchmarkDotNet.Portability; +#if NETSTANDARD2_0 +using System.Collections.Concurrent; +using System.Linq; +#endif + namespace BenchmarkDotNet.Parameters { internal class ParameterComparer : IComparer { +#if NETSTANDARD2_0 + private static readonly ConcurrentDictionary s_tupleMembersCache = new(); + + private static MemberInfo[] GetTupleMembers(Type type) + => s_tupleMembersCache.GetOrAdd(type, t => + { + var members = type.FullName.StartsWith("System.Tuple`") + ? type.GetProperties(BindingFlags.Public | BindingFlags.Instance).Cast() + : type.GetFields(BindingFlags.Public | BindingFlags.Instance).Cast(); + return members + .Where(p => p.Name.StartsWith("Item")) + .OrderBy(p => p.Name) + .ToArray(); + }); +#endif + public static readonly ParameterComparer Instance = new ParameterComparer(); public int Compare(ParameterInstances x, ParameterInstances y) { if (x == null && y == null) return 0; - if (x != null && y == null) return 1; if (x == null) return -1; + if (y == null) return 1; + for (int i = 0; i < Math.Min(x.Count, y.Count); i++) { - var compareTo = CompareValues(x[i]?.Value, y[i]?.Value); - if (compareTo != 0) - return compareTo; + var comparison = CompareValues(x[i]?.Value, y[i]?.Value); + if (comparison != 0) + { + return comparison; + } } return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo); } - private int CompareValues(T1 x, T2 y) + private static int CompareValues(T1 x, T2 y) { - // Detect IComparable implementations. - // This works for all primitive types in addition to user types that implement IComparable. - if (x != null && y != null && x.GetType() == y.GetType()) + if (x == null || y == null || x.GetType() != y.GetType()) { - if (x is IStructuralComparable xStructuralComparable) - { - if (x is Array xArr && y is Array yArr) - { - if (xArr.Rank != yArr.Rank) return xArr.Rank.CompareTo(yArr.Rank); - - for (int dim = 0; dim < xArr.Rank; dim++) - { - if (xArr.GetLength(dim) != yArr.GetLength(dim)) - return xArr.GetLength(dim).CompareTo(yArr.GetLength(dim)); - } - - // 1D, 2D, and 3D array comparison is optimized with dedicated methods - if (xArr.Rank == 1) return StructuralComparisonWithFallback(xArr, yArr); - - if (xArr.Rank == 2 && !RuntimeInformation.IsAot) - { - return (int) GetType() - .GetMethod(nameof(CompareTwoDimensionalArray), BindingFlags.NonPublic | BindingFlags.Instance) - .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) - .Invoke(this, [xArr, yArr]); - } - - if (xArr.Rank == 3 && !RuntimeInformation.IsAot) - { - return (int) GetType() - .GetMethod(nameof(CompareThreeDimensionalArray), BindingFlags.NonPublic | BindingFlags.Instance) - .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) - .Invoke(this, [xArr, yArr]); - } + return string.CompareOrdinal(x?.ToString(), y?.ToString()); + } - return CompareEnumerables(xArr, yArr); - } - else // Probably a user-defined IStructuralComparable or tuple - { - return StructuralComparisonWithFallback(xStructuralComparable, (IStructuralComparable) y); - } + if (x is IStructuralComparable xStructuralComparable) + { + try + { + return StructuralComparisons.StructuralComparer.Compare(x, y); } - else if (x is IComparable xComparable) + // https://github.com/dotnet/BenchmarkDotNet/issues/2346 + // https://github.com/dotnet/runtime/issues/66472 + // Unfortunately we can't rely on checking the exception message because it may change per current culture. + catch (ArgumentException) { - try - { - return xComparable.CompareTo(y); - } - // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. - // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 - // For now, catch and ignore the exception, and fallback to string comparison below. - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + if (TryFallbackStructuralCompareTo(x, y, out int comparison)) { + return comparison; } + // A complex user type did not handle a multi-dimensional array or tuple, just re-throw. + throw; } - else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection comparison support - { - return CompareEnumerables(xEnumerable, yEnumerable); - } + } + + if (x is IComparable xComparable) + { + // Tuples are already handled by IStructuralComparable case, if this throws, it's the user's own fault. + return xComparable.CompareTo(y); + } + + if (x is IEnumerable xEnumerable) // General collection comparison support + { + return CompareEnumerables(xEnumerable, (IEnumerable) y); } // Anything else to differentiate between objects. return string.CompareOrdinal(x?.ToString(), y?.ToString()); } - private int StructuralComparisonWithFallback(IStructuralComparable x, IStructuralComparable y) + private static bool TryFallbackStructuralCompareTo(object x, object y, out int comparison) { - try - { - return StructuralComparisons.StructuralComparer.Compare(x, y); - } - // Some types, such as Tuple and ValueTuple, have a fallible CompareTo implementation which can throw if the inner items don't implement IComparable. - // See: https://github.com/dotnet/BenchmarkDotNet/issues/2346 - // For now, catch and ignore the exception, and fallback to string comparison below. - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + // Check for multi-dimensional array and ITuple and re-try for each element recursively. + if (x is Array xArr) { - var ITuple = Type.GetType("System.Runtime.CompilerServices.ITuple"); - - if (ITuple.IsAssignableFrom(x.GetType())) + Array yArr = (Array) y; + if (xArr.Rank != yArr.Rank) { - var lengthProperty = ITuple.GetProperty("Length"); - - var xLength = (int) lengthProperty.GetValue(x); - var yLength = (int) lengthProperty.GetValue(y); + comparison = xArr.Rank.CompareTo(yArr.Rank); + return true; + } - if (xLength != yLength) return xLength.CompareTo(yLength); + for (int dim = 0; dim < xArr.Rank; dim++) + { + if (xArr.GetLength(dim) != yArr.GetLength(dim)) + { + comparison = xArr.GetLength(dim).CompareTo(yArr.GetLength(dim)); + return true; + } + } - var indexerProperty = ITuple.GetProperty("Item"); + // Common 2D and 3D arrays are specialized to avoid expensive boxing where possible. + if (!RuntimeInformation.IsAot && xArr.Rank is 2 or 3) + { + string methodName = xArr.Rank == 2 + ? nameof(CompareTwoDArray) + : nameof(CompareThreeDArray); + comparison = (int) typeof(ParameterComparer) + .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(null, [xArr, yArr]); + return true; + } - object ItemGetter(IStructuralComparable tup, int i) => indexerProperty.GetValue(tup, [i]); + // 1D arrays will only hit this code path if a nested type is a multi-dimensional array. + // 4D and larger fall back to enumerable. + comparison = CompareEnumerables(xArr, yArr); + return true; + } - var xFlatTransformed = Enumerable.Range(0, xLength).Select(i => ItemGetter(x, i).ToString()); - var yFlatTransformed = Enumerable.Range(0, xLength).Select(i => ItemGetter(y, i).ToString()); +#if NETSTANDARD2_0 + // ITuple does not exist in netstandard2.0, so we have to use reflection. ITuple does exist in net471 and newer, but the System.ValueTuple nuget package does not implement it. + string typeName = x.GetType().FullName; + if (typeName.StartsWith("System.Tuple`")) + { + comparison = CompareTuples(x, y); + return true; + } + else if (typeName.StartsWith("System.ValueTuple`")) + { + comparison = CompareValueTuples(x, y); + return true; + } +#else + if (x is System.Runtime.CompilerServices.ITuple xTuple) + { + comparison = CompareTuples(xTuple, (System.Runtime.CompilerServices.ITuple) y); + return true; + } +#endif - return CompareEnumerables(xFlatTransformed, yFlatTransformed); - } - else - { - return string.CompareOrdinal(x?.ToString(), y?.ToString()); - } + if (x is IEnumerable xEnumerable) // General collection equality support + { + comparison = CompareEnumerables(xEnumerable, (IEnumerable) y); + return true; } + + comparison = 0; + return false; } - private int StructuralComparisonWithFallback(Array x, Array y) + private static int CompareEnumerables(IEnumerable x, IEnumerable y) { + var xEnumerator = x.GetEnumerator(); try { - return StructuralComparisons.StructuralComparer.Compare(x, y); + var yEnumerator = y.GetEnumerator(); + try + { + while (xEnumerator.MoveNext()) + { + if (!yEnumerator.MoveNext()) + { + return -1; + } + int comparison = CompareValues(xEnumerator.Current, yEnumerator.Current); + if (comparison != 0) + { + return comparison; + } + } + return yEnumerator.MoveNext() ? 1 : 0; + } + finally + { + if (yEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } + } } - // Inner element type does not support comparison, ToString (GetHashCode not preferred) elements to compare collections - catch (ArgumentException ex) when (ex.Message.Contains("At least one object must implement IComparable.")) + finally { - var xFlatTransformed = x.OfType().Select(elem => elem.ToString()); - var yFlatTransformed = y.OfType().Select(elem => elem.ToString()); - return CompareEnumerables(xFlatTransformed, yFlatTransformed); + if (xEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } } } - private int CompareEnumerables(IEnumerable x, IEnumerable y) + private static int CompareTwoDArray(T1[,] arrOne, T2[,] arrTwo) { - // Use this instead of StructuralComparisons.StructuralComparer to avoid resolving the whole enumerable to object[] - - var xEnumer = x.GetEnumerator(); - var yEnumer = y.GetEnumerator(); - - bool xHasElement, yHasElement; - - // Use bitwise AND to avoid short-circuiting, which destroys this function's length checking logic - while ((xHasElement = xEnumer.MoveNext()) & (yHasElement = yEnumer.MoveNext())) + // Assumes that arrOne & arrTwo are the same length and width. + for (int i = 0; i < arrOne.GetLength(0); i++) { - int res = CompareValues(xEnumer.Current, yEnumer.Current); - - if (res != 0) return res; + for (int j = 0; j < arrOne.GetLength(1); j++) + { + var comparison = CompareValues(arrOne[i, j], arrTwo[i, j]); + if (comparison != 0) + { + return comparison; + } + } } - if (xHasElement) return 1; - if (yHasElement) return -1; - return 0; } - private int CompareTwoDimensionalArray(T1[,] arrOne, T2[,] arrTwo) + private static int CompareThreeDArray(T1[,,] arrOne, T2[,,] arrTwo) { - // Assume that arrOne & arrTwo are the same Length & Rank - + // Assumes that arrOne & arrTwo are the same length, width, and height. for (int i = 0; i < arrOne.GetLength(0); i++) { for (int j = 0; j < arrOne.GetLength(1); j++) { - var x = arrOne[i, j]; - var y = arrTwo[i, j]; + for (int k = 0; k (T1[,,] arrOne, T2[,,] arrTwo) + private static int CompareValueTuples(object x, object y) { - // Assume that arrOne & arrTwo are the same Length & Rank - - for (int i = 0; i < arrOne.GetLength(0); i++) + foreach (FieldInfo field in GetTupleMembers(x.GetType())) { - for (int j = 0; j < arrOne.GetLength(1); j++) + var comparison = CompareValues(field.GetValue(x), field.GetValue(y)); + if (comparison != 0) { - for (int k = 0; k { +#if NETSTANDARD2_0 + private static readonly ConcurrentDictionary s_tupleMembersCache = new(); + + private static MemberInfo[] GetTupleMembers(Type type) + => s_tupleMembersCache.GetOrAdd(type, t => + { + var members = type.FullName.StartsWith("System.Tuple`") + ? type.GetProperties(BindingFlags.Public | BindingFlags.Instance).Cast() + : type.GetFields(BindingFlags.Public | BindingFlags.Instance).Cast(); + return members + .Where(p => p.Name.StartsWith("Item")) + .OrderBy(p => p.Name) + .ToArray(); + }); +#endif + public static readonly ParameterEqualityComparer Instance = new ParameterEqualityComparer(); public bool Equals(ParameterInstances x, ParameterInstances y) { if (x == null && y == null) return true; - if (x != null && y == null) return false; - if (x == null) return false; + if (x == null || y == null) return false; - for (int i = 0; i < Math.Min(x.Count, y.Count); i++) - { - var isEqual = ValuesEqual(x[i]?.Value, y[i]?.Value); + if (x.Count != y.Count) return false; - if (!isEqual) return false; + for (int i = 0; i < x.Count; i++) + { + if (!ValuesEqual(x[i]?.Value, y[i]?.Value)) + { + return false; + } } return true; } - private bool ValuesEqual(T1 x, T2 y) + private static bool ValuesEqual(T1 x, T2 y) { - if (x == null && y == null) return true; + if (x == null && y == null) + { + return true; + } - if (x != null && y != null && x.GetType() == y.GetType()) + if (x == null || y == null || x.GetType() != y.GetType()) { - if (x is IStructuralEquatable xStructuralEquatable) + // The objects are of different types or one is null, they cannot be equal + return false; + } + + if (x is IStructuralEquatable xStructuralEquatable) + { + try + { + return StructuralComparisons.StructuralEqualityComparer.Equals(xStructuralEquatable, y); + } + // https://github.com/dotnet/runtime/issues/66472 + // Unfortunately we can't rely on checking the exception message because it may change per current culture. + catch (ArgumentException) { - if (x is Array xArr && y is Array yArr) + if (TryFallbackStructuralEquals(x, y, out bool equals)) { - if (xArr.Rank != yArr.Rank) return false; + return equals; + } + // A complex user type did not handle a multi-dimensional array, just re-throw. + throw; + } + } - for (int dim = 0; dim < xArr.Rank; dim++) - { - if (xArr.GetLength(dim) != yArr.GetLength(dim)) return false; - } + if (x is IEnumerable xEnumerable) // General collection equality support + { + return EnumerablesEqual(xEnumerable, (IEnumerable) y); + } - // 1D, 2D, and 3D array comparison is optimized with dedicated methods - if (xArr.Rank == 1) return StructuralEquals(xArr, yArr); + return FallbackSimpleEquals(x, y); + } - if (xArr.Rank == 2 && !RuntimeInformation.IsAot) - { - return (bool) GetType() - .GetMethod(nameof(TwoDimensionalArrayEquals), BindingFlags.NonPublic | BindingFlags.Instance) - .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) - .Invoke(this, [xArr, yArr]); - } + private static bool FallbackSimpleEquals(object x, object y) + { + if (x.Equals(y)) + { + return true; + } + // Anything else to differentiate between objects (match behavior of ParameterComparer). + return string.Equals(x.ToString(), y.ToString(), StringComparison.Ordinal); + } - if (xArr.Rank == 3 && !RuntimeInformation.IsAot) - { - return (bool) GetType() - .GetMethod(nameof(ThreeDimensionalArrayEquals), BindingFlags.NonPublic | BindingFlags.Instance) - .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) - .Invoke(this, [xArr, yArr]); - } + private static bool TryFallbackStructuralEquals(object x, object y, out bool equals) + { + // Check for multi-dimensional array and ITuple and re-try for each element recursively. + if (x is Array xArr) + { + Array yArr = (Array) y; + if (xArr.Rank != yArr.Rank) + { + equals = false; + return true; + } - return EnumerablesEqual(xArr, yArr); - } - else // Probably a user-defined IStructuralEquatable or tuple + for (int dim = 0; dim < xArr.Rank; dim++) + { + if (xArr.GetLength(dim) != yArr.GetLength(dim)) { - return StructuralEquals(xStructuralEquatable, (IStructuralEquatable) y); + equals = false; + return true; } } - else if (x is IEnumerable xEnumerable && y is IEnumerable yEnumerable) // General collection equality support - { - return EnumerablesEqual(xEnumerable, yEnumerable); - } - else + + // Common 2D and 3D arrays are specialized to avoid expensive boxing where possible. + if (!RuntimeInformation.IsAot && xArr.Rank is 2 or 3) { - return x.Equals(y); + string methodName = xArr.Rank == 2 + ? nameof(TwoDArraysEqual) + : nameof(ThreeDArraysEqual); + equals = (bool) typeof(ParameterEqualityComparer) + .GetMethod(methodName, BindingFlags.NonPublic | BindingFlags.Static) + .MakeGenericMethod(xArr.GetType().GetElementType(), yArr.GetType().GetElementType()) + .Invoke(null, [xArr, yArr]); + return true; } + + // 1D arrays will only hit this code path if a nested type is a multi-dimensional array. + // 4D and larger fall back to enumerable. + equals = EnumerablesEqual(xArr, yArr); + return true; + } + +#if NETSTANDARD2_0 + // ITuple does not exist in netstandard2.0, so we have to use reflection. ITuple does exist in net471 and newer, but the System.ValueTuple nuget package does not implement it. + string typeName = x.GetType().FullName; + if (typeName.StartsWith("System.Tuple`")) + { + equals = TuplesEqual(x, y); + return true; + } + else if (typeName.StartsWith("System.ValueTuple`")) + { + equals = ValueTuplesEqual(x, y); + return true; + } +#else + if (x is System.Runtime.CompilerServices.ITuple xTuple) + { + equals = TuplesEqual(xTuple, (System.Runtime.CompilerServices.ITuple) y); + return true; + } +#endif + + if (x is IEnumerable xEnumerable) // General collection equality support + { + equals = EnumerablesEqual(xEnumerable, (IEnumerable) y); + return true; } - // The objects are of different types or one is null, they cannot be equal + equals = false; return false; } - private bool StructuralEquals(IStructuralEquatable x, IStructuralEquatable y) + private static bool EnumerablesEqual(IEnumerable x, IEnumerable y) { - return StructuralComparisons.StructuralEqualityComparer.Equals(x, y); + var xEnumerator = x.GetEnumerator(); + try + { + var yEnumerator = y.GetEnumerator(); + try + { + while (xEnumerator.MoveNext()) + { + if (!(yEnumerator.MoveNext() && ValuesEqual(xEnumerator.Current, yEnumerator.Current))) + { + return false; + } + } + return !yEnumerator.MoveNext(); + } + finally + { + if (yEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } + } + } + finally + { + if (xEnumerator is IDisposable disposable) + { + disposable.Dispose(); + } + } } - private bool EnumerablesEqual(IEnumerable x, IEnumerable y) + private static bool TwoDArraysEqual(T1[,] arrOne, T2[,] arrTwo) { - // Use this instead of StructuralComparisons.StructuralComparer to avoid resolving the whole enumerable to object[] - - var xEnumer = x.GetEnumerator(); - var yEnumer = y.GetEnumerator(); - - bool xHasElement, yHasElement; - - // Use bitwise AND to avoid short-circuiting, which destroys this function's length checking logic - while ((xHasElement = xEnumer.MoveNext()) & (yHasElement = yEnumer.MoveNext())) + // Assumes that arrOne & arrTwo are the same length and width. + for (int i = 0; i < arrOne.GetLength(0); i++) { - bool res = ValuesEqual(xEnumer.Current, yEnumer.Current); - - if (!res) return false; + for (int j = 0; j < arrOne.GetLength(1); j++) + { + if (!ValuesEqual(arrOne[i, j], arrTwo[i, j])) + { + return false; + } + } } - if (xHasElement || yHasElement) return false; - return true; } - private bool TwoDimensionalArrayEquals(T1[,] arrOne, T2[,] arrTwo) + private static bool ThreeDArraysEqual(T1[,,] arrOne, T2[,,] arrTwo) { - // Assume that arrOne & arrTwo are the same Length & Rank - + // Assumes that arrOne & arrTwo are the same length, width, and height. for (int i = 0; i < arrOne.GetLength(0); i++) { for (int j = 0; j < arrOne.GetLength(1); j++) { - var x = arrOne[i, j]; - var y = arrTwo[i, j]; + for (int k = 0; k (T1[,,] arrOne, T2[,,] arrTwo) + private static bool ValueTuplesEqual(object x, object y) { - // Assume that arrOne & arrTwo are the same Length & Rank - - for (int i = 0; i < arrOne.GetLength(0); i++) + foreach (FieldInfo field in GetTupleMembers(x.GetType())) { - for (int j = 0; j < arrOne.GetLength(1); j++) + if (!ValuesEqual(field.GetValue(x), field.GetValue(y))) { - for (int k = 0; k