Introduce CategoryDiscoverer, fix #2306

dotnet · May 14, 2023 · 521b7fd · 521b7fd
1 parent 9602893
commit 521b7fd
Show file tree

Hide file tree

Showing 14 changed files with 266 additions and 41 deletions.
diff --git a/docs/articles/samples/IntroCategoryDiscoverer.md b/docs/articles/samples/IntroCategoryDiscoverer.md
@@ -0,0 +1,26 @@
+---
+uid: BenchmarkDotNet.Samples.IntroCategoryDiscoverer
+---
+
+## Sample: IntroCategoryDiscoverer
+
+The category discovery strategy can be overridden using an instance of `ICategoryDiscoverer`.
+
+### Source code
+
+[!code-csharp[IntroCategoryDiscoverer.cs](../../../samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs)]
+
+### Output
+
+```markdown
+| Method | Categories |     Mean | Error |
+|------- |----------- |---------:|------:|
+|    Bar |      All,B | 126.5 us |    NA |
+|    Foo |      All,F | 114.0 us |    NA |
+```
+
+### Links
+
+* The permanent link to this sample: @BenchmarkDotNet.Samples.IntroCategoryDiscoverer
+
+---
diff --git a/docs/articles/samples/toc.yml b/docs/articles/samples/toc.yml
@@ -12,6 +12,8 @@
   href: IntroCategories.md
 - name: IntroCategoryBaseline
   href: IntroCategoryBaseline.md
+- name: IntroCategoryDiscoverer
+  href: IntroCategoryDiscoverer.md
 - name: IntroColdStart
   href: IntroColdStart.md
 - name: IntroComparableComplexParam

diff --git a/samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs b/samples/BenchmarkDotNet.Samples/IntroCategoryDiscoverer.cs
@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Samples
+{
+    [DryJob]
+    [CategoriesColumn]
+    [CustomCategoryDiscoverer]
+    public class IntroCategoryDiscoverer
+    {
+        private class CustomCategoryDiscoverer : DefaultCategoryDiscoverer
+        {
+            public override string[] GetCategories(MethodInfo method)
+            {
+                var categories = new List<string>();
+                categories.AddRange(base.GetCategories(method));
+                categories.Add("All");
+                categories.Add(method.Name.Substring(0, 1));
+                return categories.ToArray();
+            }
+        }
+
+        [AttributeUsage(AttributeTargets.Class)]
+        private class CustomCategoryDiscovererAttribute : Attribute, IConfigSource
+        {
+            public CustomCategoryDiscovererAttribute()
+            {
+                Config = ManualConfig.CreateEmpty()
+                    .WithCategoryDiscoverer(new CustomCategoryDiscoverer());
+            }
+
+            public IConfig Config { get; }
+        }
+
+
+        [Benchmark]
+        public void Foo() { }
+
+        [Benchmark]
+        public void Bar() { }
+    }
+}
diff --git a/src/BenchmarkDotNet/Attributes/CategoryDiscovererAttribute.cs b/src/BenchmarkDotNet/Attributes/CategoryDiscovererAttribute.cs
@@ -0,0 +1,17 @@
+using System;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Attributes
+{
+    [AttributeUsage(AttributeTargets.Class)]
+    public class CategoryDiscovererAttribute : Attribute, IConfigSource
+    {
+        public CategoryDiscovererAttribute(bool inherit = true)
+        {
+            Config = ManualConfig.CreateEmpty().WithCategoryDiscoverer(new DefaultCategoryDiscoverer(inherit));
+        }
+
+        public IConfig Config { get; }
+    }
+}
diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs
@@ -12,6 +12,7 @@
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
 using BenchmarkDotNet.Validators;
 
@@ -66,6 +67,7 @@ public abstract class DebugConfig : IConfig
         public IEnumerable<IColumnHidingRule> GetColumnHidingRules() => Array.Empty<IColumnHidingRule>();
 
         public IOrderer Orderer => DefaultOrderer.Instance;
+        public ICategoryDiscoverer? CategoryDiscoverer => DefaultCategoryDiscoverer.Instance;
         public SummaryStyle SummaryStyle => SummaryStyle.Default;
         public ConfigUnionRule UnionRule => ConfigUnionRule.Union;
         public TimeSpan BuildTimeout => DefaultConfig.Instance.BuildTimeout;

diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs
@@ -13,6 +13,7 @@
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
 
 namespace BenchmarkDotNet.Configs
@@ -72,6 +73,7 @@ public IEnumerable<IValidator> GetValidators()
         }
 
         public IOrderer Orderer => null;
+        public ICategoryDiscoverer? CategoryDiscoverer => null;
 
         public ConfigUnionRule UnionRule => ConfigUnionRule.Union;
 

diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs
@@ -10,6 +10,7 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
 using JetBrains.Annotations;
 
@@ -30,6 +31,7 @@ public interface IConfig
         IEnumerable<IColumnHidingRule> GetColumnHidingRules();
 
         IOrderer? Orderer { get; }
+        ICategoryDiscoverer? CategoryDiscoverer { get; }
 
         SummaryStyle SummaryStyle { get; }
 
@@ -57,4 +59,4 @@ public interface IConfig
         /// </summary>
         IReadOnlyList<Conclusion> ConfigAnalysisConclusion { get; }
     }
-}
+}
diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs
@@ -14,7 +14,6 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
-using JetBrains.Annotations;
 using RunMode = BenchmarkDotNet.Diagnosers.RunMode;
 
 namespace BenchmarkDotNet.Configs
@@ -50,6 +49,7 @@ public sealed class ImmutableConfig : IConfig
             string artifactsPath,
             CultureInfo cultureInfo,
             IOrderer orderer,
+            ICategoryDiscoverer categoryDiscoverer,
             SummaryStyle summaryStyle,
             ConfigOptions options,
             TimeSpan buildTimeout,
@@ -70,6 +70,7 @@ public sealed class ImmutableConfig : IConfig
             ArtifactsPath = artifactsPath;
             CultureInfo = cultureInfo;
             Orderer = orderer;
+            CategoryDiscoverer = categoryDiscoverer;
             SummaryStyle = summaryStyle;
             Options = options;
             BuildTimeout = buildTimeout;
@@ -81,6 +82,7 @@ public sealed class ImmutableConfig : IConfig
         public CultureInfo CultureInfo { get; }
         public ConfigOptions Options { get; }
         public IOrderer Orderer { get; }
+        public ICategoryDiscoverer CategoryDiscoverer { get; }
         public SummaryStyle SummaryStyle { get; }
         public TimeSpan BuildTimeout { get; }
 

diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs
@@ -7,6 +7,7 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
 
 namespace BenchmarkDotNet.Configs
@@ -69,6 +70,7 @@ public static ImmutableConfig Create(IConfig source)
                 source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath,
                 source.CultureInfo,
                 source.Orderer ?? DefaultOrderer.Instance,
+                source.CategoryDiscoverer ?? DefaultCategoryDiscoverer.Instance,
                 source.SummaryStyle ?? SummaryStyle.Default,
                 source.Options,
                 source.BuildTimeout,

diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs
@@ -13,6 +13,7 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Validators;
 using JetBrains.Annotations;
 
@@ -51,6 +52,7 @@ public class ManualConfig : IConfig
         [PublicAPI] public string ArtifactsPath { get; set; }
         [PublicAPI] public CultureInfo CultureInfo { get; set; }
         [PublicAPI] public IOrderer Orderer { get; set; }
+        [PublicAPI] public ICategoryDiscoverer CategoryDiscoverer { get; set; }
         [PublicAPI] public SummaryStyle SummaryStyle { get; set; }
         [PublicAPI] public TimeSpan BuildTimeout { get; set; } = DefaultConfig.Instance.BuildTimeout;
 
@@ -92,6 +94,12 @@ public ManualConfig WithOrderer(IOrderer orderer)
             return this;
         }
 
+        public ManualConfig WithCategoryDiscoverer(ICategoryDiscoverer categoryDiscoverer)
+        {
+            CategoryDiscoverer = categoryDiscoverer;
+            return this;
+        }
+
         public ManualConfig WithBuildTimeout(TimeSpan buildTimeout)
         {
             BuildTimeout = buildTimeout;
@@ -247,6 +255,7 @@ public void Add(IConfig config)
             hardwareCounters.AddRange(config.GetHardwareCounters());
             filters.AddRange(config.GetFilters());
             Orderer = config.Orderer ?? Orderer;
+            CategoryDiscoverer = config.CategoryDiscoverer ?? CategoryDiscoverer;
             ArtifactsPath = config.ArtifactsPath ?? ArtifactsPath;
             CultureInfo = config.CultureInfo ?? CultureInfo;
             SummaryStyle = config.SummaryStyle ?? SummaryStyle;

diff --git a/src/BenchmarkDotNet/Running/BenchmarkConverter.cs b/src/BenchmarkDotNet/Running/BenchmarkConverter.cs
@@ -51,7 +51,8 @@ private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type type, Met
             var iterationSetupMethods = GetAttributedMethods<IterationSetupAttribute>(allMethods, "IterationSetup");
             var iterationCleanupMethods = GetAttributedMethods<IterationCleanupAttribute>(allMethods, "IterationCleanup");
 
-            var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods).ToArray();
+            var targets = GetTargets(benchmarkMethods, type, globalSetupMethods, globalCleanupMethods, iterationSetupMethods, iterationCleanupMethods,
+                configPerType).ToArray();
 
             var parameterDefinitions = GetParameterDefinitions(type);
             var parameterInstancesList = parameterDefinitions.Expand(configPerType.SummaryStyle);
@@ -115,7 +116,8 @@ private static ImmutableConfig GetFullMethodConfig(MethodInfo method, ImmutableC
             Tuple<MethodInfo, TargetedAttribute>[] globalSetupMethods,
             Tuple<MethodInfo, TargetedAttribute>[] globalCleanupMethods,
             Tuple<MethodInfo, TargetedAttribute>[] iterationSetupMethods,
-            Tuple<MethodInfo, TargetedAttribute>[] iterationCleanupMethods)
+            Tuple<MethodInfo, TargetedAttribute>[] iterationCleanupMethods,
+            IConfig config)
         {
             return targetMethods
                 .Select(methodInfo => CreateDescriptor(type,
@@ -125,7 +127,8 @@ private static ImmutableConfig GetFullMethodConfig(MethodInfo method, ImmutableC
                                                    GetTargetedMatchingMethod(methodInfo, iterationSetupMethods),
                                                    GetTargetedMatchingMethod(methodInfo, iterationCleanupMethods),
                                                    methodInfo.ResolveAttribute<BenchmarkAttribute>(),
-                                                   targetMethods));
+                                                   targetMethods,
+                                                   config));
         }
 
         private static MethodInfo GetTargetedMatchingMethod(MethodInfo benchmarkMethod, Tuple<MethodInfo, TargetedAttribute>[] methods)
@@ -152,8 +155,10 @@ private static MethodInfo GetTargetedMatchingMethod(MethodInfo benchmarkMethod,
             MethodInfo iterationSetupMethod,
             MethodInfo iterationCleanupMethod,
             BenchmarkAttribute attr,
-            MethodInfo[] targetMethods)
+            MethodInfo[] targetMethods,
+            IConfig config)
         {
+            var categoryDiscoverer = config.CategoryDiscoverer ?? DefaultCategoryDiscoverer.Instance;
             var target = new Descriptor(
                 type,
                 methodInfo,
@@ -163,7 +168,7 @@ private static MethodInfo GetTargetedMatchingMethod(MethodInfo benchmarkMethod,
                 iterationCleanupMethod,
                 attr.Description,
                 baseline: attr.Baseline,
-                categories: GetCategories(methodInfo),
+                categories: categoryDiscoverer.GetCategories(methodInfo),
                 operationsPerInvoke: attr.OperationsPerInvoke,
                 methodIndex: Array.IndexOf(targetMethods, methodInfo));
             AssertMethodHasCorrectSignature("Benchmark", methodInfo);
@@ -245,21 +250,6 @@ private static IEnumerable<ParameterInstances> GetArgumentsDefinitions(MethodInf
                 yield return SmartParamBuilder.CreateForArguments(benchmark, parameterDefinitions, valuesInfo, sourceIndex, summaryStyle);
         }
 
-        private static string[] GetCategories(MethodInfo method)
-        {
-            var attributes = new List<BenchmarkCategoryAttribute>();
-            attributes.AddRange(method.GetCustomAttributes(typeof(BenchmarkCategoryAttribute), true).OfType<BenchmarkCategoryAttribute>());
-            var type = method.ReflectedType;
-            if (type != null)
-            {
-                attributes.AddRange(type.GetTypeInfo().GetCustomAttributes(typeof(BenchmarkCategoryAttribute), true).OfType<BenchmarkCategoryAttribute>());
-                attributes.AddRange(type.GetTypeInfo().Assembly.GetCustomAttributes().OfType<BenchmarkCategoryAttribute>());
-            }
-            if (attributes.Count == 0)
-                return Array.Empty<string>();
-            return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
-        }
-
         private static ImmutableArray<BenchmarkCase> GetFilteredBenchmarks(IEnumerable<BenchmarkCase> benchmarks, IEnumerable<IFilter> filters)
             => benchmarks.Where(benchmark => filters.All(filter => filter.Predicate(benchmark))).ToImmutableArray();
 

diff --git a/src/BenchmarkDotNet/Running/DefaultCategoryDiscoverer.cs b/src/BenchmarkDotNet/Running/DefaultCategoryDiscoverer.cs
@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using BenchmarkDotNet.Attributes;
+
+namespace BenchmarkDotNet.Running
+{
+    public class DefaultCategoryDiscoverer : ICategoryDiscoverer
+    {
+        public static readonly ICategoryDiscoverer Instance = new DefaultCategoryDiscoverer();
+
+        private readonly bool inherit;
+
+        public DefaultCategoryDiscoverer(bool inherit = true)
+        {
+            this.inherit = inherit;
+        }
+
+        public virtual string[] GetCategories(MethodInfo method)
+        {
+            var attributes = new List<BenchmarkCategoryAttribute>();
+            attributes.AddRange(method.GetCustomAttributes(typeof(BenchmarkCategoryAttribute), inherit).OfType<BenchmarkCategoryAttribute>());
+            var type = method.ReflectedType;
+            if (type != null)
+            {
+                attributes.AddRange(type.GetTypeInfo().GetCustomAttributes(typeof(BenchmarkCategoryAttribute), inherit).OfType<BenchmarkCategoryAttribute>());
+                attributes.AddRange(type.GetTypeInfo().Assembly.GetCustomAttributes().OfType<BenchmarkCategoryAttribute>());
+            }
+            if (attributes.Count == 0)
+                return Array.Empty<string>();
+            return attributes.SelectMany(attr => attr.Categories).Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
+        }
+    }
+}
diff --git a/src/BenchmarkDotNet/Running/ICategoryDiscoverer.cs b/src/BenchmarkDotNet/Running/ICategoryDiscoverer.cs
@@ -0,0 +1,9 @@
+using System.Reflection;
+
+namespace BenchmarkDotNet.Running
+{
+    public interface ICategoryDiscoverer
+    {
+        string[] GetCategories(MethodInfo method);
+    }
+}