Possibility to benchmark asynchronous methods #236

src/BenchmarkDotNet.Core/Code/CodeGenerator.cs

@@ -0,0 +1,104 @@
+using System;
+using System.Globalization;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Helpers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Code
+{
+    internal static class CodeGenerator
+    {
+        internal static string Generate(Benchmark benchmark)
+        {
+            var declarationsProvider = GetDeclarationsProvider(benchmark.Target);
+
+            return new StringBuilder(ResourceHelper.LoadTemplate("BenchmarkProgram.txt"))
+               .Replace("$OperationsPerInvoke$", declarationsProvider.OperationsPerInvoke)
+               .Replace("$TargetTypeNamespace$", declarationsProvider.TargetTypeNamespace)
+               .Replace("$TargetMethodReturnTypeNamespace$", declarationsProvider.TargetMethodReturnTypeNamespace)
+               .Replace("$TargetTypeName$", declarationsProvider.TargetTypeName)
+               .Replace("$TargetMethodDelegate$", declarationsProvider.TargetMethodDelegate)
+               .Replace("$TargetMethodResultHolder$", declarationsProvider.TargetMethodResultHolder)
+               .Replace("$TargetMethodDelegateType$", declarationsProvider.TargetMethodDelegateType)
+               .Replace("$TargetMethodHoldValue$", declarationsProvider.TargetMethodHoldValue)
+               .Replace("$TargetMethodReturnType$", declarationsProvider.TargetMethodReturnType)
+               .Replace("$IdleMethodDelegateType$", declarationsProvider.IdleMethodDelegateType)
+               .Replace("$IdleMethodReturnType$", declarationsProvider.IdleMethodReturnType)
+               .Replace("$SetupMethodName$", declarationsProvider.SetupMethodName)
+               .Replace("$CleanupMethodName$", declarationsProvider.CleanupMethodName)
+               .Replace("$IdleImplementation$", declarationsProvider.IdleImplementation)
+               .Replace("$AdditionalLogic$", benchmark.Target.AdditionalLogic)
+               .Replace("$TargetBenchmarkTaskArguments$", benchmark.Job.GenerateWithDefinitions())
+               .Replace("$ParamsContent$", GetParamsContent(benchmark))
+               .ToString();
+        }
+
+        private static DeclarationsProvider GetDeclarationsProvider(Target target)
+        {
+            var method = target.Method;
+
+            if (method.ReturnType == typeof(Task))
+            {
+                return new TaskDeclarationsProvider(target);
+            }
+            if (method.ReturnType.GetTypeInfo().IsGenericType 
+                && method.ReturnType.GetTypeInfo().GetGenericTypeDefinition() == typeof(Task<>))
+            {
+                return new GenericTaskDeclarationsProvider(target, typeof(TaskMethodInvoker<>));
+            }
+            if (method.ReturnType.GetTypeInfo().IsGenericType 
+                && method.ReturnType.GetTypeInfo().GetGenericTypeDefinition() == typeof(ValueTask<>))
+            {
+                return new GenericTaskDeclarationsProvider(target, typeof(ValueTaskMethodInvoker<>));
+            }
+
+            if (method.ReturnType == typeof(void))
+            {
+                var isUsingAsyncKeyword = method.HasAttribute<AsyncStateMachineAttribute>();
+                if (isUsingAsyncKeyword)
+                {
+                    throw new NotSupportedException("async void is not supported by design");
+                }
+
+                return new VoidDeclarationsProvider(target);
+            }
+            return new NonVoidDeclarationsProvider(target);
+        }
+
+        private static string GetParamsContent(Benchmark benchmark)
+        {
+            return string.Join(
+                string.Empty,
+                benchmark.Parameters.Items.Select(
+                    parameter =>
+                        $"{(parameter.IsStatic ? "" : "instance.")}{parameter.Name} = {GetParameterValue(parameter.Value)};"));
+        }
+
+        private static string GetParameterValue(object value)
+        {
+            if (value is bool)
+                return value.ToString().ToLower();
+            if (value is string)
+                return $"\"{value}\"";
+            if (value is char)
+                return $"'{value}'";
+            if (value is float)
+                return ((float)value).ToString("G", CultureInfo.InvariantCulture) + "f";
+            if (value is double)
+                return ((double)value).ToString("G", CultureInfo.InvariantCulture) + "d";
+            if (value is decimal)
+                return ((decimal)value).ToString("G", CultureInfo.InvariantCulture) + "m";
+            if (value.GetType().GetTypeInfo().IsEnum)
+                return value.GetType().GetCorrectTypeName() + "." + value;
+            if (value is Type)
+                return "typeof(" + ((Type)value).GetCorrectTypeName() + ")";
+            return value.ToString();
+        }
+    }
+}

src/BenchmarkDotNet.Core/Code/DeclarationsProvider.cs

@@ -0,0 +1,145 @@
+using System;
+using System.Linq;
+using System.Reflection;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Code
+{
+    internal abstract class DeclarationsProvider
+    {
+        // "Setup" or "Cleanup" methods are optional, so default to an empty delegate, so there is always something that can be invoked
+        private const string EmptyAction = "() => { }";
+
+        protected readonly Target Target;
+
+        internal DeclarationsProvider(Target target)
+        {
+            Target = target;
+        }
+
+        public string OperationsPerInvoke => Target.OperationsPerInvoke.ToString();
+
+        public string TargetTypeNamespace => string.IsNullOrWhiteSpace(Target.Type.Namespace) ? string.Empty : $"using {Target.Type.Namespace};";
+
+        public string TargetTypeName => Target.Type.GetCorrectTypeName();
+
+        public string SetupMethodName => Target.SetupMethod?.Name ?? EmptyAction;
+
+        public string CleanupMethodName => Target.CleanupMethod?.Name ?? EmptyAction;
+
+        public virtual string TargetMethodDelegate => Target.Method.Name;
+
+        public abstract string TargetMethodReturnTypeNamespace { get; }
+
+        public abstract string TargetMethodReturnType { get; }
+
+        public abstract string TargetMethodResultHolder { get; }
+
+        public abstract string TargetMethodHoldValue { get; }
+
+        public abstract string TargetMethodDelegateType { get; }
+
+        public abstract string IdleMethodReturnType { get; }
+
+        public abstract string IdleMethodDelegateType { get; }
+
+        public abstract string IdleImplementation { get; }
+    }
+
+    internal class VoidDeclarationsProvider : DeclarationsProvider
+    {
+        public VoidDeclarationsProvider(Target target) : base(target) { }
+
+        public override string TargetMethodReturnTypeNamespace => string.Empty;
+
+        public override string TargetMethodReturnType => "void";
+
+        public override string TargetMethodResultHolder => string.Empty;
+
+        public override string TargetMethodHoldValue => string.Empty;
+
+        public override string TargetMethodDelegateType => "Action";
+
+        public override string IdleMethodReturnType => TargetMethodReturnType;
+
+        public override string IdleMethodDelegateType => TargetMethodDelegateType;
+
+        public override string IdleImplementation => string.Empty;
+    }
+
+    internal class NonVoidDeclarationsProvider : DeclarationsProvider
+    {
+        public NonVoidDeclarationsProvider(Target target) : base(target) { }
+
+        public override string TargetMethodReturnTypeNamespace 
+            => Target.Method.ReturnType.Namespace == "System" // As "using System;" is always included in the template, don't emit it again
+                || string.IsNullOrWhiteSpace(Target.Method.ReturnType.Namespace) 
+                    ? string.Empty 
+                    : $"using {Target.Method.ReturnType.Namespace};";
+
+        public override string TargetMethodReturnType => Target.Method.ReturnType.GetCorrectTypeName();
+
+        public override string TargetMethodResultHolder => $"private {TargetMethodReturnType} value;";
+
+        public override string TargetMethodHoldValue => "value = ";
+
+        public override string TargetMethodDelegateType => $"Func<{TargetMethodReturnType}>";
+
+        public override string IdleMethodReturnType
+            => Target.Method.ReturnType.GetTypeInfo().IsValueType
+                ? "int" // we return int because creating bigger ValueType could take longer than benchmarked method itself
+                : TargetMethodReturnType;
+
+        public override string IdleMethodDelegateType
+            => Target.Method.ReturnType.GetTypeInfo().IsValueType
+                ? "Func<int>" 
+                : $"Func<{TargetMethodReturnType}>";
+
+        public override string IdleImplementation
+            => Target.Method.ReturnType.GetTypeInfo().IsValueType
+                ? "return 0;"
+                : "return null;";
+    }
+
+    internal class TaskDeclarationsProvider : VoidDeclarationsProvider
+    {
+        public TaskDeclarationsProvider(Target target) : base(target) { }
+
+        public override string TargetMethodDelegate
+            => $"() => {{ BenchmarkDotNet.Running.TaskMethodInvoker.ExecuteBlocking({Target.Method.Name}); }}";        
+
+        public override string IdleImplementation
+            => $"BenchmarkDotNet.Running.TaskMethodInvoker.Idle();";
+    }
+
+    /// <summary>
+    /// declarations provider for <see cref="Task{TResult}" /> and <see cref="ValueTask{T}" />
+    /// </summary>
+    internal class GenericTaskDeclarationsProvider : NonVoidDeclarationsProvider
+    {
+        private const char GenericArgumentSign = '`';
+
+        private readonly string invokerFullName;
+
+        public GenericTaskDeclarationsProvider(Target target, Type invoker) : base(target)
+        {
+            invokerFullName = invoker.GetTypeInfo().FullName.Split(GenericArgumentSign).First();
+        }
+
+        public override string TargetMethodReturnType 
+            => Target.Method.ReturnType.GetTypeInfo().GetGenericArguments().Single().GetCorrectTypeName();
+
+        public override string TargetMethodDelegate
+            => $"() => {{ return {invokerFullName}<{TargetMethodReturnType}>.ExecuteBlocking({Target.Method.Name}); }}";
+
+        public override string IdleMethodReturnType => TargetMethodReturnType;
+
+        public override string IdleMethodDelegateType => TargetMethodDelegateType;
+
+        public override string IdleImplementation 
+            => $"return {invokerFullName}<{TargetMethodReturnType}>.Idle();";
+    }
+}

src/BenchmarkDotNet.Core/Running/AsyncMethodInvoker.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Threading.Tasks;
+using HideFromIntelliSense = System.ComponentModel.EditorBrowsableAttribute; // we don't want people to use it
+
+namespace BenchmarkDotNet.Running
+{
+    // if you want to rename any of these methods you need to update DeclarationsProvider's code as well
+    // ReSharper disable MemberCanBePrivate.Global
+    public static class TaskMethodInvoker
+    {
+        // can't use Task.CompletedTask here because it's new in .NET 4.6 (we target 4.5)
+        private static readonly Task Completed = Task.FromResult((object)null);
+
+        [HideFromIntelliSense(System.ComponentModel.EditorBrowsableState.Never)]
+        public static void Idle() => ExecuteBlocking(() => Completed);
+
+        [HideFromIntelliSense(System.ComponentModel.EditorBrowsableState.Never)]
+        public static void ExecuteBlocking(Func<Task> future) => future.Invoke().Wait();
+    }
+
+    public static class TaskMethodInvoker<T>
+    {
+        private static readonly Task<T> Completed = Task.FromResult(default(T));
+
+        [HideFromIntelliSense(System.ComponentModel.EditorBrowsableState.Never)]
+        public static T Idle() => ExecuteBlocking(() => Completed);
+
+        // we use .Result because it's blocking and will eventually rethrow any exception
+        [HideFromIntelliSense(System.ComponentModel.EditorBrowsableState.Never)]        
+        public static T ExecuteBlocking(Func<Task<T>> future) => future.Invoke().Result;         
+    }
+
+    public static class ValueTaskMethodInvoker<T>
+    {
+        [HideFromIntelliSense(System.ComponentModel.EditorBrowsableState.Never)]
+        public static T Idle() => ExecuteBlocking(() => new ValueTask<T>(default(T)));
+
+        [HideFromIntelliSense(System.ComponentModel.EditorBrowsableState.Never)]
+        public static T ExecuteBlocking(Func<ValueTask<T>> future) => future.Invoke().Result;
+    }
+    // ReSharper restore MemberCanBePrivate.Global
+}

src/BenchmarkDotNet.Core/Templates/BenchmarkProgram.txt

@@ -52,7 +52,7 @@ namespace BenchmarkDotNet.Autogenerated
             setupAction = $SetupMethodName$;
             cleanupAction = $CleanupMethodName$;
             idleAction = Idle;
-            targetAction = $TargetMethodName$;
+            targetAction = $TargetMethodDelegate$;
         }
 
         $TargetMethodResultHolder$