Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Single point of full config creation #565

Merged
merged 2 commits into from Oct 15, 2017
Merged

Single point of full config creation #565

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0e5bff1
Single point of full config creation (force rebuild)
ig-sinicyn Oct 8, 2017
3c79e6a
Code cleanup
ig-sinicyn Oct 15, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/BenchmarkDotNet.Core/Configs/ConfigExtensions.cs
View file
Open in desktop
Expand Up @@ -38,6 +38,11 @@ public static class ConfigExtensions
public static IConfig KeepBenchmarkFiles(this IConfig config, bool value = true) => config.With(m => m.KeepBenchmarkFiles = value);
public static IConfig RemoveBenchmarkFiles(this IConfig config) => config.KeepBenchmarkFiles(false);

public static ReadOnlyConfig AsReadOnly(this IConfig config) =>
config is ReadOnlyConfig r
? r
: new ReadOnlyConfig(config);

private static IConfig With(this IConfig config, Action<ManualConfig> addAction)
{
var manualConfig = ManualConfig.Create(config);
Expand Down
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet.Core/Configs/IConfig.cs
View file
Open in desktop
Expand Up @@ -22,7 +22,7 @@ public interface IConfig
IEnumerable<Job> GetJobs();
IEnumerable<IValidator> GetValidators();
IEnumerable<HardwareCounter> GetHardwareCounters();
IEnumerable<IFilter> GetFilters();
IEnumerable<IFilter> GetFilters();

IOrderProvider GetOrderProvider();
ISummaryStyle GetSummaryStyle();
Expand Down
54 changes: 54 additions & 0 deletions src/BenchmarkDotNet.Core/Configs/ReadOnlyConfig.cs
View file
Open in desktop
@@ -0,0 +1,54 @@
using System;
using System.Collections.Generic;

using BenchmarkDotNet.Analysers;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Filters;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Order;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Validators;

using JetBrains.Annotations;

namespace BenchmarkDotNet.Configs
{
[PublicAPI]
public class ReadOnlyConfig : IConfig
{
#region Fields & .ctor

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

private readonly IConfig config;

public ReadOnlyConfig([NotNull] IConfig config)
{
if (config == null)
throw new ArgumentNullException(nameof(config));

this.config = config;
}
#endregion

#region IConfig implementation
public IEnumerable<IColumnProvider> GetColumnProviders() => config.GetColumnProviders();
public IEnumerable<IExporter> GetExporters() => config.GetExporters();
public IEnumerable<ILogger> GetLoggers() => config.GetLoggers();
public IEnumerable<IDiagnoser> GetDiagnosers() => config.GetDiagnosers();
public IEnumerable<IAnalyser> GetAnalysers() => config.GetAnalysers();
public IEnumerable<Job> GetJobs() => config.GetJobs();
public IEnumerable<IValidator> GetValidators() => config.GetValidators();
public IEnumerable<HardwareCounter> GetHardwareCounters() => config.GetHardwareCounters();
public IEnumerable<IFilter> GetFilters() => config.GetFilters();

public IOrderProvider GetOrderProvider() => config.GetOrderProvider();
public ISummaryStyle GetSummaryStyle() => config.GetSummaryStyle();

public ConfigUnionRule UnionRule => config.UnionRule;

public bool KeepBenchmarkFiles => config.KeepBenchmarkFiles;

#endregion
}
}
34 changes: 24 additions & 10 deletions src/BenchmarkDotNet.Core/Running/BenchmarkConverter.cs
View file
Open in desktop
Expand Up @@ -14,16 +14,26 @@ namespace BenchmarkDotNet.Running
{
public static partial class BenchmarkConverter
{
public static Benchmark[] TypeToBenchmarks(Type type, IConfig config = null)
public static BenchmarkRunInfo TypeToBenchmarks(Type type, IConfig config = null)
{
config = GetFullConfig(type, config);
var fullConfig = GetFullConfig(type, config);

var allMethods = type.GetMethods();
return MethodsToBenchmarks(type, allMethods, config);
return MethodsToBenchmarksWithFullConfig(type, allMethods, fullConfig);
}

public static Benchmark[] MethodsToBenchmarks(Type containingType, MethodInfo[] methods, IConfig config = null)
public static BenchmarkRunInfo MethodsToBenchmarks(Type containingType, MethodInfo[] methods, IConfig config = null)
{
var fullConfig = GetFullConfig(containingType, config);

return MethodsToBenchmarksWithFullConfig(containingType, methods, fullConfig);
}

private static BenchmarkRunInfo MethodsToBenchmarksWithFullConfig(Type containingType, MethodInfo[] methods, ReadOnlyConfig fullConfig)
{
if (fullConfig == null)
throw new ArgumentNullException(nameof(fullConfig));

var globalSetupMethods = GetAttributedMethods<GlobalSetupAttribute>(methods, "GlobalSetup");
var globalCleanupMethods = GetAttributedMethods<GlobalCleanupAttribute>(methods, "GlobalCleanup");
var iterationSetupMethods = GetAttributedMethods<IterationSetupAttribute>(methods, "IterationSetup");
Expand All @@ -34,7 +44,7 @@ public static Benchmark[] MethodsToBenchmarks(Type containingType, MethodInfo[]
var parameterDefinitions = GetParameterDefinitions(containingType);
var parameterInstancesList = parameterDefinitions.Expand();

var rawJobs = config?.GetJobs().ToArray() ?? Array.Empty<Job>();
var rawJobs = fullConfig.GetJobs().ToArray();
if (rawJobs.IsEmpty())
rawJobs = new[] { Job.Default };
var jobs = rawJobs.Distinct().ToArray();
Expand All @@ -47,14 +57,18 @@ public static Benchmark[] MethodsToBenchmarks(Type containingType, MethodInfo[]
from parameterInstance in parameterInstancesList
select new Benchmark(target, job, parameterInstance)).ToArray();

var filters = config.GetFilters().ToList();
var filters = fullConfig.GetFilters().ToList();
benchmarks = GetFilteredBenchmarks(benchmarks, filters);

var orderProvider = config?.GetOrderProvider() ?? DefaultOrderProvider.Instance;
return orderProvider.GetExecutionOrder(benchmarks).ToArray();
var orderProvider = fullConfig.GetOrderProvider() ?? DefaultOrderProvider.Instance;

return new BenchmarkRunInfo(
orderProvider.GetExecutionOrder(benchmarks).ToArray(),
containingType,
fullConfig);
}

public static IConfig GetFullConfig(Type type, IConfig config)
public static ReadOnlyConfig GetFullConfig(Type type, IConfig config)
{
config = config ?? DefaultConfig.Instance;
if (type != null)
Expand All @@ -65,7 +79,7 @@ public static IConfig GetFullConfig(Type type, IConfig config)
foreach (var configSource in allAttributes)
config = ManualConfig.Union(config, configSource.Config);
}
return config;
return config.AsReadOnly();
}

private static IEnumerable<Target> GetTargets(
Expand Down
18 changes: 18 additions & 0 deletions src/BenchmarkDotNet.Core/Running/BenchmarkRunInfo.cs
View file
Open in desktop
@@ -0,0 +1,18 @@
using System;
using BenchmarkDotNet.Configs;

namespace BenchmarkDotNet.Running
{
public class BenchmarkRunInfo
{
public BenchmarkRunInfo(Benchmark[] benchmarks, Type type, ReadOnlyConfig config)
{
Benchmarks = benchmarks;
Type = type;
Config = config;
}
public Benchmark[] Benchmarks { get; }
public Type Type { get; }
public ReadOnlyConfig Config { get; }
}
}
28 changes: 21 additions & 7 deletions src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs
View file
Open in desktop
Expand Up @@ -9,7 +9,6 @@
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Exporters;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Filters;
using BenchmarkDotNet.Horology;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
Expand All @@ -31,10 +30,21 @@ public static class BenchmarkRunnerCore

internal static readonly IResolver DefaultResolver = new CompositeResolver(EnvResolver.Instance, InfrastructureResolver.Instance);

public static Summary Run(Benchmark[] benchmarks, IConfig config, Func<Job, IToolchain> toolchainProvider)
public static Summary Run(BenchmarkRunInfo[] benchmarkRunInfos, Func<Job, IToolchain> toolchainProvider)
{
var temp = benchmarkRunInfos.FirstOrDefault();
var benchmarkRunInfo = new BenchmarkRunInfo(

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

benchmarkRunInfos.SelectMany(i => i.Benchmarks).ToArray(),
temp?.Type,
temp?.Config);
return Run(benchmarkRunInfo, toolchainProvider);
}

public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, Func<Job, IToolchain> toolchainProvider)
{
var resolver = DefaultResolver;
config = BenchmarkConverter.GetFullConfig(benchmarks.FirstOrDefault()?.Target.Type, config);
var benchmarks = benchmarkRunInfo.Benchmarks;
var config = benchmarkRunInfo.Config;

var title = GetTitle(benchmarks);
var rootArtifactsFolderPath = GetRootArtifactsFolderPath();
Expand All @@ -47,7 +57,8 @@ public static Summary Run(Benchmark[] benchmarks, IConfig config, Func<Job, IToo

try
{
return Run(benchmarks, logger, title, config, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup);
var runInfo = new BenchmarkRunInfo(benchmarks, benchmarkRunInfo.Type, config);
return Run(runInfo, logger, title, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup);
}
finally
{
Expand All @@ -66,8 +77,11 @@ private static string GetTitle(IList<Benchmark> benchmarks)
return $"BenchmarkRun-{benchmarkRunIndex:##000}-{DateTime.Now:yyyy-MM-dd-hh-mm-ss}";
}

public static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver, List<string> artifactsToCleanup)
public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, string title, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver, List<string> artifactsToCleanup)
{
var benchmarks = benchmarkRunInfo.Benchmarks;
var config = benchmarkRunInfo.Config;

logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
Expand All @@ -84,7 +98,7 @@ public static Summary Run(Benchmark[] benchmarks, ILogger logger, string title,
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
var report = Run(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup);
var report = RunCore(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider, resolver, artifactsToCleanup);
reports.Add(report);
if (report.GetResultRuns().Any())
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
Expand Down Expand Up @@ -178,7 +192,7 @@ internal static void LogTotalTime(ILogger logger, TimeSpan time, string message
logger.WriteLineStatistic($"{message}: {time.ToFormattedTotalTime()}");
}

public static BenchmarkReport Run(Benchmark benchmark, ILogger logger, IConfig config, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver, List<string> artifactsToCleanup)
private static BenchmarkReport RunCore(Benchmark benchmark, ILogger logger, ReadOnlyConfig config, string rootArtifactsFolderPath, Func<Job, IToolchain> toolchainProvider, IResolver resolver, List<string> artifactsToCleanup)
{
var toolchain = toolchainProvider(benchmark.Job);

Expand Down
44 changes: 29 additions & 15 deletions src/BenchmarkDotNet.Core/Running/ClassicBenchmarkConverter.cs
View file
Open in desktop
@@ -1,9 +1,11 @@
#if CLASSIC
using System;
using System.CodeDom.Compiler;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.Remoting.Activation;

This comment was marked as spam.

Sign in to view

This comment was marked as spam.

Sign in to view

using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Loggers;
Expand All @@ -13,7 +15,7 @@ namespace BenchmarkDotNet.Running
{
public static partial class BenchmarkConverter
{
public static Benchmark[] UrlToBenchmarks(string url, IConfig config = null)
public static BenchmarkRunInfo[] UrlToBenchmarks(string url, IConfig config = null)
{
var logger = config?.GetCompositeLogger() ?? HostEnvironmentInfo.FallbackLogger;

Expand All @@ -29,18 +31,18 @@ public static Benchmark[] UrlToBenchmarks(string url, IConfig config = null)
if (string.IsNullOrWhiteSpace(benchmarkContent))
{
logger.WriteLineHint($"content of '{url}' is empty.");
return Array.Empty<Benchmark>();
return Array.Empty<BenchmarkRunInfo>();
}
}
catch (Exception e)
{
logger.WriteLineError("BuildException: " + e.Message);
return Array.Empty<Benchmark>();
return Array.Empty<BenchmarkRunInfo>();
}
return SourceToBenchmarks(benchmarkContent, config);
}

public static Benchmark[] SourceToBenchmarks(string source, IConfig config = null)
public static BenchmarkRunInfo[] SourceToBenchmarks(string source, IConfig config = null)
{
string benchmarkContent = source;
var cSharpCodeProvider = new CSharpCodeProvider();
Expand All @@ -66,18 +68,30 @@ public static Benchmark[] SourceToBenchmarks(string source, IConfig config = nul
var logger = config?.GetCompositeLogger() ?? HostEnvironmentInfo.FallbackLogger;

compilerResults.Errors.Cast<CompilerError>().ToList().ForEach(error => logger.WriteLineError(error.ErrorText));
return Array.Empty<Benchmark>();
return Array.Empty<BenchmarkRunInfo>();
}
return (
from type in compilerResults.CompiledAssembly.GetTypes()
from benchmark in TypeToBenchmarks(type, config)
let target = benchmark.Target
select new Benchmark(
new Target(target.Type, target.Method, target.GlobalSetupMethod, target.GlobalCleanupMethod,
target.IterationSetupMethod, target.IterationCleanupMethod,
target.MethodDisplayInfo, benchmarkContent, target.Baseline, target.Categories, target.OperationsPerInvoke),
benchmark.Job,
benchmark.Parameters)).ToArray();

var types = compilerResults.CompiledAssembly.GetTypes();

var resultBenchmarks = new List<BenchmarkRunInfo>();
foreach (var type in types)
{
var runInfo = TypeToBenchmarks(type, config);
var benchmarks = runInfo.Benchmarks.Select(b =>
{
var target = b.Target;
return new Benchmark(
new Target(target.Type, target.Method, target.GlobalSetupMethod, target.GlobalCleanupMethod,

This comment was marked as spam.

Sign in to view

target.IterationSetupMethod, target.IterationCleanupMethod,
target.MethodDisplayInfo, benchmarkContent, target.Baseline, target.Categories, target.OperationsPerInvoke),
b.Job,
b.Parameters);
});
resultBenchmarks.Add(
new BenchmarkRunInfo(benchmarks.ToArray(), runInfo.Type, runInfo.Config));
}

return resultBenchmarks.ToArray();
}

private static string GetRawUrl(string url)
Expand Down
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet.Core/Running/TypeParser.cs
View file
Open in desktop
Expand Up @@ -88,7 +88,7 @@ internal IEnumerable<TypeWithMethods> MatchingTypesWithMethods(string[] args)
var allBenchmarks = typeInfo.GetBenchmarks();
var benchmarks = allBenchmarks
.Where(method => filters.methodPredicates.All(rule => rule(method)))
.ToArray();
.ToArray();

if (benchmarks.IsEmpty())
continue;
Expand Down
1 change: 1 addition & 0 deletions src/BenchmarkDotNet.Core/Validators/ValidationParameters.cs
View file
Open in desktop
Expand Up @@ -18,5 +18,6 @@ public ValidationParameters(IReadOnlyList<Benchmark> benchmarks, IConfig config)

// to have backward compatibility for people who implemented IValidator(Benchmark[] benchmarks)
public static implicit operator ValidationParameters(Benchmark[] benchmarks) => new ValidationParameters(benchmarks, null);
public static implicit operator ValidationParameters(BenchmarkRunInfo benchmarkRunInfo) => new ValidationParameters(benchmarkRunInfo.Benchmarks, null);
}
}