Big API refactoring, Part 12

BenchmarkDotNet.IntegrationTests/Properties/AssemblyInfo.cs

@@ -1,6 +1,7 @@
 using System.Reflection;
 using System.Runtime.InteropServices;
 using BenchmarkDotNet.Properties;
+using Xunit;
 
 [assembly: AssemblyTitle(BenchmarkDotNetInfo.Title + ".IntegrationTests")]
 [assembly: AssemblyProduct(BenchmarkDotNetInfo.Title + ".IntegrationTests")]
@@ -15,4 +16,6 @@
 [assembly: AssemblyCulture("")]
 
 [assembly: ComVisible(false)]
-[assembly: Guid("74362bb1-9f64-4be5-b079-b4ac19dae5db")]
+[assembly: Guid("74362bb1-9f64-4be5-b079-b4ac19dae5db")]
+
+[assembly: CollectionBehavior(DisableTestParallelization = true)]

BenchmarkDotNet/Exporters/RPlotExporter.cs

@@ -4,6 +4,7 @@
 using System.IO;
 using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Properties;
 using BenchmarkDotNet.Reports;
 
 namespace BenchmarkDotNet.Exporters
@@ -12,14 +13,18 @@ public class RPlotExporter : IExporter
     {
         public static readonly IExporter Default = new RPlotExporter();
 
+        private static object buildScriptLock = new object();
+
         public IEnumerable<string> ExportToFiles(Summary summary)
         {
             const string scriptFileName = "BuildPlots.R";
             yield return scriptFileName;
 
             var fileNamePrefix = Path.Combine(summary.CurrentDirectory, summary.Title);
             var scriptFullPath = Path.Combine(summary.CurrentDirectory, scriptFileName);
-            File.WriteAllText(scriptFullPath, ResourceHelper.LoadTemplate(scriptFileName));
+            var script = ResourceHelper.LoadTemplate(scriptFileName).Replace("$BenchmarkDotNetVersion$", BenchmarkDotNetInfo.FullTitle);
+            lock (buildScriptLock)
+                File.WriteAllText(scriptFullPath, script);
 
             var rHome = Environment.GetEnvironmentVariable("R_HOME") ?? @"C:\Program Files\R\R-3.2.3\bin\";
             if (Directory.Exists(rHome))

BenchmarkDotNet/Properties/BenchmarkDotNetInfo.cs

@@ -7,6 +7,7 @@ public static class BenchmarkDotNetInfo
         public const string Copyright = "Copyright © Andrey Akinshin, Jon Skeet, Matt Warren 2013–2016";
         public const string Version = "0.8.2";
         public const string FullVersion = "0.8.2" + (IsDevelopVersion ? "+" : "");
+        public const string FullTitle = Title + " v" + FullVersion;
 
         public const bool IsDevelopVersion = true; // Set to false for NuGet publishing
     }

BenchmarkDotNet/Running/MethodInvoker.cs

@@ -169,8 +169,9 @@ private static IList<Measurement> RunTarget(Func<MultiInvokeInput, Measurement>
                     var measurement = multiInvoke(new MultiInvokeInput(iterationMode, iterationCounter, invokeCount));
                     measurements.Add(measurement);
                     var statistics = new Statistics(measurements.Select(m => m.Nanoseconds));
+                    var statisticsWithoutOutliers = new Statistics(statistics.WithoutOutliers());
                     if (iterationCounter >= TargetAutoMinIterationCount &&
-                        statistics.StandardError < maxAcceptableError * statistics.Mean)
+                        statisticsWithoutOutliers.StandardError < maxAcceptableError * statisticsWithoutOutliers.Mean)
                         break;
                     if (isIdle && iterationCounter >= TargetIdleAutoMaxIterationCount)
                         break;

BenchmarkDotNet/Templates/BuildPlots.R

@@ -1,11 +1,14 @@
+BenchmarkDotNetVersion <- "$BenchmarkDotNetVersion$ "
 dir.create(Sys.getenv("R_LIBS_USER"), recursive = TRUE, showWarnings = FALSE)
-list.of.packages <- c("ggplot2", "dplyr", "gdata", "tidyr")
+list.of.packages <- c("ggplot2", "dplyr", "gdata", "tidyr", "grid", "gridExtra")
 new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
 if(length(new.packages)) install.packages(new.packages, lib = Sys.getenv("R_LIBS_USER"), repos = "http://cran.rstudio.com/")
 library(ggplot2)
 library(dplyr)
 library(gdata)
 library(tidyr)
+library(grid)
+library(gridExtra)
 
 ends_with <- function(vars, match, ignore.case = TRUE) {
   if (ignore.case) 
@@ -19,6 +22,10 @@ ends_with <- function(vars, match, ignore.case = TRUE) {
   substr(vars, pmax(1, length - n + 1), length) == match
 }
 std.error <- function(x) sqrt(var(x)/length(x))
+BenchmarkDotNetVersionGrob <- textGrob(BenchmarkDotNetVersion, gp = gpar(fontface=3, fontsize=10), hjust=1, x=1)
+nicePlot <- function(p) grid.arrange(p, bottom = BenchmarkDotNetVersionGrob)
+printNice <- function(p) print(nicePlot(p))
+ggsaveNice <- function(fileName, p, ...) ggsave(fileName, plot = nicePlot(p), ...)
 
 args <- commandArgs(trailingOnly = TRUE)
 files <- if (length(args) > 0) args else list.files()[list.files() %>% ends_with("-measurements.csv")]
@@ -62,10 +69,10 @@ for (file in files) {
     geom_bar(position=position_dodge(), stat="identity")
     #geom_errorbar(aes(ymin=Value-1.96*se, ymax=Value+1.96*se), width=.2, position=position_dodge(.9))
 
-  print(benchmarkBoxplot)
-  print(benchmarkBarplot)
-  ggsave(gsub("-measurements.csv", "-boxplot.png", file), benchmarkBoxplot)
-  ggsave(gsub("-measurements.csv", "-barplot.png", file), benchmarkBarplot)
+  printNice(benchmarkBoxplot)
+  printNice(benchmarkBarplot)
+  ggsaveNice(gsub("-measurements.csv", "-boxplot.png", file), benchmarkBoxplot)
+  ggsaveNice(gsub("-measurements.csv", "-barplot.png", file), benchmarkBarplot)
 
   for (target in unique(result$TargetMethod)) {
     df <- result %>% filter(TargetMethod == target)
@@ -74,8 +81,8 @@ for (file in files) {
       ggtitle(paste(title, "/", target)) +
       xlab(paste("Time,", timeUnit)) +
       geom_density(alpha=.5)
-    print(densityPlot)
-    ggsave(gsub("-measurements.csv", paste0("-", target, "-density.png"), file), densityPlot)
+    printNice(densityPlot)
+    ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-density.png"), file), densityPlot)
 
     for (job in unique(df$Job)) {
       jobDf <- df %>% filter(Job == job)
@@ -85,11 +92,11 @@ for (file in files) {
         ylab(paste("Time,", timeUnit)) +
         geom_line() +
         geom_point()
-      print(timelinePlot)
-      ggsave(gsub("-measurements.csv", paste0("-", target, "-", job, "-timeline.png"), file), timelinePlot)
+      printNice(timelinePlot)
+      ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-", job, "-timeline.png"), file), timelinePlot)
       timelinePlotSmooth <- timelinePlot + geom_smooth()
-      print(timelinePlotSmooth)
-      ggsave(gsub("-measurements.csv", paste0("-", target, "-", job, "-timelineSmooth.png"), file), timelinePlotSmooth)
+      printNice(timelinePlotSmooth)
+      ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-", job, "-timelineSmooth.png"), file), timelinePlotSmooth)
     }
 
     timelinePlot <- ggplot(df, aes(x = MeasurementIterationIndex, y=MeasurementValue, group=Launch, color=Launch)) + 
@@ -99,10 +106,10 @@ for (file in files) {
       geom_line() +
       geom_point() +
       facet_wrap(~Job)
-    print(timelinePlot)
-    ggsave(gsub("-measurements.csv", paste0("-", target, "-facetTimeline.png"), file), timelinePlot)
+    printNice(timelinePlot)
+    ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-facetTimeline.png"), file), timelinePlot)
     timelinePlotSmooth <- timelinePlot + geom_smooth()
-    print(timelinePlotSmooth)
-    ggsave(gsub("-measurements.csv", paste0("-", target, "-facetTimelineSmooth.png"), file), timelinePlotSmooth)
+    printNice(timelinePlotSmooth)
+    ggsaveNice(gsub("-measurements.csv", paste0("-", target, "-facetTimelineSmooth.png"), file), timelinePlotSmooth)
   }
 }

BenchmarkDotNet/Toolchains/Classic/ClassicBuilder.cs

@@ -11,15 +11,20 @@ namespace BenchmarkDotNet.Toolchains.Classic
 {
     internal class ClassicBuilder : IBuilder
     {
+        private static readonly object buildLock = new object();
+
         public BuildResult Build(GenerateResult generateResult, ILogger logger)
         {
-            var projectFileName = Path.Combine(generateResult.DirectoryPath, ClassicGenerator.MainClassName + ".csproj");
-            var consoleLogger = new MsBuildConsoleLogger(logger);
-            var globalProperties = new Dictionary<string, string>();
-            var buildRequest = new BuildRequestData(projectFileName, globalProperties, null, new[] { "Build" }, null);
-            var buildParameters = new BuildParameters(new ProjectCollection()) { DetailedSummary = false, Loggers = new Microsoft.Build.Framework.ILogger[] { consoleLogger } };
-            var buildResult = BuildManager.DefaultBuildManager.Build(buildParameters, buildRequest);
-            return new BuildResult(generateResult, buildResult.OverallResult == BuildResultCode.Success, buildResult.Exception);
+            lock (buildLock)
+            {
+                var projectFileName = Path.Combine(generateResult.DirectoryPath, ClassicGenerator.MainClassName + ".csproj");
+                var consoleLogger = new MsBuildConsoleLogger(logger);
+                var globalProperties = new Dictionary<string, string>();
+                var buildRequest = new BuildRequestData(projectFileName, globalProperties, null, new[] { "Build" }, null);
+                var buildParameters = new BuildParameters(new ProjectCollection()) { DetailedSummary = false, Loggers = new Microsoft.Build.Framework.ILogger[] { consoleLogger } };
+                var buildResult = BuildManager.DefaultBuildManager.Build(buildParameters, buildRequest);
+                return new BuildResult(generateResult, buildResult.OverallResult == BuildResultCode.Success, buildResult.Exception);
+            }
         }
     }
 }