Benchmark refresh

* Our previous config used to run multiple TFMs as part of a single
  benchmark run. Removed this - a single run is now on a single TFM.
  It is still possible run older TFMs via the commandline.
* Running different versions of EF Core is now possible via build
  configs, rather than different projects. This simplifies things.
* Code common to the two EF Core projects - and to the EF6 project -
  is now in Shared.EFCore and Shared respectively.
* Integrated the new benchmark projects into EFCore.sln and removed the
  older ones.
* Other various cleanup and update tasks

EFCore.sln

@@ -125,6 +125,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ScalarFunctionSample", "sam
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StreamingSample", "samples\StreamingSample\StreamingSample.csproj", "{9C26C322-4FE2-42DC-A14E-0E1D0DCEE533}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "benchmark", "benchmark", "{293B4F79-3CB9-402A-A74C-B8108C41A7CF}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.SqlServer.Benchmarks", "benchmark\EFCore.SqlServer.Benchmarks\EFCore.SqlServer.Benchmarks.csproj", "{24D7B6D5-A22A-4734-BB55-F464C112F891}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Sqlite.Benchmarks", "benchmark\EFCore.Sqlite.Benchmarks\EFCore.Sqlite.Benchmarks.csproj", "{5A44F67E-517F-458D-B9C7-12A4DBBAD64A}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EFCore.Benchmarks.SqlServer.EF6", "benchmark\EFCore.SqlServer.EF6.Benchmarks\EFCore.Benchmarks.SqlServer.EF6.csproj", "{678AB38D-B27C-4690-A3F9-2D2488391658}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -335,6 +343,18 @@ Global
 		{9C26C322-4FE2-42DC-A14E-0E1D0DCEE533}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9C26C322-4FE2-42DC-A14E-0E1D0DCEE533}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9C26C322-4FE2-42DC-A14E-0E1D0DCEE533}.Release|Any CPU.Build.0 = Release|Any CPU
+		{24D7B6D5-A22A-4734-BB55-F464C112F891}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{24D7B6D5-A22A-4734-BB55-F464C112F891}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{24D7B6D5-A22A-4734-BB55-F464C112F891}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{24D7B6D5-A22A-4734-BB55-F464C112F891}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5A44F67E-517F-458D-B9C7-12A4DBBAD64A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5A44F67E-517F-458D-B9C7-12A4DBBAD64A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5A44F67E-517F-458D-B9C7-12A4DBBAD64A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5A44F67E-517F-458D-B9C7-12A4DBBAD64A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{678AB38D-B27C-4690-A3F9-2D2488391658}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{678AB38D-B27C-4690-A3F9-2D2488391658}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{678AB38D-B27C-4690-A3F9-2D2488391658}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{678AB38D-B27C-4690-A3F9-2D2488391658}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -391,6 +411,9 @@ Global
 		{FE9D2BEB-BA71-4770-9738-8F9F2CD31D41} = {C41FD56B-38CD-40E3-89E6-071A6C58E36C}
 		{DE3D06F2-AE6F-4405-95AC-3647F5878F72} = {C41FD56B-38CD-40E3-89E6-071A6C58E36C}
 		{9C26C322-4FE2-42DC-A14E-0E1D0DCEE533} = {C41FD56B-38CD-40E3-89E6-071A6C58E36C}
+		{24D7B6D5-A22A-4734-BB55-F464C112F891} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
+		{5A44F67E-517F-458D-B9C7-12A4DBBAD64A} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
+		{678AB38D-B27C-4690-A3F9-2D2488391658} = {293B4F79-3CB9-402A-A74C-B8108C41A7CF}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {285A5EB4-BCF4-40EB-B9E1-DF6DBCB5E705}

benchmark/Directory.Build.props

@@ -0,0 +1,11 @@
+<Project>
+  <Import Project="..\Directory.Build.props" />
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetPackageVersion)" />
+    <PackageReference Include="xunit.assert" Version="$(XunitVersion)" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="$(MicrosoftExtensionsConfigurationJsonPackageVersion)" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="$(MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion)" />
+  </ItemGroup>
+
+</Project>

benchmark/EFCore.SqlServer.Benchmarks/EFCore.SqlServer.Benchmarks.csproj

@@ -0,0 +1,41 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>net461;netcoreapp2.0;netcoreapp2.1;netcoreapp2.2;netcoreapp3.0</TargetFrameworks>
+    <RootNamespace>Microsoft.EntityFrameworkCore.Benchmarks</RootNamespace>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="$(Configuration.StartsWith('Release'))">
+    <Optimize>true</Optimize>
+  </PropertyGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release' Or '$(Configuration)' == 'Debug' ">
+    <ProjectReference Include="..\..\src\EFCore.SqlServer\EFCore.SqlServer.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release22' ">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="2.2.1" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release21' ">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="2.1.4" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(Configuration)' == 'Release20' ">
+    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="2.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="config.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Remove="BenchmarkDotNet.Artifacts\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="..\Shared\**\*.cs" Link="%(RecursiveDir)%(FileName)%(Extension)" />
+    <Compile Include="..\Shared.EFCore\**\*.cs" Link="%(RecursiveDir)%(FileName)%(Extension)" />
+  </ItemGroup>
+
+</Project>

benchmarks/test/EFCore.Benchmarks.SqlServer.Version22/Models/AdventureWorks/AdventureWorksFixture.cs → benchmark/EFCore.SqlServer.Benchmarks/Models/AdventureWorks/AdventureWorksFixture.cs

@@ -5,7 +5,7 @@ namespace Microsoft.EntityFrameworkCore.Benchmarks.Models.AdventureWorks
 {
     public static class AdventureWorksFixture
     {
-        private static string _connectionString = $"{BenchmarkConfig.Instance.BenchmarkDatabase}Database=AdventureWorks2014;";
+        private static string _connectionString = $"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database=AdventureWorks2014;";
 
         // This method is called from timed code, be careful when changing it
         public static AdventureWorksContextBase CreateContext()

benchmarks/test/EFCore.Benchmarks.SqlServer.Version22/Models/Orders/OrdersFixture.cs → benchmark/EFCore.SqlServer.Benchmarks/Models/Orders/OrdersFixture.cs

@@ -11,7 +11,7 @@ public class OrdersFixture : OrdersFixtureBase
 
         public OrdersFixture(string databaseName)
         {
-            _connectionString = $@"{BenchmarkConfig.Instance.BenchmarkDatabase}Database={databaseName};";
+            _connectionString = $@"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database={databaseName};";
         }
 
         public override OrdersContextBase CreateContext(IServiceProvider serviceProvider = null, bool disableBatching = false)

benchmark/EFCore.SqlServer.Benchmarks/Program.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using BenchmarkDotNet.Running;
+
+namespace Microsoft.EntityFrameworkCore.Benchmarks
+{
+    public static class Program
+    {
+        static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+    }
+}

benchmarks/test/EFCore.Benchmarks.Sqlite.Version21/config.json → benchmark/EFCore.SqlServer.Benchmarks/config.json

@@ -5,6 +5,6 @@
       "local": "Server=(localdb)\\mssqllocaldb;Database=Benchmarks;Trusted_Connection=True;"
     },
     "benchmarkDatabase": "Server=(localdb)\\mssqllocaldb;Trusted_Connection=True;MultipleActiveResultSets=true;",
-    "productVersion": "EF Core Sqlite 2.1"
+    "productVersion": "EF Core SqlServer"
   }
 }

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/ChangeTracker/DbSetOperationTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/ChangeTracker/DbSetOperationTests.cs

@@ -116,30 +116,24 @@ public virtual void Update()
             //       API for bulk update in EF6.x
         }
 
-        [BenchmarkJob(singleRun: true)]
-        [Description("AutoDetectChanges=True")]
+        [SingleRunJob, Description("AutoDetectChanges=True")]
         public class AddDataVariationsWithAutoDetectChangesOn : AddDataVariations
         {
             protected override bool AutoDetectChanges => true;
         }
 
-        [BenchmarkJob(singleRun: true)]
-        [Description("AutoDetectChanges=True")]
+        [SingleRunJob, Description("AutoDetectChanges=True")]
         public class ExistingDataVariationsWithAutoDetectChangesOn : ExistingDataVariations
         {
             protected override bool AutoDetectChanges => true;
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         [Description("AutoDetectChanges=False")]
         public class AddDataVariationsWithAutoDetectChangesOff : AddDataVariations
         {
             protected override bool AutoDetectChanges => false;
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         [Description("AutoDetectChanges=False")]
         public class ExistingDataVariationsWithAutoDetectChangesOff : ExistingDataVariations
         {

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/ChangeTracker/FixupTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/ChangeTracker/FixupTests.cs

@@ -122,30 +122,24 @@ public virtual void QueryChildren()
             }
         }
 
-        [BenchmarkJob(singleRun: true)]
-        [Description("AutoDetectChanges=True")]
+        [SingleRunJob, Description("AutoDetectChanges=True")]
         public class ChildVariationsWithAutoDetectChangesOn : ChildVariations
         {
             protected override bool AutoDetectChanges => true;
         }
 
-        [BenchmarkJob(singleRun: true)]
-        [Description("AutoDetectChanges=True")]
+        [SingleRunJob, Description("AutoDetectChanges=True")]
         public class ParentVariationsWithAutoDetectChangesOn : ParentVariations
         {
             protected override bool AutoDetectChanges => true;
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         [Description("AutoDetectChanges=False")]
         public class ChildVariationsWithAutoDetectChangesOff : ChildVariations
         {
             protected override bool AutoDetectChanges => false;
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         [Description("AutoDetectChanges=False")]
         public class ParentVariationsWithAutoDetectChangesOff : ParentVariations
         {

benchmark/EFCore.SqlServer.EF6.Benchmarks/ChangeTracker/SingleRunJobAttribute.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+
+namespace Microsoft.EntityFrameworkCore.Benchmarks.ChangeTracker
+{
+    [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
+    public class SingleRunJobAttribute : Attribute, IConfigSource
+    {
+        public SingleRunJobAttribute()
+        {
+            Config = ManualConfig.CreateEmpty().With(new Job()
+                .WithWarmupCount(1)
+                .WithIterationCount(1)
+                .With(RunStrategy.Monitoring));
+        }
+
+        public IConfig Config { get; }
+    }
+}

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/EFCore.Benchmarks.SqlServer.EF6.csproj → benchmark/EFCore.SqlServer.EF6.Benchmarks/EFCore.Benchmarks.SqlServer.EF6.csproj

@@ -14,12 +14,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\src\EFCore.Benchmarks\EFCore.Benchmarks.csproj" />
+    <PackageReference Include="EntityFramework" Version="6.2.0" />
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="EntityFramework" Version="6.2.0" />
-    <PackageReference Include="xunit.assert" Version="$(XunitAssertPackageVersion)" />
+    <Compile Include="..\Shared\**\*.cs" Link="%(RecursiveDir)%(FileName)%(Extension)" />
   </ItemGroup>
 
 </Project>

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Initialization/InitializationTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Initialization/InitializationTests.cs

@@ -13,8 +13,6 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Initialization
 {
-    [BenchmarkJob]
-    [MemoryDiagnoser]
     public class InitializationTests
     {
         private ColdStartSandbox _sandbox;

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Models/AdventureWorks/AdventureWorksFixture.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Models/AdventureWorks/AdventureWorksFixture.cs

@@ -5,7 +5,7 @@ namespace Microsoft.EntityFrameworkCore.Benchmarks.Models.AdventureWorks
 {
     public static class AdventureWorksFixture
     {
-        public static string ConnectionString { get; } = $"{BenchmarkConfig.Instance.BenchmarkDatabase}Database=AdventureWorks2014;";
+        public static string ConnectionString { get; } = $"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database=AdventureWorks2014;";
 
         // This method is called from timed code, be careful when changing it
         public static AdventureWorksContext CreateContext()

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Models/Orders/OrdersFixture.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Models/Orders/OrdersFixture.cs

@@ -19,7 +19,7 @@ public class OrdersFixture : OrdersFixtureSeedBase
         public OrdersFixture(string databaseName, int productCount, int customerCount,
             int ordersPerCustomer, int linesPerOrder, Action<DbContext> seedAction = null)
         {
-            _connectionString = $"{BenchmarkConfig.Instance.BenchmarkDatabase}Database={databaseName};";
+            _connectionString = $"{BenchmarkEnvironment.Instance.BenchmarkDatabase}Database={databaseName};";
             _productCount = productCount;
             _customerCount = customerCount;
             _ordersPerCustomer = ordersPerCustomer;

benchmark/EFCore.SqlServer.EF6.Benchmarks/Program.cs

@@ -0,0 +1,12 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using BenchmarkDotNet.Running;
+
+namespace Microsoft.EntityFrameworkCore.Benchmarks
+{
+    public static class Program
+    {
+        static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
+    }
+}

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Query/FuncletizationTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Query/FuncletizationTests.cs

@@ -10,8 +10,6 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Query
 {
-    [BenchmarkJob]
-    [MemoryDiagnoser]
     public class FuncletizationTests
     {
         private static readonly OrdersFixture _fixture

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Query/NavigationsQueryTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Query/NavigationsQueryTests.cs

@@ -12,8 +12,6 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Query
 {
-    [BenchmarkJob]
-    [MemoryDiagnoser]
     public class NavigationsQueryTests
     {
         private AdventureWorksContext _context;

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Query/QueryCompilationTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Query/QueryCompilationTests.cs

@@ -10,8 +10,6 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Query
 {
-    [BenchmarkJob]
-    [MemoryDiagnoser]
     public class QueryCompilationTests
     {
         private static readonly OrdersFixture _fixture

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Query/RawSqlQueryTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Query/RawSqlQueryTests.cs

@@ -11,8 +11,6 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Query
 {
-    [BenchmarkJob]
-    [MemoryDiagnoser]
     public class RawSqlQueryTests
     {
         private static readonly OrdersFixture _fixture

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/Query/SimpleQueryTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/Query/SimpleQueryTests.cs

@@ -12,8 +12,6 @@
 
 namespace Microsoft.EntityFrameworkCore.Benchmarks.Query
 {
-    [BenchmarkJob]
-    [MemoryDiagnoser]
     public class SimpleQueryTests
     {
         private static readonly OrdersFixture _fixture

benchmarks/test/EFCore.Benchmarks.SqlServer.EF6/UpdatePipeline/SimpleUpdatePipelineTests.cs → benchmark/EFCore.SqlServer.EF6.Benchmarks/UpdatePipeline/SimpleUpdatePipelineTests.cs

@@ -59,8 +59,6 @@ public async Task UpdatePipeline()
             }
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         public class Insert : UpdatePipelineBase
         {
             [IterationSetup]
@@ -73,8 +71,6 @@ public override void InitializeContext()
             }
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         public class Update : UpdatePipelineBase
         {
             [IterationSetup]
@@ -89,8 +85,6 @@ public override void InitializeContext()
             }
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         public class Delete : UpdatePipelineBase
         {
             [IterationSetup]
@@ -102,8 +96,6 @@ public override void InitializeContext()
             }
         }
 
-        [BenchmarkJob]
-        [MemoryDiagnoser]
         public class Mixed : UpdatePipelineBase
         {
             [IterationSetup]