Simply run the runner, with ETW collection.

We also remove ETW collection from xunit.performance.dll. With this change, running tests in any "normal" way (VS, console runner, etc.) will not produce a .etl log. To get a log, use xunit.performance.run.exe. For example: xunit.performance.run.exe MyTests.dll -runner xunit.console.exe Fixes #4.
microsoft · Aug 12, 2015 · cf4d71e · cf4d71e
1 parent 0ef7256
commit cf4d71e
Show file tree

Hide file tree

Showing 11 changed files with 85 additions and 36 deletions.
diff --git a/ProcDomain/ProcDomain.cs b/ProcDomain/ProcDomain.cs
@@ -291,8 +291,6 @@ private Process CreateDomainProcess(string executablePath, bool runElevated)
 
             var proc = this._process = Process.Start(startInfo);
 
-            Console.WriteLine(proc.Handle);
-
             return proc;
         }
     }

diff --git a/xunit.performance/ETWLogging.cs → xunit.performance.run/ETWLogging.cs b/xunit.performance/ETWLogging.cs → xunit.performance.run/ETWLogging.cs
@@ -3,6 +3,7 @@
 using Microsoft.Diagnostics.Tracing.Parsers.Clr;
 using Microsoft.Diagnostics.Tracing.Session;
 using Microsoft.ProcessDomain;
+using Microsoft.Xunit.Performance.Internal;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.Tracing;

diff --git a/xunit.performance.run/Program.cs b/xunit.performance.run/Program.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -11,13 +12,56 @@ namespace Microsoft.Xunit.Performance
 {
     class Program
     {
-        static void Main(string[] args)
+        static int Main(string[] args)
         {
             var project = ParseCommandLine(args);
 
-            foreach (var test in DiscoverTests(project.Assemblies, project.Filters))
+            //foreach (var test in DiscoverTests(project.Assemblies, project.Filters))
+            //{
+            //    Console.WriteLine(test.DisplayName);
+            //}
+
+            var runnerArgs = new StringBuilder();
+            foreach (var assembly in project.Assemblies)
+            {
+                runnerArgs.Append(assembly.AssemblyFilename);
+                runnerArgs.Append(" ");
+            }
+
+            foreach (var trait in project.Filters.IncludedTraits)
+                foreach (var traitVal in trait.Value)
+                    runnerArgs.Append($"-trait \"{trait.Key}={traitVal}\" ");
+
+            foreach (var trait in project.Filters.ExcludedTraits)
+                foreach (var traitVal in trait.Value)
+                    runnerArgs.Append($"-notrait \"{trait.Key}={traitVal}\" ");
+
+            foreach (var method in project.Filters.IncludedMethods)
+                runnerArgs.Append($"-method \"{method}\" ");
+
+            foreach (var includeClass in project.Filters.IncludedClasses)
+                runnerArgs.Append($"-class \"{includeClass}\" ");
+
+            runnerArgs.Append("-nologo -verbose");
+
+            using (ETWLogging.StartAsync(Path.Combine(project.EtlDirectory, project.RunName + ".etl")).Result)
             {
-                Console.WriteLine(test.DisplayName);
+                Environment.SetEnvironmentVariable("XUNIT_PERFORMANCE_RUN_ID", project.RunName);
+
+                ProcessStartInfo startInfo = new ProcessStartInfo()
+                {
+                    FileName = project.RunnerCommand,
+                    Arguments = runnerArgs.ToString(),
+                    CreateNoWindow = false,
+                    UseShellExecute = false,
+                };
+
+                using (var proc = Process.Start(startInfo))
+                {
+                    proc.EnableRaisingEvents = true;
+                    proc.WaitForExit();
+                    return proc.ExitCode;
+                }
             }
         }
 
@@ -163,6 +207,15 @@ private static XunitPerformanceProject ParseCommandLine(string[] args)
                     if (option.Value == null)
                         throw new ArgumentException("missing argument for -runName");
                 }
+                else if (optionName == "etldir")
+                {
+                    if (option.Value == null)
+                        throw new ArgumentException("missing argument for -etlDir");
+                    if (!Directory.Exists(option.Value))
+                        throw new ArgumentException($"directory not found: {option.Value}");
+
+                    project.EtlDirectory = option.Value;
+                }
                 else
                 {
                     if (option.Value == null)

diff --git a/xunit.performance.run/XunitPerformanceProject.cs b/xunit.performance.run/XunitPerformanceProject.cs
@@ -26,6 +26,6 @@ public string BaselineRunnerCommand
 
         public string RunName { get; set; } = Environment.MachineName + "-" + DateTimeOffset.UtcNow.ToString("yyyy-MM-dd-HH-mm-ss");
 
-        public string EtlDirectory { get; set; }
+        public string EtlDirectory { get; set; } = ".";
     }
 }
diff --git a/xunit.performance.run/packages.config b/xunit.performance.run/packages.config
@@ -1,10 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="Microsoft.Diagnostics.Tracing.TraceEvent" version="1.0.35" targetFramework="net46" />
   <package id="xunit" version="2.1.0-beta4-build3109" targetFramework="net46" />
   <package id="xunit.abstractions" version="2.0.0" targetFramework="net46" />
   <package id="xunit.assert" version="2.1.0-beta4-build3109" targetFramework="net46" />
   <package id="xunit.core" version="2.1.0-beta4-build3109" targetFramework="net46" />
   <package id="xunit.extensibility.core" version="2.1.0-beta4-build3109" targetFramework="net46" />
   <package id="xunit.extensibility.execution" version="2.1.0-beta4-build3109" targetFramework="net46" />
+  <package id="xunit.runner.console" version="2.1.0-beta4-build3109" targetFramework="net46" />
   <package id="xunit.runner.utility" version="2.1.0-beta4-build3109" targetFramework="net46" />
 </packages>
diff --git a/xunit.performance.run/xunit.performance.run.csproj b/xunit.performance.run/xunit.performance.run.csproj
@@ -36,6 +36,10 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="Microsoft.Diagnostics.Tracing.TraceEvent, Version=1.0.35.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\lib\net40\Microsoft.Diagnostics.Tracing.TraceEvent.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
@@ -67,6 +71,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="ConsoleDiagnosticsMessageVisitor.cs" />
+    <Compile Include="ETWLogging.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="TestCaseDisoveryVisitor.cs" />
@@ -76,13 +81,29 @@
     <None Include="App.config" />
     <None Include="packages.config" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\ProcDomain\ProcDomain.csproj">
+      <Project>{905abe0d-b057-4218-a56b-9c25a9dca6e1}</Project>
+      <Name>ProcDomain</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\xunit.performance.logger\xunit.performance.logger.csproj">
+      <Project>{b7b0a4d7-ef9d-4ced-954a-f6e69e30ce27}</Project>
+      <Name>xunit.performance.logger</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\xunit.performance\xunit.performance.csproj">
+      <Project>{67071a0f-fe22-4eea-86e5-419e9eb4c1bd}</Project>
+      <Name>xunit.performance</Name>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
     <Error Condition="!Exists('..\packages\xunit.core.2.1.0-beta4-build3109\build\portable-net45+netcore45+wp8+wpa81\xunit.core.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\xunit.core.2.1.0-beta4-build3109\build\portable-net45+netcore45+wp8+wpa81\xunit.core.props'))" />
+    <Error Condition="!Exists('..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets'))" />
   </Target>
+  <Import Project="..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets" Condition="Exists('..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets')" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">

diff --git a/xunit.performance/BenchmarkConfiguration.cs b/xunit.performance/BenchmarkConfiguration.cs
@@ -9,7 +9,6 @@ namespace Microsoft.Xunit.Performance
     static class BenchmarkConfiguration
     {
         public static readonly string RunId = Environment.GetEnvironmentVariable("XUNIT_PERFORMANCE_RUN_ID") ?? Environment.MachineName + ":" + DateTimeOffset.UtcNow.ToString("u");
-        public static readonly string ETLPath = Environment.GetEnvironmentVariable("XUNIT_PERFORMANCE_ETL_PATH") ?? "xunit.performance.etl";
         public static readonly int MaxIteration = int.Parse(Environment.GetEnvironmentVariable("XUNIT_PERFORMANCE_MAX_ITERATION") ?? "1000");
         public static readonly int MaxTotalMilliseconds = int.Parse(Environment.GetEnvironmentVariable("XUNIT_PERFORMANCE_MAX_TOTAL_MILLISECONDS") ?? "1000");
     }

diff --git a/xunit.performance/BenchmarkEventSource.cs b/xunit.performance/BenchmarkEventSource.cs
@@ -1,10 +1,10 @@
 using System;
 using System.Diagnostics.Tracing;
 
-namespace Microsoft.Xunit.Performance
+namespace Microsoft.Xunit.Performance.Internal
 {
     [EventSource(Name = "Microsoft-Xunit-Benchmark", Guid = "A3B447A8-6549-4158-9BAD-76D442A47061")]
-    internal sealed class BenchmarkEventSource : EventSource
+    public sealed class BenchmarkEventSource : EventSource
     {
         public class Tasks
         {

diff --git a/xunit.performance/BenchmarkTestInvoker.cs b/xunit.performance/BenchmarkTestInvoker.cs
@@ -1,4 +1,5 @@
-using System;
+using Microsoft.Xunit.Performance.Internal;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
@@ -12,8 +13,6 @@ namespace Microsoft.Xunit.Performance
 {
     internal class BenchmarkTestInvoker : XunitTestInvoker
     {
-        static bool _initialized;
-        static IDisposable _etwLogger; // just to keep the logger rooted, so it doesn't get finalized during the run
         static SemaphoreSlim _semaphore = new SemaphoreSlim(1);
 
         public BenchmarkTestInvoker(ITest test,
@@ -37,12 +36,6 @@ protected override async Task<decimal> InvokeTestMethodAsync(object testClassIns
             await _semaphore.WaitAsync();
             try
             {
-                if (!_initialized)
-                {
-                    _etwLogger = await ETWLogging.StartAsync(BenchmarkConfiguration.ETLPath);
-                    _initialized = true;
-                }
-
                 return await base.InvokeTestMethodAsync(testClassInstance);
             }
             finally

diff --git a/xunit.performance/packages.config b/xunit.performance/packages.config
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Diagnostics.Tracing.TraceEvent" version="1.0.35" targetFramework="net452" />
   <package id="xunit" version="2.1.0-beta4-build3109" targetFramework="net452" />
   <package id="xunit.abstractions" version="2.0.0" targetFramework="net452" />
   <package id="xunit.assert" version="2.1.0-beta4-build3109" targetFramework="net452" />

diff --git a/xunit.performance/xunit.performance.csproj b/xunit.performance/xunit.performance.csproj
@@ -36,10 +36,6 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Microsoft.Diagnostics.Tracing.TraceEvent, Version=1.0.35.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
-      <HintPath>..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\lib\net40\Microsoft.Diagnostics.Tracing.TraceEvent.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.Numerics" />
@@ -77,31 +73,18 @@
     <Compile Include="BenchmarkTestInvoker.cs" />
     <Compile Include="BenchmarkTestRunner.cs" />
     <Compile Include="BenchmarkTraitDiscoverer.cs" />
-    <Compile Include="ETWLogging.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
   </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\ProcDomain\ProcDomain.csproj">
-      <Project>{905abe0d-b057-4218-a56b-9c25a9dca6e1}</Project>
-      <Name>ProcDomain</Name>
-    </ProjectReference>
-    <ProjectReference Include="..\xunit.performance.logger\xunit.performance.logger.csproj">
-      <Project>{b7b0a4d7-ef9d-4ced-954a-f6e69e30ce27}</Project>
-      <Name>xunit.performance.logger</Name>
-    </ProjectReference>
-  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
     <PropertyGroup>
       <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
     </PropertyGroup>
-    <Error Condition="!Exists('..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets'))" />
     <Error Condition="!Exists('..\packages\xunit.core.2.1.0-beta4-build3109\build\portable-net45+netcore45+wp8+wpa81\xunit.core.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\xunit.core.2.1.0-beta4-build3109\build\portable-net45+netcore45+wp8+wpa81\xunit.core.props'))" />
   </Target>
-  <Import Project="..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets" Condition="Exists('..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.35\build\Microsoft.Diagnostics.Tracing.TraceEvent.targets')" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">