Implement file logging with default xunit tests

.appveyor.yml

@@ -12,6 +12,7 @@ environment:
   global:
     DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     DOTNET_CLI_TELEMETRY_OPTOUT: 1
+    ASPNETCORE_TEST_LOG_DIR: "$APPVEYOR_BUILD_FOLDER\\artifacts\\logs"
 test: 'off'
 deploy: 'off'
 os: Visual Studio 2017

.travis.yml

@@ -5,6 +5,7 @@ env:
   global:
   - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
   - DOTNET_CLI_TELEMETRY_OPTOUT: 1
+  - ASPNETCORE_TEST_LOG_DIR: "$APPVEYOR_BUILD_FOLDER\\artifacts\\logs"
 mono: none
 os:
 - linux

NuGetPackageVerifier.json

@@ -2,6 +2,15 @@
     "Default": {
         "rules": [
             "DefaultCompositeRule"
-        ]
+        ],
+        "packages": {
+            "Microsoft.Extensions.Logging.Testing": {
+                "Exclusions": {
+                    "BUILD_ITEMS_FRAMEWORK": {
+                        "*": "Props file intentionally targets any framework since the content is the same for both netstandard2.0 and net461."
+                    }
+                }
+            }
+        }
     }
 }

build/dependencies.props

@@ -4,31 +4,32 @@
   </PropertyGroup>
   <PropertyGroup Label="Package Versions">
     <InternalAspNetCoreSdkPackageVersion>2.1.0-preview3-17004</InternalAspNetCoreSdkPackageVersion>
-    <MicrosoftAspNetCoreTestingPackageVersion>2.1.0-preview3-32176</MicrosoftAspNetCoreTestingPackageVersion>
+    <MicrosoftAspNetCoreTestingPackageVersion>2.1.0-preview3-32192</MicrosoftAspNetCoreTestingPackageVersion>
     <MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>2.6.1</MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>
-    <MicrosoftExtensionsConfigurationAbstractionsPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsConfigurationAbstractionsPackageVersion>
-    <MicrosoftExtensionsConfigurationBinderPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsConfigurationBinderPackageVersion>
-    <MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>
-    <MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>
-    <MicrosoftExtensionsConfigurationJsonPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsConfigurationJsonPackageVersion>
-    <MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>
-    <MicrosoftExtensionsDependencyInjectionPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsDependencyInjectionPackageVersion>
-    <MicrosoftExtensionsDependencyModelPackageVersion>2.1.0-preview2-26403-06</MicrosoftExtensionsDependencyModelPackageVersion>
-    <MicrosoftExtensionsFileProvidersPhysicalPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsFileProvidersPhysicalPackageVersion>
-    <MicrosoftExtensionsOptionsConfigurationExtensionsPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsOptionsConfigurationExtensionsPackageVersion>
-    <MicrosoftExtensionsOptionsPackageVersion>2.1.0-preview3-32176</MicrosoftExtensionsOptionsPackageVersion>
+    <MicrosoftExtensionsConfigurationAbstractionsPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsConfigurationAbstractionsPackageVersion>
+    <MicrosoftExtensionsConfigurationBinderPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsConfigurationBinderPackageVersion>
+    <MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>
+    <MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsConfigurationFileExtensionsPackageVersion>
+    <MicrosoftExtensionsConfigurationJsonPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsConfigurationJsonPackageVersion>
+    <MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>
+    <MicrosoftExtensionsDependencyInjectionPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsDependencyInjectionPackageVersion>
+    <MicrosoftExtensionsDependencyModelPackageVersion>2.1.0-preview2-26406-04</MicrosoftExtensionsDependencyModelPackageVersion>
+    <MicrosoftExtensionsFileProvidersPhysicalPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsFileProvidersPhysicalPackageVersion>
+    <MicrosoftExtensionsOptionsConfigurationExtensionsPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsOptionsConfigurationExtensionsPackageVersion>
+    <MicrosoftExtensionsOptionsPackageVersion>2.1.0-preview3-32192</MicrosoftExtensionsOptionsPackageVersion>
     <MicrosoftNETCoreApp20PackageVersion>2.0.0</MicrosoftNETCoreApp20PackageVersion>
-    <MicrosoftNETCoreApp21PackageVersion>2.1.0-preview2-26403-06</MicrosoftNETCoreApp21PackageVersion>
+    <MicrosoftNETCoreApp21PackageVersion>2.1.0-preview2-26406-04</MicrosoftNETCoreApp21PackageVersion>
     <MicrosoftNETTestSdkPackageVersion>15.6.1</MicrosoftNETTestSdkPackageVersion>
     <MoqPackageVersion>4.7.49</MoqPackageVersion>
     <NewtonsoftJsonPackageVersion>11.0.2</NewtonsoftJsonPackageVersion>
     <SerilogExtensionsLoggingPackageVersion>1.4.0</SerilogExtensionsLoggingPackageVersion>
     <SerilogSinksFilePackageVersion>3.2.0</SerilogSinksFilePackageVersion>
-    <SystemDiagnosticsEventLogPackageVersion>4.5.0-preview2-26403-05</SystemDiagnosticsEventLogPackageVersion>
-    <SystemReflectionMetadataPackageVersion>1.6.0-preview2-26403-05</SystemReflectionMetadataPackageVersion>
-    <SystemValueTuplePackageVersion>4.5.0-preview2-26403-05</SystemValueTuplePackageVersion>
+    <SystemDiagnosticsEventLogPackageVersion>4.5.0-preview2-26406-04</SystemDiagnosticsEventLogPackageVersion>
+    <SystemReflectionMetadataPackageVersion>1.6.0-preview2-26406-04</SystemReflectionMetadataPackageVersion>
+    <SystemValueTuplePackageVersion>4.5.0-preview2-26406-04</SystemValueTuplePackageVersion>
     <XunitAbstractionsPackageVersion>2.0.1</XunitAbstractionsPackageVersion>
     <XunitAssertPackageVersion>2.3.1</XunitAssertPackageVersion>
+    <XunitExtensibilityExecutionPackageVersion>2.3.1</XunitExtensibilityExecutionPackageVersion>
     <XunitPackageVersion>2.3.1</XunitPackageVersion>
     <XunitRunnerVisualStudioPackageVersion>2.4.0-beta.1.build3945</XunitRunnerVisualStudioPackageVersion>
   </PropertyGroup>

src/Microsoft.Extensions.Logging.Testing/AssemblyTestLog.cs

@@ -5,9 +5,11 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Text;
 using Microsoft.Extensions.DependencyInjection;
 using Serilog;
 using Serilog.Extensions.Logging;
@@ -20,6 +22,14 @@ public class AssemblyTestLog : IDisposable
         public static readonly string OutputDirectoryEnvironmentVariableName = "ASPNETCORE_TEST_LOG_DIR";
         private static readonly string LogFileExtension = ".log";
         private static readonly int MaxPathLength = 245;
+        private static char[] InvalidFileChars = new char[]
+        {
+            '\"', '<', '>', '|', '\0',
+            (char)1, (char)2, (char)3, (char)4, (char)5, (char)6, (char)7, (char)8, (char)9, (char)10,
+            (char)11, (char)12, (char)13, (char)14, (char)15, (char)16, (char)17, (char)18, (char)19, (char)20,
+            (char)21, (char)22, (char)23, (char)24, (char)25, (char)26, (char)27, (char)28, (char)29, (char)30,
+            (char)31, ':', '*', '?', '\\', '/', ' ', (char)127
+        };
 
         private static readonly object _lock = new object();
         private static readonly Dictionary<Assembly, AssemblyTestLog> _logs = new Dictionary<Assembly, AssemblyTestLog>();
@@ -42,9 +52,12 @@ private AssemblyTestLog(ILoggerFactory globalLoggerFactory, ILogger globalLogger
         public IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, [CallerMemberName] string testName = null) =>
             StartTestLog(output, className, out loggerFactory, LogLevel.Debug, testName);
 
-        public IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, [CallerMemberName] string testName = null)
+        public IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, [CallerMemberName] string testName = null) =>
+            StartTestLog(output, className, out loggerFactory, minLogLevel, out var _, testName);
+
+        internal IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, LogLevel minLogLevel, out string resolvedTestName, [CallerMemberName] string testName = null)
         {
-            var serviceProvider = CreateLoggerServices(output, className, minLogLevel, testName);
+            var serviceProvider = CreateLoggerServices(output, className, minLogLevel, out resolvedTestName, testName);
             var factory = serviceProvider.GetRequiredService<ILoggerFactory>();
             loggerFactory = factory;
             var logger = loggerFactory.CreateLogger("TestLifetime");
@@ -72,11 +85,13 @@ public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string class
 
         public ILoggerFactory CreateLoggerFactory(ITestOutputHelper output, string className, LogLevel minLogLevel, [CallerMemberName] string testName = null)
         {
-            return CreateLoggerServices(output, className, minLogLevel, testName).GetRequiredService<ILoggerFactory>();
+            return CreateLoggerServices(output, className, minLogLevel, out var _, testName).GetRequiredService<ILoggerFactory>();
         }
 
-        public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string className, LogLevel minLogLevel, [CallerMemberName] string testName = null)
+        public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string className, LogLevel minLogLevel, out string normalizedTestName, [CallerMemberName] string testName = null)
         {
+            normalizedTestName = string.Empty;
+
             // Try to shorten the class name using the assembly name
             if (className.StartsWith(_assemblyName + "."))
             {
@@ -87,6 +102,7 @@ public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string cl
             if (!string.IsNullOrEmpty(_baseDirectory))
             {
                 var testOutputDirectory = Path.Combine(GetAssemblyBaseDirectory(_assemblyName, _baseDirectory), className);
+                testName = RemoveIllegalFileChars(testName);
 
                 if (testOutputDirectory.Length + testName.Length + LogFileExtension.Length >= MaxPathLength)
                 {
@@ -118,11 +134,13 @@ public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string cl
                         if (!File.Exists(testOutputFile))
                         {
                             _globalLogger.LogWarning($"To resolve log file collision, the enumerated file {testOutputFile} will be used.");
+                            testName = $"{testName}.{i}";
                             break;
                         }
                     }
                 }
 
+                normalizedTestName = testName;
                 serilogLoggerProvider = ConfigureFileLogging(testOutputFile);
             }
 
@@ -234,6 +252,17 @@ private static SerilogLoggerProvider ConfigureFileLogging(string fileName)
             return new SerilogLoggerProvider(serilogger, dispose: true);
         }
 
+        private static string RemoveIllegalFileChars(string s)
+        {
+            var sb = new StringBuilder();
+
+            foreach (var c in s)
+            {
+                sb.Append(InvalidFileChars.Contains(c) ? '_' : c);
+            }
+            return sb.ToString();
+        }
+
         public void Dispose()
         {
             (_serviceProvider as IDisposable)?.Dispose();

src/Microsoft.Extensions.Logging.Testing/LoggedTest.cs

@@ -4,24 +4,50 @@
 using System;
 using System.Reflection;
 using System.Runtime.CompilerServices;
+using Microsoft.Extensions.DependencyInjection;
 using Xunit.Abstractions;
 
 namespace Microsoft.Extensions.Logging.Testing
 {
     public abstract class LoggedTest
     {
-        private readonly ITestOutputHelper _output;
+        private ILoggerFactory _loggerFactory;
 
-        public LoggedTest(ITestOutputHelper output)
+        // Obsolete but keeping for back compat
+        public LoggedTest(ITestOutputHelper output = null)
         {
-            _output = output;
+            TestOutputHelper = output;
         }
 
+        // Internal for testing
+        internal string TestMethodTestName { get; set; }
+
+        public ILogger Logger { get; set; }
+
+        public ILoggerFactory LoggerFactory
+        {
+            get
+            {
+                return _loggerFactory;
+            }
+            set
+            {
+                _loggerFactory = value;
+                AddTestLogging = services => services.AddSingleton(_loggerFactory);
+            }
+        }
+
+        public ITestOutputHelper TestOutputHelper { get; set; }
+
+        public ITestSink TestSink { get; set; }
+
+        public Action<IServiceCollection> AddTestLogging { get; private set; } = services => { };
+
         public IDisposable StartLog(out ILoggerFactory loggerFactory, [CallerMemberName] string testName = null) => StartLog(out loggerFactory, LogLevel.Information, testName);
 
         public IDisposable StartLog(out ILoggerFactory loggerFactory, LogLevel minLogLevel, [CallerMemberName] string testName = null)
         {
-            return AssemblyTestLog.ForAssembly(GetType().GetTypeInfo().Assembly).StartTestLog(_output, GetType().FullName, out loggerFactory, minLogLevel, testName);
+            return AssemblyTestLog.ForAssembly(GetType().GetTypeInfo().Assembly).StartTestLog(TestOutputHelper, GetType().FullName, out loggerFactory, minLogLevel, testName);
         }
     }
 }

src/Microsoft.Extensions.Logging.Testing/Microsoft.Extensions.Logging.Testing.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <Description>Helpers for writing tests that use Microsoft.Extensions.Logging. Contains null implementations of the abstractions that do nothing, as well as test implementations that are observable.</Description>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFrameworks>netstandard2.0;net461</TargetFrameworks>
     <NoWarn>$(NoWarn);CS1591</NoWarn>
     <PackageTags>$(PackageTags);testing</PackageTags>
     <EnableApiCheck>false</EnableApiCheck>
@@ -14,11 +14,16 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.AspNetCore.Testing" Version="$(MicrosoftAspNetCoreTestingPackageVersion)" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="$(MicrosoftExtensionsDependencyInjectionPackageVersion)" />
     <PackageReference Include="xunit.abstractions" Version="$(XunitAbstractionsPackageVersion)" />
+    <PackageReference Include="xunit.extensibility.execution" Version="$(XunitExtensibilityExecutionPackageVersion)" />
     <PackageReference Include="xunit.assert" Version="$(XunitAssertPackageVersion)" />
     <PackageReference Include="Serilog.Extensions.Logging" Version="$(SerilogExtensionsLoggingPackageVersion)" />
     <PackageReference Include="Serilog.Sinks.File" Version="$(SerilogSinksFilePackageVersion)" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="build\**\*.props" PackagePath="%(Identity)" />
+  </ItemGroup>
 </Project>

src/Microsoft.Extensions.Logging.Testing/Properties/AssemblyInfo.cs

@@ -0,0 +1,6 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("Microsoft.Extensions.Logging.Testing.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100f33a29044fa9d740c9b3213a93e57c84b472c84e0b8a0e1ae48e67a9f8f6de9d5f7f3d52ac23e48ac51801f1dc950abe901da34d2a9e3baadb141a17c77ef3c565dd5ee5054b91cf63bb3c6ab83f72ab3aafe93d0fc3c2348b764fafb0b1c0733de51459aeab46580384bf9d74c4e28164b7cde247f891ba07891c9d872ad2bb")]

src/Microsoft.Extensions.Logging.Testing/TestLoggerProvider.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Microsoft.Extensions.Logging.Testing
+{
+    public class TestLoggerProvider : ILoggerProvider
+    {
+        private readonly ITestSink _sink;
+
+        public TestLoggerProvider(ITestSink sink)
+        {
+            _sink = sink;
+        }
+
+        public ILogger CreateLogger(string categoryName)
+        {
+            return new TestLogger(categoryName, _sink, enabled: true);
+        }
+
+        public void Dispose()
+        {
+        }
+    }
+}

src/Microsoft.Extensions.Logging.Testing/Xunit/LogLevelAttribute.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace Microsoft.Extensions.Logging.Testing
+{
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class LogLevelAttribute : Attribute
+    {
+        public LogLevelAttribute(LogLevel logLevel)
+        {
+            LogLevel = logLevel;
+        }
+
+        public LogLevel LogLevel { get; }
+    }
+}

src/Microsoft.Extensions.Logging.Testing/Xunit/LoggedConditionalFactDiscoverer.cs

@@ -0,0 +1,28 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.AspNetCore.Testing.xunit;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.Extensions.Logging.Testing
+{
+    public class LoggedConditionalFactDiscoverer : LoggedFactDiscoverer
+    {
+        private readonly IMessageSink _diagnosticMessageSink;
+
+        public LoggedConditionalFactDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink)
+        {
+            _diagnosticMessageSink = diagnosticMessageSink;
+        }
+
+        protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
+        {
+            var skipReason = testMethod.EvaluateSkipConditions();
+            return skipReason != null
+                ? new SkippedTestCase(skipReason, _diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod)
+                : base.CreateTestCase(discoveryOptions, testMethod, factAttribute);
+        }
+
+    }
+}

src/Microsoft.Extensions.Logging.Testing/Xunit/LoggedConditionalTheoryDiscoverer.cs

@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Testing.xunit;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.Extensions.Logging.Testing
+{
+    public class LoggedConditionalTheoryDiscoverer : LoggedTheoryDiscoverer
+    {
+        public LoggedConditionalTheoryDiscoverer(IMessageSink diagnosticMessageSink)
+            : base(diagnosticMessageSink)
+        {
+        }
+
+        protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(
+            ITestFrameworkDiscoveryOptions discoveryOptions,
+            ITestMethod testMethod,
+            IAttributeInfo theoryAttribute)
+        {
+            var skipReason = testMethod.EvaluateSkipConditions();
+            return skipReason != null
+               ? new[] { new SkippedTestCase(skipReason, DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod) }
+               : base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute);
+        }
+
+        protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(
+            ITestFrameworkDiscoveryOptions discoveryOptions,
+            ITestMethod testMethod, IAttributeInfo theoryAttribute,
+            object[] dataRow)
+        {
+            var skipReason = testMethod.EvaluateSkipConditions();
+            return skipReason != null
+                ? base.CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason)
+                : base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow);
+        }
+
+    }
+}