diff --git a/src/Mvc/test/Mvc.FunctionalTests/ErrorPageTests.cs b/src/Mvc/test/Mvc.FunctionalTests/ErrorPageTests.cs index cc74a6244d72..d09f99af276b 100644 --- a/src/Mvc/test/Mvc.FunctionalTests/ErrorPageTests.cs +++ b/src/Mvc/test/Mvc.FunctionalTests/ErrorPageTests.cs @@ -18,7 +18,7 @@ namespace Microsoft.AspNetCore.Mvc.FunctionalTests; /// /// Functional test to verify the error reporting of Razor compilation by diagnostic middleware. /// -public class ErrorPageTests : IClassFixture>, IDisposable +public class ErrorPageTests : IClassFixture> { private static readonly string PreserveCompilationContextMessage = HtmlEncoder.Default.Encode( "One or more compilation references may be missing. " + @@ -183,9 +183,4 @@ public async Task AggregateException_FlattensInnerExceptions() Assert.Contains(nullReferenceException, content); Assert.Contains(indexOutOfRangeException, content); } - - public void Dispose() - { - _assemblyTestLog.Dispose(); - } } diff --git a/src/Testing/src/AssemblyTestLog.cs b/src/Testing/src/AssemblyTestLog.cs index 9ca775cd45ee..027e41f53c62 100644 --- a/src/Testing/src/AssemblyTestLog.cs +++ b/src/Testing/src/AssemblyTestLog.cs @@ -20,20 +20,21 @@ namespace Microsoft.AspNetCore.Testing; -public class AssemblyTestLog : IDisposable +public class AssemblyTestLog : IAcceptFailureReports, IDisposable { private const string MaxPathLengthEnvironmentVariableName = "ASPNETCORE_TEST_LOG_MAXPATH"; private const string LogFileExtension = ".log"; private static readonly int MaxPathLength = GetMaxPathLength(); - private static readonly object _lock = new object(); - private static readonly Dictionary _logs = new Dictionary(); + private static readonly object _lock = new(); + private static readonly Dictionary _logs = new(); private readonly ILoggerFactory _globalLoggerFactory; private readonly ILogger _globalLogger; private readonly string _baseDirectory; private readonly Assembly _assembly; private readonly IServiceProvider _serviceProvider; + private bool _testFailureReported; private static int GetMaxPathLength() { @@ -51,6 +52,9 @@ private AssemblyTestLog(ILoggerFactory globalLoggerFactory, ILogger globalLogger _serviceProvider = serviceProvider; } + // internal for testing + internal bool OnCI { get; set; } = SkipOnCIAttribute.OnCI(); + [SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "Required to maintain compatibility")] public IDisposable StartTestLog(ITestOutputHelper output, string className, out ILoggerFactory loggerFactory, [CallerMemberName] string testName = null) => StartTestLog(output, className, out loggerFactory, LogLevel.Debug, testName); @@ -176,7 +180,8 @@ public IServiceProvider CreateLoggerServices(ITestOutputHelper output, string cl return serviceCollection.BuildServiceProvider(); } - public static AssemblyTestLog Create(Assembly assembly, string baseDirectory) + // internal for testing. Expectation is AspNetTestAssembly runner calls ForAssembly() first for every Assembly. + internal static AssemblyTestLog Create(Assembly assembly, string baseDirectory) { var logStart = DateTimeOffset.UtcNow; SerilogLoggerProvider serilogLoggerProvider = null; @@ -218,12 +223,20 @@ public static AssemblyTestLog ForAssembly(Assembly assembly) { if (!_logs.TryGetValue(assembly, out var log)) { - var baseDirectory = TestFileOutputContext.GetOutputDirectory(assembly); + var stackTrace = Environment.StackTrace; + if (!stackTrace.Contains( + "Microsoft.AspNetCore.Testing" +#if NETCOREAPP + , StringComparison.Ordinal +#endif + )) + { + throw new InvalidOperationException($"Unexpected initial {nameof(ForAssembly)} caller."); + } - log = Create(assembly, baseDirectory); - _logs[assembly] = log; + var baseDirectory = TestFileOutputContext.GetOutputDirectory(assembly); - // Try to clear previous logs, continue if it fails. + // Try to clear previous logs, continue if it fails. Do this before creating new global logger. var assemblyBaseDirectory = TestFileOutputContext.GetAssemblyBaseDirectory(assembly); if (!string.IsNullOrEmpty(assemblyBaseDirectory) && !TestFileOutputContext.GetPreserveExistingLogsInOutput(assembly)) @@ -232,13 +245,24 @@ public static AssemblyTestLog ForAssembly(Assembly assembly) { Directory.Delete(assemblyBaseDirectory, recursive: true); } - catch { } + catch + { + } } + + log = Create(assembly, baseDirectory); + _logs[assembly] = log; } + return log; } } + public void ReportTestFailure() + { + _testFailureReported = true; + } + private static TestFrameworkFileLoggerAttribute GetFileLoggerAttribute(Assembly assembly) => assembly.GetCustomAttribute() ?? throw new InvalidOperationException($"No {nameof(TestFrameworkFileLoggerAttribute)} found on the assembly {assembly.GetName().Name}. " @@ -268,10 +292,28 @@ private static SerilogLoggerProvider ConfigureFileLogging(string fileName, DateT return new SerilogLoggerProvider(serilogger, dispose: true); } - public void Dispose() + void IDisposable.Dispose() { (_serviceProvider as IDisposable)?.Dispose(); _globalLoggerFactory.Dispose(); + + // Clean up if no tests failed and we're not running local tests. (Ignoring tests of this class, OnCI is + // true on both build and Helix agents.) In particular, remove the directory containing the global.log + // file. All test class log files for this assembly are in subdirectories of this location. + if (!_testFailureReported && + OnCI && + _baseDirectory is not null && + Directory.Exists(_baseDirectory)) + { + try + { + Directory.Delete(_baseDirectory, recursive: true); + } + catch + { + // Best effort. Ignore problems deleting locked logged files. + } + } } private class AssemblyLogTimestampOffsetEnricher : ILogEventEnricher diff --git a/src/Testing/src/AssemblyTestLogFixtureAttribute.cs b/src/Testing/src/AssemblyTestLogFixtureAttribute.cs new file mode 100644 index 000000000000..e4a4452cd458 --- /dev/null +++ b/src/Testing/src/AssemblyTestLogFixtureAttribute.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.AspNetCore.Testing; + +public class AssemblyTestLogFixtureAttribute : AssemblyFixtureAttribute +{ + public AssemblyTestLogFixtureAttribute() : base(typeof(AssemblyTestLog)) + { + } +} diff --git a/src/Testing/src/build/Microsoft.AspNetCore.Testing.props b/src/Testing/src/build/Microsoft.AspNetCore.Testing.props index 063e9094d172..47d06dfef7a7 100644 --- a/src/Testing/src/build/Microsoft.AspNetCore.Testing.props +++ b/src/Testing/src/build/Microsoft.AspNetCore.Testing.props @@ -11,8 +11,8 @@ + BeforeTargets="GetAssemblyAttributes" + Condition="'$(GenerateLoggingTestingAssemblyAttributes)' != 'false'"> true false @@ -24,6 +24,7 @@ <_Parameter2>Microsoft.AspNetCore.Testing + <_Parameter1>$(PreserveExistingLogsInOutput) <_Parameter2>$(TargetFramework) diff --git a/src/Testing/src/xunit/AspNetTestAssemblyRunner.cs b/src/Testing/src/xunit/AspNetTestAssemblyRunner.cs index 0d99ffcff58b..5d149fd48a93 100644 --- a/src/Testing/src/xunit/AspNetTestAssemblyRunner.cs +++ b/src/Testing/src/xunit/AspNetTestAssemblyRunner.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -14,7 +15,7 @@ namespace Microsoft.AspNetCore.Testing; public class AspNetTestAssemblyRunner : XunitTestAssemblyRunner { - private readonly Dictionary _assemblyFixtureMappings = new Dictionary(); + private readonly Dictionary _assemblyFixtureMappings = new(); public AspNetTestAssemblyRunner( ITestAssembly testAssembly, @@ -26,6 +27,9 @@ public AspNetTestAssemblyRunner( { } + // internal for testing + internal IEnumerable Fixtures => _assemblyFixtureMappings.Values; + protected override async Task AfterTestAssemblyStartingAsync() { await base.AfterTestAssemblyStartingAsync().ConfigureAwait(false); @@ -33,8 +37,8 @@ protected override async Task AfterTestAssemblyStartingAsync() // Find all the AssemblyFixtureAttributes on the test assembly await Aggregator.RunAsync(async () => { - var fixturesAttributes = ((IReflectionAssemblyInfo)TestAssembly.Assembly) - .Assembly + var assembly = ((IReflectionAssemblyInfo)TestAssembly.Assembly).Assembly; + var fixturesAttributes = assembly .GetCustomAttributes(typeof(AssemblyFixtureAttribute), false) .Cast() .ToList(); @@ -42,15 +46,30 @@ await Aggregator.RunAsync(async () => // Instantiate all the fixtures foreach (var fixtureAttribute in fixturesAttributes) { - var ctorWithDiagnostics = fixtureAttribute.FixtureType.GetConstructor(new[] { typeof(IMessageSink) }); object instance = null; - if (ctorWithDiagnostics != null) + var staticCreator = fixtureAttribute.FixtureType.GetMethod( + name: "ForAssembly", + bindingAttr: BindingFlags.Public | BindingFlags.Static, + binder: null, + types: new[] { typeof(Assembly) }, + modifiers: null); + if (staticCreator is null) { - instance = Activator.CreateInstance(fixtureAttribute.FixtureType, DiagnosticMessageSink); + var ctorWithDiagnostics = fixtureAttribute + .FixtureType + .GetConstructor(new[] { typeof(IMessageSink) }); + if (ctorWithDiagnostics is null) + { + instance = Activator.CreateInstance(fixtureAttribute.FixtureType); + } + else + { + instance = Activator.CreateInstance(fixtureAttribute.FixtureType, DiagnosticMessageSink); + } } else { - instance = Activator.CreateInstance(fixtureAttribute.FixtureType); + instance = staticCreator.Invoke(obj: null, parameters: new[] { assembly }); } _assemblyFixtureMappings[fixtureAttribute.FixtureType] = instance; @@ -66,12 +85,12 @@ await Aggregator.RunAsync(async () => protected override async Task BeforeTestAssemblyFinishedAsync() { // Dispose fixtures - foreach (var disposable in _assemblyFixtureMappings.Values.OfType()) + foreach (var disposable in Fixtures.OfType()) { Aggregator.Run(disposable.Dispose); } - foreach (var disposable in _assemblyFixtureMappings.Values.OfType()) + foreach (var disposable in Fixtures.OfType()) { await Aggregator.RunAsync(disposable.DisposeAsync).ConfigureAwait(false); } @@ -79,12 +98,13 @@ protected override async Task BeforeTestAssemblyFinishedAsync() await base.BeforeTestAssemblyFinishedAsync().ConfigureAwait(false); } - protected override Task RunTestCollectionAsync( + protected override async Task RunTestCollectionAsync( IMessageBus messageBus, ITestCollection testCollection, IEnumerable testCases, CancellationTokenSource cancellationTokenSource) - => new AspNetTestCollectionRunner( + { + var runSummary = await new AspNetTestCollectionRunner( _assemblyFixtureMappings, testCollection, testCases, @@ -92,5 +112,17 @@ protected override Task RunTestCollectionAsync( messageBus, TestCaseOrderer, new ExceptionAggregator(Aggregator), - cancellationTokenSource).RunAsync(); + cancellationTokenSource) + .RunAsync() + .ConfigureAwait(false); + if (runSummary.Failed != 0) + { + foreach (var fixture in Fixtures.OfType()) + { + fixture.ReportTestFailure(); + } + } + + return runSummary; + } } diff --git a/src/Testing/src/xunit/IAcceptFailureReports.cs b/src/Testing/src/xunit/IAcceptFailureReports.cs new file mode 100644 index 000000000000..30ca366b3f1b --- /dev/null +++ b/src/Testing/src/xunit/IAcceptFailureReports.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.AspNetCore.Testing; + +internal interface IAcceptFailureReports +{ + void ReportTestFailure(); +} diff --git a/src/Testing/test/AspNetTestAssemblyRunnerTest.cs b/src/Testing/test/AspNetTestAssemblyRunnerTest.cs new file mode 100644 index 000000000000..dbce4c69bb82 --- /dev/null +++ b/src/Testing/test/AspNetTestAssemblyRunnerTest.cs @@ -0,0 +1,219 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Reflection; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.AspNetCore.Testing; + +public class AspNetTestAssemblyRunnerTest +{ + private const int NotCalled = -1; + + [Fact] + public async Task ForAssemblyHasHigherPriorityThanConstructors() + { + var runner = TestableAspNetTestAssemblyRunner.Create(typeof(TestAssemblyFixtureWithAll)); + + await runner.AfterTestAssemblyStartingAsync_Public(); + + Assert.NotNull(runner.Fixtures); + var fixtureObject = Assert.Single(runner.Fixtures); + var fixture = Assert.IsType(fixtureObject); + Assert.False(fixture.ConstructorWithMessageSinkCalled); + Assert.True(fixture.ForAssemblyCalled); + Assert.False(fixture.ParameterlessConstructorCalled); + } + + [Fact] + public async Task ConstructorWithMessageSinkHasHigherPriorityThanParameterlessConstructor() + { + var runner = TestableAspNetTestAssemblyRunner.Create(typeof(TestAssemblyFixtureWithMessageSink)); + + await runner.AfterTestAssemblyStartingAsync_Public(); + + Assert.NotNull(runner.Fixtures); + var fixtureObject = Assert.Single(runner.Fixtures); + var fixture = Assert.IsType(fixtureObject); + Assert.True(fixture.ConstructorWithMessageSinkCalled); + Assert.False(fixture.ParameterlessConstructorCalled); + } + + [Fact] + public async Task CalledInExpectedOrder_SuccessWithDispose() + { + var runner = TestableAspNetTestAssemblyRunner.Create(typeof(TextAssemblyFixtureWithDispose)); + + var runSummary = await runner.RunAsync(); + + Assert.NotNull(runSummary); + Assert.Equal(0, runSummary.Failed); + Assert.Equal(0, runSummary.Skipped); + Assert.Equal(1, runSummary.Total); + + Assert.NotNull(runner.Fixtures); + var fixtureObject = Assert.Single(runner.Fixtures); + var fixture = Assert.IsType(fixtureObject); + Assert.Equal(NotCalled, fixture.ReportTestFailureCalledAt); + Assert.Equal(0, fixture.DisposeCalledAt); + } + + [Fact] + public async Task CalledInExpectedOrder_FailedWithDispose() + { + var runner = TestableAspNetTestAssemblyRunner.Create( + typeof(TextAssemblyFixtureWithDispose), + failTestCase: true); + + var runSummary = await runner.RunAsync(); + + Assert.NotNull(runSummary); + Assert.Equal(1, runSummary.Failed); + Assert.Equal(0, runSummary.Skipped); + Assert.Equal(1, runSummary.Total); + + Assert.NotNull(runner.Fixtures); + var fixtureObject = Assert.Single(runner.Fixtures); + var fixture = Assert.IsType(fixtureObject); + Assert.Equal(0, fixture.ReportTestFailureCalledAt); + Assert.Equal(1, fixture.DisposeCalledAt); + } + + [Fact] + public async Task CalledInExpectedOrder_SuccessWithAsyncDispose() + { + var runner = TestableAspNetTestAssemblyRunner.Create(typeof(TestAssemblyFixtureWithAsyncDispose)); + + var runSummary = await runner.RunAsync(); + + Assert.NotNull(runSummary); + Assert.Equal(0, runSummary.Failed); + Assert.Equal(0, runSummary.Skipped); + Assert.Equal(1, runSummary.Total); + + Assert.NotNull(runner.Fixtures); + var fixtureObject = Assert.Single(runner.Fixtures); + var fixture = Assert.IsType(fixtureObject); + Assert.Equal(0, fixture.InitializeAsyncCalledAt); + Assert.Equal(NotCalled, fixture.ReportTestFailureCalledAt); + Assert.Equal(1, fixture.AsyncDisposeCalledAt); + } + + [Fact] + public async Task CalledInExpectedOrder_FailedWithAsyncDispose() + { + var runner = TestableAspNetTestAssemblyRunner.Create( + typeof(TestAssemblyFixtureWithAsyncDispose), + failTestCase: true); + + var runSummary = await runner.RunAsync(); + + Assert.NotNull(runSummary); + Assert.Equal(1, runSummary.Failed); + Assert.Equal(0, runSummary.Skipped); + Assert.Equal(1, runSummary.Total); + + Assert.NotNull(runner.Fixtures); + var fixtureObject = Assert.Single(runner.Fixtures); + var fixture = Assert.IsType(fixtureObject); + Assert.Equal(0, fixture.InitializeAsyncCalledAt); + Assert.Equal(1, fixture.ReportTestFailureCalledAt); + Assert.Equal(2, fixture.AsyncDisposeCalledAt); + } + + private class TestAssemblyFixtureWithAll + { + private TestAssemblyFixtureWithAll(bool forAssemblyCalled) + { + ForAssemblyCalled = forAssemblyCalled; + } + + public TestAssemblyFixtureWithAll() + { + ParameterlessConstructorCalled = true; + } + + public TestAssemblyFixtureWithAll(IMessageSink messageSink) + { + ConstructorWithMessageSinkCalled = true; + } + + public static TestAssemblyFixtureWithAll ForAssembly(Assembly assembly) + { + return new TestAssemblyFixtureWithAll(forAssemblyCalled: true); + } + + public bool ParameterlessConstructorCalled { get; } + + public bool ConstructorWithMessageSinkCalled { get; } + + public bool ForAssemblyCalled { get; } + } + + private class TestAssemblyFixtureWithMessageSink + { + public TestAssemblyFixtureWithMessageSink() + { + ParameterlessConstructorCalled = true; + } + + public TestAssemblyFixtureWithMessageSink(IMessageSink messageSink) + { + ConstructorWithMessageSinkCalled = true; + } + + public bool ParameterlessConstructorCalled { get; } + + public bool ConstructorWithMessageSinkCalled { get; } + } + + private class TextAssemblyFixtureWithDispose : IAcceptFailureReports, IDisposable + { + private int _position; + + public int ReportTestFailureCalledAt { get; private set; } = NotCalled; + + public int DisposeCalledAt { get; private set; } = NotCalled; + + void IAcceptFailureReports.ReportTestFailure() + { + ReportTestFailureCalledAt = _position++; + } + + void IDisposable.Dispose() + { + DisposeCalledAt = _position++; + } + } + + private class TestAssemblyFixtureWithAsyncDispose : IAcceptFailureReports, IAsyncLifetime + { + private int _position; + + public int InitializeAsyncCalledAt { get; private set; } = NotCalled; + + public int ReportTestFailureCalledAt { get; private set; } = NotCalled; + + public int AsyncDisposeCalledAt { get; private set; } = NotCalled; + + Task IAsyncLifetime.InitializeAsync() + { + InitializeAsyncCalledAt = _position++; + return Task.CompletedTask; + } + + void IAcceptFailureReports.ReportTestFailure() + { + ReportTestFailureCalledAt = _position++; + } + + Task IAsyncLifetime.DisposeAsync() + { + AsyncDisposeCalledAt = _position++; + return Task.CompletedTask; + } + } +} diff --git a/src/Testing/test/AssemblyTestLogTests.cs b/src/Testing/test/AssemblyTestLogTests.cs index c31c6e404b29..74306861929a 100644 --- a/src/Testing/test/AssemblyTestLogTests.cs +++ b/src/Testing/test/AssemblyTestLogTests.cs @@ -4,24 +4,16 @@ using System; using System.IO; using System.Linq; -using System.Reflection; using System.Text.RegularExpressions; using System.Threading.Tasks; -using Microsoft.AspNetCore.Testing; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Testing.Tests; using Xunit; -namespace Microsoft.Extensions.Logging.Testing.Tests; +namespace Microsoft.AspNetCore.Testing; public class AssemblyTestLogTests : LoggedTest { - private static readonly Assembly ThisAssembly = typeof(AssemblyTestLogTests).GetTypeInfo().Assembly; - private static readonly string ThisAssemblyName = ThisAssembly.GetName().Name; - private static readonly string TFM = ThisAssembly - .GetCustomAttributes() - .OfType() - .FirstOrDefault() - .TargetFramework; - [Fact] public void FunctionalLogs_LogsPreservedFromNonQuarantinedTest() { @@ -38,16 +30,52 @@ public void FunctionalLogs_LogsPreservedFromQuarantinedTest() public void ForAssembly_ReturnsSameInstanceForSameAssembly() { Assert.Same( - AssemblyTestLog.ForAssembly(ThisAssembly), - AssemblyTestLog.ForAssembly(ThisAssembly)); + AssemblyTestLog.ForAssembly(TestableAssembly.ThisAssembly), + AssemblyTestLog.ForAssembly(TestableAssembly.ThisAssembly)); } + [Fact] + public Task ForAssemblyWritesToAssemblyBaseDirectory() => + RunTestLogFunctionalTest((tempDir) => + { + var logger = LoggerFactory.CreateLogger("Test"); + + var assembly = TestableAssembly.Create(typeof(AssemblyTestLog), logDirectory: tempDir); + var assemblyName = assembly.GetName().Name; + var testName = $"{TestableAssembly.TestClassName}.{TestableAssembly.TestMethodName}"; + + var tfmPath = Path.Combine(tempDir, assemblyName, TestableAssembly.TFM); + var globalLogPath = Path.Combine(tfmPath, "global.log"); + var testLog = Path.Combine(tfmPath, TestableAssembly.TestClassName, $"{testName}.log"); + + using var testAssemblyLog = AssemblyTestLog.ForAssembly(assembly); + testAssemblyLog.OnCI = true; + logger.LogInformation("Created test log in {baseDirectory}", tempDir); + + using (testAssemblyLog.StartTestLog( + output: null, + className: $"{assemblyName}.{TestableAssembly.TestClassName}", + loggerFactory: out var testLoggerFactory, + minLogLevel: LogLevel.Trace, + testName: testName)) + { + var testLogger = testLoggerFactory.CreateLogger("TestLogger"); + testLogger.LogInformation("Information!"); + testLogger.LogTrace("Trace!"); + } + + Assert.True(File.Exists(globalLogPath), $"Expected global log file {globalLogPath} to exist."); + Assert.True(File.Exists(testLog), $"Expected test log file {testLog} to exist."); + + logger.LogInformation("Finished test log in {baseDirectory}", tempDir); + }); + [Fact] public void TestLogWritesToITestOutputHelper() { var output = new TestTestOutputHelper(); - using var assemblyLog = AssemblyTestLog.Create(ThisAssembly, baseDirectory: null); + using var assemblyLog = AssemblyTestLog.Create(TestableAssembly.ThisAssembly, baseDirectory: null); using (assemblyLog.StartTestLog(output, "NonExistant.Test.Class", out var loggerFactory)) { var logger = loggerFactory.CreateLogger("TestLogger"); @@ -73,7 +101,9 @@ public Task TestLogEscapesIllegalFileNames() => var illegalTestName = "T:e/s//t"; var escapedTestName = "T_e_s_t"; - using var testAssemblyLog = AssemblyTestLog.Create(ThisAssembly, baseDirectory: tempDir); + using var testAssemblyLog = AssemblyTestLog.Create( + TestableAssembly.ThisAssembly, + baseDirectory: tempDir); using var disposable = testAssemblyLog.StartTestLog( output: null, className: "FakeTestAssembly.FakeTestClass", @@ -93,13 +123,16 @@ public Task TestLogWritesToGlobalLogFile() => // but it's also testing the test logging facility. So this is pretty meta ;) var logger = LoggerFactory.CreateLogger("Test"); - using (var testAssemblyLog = AssemblyTestLog.Create(ThisAssembly, baseDirectory: tempDir)) + using (var testAssemblyLog = AssemblyTestLog.Create( + TestableAssembly.ThisAssembly, + baseDirectory: tempDir)) { + testAssemblyLog.OnCI = false; logger.LogInformation("Created test log in {baseDirectory}", tempDir); using (testAssemblyLog.StartTestLog( output: null, - className: $"{ThisAssemblyName}.FakeTestClass", + className: $"{TestableAssembly.ThisAssemblyName}.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) @@ -112,8 +145,17 @@ public Task TestLogWritesToGlobalLogFile() => logger.LogInformation("Finished test log in {baseDirectory}", tempDir); - var globalLogPath = Path.Combine(tempDir, ThisAssemblyName, TFM, "global.log"); - var testLog = Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass", "FakeTestName.log"); + var globalLogPath = Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "global.log"); + var testLog = Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "FakeTestClass", + "FakeTestName.log"); Assert.True(File.Exists(globalLogPath), $"Expected global log file {globalLogPath} to exist"); Assert.True(File.Exists(testLog), $"Expected test log file {testLog} to exist"); @@ -134,19 +176,114 @@ [OFFSET] [TestLifetime] [Information] Finished test FakeTestName in DURATION ", testLogContent, ignoreLineEndingDifferences: true); }); + [Fact] + public Task TestLogCleansLogFiles_AfterSuccessfulRun() => + RunTestLogFunctionalTest((tempDir) => + { + var logger = LoggerFactory.CreateLogger("Test"); + var globalLogPath = Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "global.log"); + var testLog = Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "FakeTestClass", + "FakeTestName.log"); + + using (var testAssemblyLog = AssemblyTestLog.Create( + TestableAssembly.ThisAssembly, + baseDirectory: tempDir)) + { + testAssemblyLog.OnCI = true; + logger.LogInformation("Created test log in {baseDirectory}", tempDir); + + using (testAssemblyLog.StartTestLog( + output: null, + className: $"{TestableAssembly.ThisAssemblyName}.FakeTestClass", + loggerFactory: out var testLoggerFactory, + minLogLevel: LogLevel.Trace, + testName: "FakeTestName")) + { + var testLogger = testLoggerFactory.CreateLogger("TestLogger"); + testLogger.LogInformation("Information!"); + testLogger.LogTrace("Trace!"); + } + + Assert.True(File.Exists(globalLogPath), $"Expected global log file {globalLogPath} to exist."); + Assert.True(File.Exists(testLog), $"Expected test log file {testLog} to exist."); + } + + logger.LogInformation("Finished test log in {baseDirectory}", tempDir); + + Assert.True(!File.Exists(globalLogPath), $"Expected no global log file {globalLogPath} to exist."); + Assert.True(!File.Exists(testLog), $"Expected no test log file {testLog} to exist."); + }); + + [Fact] + public Task TestLogDoesNotCleanLogFiles_AfterFailedRun() => + RunTestLogFunctionalTest((tempDir) => + { + var logger = LoggerFactory.CreateLogger("Test"); + var globalLogPath = Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "global.log"); + var testLog = Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "FakeTestClass", + "FakeTestName.log"); + + using (var testAssemblyLog = AssemblyTestLog.Create( + TestableAssembly.ThisAssembly, + baseDirectory: tempDir)) + { + testAssemblyLog.OnCI = true; + logger.LogInformation("Created test log in {baseDirectory}", tempDir); + + using (testAssemblyLog.StartTestLog( + output: null, + className: $"{TestableAssembly.ThisAssemblyName}.FakeTestClass", + loggerFactory: out var testLoggerFactory, + minLogLevel: LogLevel.Trace, + testName: "FakeTestName")) + { + var testLogger = testLoggerFactory.CreateLogger("TestLogger"); + testLogger.LogInformation("Information!"); + testLogger.LogTrace("Trace!"); + } + + testAssemblyLog.ReportTestFailure(); + } + + logger.LogInformation("Finished test log in {baseDirectory}", tempDir); + + Assert.True(File.Exists(globalLogPath), $"Expected global log file {globalLogPath} to exist."); + Assert.True(File.Exists(testLog), $"Expected test log file {testLog} to exist."); + }); + [Fact] public Task TestLogTruncatesTestNameToAvoidLongPaths() => RunTestLogFunctionalTest((tempDir) => { - var longTestName = new string('0', 50) + new string('1', 50) + new string('2', 50) + new string('3', 50) + new string('4', 50); + var longTestName = new string('0', 50) + new string('1', 50) + new string('2', 50) + + new string('3', 50) + new string('4', 50); var logger = LoggerFactory.CreateLogger("Test"); - using (var testAssemblyLog = AssemblyTestLog.Create(ThisAssembly, baseDirectory: tempDir)) + using (var testAssemblyLog = AssemblyTestLog.Create( + TestableAssembly.ThisAssembly, + baseDirectory: tempDir)) { + testAssemblyLog.OnCI = false; logger.LogInformation("Created test log in {baseDirectory}", tempDir); using (testAssemblyLog.StartTestLog( output: null, - className: $"{ThisAssemblyName}.FakeTestClass", + className: $"{TestableAssembly.ThisAssemblyName}.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: longTestName)) @@ -157,14 +294,21 @@ public Task TestLogTruncatesTestNameToAvoidLongPaths() => logger.LogInformation("Finished test log in {baseDirectory}", tempDir); - var testLogFiles = new DirectoryInfo(Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass")).EnumerateFiles(); + var testLogFiles = new DirectoryInfo( + Path.Combine(tempDir, TestableAssembly.ThisAssemblyName, TestableAssembly.TFM, "FakeTestClass")) + .EnumerateFiles(); var testLog = Assert.Single(testLogFiles); var testFileName = Path.GetFileNameWithoutExtension(testLog.Name); // The first half of the file comes from the beginning of the test name passed to the logger - Assert.Equal(longTestName.Substring(0, testFileName.Length / 2), testFileName.Substring(0, testFileName.Length / 2)); + Assert.Equal( + longTestName.Substring(0, testFileName.Length / 2), + testFileName.Substring(0, testFileName.Length / 2)); + // The last half of the file comes from the ending of the test name passed to the logger - Assert.Equal(longTestName.Substring(longTestName.Length - testFileName.Length / 2, testFileName.Length / 2), testFileName.Substring(testFileName.Length - testFileName.Length / 2, testFileName.Length / 2)); + Assert.Equal( + longTestName.Substring(longTestName.Length - testFileName.Length / 2, testFileName.Length / 2), + testFileName.Substring(testFileName.Length - testFileName.Length / 2, testFileName.Length / 2)); }); [Fact] @@ -172,15 +316,18 @@ public Task TestLogEnumerateFilenamesToAvoidCollisions() => RunTestLogFunctionalTest((tempDir) => { var logger = LoggerFactory.CreateLogger("Test"); - using (var testAssemblyLog = AssemblyTestLog.Create(ThisAssembly, baseDirectory: tempDir)) + using (var testAssemblyLog = AssemblyTestLog.Create( + TestableAssembly.ThisAssembly, + baseDirectory: tempDir)) { + testAssemblyLog.OnCI = false; logger.LogInformation("Created test log in {baseDirectory}", tempDir); for (var i = 0; i < 10; i++) { using (testAssemblyLog.StartTestLog( output: null, - className: $"{ThisAssemblyName}.FakeTestClass", + className: $"{TestableAssembly.ThisAssemblyName}.FakeTestClass", loggerFactory: out var testLoggerFactory, minLogLevel: LogLevel.Trace, testName: "FakeTestName")) @@ -193,12 +340,22 @@ public Task TestLogEnumerateFilenamesToAvoidCollisions() => logger.LogInformation("Finished test log in {baseDirectory}", tempDir); // The first log file exists - Assert.True(File.Exists(Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass", "FakeTestName.log"))); + Assert.True(File.Exists(Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "FakeTestClass", + "FakeTestName.log"))); // Subsequent files exist for (var i = 0; i < 9; i++) { - Assert.True(File.Exists(Path.Combine(tempDir, ThisAssemblyName, TFM, "FakeTestClass", $"FakeTestName.{i}.log"))); + Assert.True(File.Exists(Path.Combine( + tempDir, + TestableAssembly.ThisAssemblyName, + TestableAssembly.TFM, + "FakeTestClass", + $"FakeTestName.{i}.log"))); } }); diff --git a/src/Testing/test/TestableAspNetTestAssemblyRunner.cs b/src/Testing/test/TestableAspNetTestAssemblyRunner.cs new file mode 100644 index 000000000000..17f9373b34c2 --- /dev/null +++ b/src/Testing/test/TestableAspNetTestAssemblyRunner.cs @@ -0,0 +1,105 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Threading.Tasks; +using Moq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.AspNetCore.Testing; + +public class TestableAspNetTestAssemblyRunner : AspNetTestAssemblyRunner +{ + private TestableAspNetTestAssemblyRunner( + ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) : base( + testAssembly, + testCases, + diagnosticMessageSink, + executionMessageSink, + executionOptions) + { + } + + public static TestableAspNetTestAssemblyRunner Create(Type fixtureType, bool failTestCase = false) + { + var assembly = TestableAssembly.Create(fixtureType, failTestCase: failTestCase); + var testAssembly = GetTestAssembly(assembly); + var testCase = GetTestCase(assembly, testAssembly); + + return new TestableAspNetTestAssemblyRunner( + testAssembly, + new[] { testCase }, + diagnosticMessageSink: new NullMessageSink(), + executionMessageSink: new NullMessageSink(), + executionOptions: Mock.Of()); + + // Do not call Xunit.Sdk.Reflector.Wrap(assembly) because it uses GetTypes() and that method + // throws NotSupportedException for a dynamic assembly. + IAssemblyInfo GetAssemblyInfo(Assembly assembly) + { + var testAssemblyName = assembly.GetName().Name; + var assemblyInfo = new Mock(); + assemblyInfo.SetupGet(r => r.Assembly).Returns(assembly); + assemblyInfo.SetupGet(r => r.Name).Returns(testAssemblyName); + assemblyInfo + .SetupGet(r => r.AssemblyPath) + .Returns(Path.Combine(Directory.GetCurrentDirectory(), $"{testAssemblyName}.dll")); + + foreach (var attribute in CustomAttributeData.GetCustomAttributes(assembly)) + { + var attributeInfo = Reflector.Wrap(attribute); + var attributeName = attribute.AttributeType.AssemblyQualifiedName; + assemblyInfo + .Setup(r => r.GetCustomAttributes(attributeName)) + .Returns(new[] { attributeInfo }); + } + + var typeInfo = Reflector.Wrap(assembly.GetType(TestableAssembly.TestClassName)); + assemblyInfo.Setup(r => r.GetType(TestableAssembly.TestClassName)).Returns(typeInfo); + assemblyInfo.Setup(r => r.GetTypes(It.IsAny())).Returns(new[] { typeInfo }); + + return assemblyInfo.Object; + } + + ITestAssembly GetTestAssembly(Assembly assembly) + { + var assemblyInfo = GetAssemblyInfo(assembly); + + return new TestAssembly(assemblyInfo); + } + + IXunitTestCase GetTestCase(Assembly assembly, ITestAssembly testAssembly) + { + var testAssemblyName = assembly.GetName().Name; + var testCollection = new TestCollection( + testAssembly, + collectionDefinition: null, + displayName: $"Mock collection for '{testAssemblyName}'."); + + var type = assembly.GetType(TestableAssembly.TestClassName); + var testClass = new TestClass(testCollection, Reflector.Wrap(type)); + var method = type.GetMethod(TestableAssembly.TestMethodName); + var methodInfo = Reflector.Wrap(method); + var testMethod = new TestMethod(testClass, methodInfo); + + return new XunitTestCase( + diagnosticMessageSink: new NullMessageSink(), + defaultMethodDisplay: TestMethodDisplay.ClassAndMethod, + defaultMethodDisplayOptions: TestMethodDisplayOptions.None, + testMethod: testMethod); + } + } + + public Task AfterTestAssemblyStartingAsync_Public() + { + return base.AfterTestAssemblyStartingAsync(); + } +} diff --git a/src/Testing/test/TestableAssembly.cs b/src/Testing/test/TestableAssembly.cs new file mode 100644 index 000000000000..fd4b557cca12 --- /dev/null +++ b/src/Testing/test/TestableAssembly.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using Xunit; + +namespace Microsoft.AspNetCore.Testing; + +/* Creates a very simple dynamic assembly containing + * + * [Assembly: TestFramework( + * typeName: "Microsoft.AspNetCore.Testing.AspNetTestFramework", + * assemblyName: "Microsoft.AspNetCore.Testing")] + * [assembly: AssemblyFixture(typeof({fixtureType}))] + * [assembly: TestOutputDirectory( + * preserveExistingLogsInOutput: "false", + * targetFramework: TFM, + * baseDirectory: logDirectory)] // logdirectory is passed into Create(...). + * + * public class MyTestClass + * { + * public MyTestClass() { } + * + * [Fact] + * public MyTestMethod() + * { + * if (failTestCase) // Not exactly; condition checked during generation. + * { + * Assert.True(condition: false); + * } + * } + * } + */ +public static class TestableAssembly +{ + public static readonly Assembly ThisAssembly = typeof(TestableAssembly).GetTypeInfo().Assembly; + public static readonly string ThisAssemblyName = ThisAssembly.GetName().Name; + + private static readonly TestOutputDirectoryAttribute ThisOutputDirectoryAttribute = + ThisAssembly.GetCustomAttributes().OfType().FirstOrDefault(); + public static readonly string BaseDirectory = ThisOutputDirectoryAttribute.BaseDirectory; + public static readonly string TFM = ThisOutputDirectoryAttribute.TargetFramework; + + public const string TestClassName = "MyTestClass"; + public const string TestMethodName = "MyTestMethod"; + + public static Assembly Create(Type fixtureType, string logDirectory = null, bool failTestCase = false) + { + var frameworkConstructor = typeof(TestFrameworkAttribute) + .GetConstructor(new[] { typeof(string), typeof(string) }); + var frameworkBuilder = new CustomAttributeBuilder( + frameworkConstructor, + new[] { "Microsoft.AspNetCore.Testing.AspNetTestFramework", "Microsoft.AspNetCore.Testing" }); + + var fixtureConstructor = typeof(AssemblyFixtureAttribute).GetConstructor(new[] { typeof(Type) }); + var fixtureBuilder = new CustomAttributeBuilder(fixtureConstructor, new[] { fixtureType }); + + var outputConstructor = typeof(TestOutputDirectoryAttribute).GetConstructor( + new[] { typeof(string), typeof(string), typeof(string) }); + var outputBuilder = new CustomAttributeBuilder(outputConstructor, new[] { "false", TFM, logDirectory }); + + var testAssemblyName = $"Test{Guid.NewGuid():n}"; + var assemblyName = new AssemblyName(testAssemblyName); + var assembly = AssemblyBuilder.DefineDynamicAssembly( + assemblyName, + AssemblyBuilderAccess.Run, + new[] { frameworkBuilder, fixtureBuilder, outputBuilder }); + + var module = assembly.DefineDynamicModule(testAssemblyName); + var type = module.DefineType(TestClassName, TypeAttributes.Public); + type.DefineDefaultConstructor(MethodAttributes.Public); + + var method = type.DefineMethod(TestMethodName, MethodAttributes.Public); + var factConstructor = typeof(FactAttribute).GetConstructor(Array.Empty()); + var factBuilder = new CustomAttributeBuilder(factConstructor, Array.Empty()); + method.SetCustomAttribute(factBuilder); + + var generator = method.GetILGenerator(); + if (failTestCase) + { + // Assert.True(condition: false); + generator.Emit(OpCodes.Ldc_I4_0); + var trueInfo = typeof(Assert).GetMethod("True", new[] { typeof(bool) }); + generator.EmitCall(OpCodes.Call, trueInfo, optionalParameterTypes: null); + } + + generator.Emit(OpCodes.Ret); + type.CreateType(); + + return assembly; + } +}