/
VsExecutionSink.cs
306 lines (248 loc) · 12.9 KB
/
VsExecutionSink.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
using Xunit.Abstractions;
using VsTestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult;
using VsTestResultMessage = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResultMessage;
namespace Xunit.Runner.VisualStudio
{
public class VsExecutionSink : TestMessageSink, IExecutionSink, IDisposable
{
readonly Func<bool> cancelledThunk;
readonly ITestFrameworkExecutionOptions executionOptions;
readonly LoggerHelper logger;
readonly IMessageSinkWithTypes innerSink;
readonly ITestExecutionRecorder recorder;
readonly Dictionary<string, TestCase> testCasesMap;
public VsExecutionSink(IMessageSinkWithTypes innerSink,
ITestExecutionRecorder recorder,
LoggerHelper logger,
Dictionary<string, TestCase> testCasesMap,
ITestFrameworkExecutionOptions executionOptions,
Func<bool> cancelledThunk)
{
this.innerSink = innerSink;
this.recorder = recorder;
this.logger = logger;
this.testCasesMap = testCasesMap;
this.executionOptions = executionOptions;
this.cancelledThunk = cancelledThunk;
ExecutionSummary = new ExecutionSummary();
Diagnostics.ErrorMessageEvent += HandleErrorMessage;
Execution.TestAssemblyCleanupFailureEvent += HandleTestAssemblyCleanupFailure;
Execution.TestAssemblyFinishedEvent += HandleTestAssemblyFinished;
Execution.TestCaseCleanupFailureEvent += HandleTestCaseCleanupFailure;
Execution.TestCaseFinishedEvent += HandleTestCaseFinished;
Execution.TestCaseStartingEvent += HandleTestCaseStarting;
Execution.TestClassCleanupFailureEvent += HandleTestClassCleanupFailure;
Execution.TestCleanupFailureEvent += HandleTestCleanupFailure;
Execution.TestCollectionCleanupFailureEvent += HandleTestCollectionCleanupFailure;
Execution.TestFailedEvent += HandleTestFailed;
Execution.TestMethodCleanupFailureEvent += HandleTestMethodCleanupFailure;
Execution.TestPassedEvent += HandleTestPassed;
Execution.TestSkippedEvent += HandleTestSkipped;
}
public ExecutionSummary ExecutionSummary { get; private set; }
public ManualResetEvent Finished { get; } = new ManualResetEvent(initialState: false);
public override void Dispose()
{
((IDisposable)Finished).Dispose();
base.Dispose();
}
TestCase FindTestCase(ITestCase testCase)
{
if (testCasesMap.TryGetValue(testCase.UniqueID, out var result))
return result;
logger.LogError(testCase, "Result reported for unknown test case: {0}", testCase.DisplayName);
return null;
}
private void TryAndReport(string actionDescription, ITestCase testCase, Action action)
{
try
{
action();
}
catch (Exception ex)
{
logger.LogError(testCase, "Error occured while {0} for test case {1}: {2}", actionDescription, testCase.DisplayName, ex);
}
}
void HandleCancellation(MessageHandlerArgs args)
{
if (cancelledThunk())
args.Stop();
}
void HandleErrorMessage(MessageHandlerArgs<IErrorMessage> args)
{
ExecutionSummary.Errors++;
logger.LogError("Catastrophic failure: {0}", ExceptionUtility.CombineMessages(args.Message));
HandleCancellation(args);
}
void HandleTestAssemblyFinished(MessageHandlerArgs<ITestAssemblyFinished> args)
{
var assemblyFinished = args.Message;
ExecutionSummary.Failed = assemblyFinished.TestsFailed;
ExecutionSummary.Skipped = assemblyFinished.TestsSkipped;
ExecutionSummary.Time = assemblyFinished.ExecutionTime;
ExecutionSummary.Total = assemblyFinished.TestsRun;
Finished.Set();
HandleCancellation(args);
}
void HandleTestFailed(MessageHandlerArgs<ITestFailed> args)
{
var testFailed = args.Message;
var result = MakeVsTestResult(TestOutcome.Failed, testFailed);
if (result != null)
{
result.ErrorMessage = ExceptionUtility.CombineMessages(testFailed);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
TryAndReport("RecordResult (Fail)", testFailed.TestCase, () => recorder.RecordResult(result));
}
else
logger.LogWarning(testFailed.TestCase, "(Fail) Could not find VS test case for {0} (ID = {1})", testFailed.TestCase.DisplayName, testFailed.TestCase.UniqueID);
HandleCancellation(args);
}
void HandleTestPassed(MessageHandlerArgs<ITestPassed> args)
{
var testPassed = args.Message;
var result = MakeVsTestResult(TestOutcome.Passed, testPassed);
if (result != null)
TryAndReport("RecordResult (Pass)", testPassed.TestCase, () => recorder.RecordResult(result));
else
logger.LogWarning(testPassed.TestCase, "(Pass) Could not find VS test case for {0} (ID = {1})", testPassed.TestCase.DisplayName, testPassed.TestCase.UniqueID);
HandleCancellation(args);
}
void HandleTestSkipped(MessageHandlerArgs<ITestSkipped> args)
{
var testSkipped = args.Message;
var result = MakeVsTestResult(TestOutcome.Skipped, testSkipped);
if (result != null)
TryAndReport("RecordResult (Skip)", testSkipped.TestCase, () => recorder.RecordResult(result));
else
logger.LogWarning(testSkipped.TestCase, "(Skip) Could not find VS test case for {0} (ID = {1})", testSkipped.TestCase.DisplayName, testSkipped.TestCase.UniqueID);
HandleCancellation(args);
}
void HandleTestCaseStarting(MessageHandlerArgs<ITestCaseStarting> args)
{
var testCaseStarting = args.Message;
var vsTestCase = FindTestCase(testCaseStarting.TestCase);
if (vsTestCase != null)
TryAndReport("RecordStart", testCaseStarting.TestCase, () => recorder.RecordStart(vsTestCase));
else
logger.LogWarning(testCaseStarting.TestCase, "(Starting) Could not find VS test case for {0} (ID = {1})", testCaseStarting.TestCase.DisplayName, testCaseStarting.TestCase.UniqueID);
HandleCancellation(args);
}
void HandleTestCaseFinished(MessageHandlerArgs<ITestCaseFinished> args)
{
var testCaseFinished = args.Message;
var vsTestCase = FindTestCase(testCaseFinished.TestCase);
if (vsTestCase != null)
TryAndReport("RecordEnd", testCaseFinished.TestCase, () => recorder.RecordEnd(vsTestCase, GetAggregatedTestOutcome(testCaseFinished)));
else
logger.LogWarning(testCaseFinished.TestCase, "(Finished) Could not find VS test case for {0} (ID = {1})", testCaseFinished.TestCase.DisplayName, testCaseFinished.TestCase.UniqueID);
HandleCancellation(args);
}
void HandleTestAssemblyCleanupFailure(MessageHandlerArgs<ITestAssemblyCleanupFailure> args)
{
ExecutionSummary.Errors++;
var cleanupFailure = args.Message;
WriteError($"Test Assembly Cleanup Failure ({cleanupFailure.TestAssembly.Assembly.AssemblyPath})", cleanupFailure, cleanupFailure.TestCases);
HandleCancellation(args);
}
void HandleTestCaseCleanupFailure(MessageHandlerArgs<ITestCaseCleanupFailure> args)
{
ExecutionSummary.Errors++;
var cleanupFailure = args.Message;
WriteError($"Test Case Cleanup Failure ({cleanupFailure.TestCase.DisplayName})", cleanupFailure, cleanupFailure.TestCases);
HandleCancellation(args);
}
void HandleTestClassCleanupFailure(MessageHandlerArgs<ITestClassCleanupFailure> args)
{
ExecutionSummary.Errors++;
var cleanupFailure = args.Message;
WriteError($"Test Class Cleanup Failure ({cleanupFailure.TestClass.Class.Name})", cleanupFailure, cleanupFailure.TestCases);
HandleCancellation(args);
}
void HandleTestCollectionCleanupFailure(MessageHandlerArgs<ITestCollectionCleanupFailure> args)
{
ExecutionSummary.Errors++;
var cleanupFailure = args.Message;
WriteError($"Test Collection Cleanup Failure ({cleanupFailure.TestCollection.DisplayName})", cleanupFailure, cleanupFailure.TestCases);
HandleCancellation(args);
}
void HandleTestCleanupFailure(MessageHandlerArgs<ITestCleanupFailure> args)
{
ExecutionSummary.Errors++;
var cleanupFailure = args.Message;
WriteError($"Test Cleanup Failure ({cleanupFailure.Test.DisplayName})", cleanupFailure, cleanupFailure.TestCases);
HandleCancellation(args);
}
void HandleTestMethodCleanupFailure(MessageHandlerArgs<ITestMethodCleanupFailure> args)
{
ExecutionSummary.Errors++;
var cleanupFailure = args.Message;
WriteError($"Test Method Cleanup Failure ({cleanupFailure.TestMethod.Method.Name})", cleanupFailure, cleanupFailure.TestCases);
HandleCancellation(args);
}
void WriteError(string failureName, IFailureInformation failureInfo, IEnumerable<ITestCase> testCases)
{
foreach (var testCase in testCases)
{
var result = MakeVsTestResult(TestOutcome.Failed, testCase, testCase.DisplayName);
if (result != null)
{
result.ErrorMessage = $"[{failureName}]: {ExceptionUtility.CombineMessages(failureInfo)}";
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(failureInfo);
TryAndReport("RecordEnd (Failure)", testCase, () => recorder.RecordEnd(result.TestCase, result.Outcome));
TryAndReport("RecordResult (Failure)", testCase, () => recorder.RecordResult(result));
}
else
logger.LogWarning(testCase, "(Failure) Could not find VS test case for {0} (ID = {1})", testCase.DisplayName, testCase.UniqueID);
}
}
VsTestResult MakeVsTestResult(TestOutcome outcome, ITestResultMessage testResult)
=> MakeVsTestResult(outcome, testResult.TestCase, testResult.Test.DisplayName, (double)testResult.ExecutionTime, testResult.Output);
VsTestResult MakeVsTestResult(TestOutcome outcome, ITestSkipped skippedResult)
=> MakeVsTestResult(outcome, skippedResult.TestCase, skippedResult.Test.DisplayName, (double)skippedResult.ExecutionTime, skippedResult.Reason);
VsTestResult MakeVsTestResult(TestOutcome outcome, ITestCase testCase, string displayName, double executionTime = 0.0, string output = null)
{
var vsTestCase = FindTestCase(testCase);
if (vsTestCase == null)
return null;
var result = new VsTestResult(vsTestCase)
{
#if !WINDOWS_UAP
ComputerName = Environment.MachineName,
#endif
DisplayName = displayName,
Duration = TimeSpan.FromSeconds(executionTime),
Outcome = outcome,
};
// Work around VS considering a test "not run" when the duration is 0
if (result.Duration.TotalMilliseconds == 0)
result.Duration = TimeSpan.FromMilliseconds(1);
if (!string.IsNullOrEmpty(output))
result.Messages.Add(new VsTestResultMessage(VsTestResultMessage.StandardOutCategory, output));
return result;
}
TestOutcome GetAggregatedTestOutcome(ITestCaseFinished testCaseFinished)
{
if (testCaseFinished.TestsRun == 0)
return TestOutcome.NotFound;
else if (testCaseFinished.TestsFailed > 0)
return TestOutcome.Failed;
else if (testCaseFinished.TestsSkipped > 0)
return TestOutcome.Skipped;
else
return TestOutcome.Passed;
}
public override bool OnMessageWithTypes(IMessageSinkMessage message, HashSet<string> messageTypes)
{
var result = innerSink.OnMessageWithTypes(message, messageTypes);
return base.OnMessageWithTypes(message, messageTypes) && result;
}
}
}