-
Notifications
You must be signed in to change notification settings - Fork 0
/
TestManager.cs
268 lines (238 loc) · 10.4 KB
/
TestManager.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
namespace Arktos.WinBert.Testing
{
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using AppDomainToolkit;
using Arktos.WinBert.Analysis;
using Arktos.WinBert.Differencing;
using Arktos.WinBert.Xml;
using System.IO;
/// <summary>
/// This base class contains some basic implementations that TestManager derived classes will find useful along with
/// the WinBert pipeline stack.
/// </summary>
/// <remarks>
/// To implement a functional test manager, you must override two specific methods: GenerateTests and RunTests. You may choose
/// to implement InstrumentTests if you wish, but an implementation is provided that performs no instrumentation by default. The
/// default implementation of the ExecuteStack method will run the pipeline in paralell for both the current and previous builds that
/// are supplied to the Run method.
/// </remarks>
public abstract class TestManager : ITestManager
{
#region Fields & Constants
private readonly IBehavioralAnalyzer defaultAnalyzer;
#endregion
#region Constructors & Destructors
/// <summary>
/// Initializes a new instance of the TestManager class.
/// </summary>
/// <param name="config">
/// The configuration.
/// </param>
public TestManager(WinBertConfig config)
{
if (config == null)
{
throw new ArgumentNullException("config");
}
this.Config = config;
this.defaultAnalyzer = new BertBehavioralAnalyzer();
}
#endregion
#region Properties
/// <summary>
/// Gets the currently loaded configuration object.
/// </summary>
public WinBertConfig Config { get; private set; }
#endregion
#region Public Methods
/// <summary>
/// This method should generate tests for the target assembly and return a target to the assembly containing said tests.
/// </summary>
/// <remarks>
/// No default implementation can exists for this method, so sub-classes must override it and create the test assemblies.
/// </remarks>
/// <param name="target">
/// The target to generate tests for.
/// </param>
/// <param name="validTypeNames">
/// A list of valid type names. If applicable, tests should only be generated for these types.
/// </param>
/// <returns>
/// An ITestTarget implementation pointing to the assembly containing the generated tests along with the original target assembly.
/// </returns>
public abstract IAssemblyTarget GenerateTests(IAssemblyTarget target, IEnumerable<string> validTypeNames);
/// <summary>
/// Instruments the target assembly and it's tests. Overrides may choose to instrument the tests, the original assembly or
/// both if desired.
/// </summary>
/// <remarks>
/// The default implementation of this method performs no instrumentation, that is it simply returns the passed in test target.
/// </remarks>
/// <param name="target">
/// The target assembly.
/// </param>
/// <param name="tests">
/// The tests to instrument.
/// </param>
/// <returns>
/// A new test target replacing the original assemblies with instrumented ones.
/// </returns>
public abstract ITestTarget InstrumentTests(ITestTarget toInstrument);
/// <summary>
/// Executes tests.
/// </summary>
/// <remarks>
/// No default implementation can exist for this method as this class will have no idea as to what the generated
/// tests will look like.
/// </remarks>
/// <param name="target">
/// The target assembly to execute the tests against.
/// </param>
/// <param name="tests">
/// The assembly target containing the tests to execute and the corresponding assembly under test.
/// </param>
/// <returns>
/// A test run result implementation.
/// </returns>
public abstract ITestRunResult RunTests(IAssemblyTarget target, IAssemblyTarget tests);
/// <inheritdoc />
/// <remarks>
/// This default implementation will perform a diff against the two builds using default differencing mechanisms
/// and then execute the WinBert stack on the difference result if a different is detected. You may override this method
/// if needed, but only in rare occasions when you need a hook into the differencing pipeline should this be the case.
/// </remarks>
public AnalysisResult Run(Build previous, Build current)
{
AnalysisResult result = null;
var diff = this.Diff(previous, current);
if (diff != null && diff.AreDifferences)
{
result = this.ExecuteStack(diff);
}
return result;
}
/// <summary>
/// Executes the entire WinBert stack.
/// </summary>
/// <remarks>
/// Override this mehod for complete control over the way the TestManager executes each phase of the WinBert pipeline.
/// Be aware that this method invokes test generation, instrumentation, and execution in a paralell fashion.
/// </remarks>
/// <param name="diff">
/// The difference result.
/// </param>
/// <returns>
/// An AnalysisResult.
/// </returns>
protected virtual AnalysisResult ExecuteStack(IAssemblyDifference diff)
{
if (diff == null)
{
throw new ArgumentNullException("diff");
}
// Invoke two versions of the stack simultaneously
ITestRunResult oldAssemblyResults = null, newAssemblyResults = null;
var typeNames = diff.TypeDifferences.Select(x => x.Name);
var tests = this.GenerateTests(diff.NewAssemblyTarget, typeNames);
var instrumented = this.InstrumentTests(TestTarget.Create(diff.OldAssemblyTarget, diff.NewAssemblyTarget, tests));
// Execute tests in parallel.
Parallel.Invoke(
() =>
{
newAssemblyResults = this.RunTests(instrumented.TargetNewAssembly, instrumented.TestAssembly);
},
() =>
{
oldAssemblyResults = this.RunTests(instrumented.TargetOldAssembly, instrumented.TestAssembly);
});
// Perform analysis and we're done.
return this.Analyze(diff, oldAssemblyResults, newAssemblyResults);
}
/// <summary>
/// Performs analysis on the target test run results. Override this if you need to do anything special while generating
/// the analysis restuls or if you use a specialized representation during instrumentation else just use the default
/// implementation. It will use a standard BERT analyzer to load up the analysis results log files and perform a regression
/// test analysis.
/// </summary>
/// <param name="diff">
/// The difference between assemblies.
/// </param>
/// <param name="previousResults">
/// The test results for the previous build.
/// </param>
/// <param name="currentResults">
/// The test results for the current build.
/// </param>
/// <returns>
/// An analysis result.
/// </returns>
protected virtual AnalysisResult Analyze(IAssemblyDifference diff, ITestRunResult previousResults, ITestRunResult currentResults)
{
if (diff == null)
{
throw new ArgumentNullException("diff");
}
if (previousResults == null)
{
throw new ArgumentNullException("previousResults");
}
if (currentResults == null)
{
throw new ArgumentNullException("currentResults");
}
return this.defaultAnalyzer.Analyze(diff, previousResults, currentResults);
}
/// <summary>
/// Performs a diff in another application domain. The method by which this diff occurs should likely
/// never change, so the method is considered sealed. Derived classes should always call this to get
/// the differences between the two assemblies in question.
/// </summary>
/// <param name="previous">
/// The previous build.
/// </param>
/// <param name="current">
/// The current build.
/// </param>
/// <returns>
/// The assembly difference result.
/// </returns>
protected IAssemblyDifference Diff(Build previous, Build current)
{
if (previous == null)
{
throw new ArgumentNullException("previous");
}
if (current == null)
{
throw new ArgumentNullException("current");
}
// Boot up another application domain.
using (var diffEnv = AppDomainContext.Create())
{
// Pull in paths of the targets in case of dependencies.
diffEnv.RemoteResolver.AddProbePaths(
Path.GetDirectoryName(previous.AssemblyPath),
Path.GetDirectoryName(current.AssemblyPath));
// Execute the diff in another application domain.
return RemoteFunc.Invoke(
diffEnv.Domain,
this.Config.DiffIgnoreList,
previous.AssemblyPath,
current.AssemblyPath,
(ignoreTargets, previousTargetPath, currentTargetPath) =>
{
// Fire up an assembly loader.
var loader = new AssemblyLoader();
var oldAssembly = loader.LoadAssembly(LoadMethod.LoadFile, previousTargetPath);
var newAssembly = loader.LoadAssembly(LoadMethod.LoadFile, currentTargetPath);
var differ = new AssemblyDiffer(ignoreTargets);
return differ.Diff(oldAssembly, newAssembly);
});
}
}
#endregion
}
}