/
Run.cs
484 lines (455 loc) · 22.4 KB
/
Run.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
#region Copyright & licence
// This file is part of NinjaTurtles.
//
// NinjaTurtles is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// NinjaTurtles is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with NinjaTurtles. If not, see <http://www.gnu.org/licenses/>.
//
// Copyright (C) 2012-14 David Musgrove and others.
#endregion
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Security.Principal;
using System.Xml;
using System.Xml.Xsl;
using Mono.Cecil;
using NinjaTurtlesMutation.Console.Options;
using NinjaTurtlesMutation.Console.Reporting;
namespace NinjaTurtlesMutation.Console.Commands
{
internal class Run : Command
{
private string _testAssemblyLocation;
private string _message;
private IDictionary<string, string> _testMethods;
private readonly MutationTestingReportSummary _report = new MutationTestingReportSummary();
private Output _outputOption;
private readonly TextWriter _originalOut = System.Console.Out;
private string _outputPath;
protected override string HelpText
{
get { return @"usage: ntm run [<options>] TEST_ASSEMBLY
Runs all mutation tests found in the specified test assembly.
Options:
--class [-c] : Specifies the full type name of the class for which
mutation testing should be applied. If no accompanying
method name is specified, all methods in the class are
identified, and mutation testing applied for each of
them.
--detach-bench : Run the tests benchmarks in a detached process, reset
between each benchmark.
--format [-f] : Specifies the format for the output file specified by
the --output option. By default, this will be XML, but
HTML can be specified here to transform the results
into displayable format.
--killtime-factor : Set, compared to the benchmark, how much time a runner
will wait before marking a mutation killed. Default at 2
--max-busy : Define how many test runners can run simultaneously.
--namespace [-N] : Specifies the namespace class for which mutation testing
should be applied. All classes and method under that
namespace are identified, and mutation testing is applied
for each of them.
--no-pretest : Skip the first test pass. If some tests are faulty, the
number of dead mutant may be superior than normal.
--onetime-run : Close the tests runners after each run. Use this if you
need to be sure each tests run are made in a new process.
--output [-o] : Specifies the name of a file to receive the mutation
testing output. This file will be deleted if it already
exists.
--parallelization [-p] : Set the number of test runner to use. Default value is 8.
--success-score : Set the minimum mutation score at which the test pass
--turtle-types [-T] KEYS : Set the kind of mutation operators to apply.
A = Arithmetic operator rotation
T = Bitwise operator rotation
B = Branch condition rotation
C = Conditional boundary switch
S = Sequence point deletion
R = Variable read substitution
W = Variable write substitution
Arguments:
TEST_ASSEMBLY : The file name of the test assembly to inspect, which
should be in the current working directory.
Example:
ntm run -N NinjaTurtlesMutation
-o NinjaTurtlesMutation.html -f HTML
NinjaTurtlesMutation.Tests.dll
This command will identify all classes and methods under NinjaTurtlesMutation
namespace, and use unit tests from NinjaTurtlesMutation.Tests.dll to perform
mutation testing of these methods. The resulting output will be transformed
to HTML and saved to the file NinjaTurtlesMutation.html.";
}
}
public override bool Validate()
{
if (!base.Validate())
{
return false;
}
if (Options.Arguments.Count == 0)
{
using (new OutputWriterErrorHighlight())
{
OutputWriter.WriteLine(
OutputVerbosity.Quiet,
@"The 'run' command must take the path to a test assembly as an argument.");
}
return false;
}
_testAssemblyLocation = Path.Combine(Environment.CurrentDirectory, Options.Arguments[0]);
if (!File.Exists(_testAssemblyLocation))
{
using (new OutputWriterErrorHighlight())
{
OutputWriter.WriteLine(
OutputVerbosity.Quiet,
@"Test assembly '{0}' does not exist in the current working directory.",
Options.Arguments[0]);
return false;
}
}
_testMethods = TestUtils.NUnit.GetTestsNameDictionnaryTranslation(AssemblyDefinition.ReadAssembly(_testAssemblyLocation));
if (_testMethods.Count == 0)
{
using (new OutputWriterErrorHighlight())
{
OutputWriter.WriteLine(
OutputVerbosity.Quiet,
@"Not test found in the assembly {0}", _testAssemblyLocation);
return false;
}
}
var isAdmin = new WindowsPrincipal(WindowsIdentity.GetCurrent()).IsInRole(WindowsBuiltInRole.Administrator);
if (!isAdmin)
{
using (new OutputWriterErrorHighlight())
{
OutputWriter.WriteLine(
OutputVerbosity.Quiet,
@"The 'run' command need admin privilege to create symlink.");
}
return false;
}
if (Options.Options.Any(o => o is NoPreTest))
return true;
var benchmark = new TestsBenchmark(_testAssemblyLocation);
System.Console.Write("Checking if tests pass...");
benchmark.LaunchBenchmark(Options.Options.Any(o => o is DetachBench));
var originalSourcesPassTests = benchmark.TestsPass;
if (!originalSourcesPassTests)
{
System.Console.WriteLine("FAIL");
using (new OutputWriterErrorHighlight())
{
OutputWriter.WriteLine(
OutputVerbosity.Quiet,
@"The original source code tests fail. All your tests must pass in order to start mutation testing.");
}
return false;
}
System.Console.WriteLine("OK");
return true;
}
public override bool Execute()
{
using (var stream = new MemoryStream())
using (var writer = new StreamWriter(stream))
{
ConfigureOutput(writer);
var runnerMethod = ConfigureRun();
var successThreshold = Options.Options.OfType<SuccessThreshold>().SingleOrDefault();
var successThresholdValue = successThreshold != null ? successThreshold.MinScore : 1;
var parallelLevel = Options.Options.OfType<ParallelLevel>().SingleOrDefault();
var parallelValue = parallelLevel != null ? parallelLevel.ParallelValue : 8;
var maxBusyRunners = Options.Options.OfType<MaxBusyRunner>().SingleOrDefault();
var maxBusyRunnersValue = maxBusyRunners != null ? maxBusyRunners.MaxBusyRunnersValue : parallelValue;
var oneTimeRunners = Options.Options.Any(o => o is OneTimeRunners);
var killTimeFactor = Options.Options.OfType<KillTimeFactor>().SingleOrDefault();
var killTimeFactorValue = killTimeFactor != null ? killTimeFactor.Factor : 2;
using (var dispatcher = new TestsDispatcher(parallelValue, maxBusyRunnersValue, oneTimeRunners, killTimeFactorValue))
runnerMethod(dispatcher);
RestoreOutput();
var score = _report.GetMutationScore();
var result = score >= successThresholdValue;
ReportResult(result, _report);
return result;
}
}
private bool RunMutationTestsForClass(TestsDispatcher dispatcher)
{
string targetClass = Options.Options.OfType<TargetClass>().Single().ClassName;
var testAssembly = Assembly.LoadFrom(_testAssemblyLocation);
var matchedType = TypeResolver.ResolveTypeFromReferences(testAssembly, targetClass);
if (matchedType == null)
{
_message = String.Format(@"Unknown type '{0}'", targetClass);
return false;
}
var assemblyDefinition = AssemblyDefinition.ReadAssembly(matchedType.Assembly.Location);
var targetType = assemblyDefinition.MainModule.Types.FirstOrDefault(t => t.FullName == targetClass);
bool result = true;
foreach (var methodInfo in targetType.Methods
.Where(m => m.HasBody && m.Name != Methods.STATIC_CONSTRUCTOR))
{
string targetMethod = methodInfo.Name;
string methodReturnType = methodInfo.ReturnType.FullName;
var methodsGenerics = methodInfo.GenericParameters.ToArray();
var parameterTypes = methodInfo.Parameters.Select(p => p.ParameterType).ToArray();
bool runResultBuf = RunTests(matchedType.Assembly.Location, targetClass, methodReturnType, targetMethod, methodsGenerics, parameterTypes, dispatcher);
result &= runResultBuf;
}
if (String.IsNullOrEmpty(_message))
{
_message = String.Format(
@"Mutation testing {0}",
result ? "passed" : "failed");
}
return result;
}
private bool RunMutationTestsForClassAndMethod(TestsDispatcher dispatcher)
{
string targetClass = Options.Options.OfType<TargetClass>().Single().ClassName;
var testAssembly = Assembly.LoadFrom(_testAssemblyLocation);
var matchedType = TypeResolver.ResolveTypeFromReferences(testAssembly, targetClass);
if (matchedType == null)
{
_message = String.Format(@"Unknown type '{0}'", targetClass);
return false;
}
string targetMethod = Options.Options.OfType<TargetMethod>().Single().MethodName;
var typeOptions = Options.Options.OfType<ParameterType>().Select(p => p.ResolvedType).ToArray();
var result =
typeOptions.Any()
? RunTests(matchedType.Assembly.Location, targetClass, targetMethod, dispatcher, typeOptions)
: RunTests(matchedType.Assembly.Location, targetClass, targetMethod, dispatcher);
if (String.IsNullOrEmpty(_message))
{
_message = String.Format(
@"Mutation testing {0}",
result ? "passed" : "failed");
}
return result;
}
private bool RunMutationTestsForAllClassAndMethods(TestsDispatcher dispatcher)
{
bool result = true;
var nspace = Options.Options.OfType<TargetNamespace>().Single().NamespaceName;
var testAssembly = Assembly.LoadFrom(_testAssemblyLocation);
var matchedTypes = TypeResolver.ResolveNamespaceTypesFromReferences(testAssembly, nspace);
System.Console.WriteLine("testassembly : [{0}], matched types : [[{1}]]", testAssembly, String.Join("], [", matchedTypes.Select(t => t.FullName))); ////////////////
System.Console.WriteLine("{0} types matched under {1}", matchedTypes.Length, nspace); ////////////////////
if (matchedTypes.Length == 0)
{
_message = String.Format(@"No types found under {0}", nspace);
return false;
}
foreach (var type in matchedTypes)
{
var resultBuf = RunMutationTestsForType(type, type.FullName, dispatcher);
result &= resultBuf;
}
if (!String.IsNullOrEmpty(_message))
return result;
_message = String.Format(@"Mutation testing {0}", result ? "passed" : "failed");
return result;
}
private bool RunMutationTestsForType(Type type, string targetClass, TestsDispatcher dispatcher)
{
bool result = true;
var assemblyDefinition = AssemblyDefinition.ReadAssembly(type.Assembly.Location);
var targetType = assemblyDefinition.MainModule.Types.FirstOrDefault(t => t.FullName == targetClass);
foreach (var methodInfo in targetType.Methods.Where(m => m.HasBody && m.Name != Methods.STATIC_CONSTRUCTOR))
{
string targetMethod = methodInfo.Name;
string methodReturnType = methodInfo.ReturnType.FullName;
var methodsGenerics = methodInfo.GenericParameters.ToArray();
var parameterTypes = methodInfo.Parameters.Select(p => p.ParameterType).ToArray();
bool runResultBuf = RunTests(type.Assembly.Location, targetClass, methodReturnType, targetMethod, methodsGenerics, parameterTypes, dispatcher);
result &= runResultBuf;
}
return result;
}
private bool RunTests(string targetAssemblyLocation, string targetClass, string returnType, string targetMethod, GenericParameter[] methodGenerics, TypeReference[] parameterTypes, TestsDispatcher dispatcher)
{
var parameterList = parameterTypes == null || parameterTypes.Length == 0
? null
: String.Join(", ", parameterTypes.Select(t => t.Name).ToArray());
OutputMethod(targetClass, targetMethod, parameterList);
MutationTest mutationTest =
parameterTypes == null
? (MutationTest)MutationTestBuilder.For(targetAssemblyLocation, targetClass, returnType, targetMethod, methodGenerics, dispatcher, _testMethods)
: (MutationTest)MutationTestBuilder.For(targetAssemblyLocation, targetClass, returnType, targetMethod, methodGenerics, dispatcher, _testMethods,parameterTypes);
mutationTest.TestAssemblyLocation = _testAssemblyLocation;
var result = BuildAndRunMutationTest(mutationTest);
return result;
}
private bool RunTests(string targetAssemblyLocation, string targetClass, string targetMethod, TestsDispatcher dispatcher, Type[] parameterTypes = null)
{
var parameterList = parameterTypes == null || parameterTypes.Length == 0
? null
: String.Join(", ", parameterTypes.Select(t => t.Name).ToArray());
OutputMethod(targetClass, targetMethod, parameterList);
MutationTest mutationTest =
parameterTypes == null
? (MutationTest)MutationTestBuilder.For(targetAssemblyLocation, targetClass, targetMethod, dispatcher, _testMethods)
: (MutationTest)MutationTestBuilder.For(targetAssemblyLocation, targetClass, targetMethod, dispatcher, _testMethods,parameterTypes);
mutationTest.TestAssemblyLocation = _testAssemblyLocation;
var result = BuildAndRunMutationTest(mutationTest);
return result;
}
private bool BuildAndRunMutationTest(MutationTest mutationTest)
{
var outputOption = Options.Options.OfType<Output>().FirstOrDefault();
if (outputOption != null)
{
string outputPath = Path.Combine(Environment.CurrentDirectory, outputOption.FileName);
mutationTest.MergeReportTo(outputPath);
}
var turtlesOption = Options.Options.OfType<TurtlesTypes>().FirstOrDefault();
if (turtlesOption != null)
mutationTest.With(turtlesOption.Types);
bool result = false;
try
{
mutationTest.Run(Options.Options.Any(o => o is DetachBench));
result = true;
}
catch (MutationTestFailureException)
{
}
catch (Exception ex)
{
System.Console.Error.WriteLine("An error occur during mutation test of {0}:\n{1}\nThe corresponding report was not merged", mutationTest.TargetMethod, ex);
return false;
}
_report.MergeMutationTestReport(mutationTest.Report);
return result;
}
private Func<TestsDispatcher, bool> ConfigureRun()
{
if (Options.Options.Any(o => o is TargetNamespace))
return RunMutationTestsForAllClassAndMethods;
if (Options.Options.Any(o => o is TargetClass) && Options.Options.Any(o => o is TargetMethod))
return RunMutationTestsForClassAndMethod;
return RunMutationTestsForClass;
}
private void ConfigureOutput(StreamWriter sinkStream)
{
_outputPath = "";
_outputOption = Options.Options.OfType<Output>().FirstOrDefault();
if (_outputOption != null)
{
_outputPath = Path.Combine(Environment.CurrentDirectory, _outputOption.FileName);
if (File.Exists(_outputPath)) File.Delete(_outputPath);
}
if (!Options.Options.Any(o => o is Verbose))
{
System.Console.SetOut(sinkStream);
}
}
private void RestoreOutput()
{
if (!Options.Options.Any(o => o is Verbose))
{
System.Console.SetOut(_originalOut);
}
}
private void ReportResult(bool result, MutationTestingReportSummary reportSummary = null)
{
var formatOption = Options.Options.OfType<Format>().FirstOrDefault();
if (_outputOption != null && formatOption != null && formatOption.OutputFormat == "HTML")
{
FormatOutput(_outputPath);
}
OutputWriter.WriteLine();
var statusColor = result
? ConsoleColor.Green
: ConsoleColor.Red;
using (new OutputWriterHighlight(statusColor))
{
if (reportSummary != null)
OutputWriter.WriteLine(reportSummary.ToString());
OutputWriter.WriteLine(_message);
}
}
private static void FormatOutput(string outputPath)
{
if (!File.Exists(outputPath)) return;
string tempPath = Path.GetTempFileName();
File.Delete(tempPath);
File.Move(outputPath, tempPath);
var xslt = new XslCompiledTransform();
var resolver = new EmbeddedResourceResolver();
xslt.Load("ReportXslt.xslt", XsltSettings.TrustedXslt, resolver);
using (var reader = XmlReader.Create(tempPath))
{
xslt.Transform(reader, XmlWriter.Create(outputPath));
}
File.Delete(tempPath);
}
private static void OutputMethod(string targetClass, string targetMethod, string parameterList)
{
if (String.IsNullOrEmpty(parameterList))
{
OutputWriter.WriteLine(
@"Running mutation tests for {0}.{1}",
targetClass,
targetMethod);
}
else
{
OutputWriter.WriteLine(
@"Running mutation tests for {0}.{1}({2})",
targetClass,
targetMethod,
parameterList);
}
}
private bool RunAllMutationTestsInAssembly()
{
var testAssembly = Assembly.LoadFrom(_testAssemblyLocation);
int tests = 0;
int failures = 0;
foreach (var type in testAssembly.GetTypes())
{
if (!type.IsPublic) continue;
var mutationTests = type.GetMethods()
.Where(m => m.GetCustomAttributes(true).Any(a => a.GetType() == typeof(MutationTestAttribute)));
if (mutationTests.Any())
{
var testFixtureInstance = Activator.CreateInstance(type);
foreach (var mutationTest in mutationTests)
{
tests++;
try
{
mutationTest.Invoke(testFixtureInstance, null);
}
catch (MutationTestFailureException)
{
failures++;
}
OutputWriter.Write(".");
}
}
}
OutputWriter.WriteLine();
_message = String.Format(
@"Mutation test summary: {0} passed, {1} failed",
tests - failures,
failures);
return tests > 0 && failures == 0;
}
}
}