-
Notifications
You must be signed in to change notification settings - Fork 2
/
Runner.scala
1562 lines (1459 loc) · 84.4 KB
/
Runner.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* Copyright 2001-2013 Artima, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.scalatest.tools
import org.scalatest._
import java.net.URL
import java.net.MalformedURLException
import java.net.URLClassLoader
import java.io.File
import java.io.IOException
import javax.swing.SwingUtilities
import java.util.concurrent.ArrayBlockingQueue
import java.util.regex.Pattern
import java.util.concurrent.Semaphore
import org.scalatest.events._
import java.util.concurrent.Executors
import java.util.concurrent.ExecutorService
import java.util.concurrent.ThreadFactory
import SuiteDiscoveryHelper._
import org.scalatest.time.Span
import org.scalatest.time.Seconds
import org.scalatest.time.Millis
import java.util.concurrent.atomic.AtomicInteger
import Suite.{mergeMap, CHOSEN_STYLES, SELECTED_TAG}
import ArgsParser._
import org.scalactic.Requirements._
import org.scalatest.prop.Seed
/*
Command line args:
a - archive count for dashboard reporter --archive
A - select previously failed and/or canceled tests to rerun --again
b - run a testNG test (b for Beust) --testng
B -
c - parallel execution (--parallel) --parallel (deprecated, will be custom reporter)
C - custom reporter (temporarily)
d - dashboard reporter --dashboard
D - config map pair -D
e - standard error reporter (--stderr-reporter) --stderr
E -
f - file reporter --file
F - Span scale factor
g - graphical reporter --graphical
G -
h - HTML Reporter --html
H -
i - this one is used for the Suite ID --suiteId
I -
j - currently JUnit directly (can drop and use WrapWith) --junit
J -
k - socket reporter XML
K - socket reporter binary
l - tags to exclude --exclude
L -
m - members only path --members
M - memory reporter --memory (records failed tests in a file, so they can be rerun with A)
n - tags to include --include
N -
o - standard out reporter
O -
p - space-separated runpath (currently deprecated, will be parallel execution)
P - parallel execution (temporarily)
q - Suffix? -q Spec will only look at class files whose name ends in Spec
Q - equalivalent to -q Suite -q Spec
r - custom reporter (currently deprecated, will be runpath)
R - space-separated runpath (temporarily)
s - suite class name (to become a glob)
S - initial seed for Randomizer.default
t - test name
T - sorting timeout --sorting-timeout
u - JUnit XML reporter
U -
v - ScalaTest version number (also -version and --version)
V -
w - wildcard path
W - slowpoke detector (with two integral args, for timeout (delay) and interval (period), in seconds)
x - save for ScalaTest native XML
X -
y - sets org.scalatest.chosenstyle -y FunSpec or -y "FunSpec FunSuite" DEACTIVATED IN 3.1.0
-Y for after -h to set a custom style sheet
z - test name wildcard
Z -
StringReporter configuration params:
A - drop AlertProvided events
B - drop NoteProvided events
C - drop TestSucceeded events
D - show all durations
E - drop TestPending events
F - show full stack traces
*G - reminder with full stack traces
H - drop SuiteStarting events
*I - Reminder without stack traces
J
*K - exclude TestCanceled events from reminder
L - drop SuiteCompleted events
M - drop MarkupProvided events
N - drop TestStarting events
O - drop InfoProvided events
P - drop ScopeOpened events
Q - drop ScopeClosed events
R - drop ScopePending events
S - show short stack traces
*T - reminder with short stack traces
U - unformatted mode
V
W - without color
X - drop TestIgnored events
Z
*/
private[tools] case class SuiteConfig(suite: Suite, dynaTags: DynaTags, requireSelectedTag: Boolean, excludeNestedSuites: Boolean)
private[scalatest] case class ConcurrentConfig(numThreads: Int, enableSuiteSortingReporter: Boolean)
private[tools] case class SlowpokeConfig(delayInMillis: Long, periodInMillis: Long)
/**
* Application that runs a suite of tests.
*
* <p>
* Note: this application offers the full range of ScalaTest features via command line arguments described below. If you just want
* to run a suite of tests from the command line and see results on the standard output, you may prefer to use <a href="../run$.html">ScalaTest's simple runner</a>.
* </p>
*
* <p>
* The basic form of a <code>Runner</code> invocation is:
* </p>
*
* <pre class="stExamples">
* scala [-cp scalatest-<version>.jar:...] org.scalatest.tools.Runner [arguments]
* </pre>
*
* <p>
* The arguments <code>Runner</code> accepts are described in the following table:
* </p>
*
* <table style="border-collapse: collapse; border: 1px solid black">
* <tr><th style="background-color: #CCCCCC; border-width: 1px; padding: 3px; text-align: center; border: 1px solid black">argument</th><th style="background-color: #CCCCCC; border-width: 1px; padding: 3px; text-align: center; border: 1px solid black">description</th><th style="background-color: #CCCCCC; border-width: 1px; padding: 3px; text-align: center; border: 1px solid black">example</th></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-D<em>key</em>=<em>value</em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">defines a key/value pair for the <a href="#configMapSection"><em>config map</em></a></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-DmaxConnections=100</code></a></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-R <em><runpath elements></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">the <a href="#specifyingARunpath">specifies the <em>runpath</em></a> from which tests classes will be<br/>discovered and loaded (Note: only one <code>-R</code> allowed)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><em>Unix</em>: <code>-R target/classes:target/generated/classes</code><br/><em>Windows</em>: <code>-R target\classes;target\generated\classes</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-n <em><tag name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><a href="#specifyingTagsToIncludeAndExclude">specifies a tag to include</a> (Note: only one tag name allowed per <code>-n</code>)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-n UnitTests -n FastTests</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-l <em><tag name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><a href="#specifyingTagsToIncludeAndExclude">specifies a tag to exclude</a> (Note: only one tag name allowed per <code>-l</code>)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-l SlowTests -l PerfTests</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-P<em>[CS][integer thread count]</em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><a href="#executingSuitesInParallel">specifies a parallel run</a>, with optional cached thread pool, suite sorting and thread count<br/>(Note: only one <code>-P</code> allowed)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-P</code>, <code>-PS</code>, <code>-PS8</code>, <code>-P8</code>, <code>-PC</code> <em>or</em> <code>-PCS</code>. Negative or zero number of thread count is not allowed.</td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-s <em><suite class name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">specifies a <a href="#executingSuites">suite class</a> to run</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-s com.company.project.StackSpec</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-m <em><members-only package></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">requests that suites that are <a href="#membersOnlyWildcard">direct members of the specified package</a><br/> be discovered and run</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-m com.company.project</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-w <em><wildcard package></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">requests that suites that are <a href="#membersOnlyWildcard">members of the specified package or its subpackages</a><br/>be discovered and run</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-w com.company.project</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-q <em><suffixes></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">specify <a href="#specifyingSuffixesToDiscover">suffixes to discover</a></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-q Spec -q Suite</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-Q</code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">discover only classes whose names end with <code>Spec</code> or <code>Suite</code><br/>(or other suffixes specified by <code>-q</code>)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-Q</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-j <em><JUnit class name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">instantiate and run a <a href="#specifyingJUnitTests">JUnit test class</a></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-j StackTestClass</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-b <em><TestNG XML file></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">run <a href="#specifyingTestNGXML">TestNG tests</a> using the specified TestNG XML file</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-b testng.xml</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-F <em><span scale factor></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">a factor by which to <a href="#scalingTimeSpans">scale time spans</a><br/>(Note: only one <code>-F</code> is allowed)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-F 10</code> <em>or</em> <code>-F 2.5</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-T <em><sorting timeout></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">specifies a integer timeout (in seconds) for sorting the events of<br/>parallel runs back into sequential order</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-T 5</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-i <em><suite ID></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">specifies a <a href="#selectingSuitesAndTests">suite to run by ID</a> (Note: must follow <code>-s</code>, <br/>and is intended to be used primarily by tools such as IDEs.)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-i com.company.project.FileSpec-file1.txt</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-t <em><test name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><a href="#selectingSuitesAndTests">select the test</a> with the specified name</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-t "An empty Stack should complain when popped"</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-z <em><test name substring></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><a href="#selectingSuitesAndTests">select tests</a> whose names include the specified substring</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-z "popped"</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-g<em>[NCXEHLOPQMD]</em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">select the graphical reporter</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-g</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-f<em>[NCXEHLOPQMDWSFU] <filename></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">select the file reporter</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-f output.txt</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-u <em><directory name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">select the JUnit XML reporter</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-u target/junitxmldir</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-h <em><directory name></em> [-Y <em><css file name></em>]</code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">select the HTML reporter, optionally including the specified CSS file</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-h target/htmldir -Y src/main/html/customStyles.css</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-v</code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">print the ScalaTest version</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-v</code> <em>or, also</em> <code>-version</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-o<em>[NCXEHLOPQMDWSFU]</em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">select the standard output reporter</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-o</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-e<em>[NCXEHLOPQMDWSFU]</em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">select the standard error reporter</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-e</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-C<em>[NCXEHLOPQMD] <reporter class></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">select a custom reporter</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-C com.company.project.BarReporter</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-M <em><file name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">memorize failed and canceled tests in a file, so they can be rerun with -A (again)</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-M rerun.txt</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-A <em><file name></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">used in conjunction with -M (momento) to select previously failed<br/>and canceled tests to rerun again</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-A rerun.txt</code></td></tr>
* <tr><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-W <em><delay></em> <em><period></em></code></td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center">requests <a href="#slowpokeNotifications">notifications of <em>slowpoke</em> tests</a>, tests that have been running<br/>longer than <em>delay</em> seconds, every <em>period</em> seconds.</td><td style="border-width: 1px; padding: 3px; border: 1px solid black; text-align: center"><code>-W 60 60</code></td></tr>
* </table>
*
* <p>
* The simplest way to start <code>Runner</code> is to specify the directory containing your compiled tests as the sole element of the runpath, for example:
* </p>
*
* <pre class="stExamples">scala -classpath scalatest-<version>.jar org.scalatest.tools.Runner -R compiled_tests</pre>
*
* <p>
* Given the previous command, <code>Runner</code> will discover and execute all <code>Suite</code>s in the <code>compiled_tests</code> directory and its subdirectories,
* and show results in graphical user interface (GUI).
* </p>
*
* <a name="executingSuites"></a>
* <h2>Executing suites</h2>
*
* <p>
* Each <code>-s</code> argument must be followed by one and only one fully qualified class name. The class must either extend <code>Suite</code> and
* have a public, no-arg constructor, or be annotated by a valid <code>WrapWith</code> annotation.
* </p>
*
* <a name="configMapSection"></a>
* <h2>Specifying the config map</h2>
*
* <p>
* A <em>config map</em> contains pairs consisting of a string key and a value that may be of any type. (Keys that start with
* "org.scalatest." are reserved for ScalaTest. Configuration values that are themselves strings may be specified on the
* <code>Runner</code> command line.
* Each configuration pair is denoted with a "-D", followed immediately by the key string, an "=", and the value string.
* For example:
* </p>
*
* <pre class="stExamples">-Ddbname=testdb -Dserver=192.168.1.188</pre>
*
* <a name="specifyingARunpath"></a>
* <h2>Specifying a runpath</h2>
*
* <p>
* A runpath is the list of filenames, directory paths, and/or URLs that <code>Runner</code>
* uses to load classes for the running test. If runpath is specified, <code>Runner</code> creates
* a custom class loader to load classes available on the runpath.
* The graphical user interface reloads the test classes anew for each run
* by creating and using a new instance of the custom class loader for each run.
* The classes that comprise the test may also be made available on
* the classpath, in which case no runpath need be specified.
* </p>
*
* <p>
* The runpath is specified with the <b>-R</b> option. The <b>-R</b> must be followed by a space,
* a double quote (<code>"</code>), a white-space-separated list of
* paths and URLs, and a double quote. If specifying only one element in the runpath, you can leave off
* the double quotes, which only serve to combine a white-space separated list of strings into one
* command line argument. If you have path elements that themselves have a space in them, you must
* place a backslash (\) in front of the space. Here's an example:
* </p>
*
* <pre class="stExamples">-R "serviceuitest-1.1beta4.jar myjini http://myhost:9998/myfile.jar target/class\ files"</pre>
*
* <a name="specifyingReporters"></a>
* <h2>Specifying reporters</h2>
*
* <p>
* Reporters can be specified on the command line in any of the following
* ways:
* </p>
*
* <ul>
* <li> <code><b>-g[configs...]</b></code> - causes display of a graphical user interface that allows
* tests to be run and results to be investigated</li>
* <li> <code><b>-f[configs...] <filename></b></code> - causes test results to be written to
* the named file</li>
* <li> <code><b>-u <directory></b></code> - causes test results to be written to
* junit-style xml files in the named directory</li>
* <li> <code><b>-h <directory> [-Y <CSS file>]</b></code> - causes test results to be written to
* HTML files in the named directory, optionally included the specified CSS file</li>
* <li> <code><b>-a <number of files to archive></b></code> - causes specified number of old
* summary and durations files to be archived (in summaries/ and durations/ subdirectories)
* for dashboard reporter (default is two)</li>
* <li> <code><b>-o[configs...]</b></code> - causes test results to be written to
* the standard output</li>
* <li> <code><b>-e[configs...]</b></code> - causes test results to be written to
* the standard error</li>
* <li> <code><b>-k <host> <port></b></code> - causes test results to be written to
* socket in the named host and port number, using XML format</li>
* <li> <code><b>-K <host> <port></b></code> - causes test results to be written to
* socket in the named host and port number, using Java object binary format</li>
* <li> <code><b>-C[configs...] <reporterclass></b></code> - causes test results to be reported to
* an instance of the specified fully qualified <code>Reporter</code> class name</li>
* </ul>
*
* <p>
* The <code><b>[configs...]</b></code> parameter, which is used to configure reporters, is described in the next section.
* </p>
*
* <p>
* The <code><b>-C</b></code> option causes the reporter specified in
* <code><b><reporterclass></b></code> to be
* instantiated.
* Each reporter class specified with a <b>-C</b> option must be public, implement
* <code>org.scalatest.Reporter</code>, and have a public no-arg constructor.
* Reporter classes must be specified with fully qualified names.
* The specified reporter classes may be
* deployed on the classpath. If a runpath is specified with the
* <code>-R</code> option, the specified reporter classes may also be loaded from the runpath.
* All specified reporter classes will be loaded and instantiated via their no-arg constructor.
* </p>
*
* <p>
* For example, to run a suite named <code>MySuite</code> from the <code>mydir</code> directory
* using two reporters, the graphical reporter and a file reporter
* writing to a file named <code>"test.out"</code>, you would type:
* </p>
*
* <pre class="stExamples">java -jar scalatest.jar -R mydir <b>-g -f test.out</b> -s MySuite</pre>
*
* <p>
* The <code><b>-g</b></code>, <code><b>-o</b></code>, or <code><b>-e</b></code> options can
* appear at most once each in any single command line.
* Multiple appearances of <code><b>-f</b></code> and <code><b>-C</b></code> result in multiple reporters
* unless the specified <code><b><filename></b></code> or <code><b><reporterclass></b></code> is
* repeated. If any of <code><b>-g</b></code>, <code><b>-o</b></code>, <code><b>-e</b></code>,
* <code><b><filename></b></code> or <code><b><reporterclass></b></code> are repeated on
* the command line, the <code>Runner</code> will print an error message and not run the tests.
* </p>
*
* <p>
* <code>Runner</code> adds the reporters specified on the command line to a <em>dispatch reporter</em>,
* which will dispatch each method invocation to each contained reporter. <code>Runner</code> will pass
* the dispatch reporter to executed suites. As a result, every
* specified reporter will receive every report generated by the running suite of tests.
* If no reporters are specified, a graphical
* runner will be displayed that provides a graphical report of
* executed suites.
* </p>
*
* <a name="configuringReporters"></a>
* <h2>Configuring reporters</h2>
*
* <p>
* Each reporter option on the command line can include configuration characters. Configuration characters
* are specified immediately following the <code><b>-g</b></code>, <code><b>-o</b></code>,
* <code><b>-e</b></code>, <code><b>-f</b></code>, or <code><b>-C</b></code>. The following configuration
* characters, which cause reports to be dropped, are valid for any reporter:
* </p>
*
* <ul>
* <li> <code><b>N</b></code> - drop <code>TestStarting</code> events</li>
* <li> <code><b>C</b></code> - drop <code>TestSucceeded</code> events</li>
* <li> <code><b>X</b></code> - drop <code>TestIgnored</code> events</li>
* <li> <code><b>E</b></code> - drop <code>TestPending</code> events</li>
* <li> <code><b>H</b></code> - drop <code>SuiteStarting</code> events</li>
* <li> <code><b>L</b></code> - drop <code>SuiteCompleted</code> events</li>
* <li> <code><b>O</b></code> - drop <code>InfoProvided</code> events</li>
* <li> <code><b>P</b></code> - drop <code>ScopeOpened</code> events</li>
* <li> <code><b>Q</b></code> - drop <code>ScopeClosed</code> events</li>
* <li> <code><b>R</b></code> - drop <code>ScopePending</code> events</li>
* <li> <code><b>M</b></code> - drop <code>MarkupProvided</code> events</li>
* </ul>
*
* <p>
* A dropped event will not be delivered to the reporter at all. So the reporter will not know about it and therefore not
* present information about the event in its report. For example, if you specify <code>-oN</code>, the standard output reporter
* will never receive any <code>TestStarting</code> events and will therefore never report them. The purpose of these
* configuration parameters is to allow users to selectively remove events they find add clutter to the report without
* providing essential information.
* </p>
*
* <p>
* The following three reporter configuration parameters may additionally be used on standard output (-o), standard error (-e),
* and file (-f) reporters:
* </p>
*
* <ul>
* <li> <code><b>W</b></code> - without color</li>
* <li> <code><b>D</b></code> - show all durations</li>
* <li> <code><b>S</b></code> - show short stack traces</li>
* <li> <code><b>F</b></code> - show full stack traces</li>
* <li> <code><b>U</b></code> - unformatted mode</li>
* <li> <code><b>I</b></code> - show reminder of failed and canceled tests without stack traces</li>
* <li> <code><b>T</b></code> - show reminder of failed and canceled tests with short stack traces</li>
* <li> <code><b>G</b></code> - show reminder of failed and canceled tests with full stack traces</li>
* <li> <code><b>K</b></code> - exclude <code>TestCanceled</code> events from reminder</li>
* </ul>
*
* <p>
* If you specify a W, D, S, F, U, R, T, G, or K for any reporter other than standard output, standard error, or file reporters, <code>Runner</code>
* will complain with an error message and not perform the run.
* </p>
*
* <p>
* Configuring a standard output, error, or file reporter with <code>D</code> will cause that reporter to
* print a duration for each test and suite. When running in the default mode, a duration will only be printed for
* the entire run.
* </p>
*
* <p>
* Configuring a standard output, error, or file reporter with <code>F</code> will cause that reporter to print full stack traces for all exceptions,
* including <code>TestFailedExceptions</code>. Every <code>TestFailedException</code> contains a stack depth of the
* line of test code that failed so that users won't need to search through a stack trace to find it. When running in the default,
* mode, these reporters will only show full stack traces when other exceptions are thrown, such as an exception thrown
* by production code. When a <code>TestFailedException</code> is thrown in default mode, only the source filename and
* line number of the line of test code that caused the test to fail are printed along with the error message, not the full stack
* trace.
* </p>
*
* <p>
* The 'U' unformatted configuration removes some formatting from the output and adds verbosity.
* The purpose of unformatted (or, "ugly") mode is to facilitate debugging of parallel runs. If you have
* tests that fail or hang during parallel runs, but succeed when run sequentially, unformatted mode can help.
* In unformatted mode, you can see exactly what is happening when it is happening. Rather than attempting to make the output
* look as pretty and human-readable as possible, unformatted mode will just print out verbose information about each event
* as it arrives, helping you track down the problem
* you are trying to debug.
* </p>
*
* <p>
* By default, a standard output, error, or file reporter inserts ansi escape codes into the output printed to change and later reset
* terminal colors. Information printed as a result of run starting, completed, and stopped events
* is printed in cyan. Information printed as a result of ignored or pending test events is shown in yellow. Information printed
* as a result of test failed, suite aborted, or run aborted events is printed in red. All other information is printed in green.
* The purpose of these colors is to facilitate speedy reading of the output, especially the finding of failed tests, which can
* get lost in a sea of passing tests. Configuring a standard output, error, or file reporter into without-color mode (<code>W</code>) will
* turn off this behavior. No ansi codes will be inserted.
* </p>
*
* <p>
* The <code>R</code>, <code>T</code>, and <code>G</code> options enable "reminders" of failed and, optionally, canceled tests to be printed
* at the end of the summary. This minimizes or eliminates the need to search and scroll backwards to find out what tests failed or were canceled.
* For large test suites, the actual failure message could have scrolled off the top of the buffer, making it otherwise impossible
* to see what failed. You can configure the detail level of the stack trace for regular reports of failed and canceled tests independently
* from that of reminders. To set the detail level for regular reports, use <code>S</code> for short stack traces, <code>F</code> for
* full stack traces, or nothing for the default of no stack trace. To set the detail level for reminder reports, use <code>T</code> for
* reminders with short stack traces, <code>G</code> for reminders with full stack traces in reminders, or <code>R</code> for reminders
* with no stack traces. If you wish to exclude reminders of canceled tests, <em>i.e.</em>, only see reminders of failed tests, specify
* <code>K</code> along with one of <code>R</code>, <code>T</code>, or <code>G</code>, as in <code>"-oRK"</code>.
* </p>
*
* <p>
* For example, to run a suite using two reporters, the graphical reporter configured to present every reported event
* and a standard error reporter configured to present everything but test starting, test succeeded, test ignored, test
* pending, suite starting, suite completed, and info provided events, you would type:
* </p>
*
* <p>
* <code>scala -classpath scalatest-<version>.jar -R mydir <strong>-g -eNDXEHLO</strong> -s MySuite</code>
* </p>
*
* <p>
* Note that no white space is allowed between the reporter option and the initial configuration
* parameters. So <code>"-e NDXEHLO"</code> will not work,
* <code>"-eNDXEHLO"</code> will work.
* </p>
*
* <a name="specifyingTagsToIncludeAndExclude"></a>
* <h2>Specifying tags to include and exclude</h2>
*
* <p>
* You can specify tag names of tests to include or exclude from a run. To specify tags to include,
* use <code>-n</code> followed by a white-space-separated list of tag names to include, surrounded by
* double quotes. (The double quotes are not needed if specifying just one tag.) Similarly, to specify tags
* to exclude, use <code>-l</code> followed by a white-space-separated
* list of tag names to exclude, surrounded by double quotes. (As before, the double quotes are not needed
* if specifying just one tag.) If tags to include is not specified, then all tests
* except those mentioned in the tags to exclude (and in the <code>org.scalatest.Ignore</code> tag), will be executed.
* (In other words, the absence of a <code>-n</code> option is like a wildcard, indicating all tests be included.)
* If tags to include is specified, then only those tests whose tags are mentioned in the argument following <code>-n</code>
* and not mentioned in the tags to exclude, will be executed. For more information on test tags, see
* the <a href="../Suite.html">documentation for <code>Suite</code></a>. Here are some examples:
* </p>
*
* <p>
* <ul>
* <li><code>-n CheckinTests</code></li>
* <li><code>-n FunctionalTests -l org.scalatest.tags.Slow</code></li>
* <li><code>-n "CheckinTests FunctionalTests" -l "org.scalatest.tags.Slow org.scalatest.tags.Network"</code></li>
* </ul>
* </p>
*
* <a name="specifyingSuffixesToDiscover"></a>
* <h2>Specifying suffixes to discover</h2>
*
* <p>
* You can specify suffixes of <code>Suite</code> names to discover. To specify suffixes to discover,
* use <code>-q</code> followed by a vertical-bar-separated list of suffixes to discover, surrounded by
* double quotes. (The double quotes are not needed if specifying just one suffix.) Or you can specify
* them individually using multiple -q's.
* If suffixes to discover is not specified, then all suffixes are considered.
* If suffixes is specified, then only those Suites whose class names end in one of the specified suffixes
* will be considered during discovery. Here are some examples:
* </p>
*
* <p>
* <ul>
* <li><code>-q Spec</code></li>
* <li><code>-q "Spec|Suite"</code></li>
* <li><code>-q Spec -q Suite</code></li>
* </ul>
* </p>
*
* <p>
* Option -Q can be used to specify a default set of suffixes "Spec|Suite". If you specify both -Q and -q, you'll get Spec
* and Suite in addition to the other suffix or suffixes you specify with -q.
* </p>
*
* <p>
* Specifying suffixes can speed up the discovery process because class files with names not ending the specified suffixes
* can be immediately disqualified, without needing to load and inspect them to see if they either extend <code>Suite</code>
* and declare a public, no-arg constructor, or are annotated with <code>WrapWith</code>.
* </p>
*
* <a name="executingSuitesInParallel"></a>
* <h2>Executing <code>Suite</code>s in parallel</h2>
*
* <p>
* With the proliferation of multi-core architectures, and the often parallelizable nature of tests, it is useful to be able to run
* tests in parallel. If you include <code>-P</code> on the command line, <code>Runner</code> will pass a <code>Distributor</code> to
* the <code>Suite</code>s you specify with <code>-s</code>. <code>Runner</code> will set up a thread pool to execute any <code>Suite</code>s
* passed to the <code>Distributor</code>'s <code>put</code> method in parallel. Trait <code>Suite</code>'s implementation of
* <code>runNestedSuites</code> will place any nested <code>Suite</code>s into this <code>Distributor</code>. Thus, if you have a <code>Suite</code>
* of tests that must be executed sequentially, you should override <code>runNestedSuites</code> as described in the <a href="../Distributor.html">documentation for <code>Distributor</code></a>.
* </p>
*
* <p>
* The <code>-P</code> option may optionally be appended with a number (e.g.
* "<code>-P10</code>" -- no intervening space) to specify the number of
* threads to be created in the thread pool. If no number (or 0) is
* specified, the number of threads will be decided based on the number of
* processors available.
* </p>
*
* <a name="specifyingSuites"></a>
* <h2>Specifying <code>Suite</code>s</h2>
*
* <p>
* Suites are specified on the command line with a <b>-s</b> followed by the fully qualified
* name of a <code>Suite</code> subclass, as in:
* </p>
*
* <pre class="stExamples">-s com.artima.serviceuitest.ServiceUITestkit</pre>
*
* <p>
* Each specified suite class must be public, a subclass of
* <code>org.scalatest.Suite</code>, and contain a public no-arg constructor.
* <code>Suite</code> classes must be specified with fully qualified names.
* The specified <code>Suite</code> classes may be
* loaded from the classpath. If a runpath is specified with the
* <code>-R</code> option, specified <code>Suite</code> classes may also be loaded from the runpath.
* All specified <code>Suite</code> classes will be loaded and instantiated via their no-arg constructor.
* </p>
*
* <p>
* The runner will invoke <code>execute</code> on each instantiated <code>org.scalatest.Suite</code>,
* passing in the dispatch reporter to each <code>execute</code> method.
* </p>
*
* <p>
* <code>Runner</code> is intended to be used from the command line. It is included in <code>org.scalatest</code>
* package as a convenience for the user. If this package is incorporated into tools, such as IDEs, which take
* over the role of runner, object <code>org.scalatest.tools.Runner</code> may be excluded from that implementation of the package.
* All other public types declared in package <code>org.scalatest.tools.Runner</code> should be included in any such usage, however,
* so client software can count on them being available.
* </p>
*
* <a name="membersOnlyWildcard"></a>
* <h2>Specifying "members-only" and "wildcard" <code>Suite</code> paths</h2>
*
* <p>
* If you specify <code>Suite</code> path names with <code>-m</code> or <code>-w</code>, <code>Runner</code> will automatically
* discover and execute accessible <code>Suite</code>s in the runpath that are either a member of (in the case of <code>-m</code>)
* or enclosed by (in the case of <code>-w</code>) the specified path. As used in this context, a <em>path</em> is a portion of a fully qualified name.
* For example, the fully qualifed name <code>com.example.webapp.MySuite</code> contains paths <code>com</code>, <code>com.example</code>, and <code>com.example.webapp</code>.
* The fully qualifed name <code>com.example.webapp.MyObject.NestedSuite</code> contains paths <code>com</code>, <code>com.example</code>,
* <code>com.example.webapp</code>, and <code>com.example.webapp.MyObject</code>.
* An <em>accessible <code>Suite</code></em> is a public class that extends <code>org.scalatest.Suite</code>
* and defines a public no-arg constructor. Note that <code>Suite</code>s defined inside classes and traits do not have no-arg constructors,
* and therefore won't be discovered. <code>Suite</code>s defined inside singleton objects, however, do get a no-arg constructor by default, thus
* they can be discovered.
* </p>
*
* <p>
* For example, if you specify <code>-m com.example.webapp</code>
* on the command line, and you've placed <code>com.example.webapp.RedSuite</code> and <code>com.example.webapp.BlueSuite</code>
* on the runpath, then <code>Runner</code> will instantiate and execute both of those <code>Suite</code>s. The difference
* between <code>-m</code> and <code>-w</code> is that for <code>-m</code>, only <code>Suite</code>s that are direct members of the named path
* will be discovered. For <code>-w</code>, any <code>Suite</code>s whose fully qualified
* name begins with the specified path will be discovered. Thus, if <code>com.example.webapp.controllers.GreenSuite</code>
* exists on the runpath, invoking <code>Runner</code> with <code>-w com.example.webapp</code> will cause <code>GreenSuite</code>
* to be discovered, because its fully qualifed name begins with <code>"com.example.webapp"</code>. But if you invoke <code>Runner</code>
* with <code>-m com.example.webapp</code>, <code>GreenSuite</code> will <em>not</em> be discovered because it is directly
* a member of <code>com.example.webapp.controllers</code>, not <code>com.example.webapp</code>.
* </p>
*
* <p>
* If you specify no <code>-s</code>, <code>-m</code>, or <code>-w</code> arguments on the command line to <code>Runner</code>, it will discover and execute all accessible <code>Suite</code>s
* in the runpath.
* </p>
*
* <a name="selectingSuitesAndTests"></a>
* <h2>Selecting suites and tests</h2>
*
* <p>
* <code>Runner</code> accepts three arguments that facilitate selecting suites and tests: <code>-i</code>, <code>-t</code>, and </code>-z</code>.
* The <code>-i</code> option enables a suite to be selected by suite ID. This argument is intended to allow tools such as IDEs or build tools to
* rerun specific tests or suites from information included in the results of a previous run. A <code>-i</code> must follow a <code>-s</code>
* that specifies a class with a public, no-arg constructor. The <code>-i</code> parameter can be used, for example, to rerun a nested suite that
* declares no zero-arg constructor, which was created by containing suite that does declare a no-arg constructor. In this case, <code>-s</code> would be
* used to specify the class ScalaTest can instantiate directly, the containing suite that has a public, no-arg constructor, and <code>-i</code> would be
* used to select the desired nested suite. One important use case for <code>-i</code> is to enable such a nested suite that aborted during the previous run
* to be rerun. <!-- TODO: Need to point them to more info, maybe in SuiteMixin's rerunner method description? -->
* </p>
*
* <p>
* The <code>-t</code> argument allows a test to be selected by its (complete) test name. Like <code>-i</code>, the <code>-t</code> argument is primarily intented
* to be used by tools such as IDEs or build tools, to rerun selected tests based on information obtained from the results of a previous run.
* For example, <code>-t</code> could be used to rerun a test that failed in the previous run.
* The <code>-t</code> argument can be used directly by users, but because descriptive test names are usually rather long, the <code>-z</code> argument (described next), will
* usually be a more practical choice for users. If a <code>-t</code> follows either <code>-s</code> or <code>-i</code>, then it only applies to the suite
* identified. If it is specified independent of a <code>-s</code> or <code>-i</code>, then discovery is performed to find all Suites containing the test name.
* </p>
*
* <p>
* The <code>-z</code> option allows tests to be selected by a simplified wildcard: any test whose name includes the substring specified after <code>-z</code>
* will be selected. For example, <code>-z popped</code> would select tests named <code>"An empty stack should complain when popped"</code> and <code>"A non-empty stack
* should return the last-pushed value when popped</code>, but not <code>"An empty stack should be empty"</code>. In short, <code>-z popped</code> would select any
* tests whose name includes the substring <code>"popped"</code>, and not select any tests whose names don't include <code>"popped"</code>. This simplified
* approach to test name wildcards, which was suggested by Mathias Doenitz, works around the difficulty of finding an actual wildcard character that will work
* reliably on different operating systems. Like <code>-t</code>, if <code>-z</code> follows <code>-s</code> or <code>-i</code>, then it only applies to the Suite specified. Otherwise discovery is performed to find all Suites containing test names that include the substring.
* </p>
*
* <a name="scalingTimeSpans"></a>
* <h2>Specifying a span scale factor</h2>
*
* <p>
* If you specify a integer or floating point <em>span scale factor</em> with <code>-F</code>, trait <a href="../concurrent/ScaledTimeSpans.html"><code>ScaledTimeSpans</code></a>
* trait will return the specified value from its implementation of <code>spanScaleFactor</code>. This allows you to tune the "patience" of a run (how long to wait
* for asynchronous operations) from the command line. For more information, see the documentation for trait <a href="../concurrent/ScaledTimeSpans.html"><code>ScaledTimeSpans</code></a>.
* </p>
*
* <a name="specifyingTestNGXML"></a>
* <h2>Specifying TestNG XML config file paths</h2>
*
* <p>
* If you specify one or more file paths with <code>-b</code> (b for Beust, the last name of TestNG's creator), <code>Runner</code> will create a <code>org.scalatest.testng.TestNGWrapperSuite</code>,
* passing in a <code>List</code> of the specified paths. When executed, the <code>TestNGWrapperSuite</code> will create one <code>TestNG</code> instance
* and pass each specified file path to it for running. If you include <code>-b</code> arguments, you must include TestNG's jar file on the class path or runpath.
* The <code>-b</code> argument will enable you to run existing <code>TestNG</code> tests, including tests written in Java, as part of a ScalaTest run.
* You need not use <code>-b</code> to run suites written in Scala that extend <code>TestNGSuite</code>. You can simply run such suites with
* <code>-s</code>, <code>-m</code>, or </code>-w</code> parameters.
* </p>
*
* <a name="specifyingJUnitTests"></a>
* <h2>Specifying JUnit tests</h2>
*
* <p>
* JUnit tests, including ones written in Java, may be run by specifying
* <code>-j classname</code>, where the classname is a valid JUnit class
* such as a TestCase, TestSuite, or a class implementing a static suite()
* method returning a TestSuite. </p>
* <p>
* To use this option you must include a JUnit jar file on your classpath.
* </p>
*
* <a name="memorizingAndRerunning"> </a>
* <h2>Memorizing and rerunning failed and canceled tests</h2>
*
* <p>
* You can memorize failed and canceled tests using <code>-M</code>:
* </p>
*
* <pre class="stHighlighted">
* -M failed-canceled.txt
* </pre>
*
* All failed and canceled tests will be memorized in <code>failed-canceled.txt</code>, to rerun them again, you use <code>-A</code>:
*
* <pre class="stHighlighted">
* -A failed-canceled.txt
* </pre>
*
* <a name="slowpokeNotifications"> </a>
* <h2>Slowpoke notifications</h2>
*
* <p>
* You can request to recieve periodic notifications of <em>slowpokes</em>, tests that have been running longer than a given amount of time, specified in
* seconds by the first integer after <code>-W</code>, the <em>delay</em>.
* You specify the period between slowpoke notifications in seconds with the second integer after <code>-W</code>, the <em>period</em>. Thus to receive
* notifications very minute of tests that have been running longer than two minutes, you'd use:
* </p>
*
* <pre class="stGray">
* <code>-W 120 60</code>
* </pre>
*
* <p>
* Slowpoke notifications will be sent via <a href="../events/AlertProvided.html"><code>AlertProvided</code></a> events. The standard out reporter, for example,
* will report such notifications like:
* </p>
*
* <pre class="stREPL">
* <span class="stYellow">*** Test still running after 2 minutes, 13 seconds: suite name: ExampleSpec, test name: An egg timer should take 10 minutes.</span>
* </pre>
*
* @author Bill Venners
* @author George Berger
* @author Josh Cough
* @author Chee Seng
*/
object Runner {
private val RUNNER_JFRAME_START_X: Int = 150
private val RUNNER_JFRAME_START_Y: Int = 100
@volatile private[scalatest] var spanScaleFactor: Double = 1.0
private final val DefaultNumFilesToArchive = 2
// TO
// We always include a PassFailReporter on runs in order to determine
// whether or not all tests passed.
//
// The thread that calls Runner.run() will either start a GUI, if a graphic
// reporter was requested, or just run the tests itself. If a GUI is started,
// an event handler thread will get going, and it will start a RunnerThread,
// which will actually do the running. The GUI can repeatedly start RunnerThreads
// and RerunnerThreads, until the GUI is closed. If -P is specified, that means
// the tests should be run concurrently, which in turn means a Distributor will
// be passed to the execute method of the Suites, which will in turn populate
// it with its nested suites instead of executing them directly in the same
// thread. The Distributor works in conjunction with a pool of threads that
// will take suites out of the distributor queue and execute them. The DispatchReporter
// will serialize all reports via an actor, which because that actor uses receive
// not react, will have its own thread. So the DispatchReporter actor's thread will
// be the one that actually invokes TestFailed, RunAborted, etc., on this PassFailReporter.
// The thread that invoked Runner.run(), will be the one that calls allTestsPassed.
//
// The thread that invoked Runner.run() will be the one to instantiate the PassFailReporter
// and in its primary constructor acquire the single semaphore permit. This permit will
// only be released by the DispatchReporter's actor thread when a runAborted, runStopped,
// or runCompleted is invoked. allTestsPassed will block until it can reacquire the lone
// semaphore permit. Thus, a PassFailReporter can just be used for one run, then it is
// spent. A new PassFailReporter is therefore created each time the Runner.run() method is invoked.
//
private class PassFailReporter extends Reporter {
@volatile private var failedAbortedOrStopped = false
private val runDoneSemaphore = new Semaphore(1)
runDoneSemaphore.acquire()
override def apply(event: Event): Unit = {
event match {
case _: TestFailed =>
failedAbortedOrStopped = true
case _: RunAborted =>
failedAbortedOrStopped = true
runDoneSemaphore.release()
case _: SuiteAborted =>
failedAbortedOrStopped = true
case _: RunStopped =>
failedAbortedOrStopped = true
runDoneSemaphore.release()
case _: RunCompleted =>
runDoneSemaphore.release()
case _ =>
}
}
def allTestsPassed = {
runDoneSemaphore.acquire()
!failedAbortedOrStopped
}
}
// TODO: I don't think I'm enforcing that properties can't start with "org.scalatest"
// TODO: I don't think I'm handling rejecting multiple -f/-C with the same arg. -f fred.txt -f fred.txt should
// fail, as should -C MyReporter -C MyReporter. I'm failing on -o -o, -g -g, and -e -e, but the error messages
// could indeed be nicer.
/**
* Runs a suite of tests, with optional GUI. See the main documentation for this singleton object for the details.
*/
def main(args: Array[String]): Unit = {
// println("FOR DEBUGGING, THESE ARE THE ARGS PASSED TO main(): " + args.mkString(" "))
Thread.currentThread.setName("ScalaTest-main")
val result =
if (args.contains("-v") || args.contains("--version")) {
val version = org.scalatest.ScalaTestVersions.ScalaTestVersion
val scalaVersion = org.scalatest.ScalaTestVersions.BuiltForScalaVersion
println("ScalaTest " + version + " (Built for Scala " + scalaVersion + ")")
runOptionallyWithPassFailReporter(args.filter(arg => arg != "-v" && arg != "--version"), true)
}
else
runOptionallyWithPassFailReporter(args, true)
if (result)
System.exit(0)
else
System.exit(1)
}
/**
* Runs a suite of tests, with optional GUI. See the main documentation for this singleton object for the details.
* The difference between this method and <code>main</code> is simply that this method will block until the run
* has completed, aborted, or been stopped, and return <code>true</code> if all tests executed and passed. In other
* words, if any test fails, or if any suite aborts, or if the run aborts or is stopped, this method will
* return <code>false</code>. This value is used, for example, by the ScalaTest ant task to determine whether
* to continue the build if <code>haltOnFailure</code> is set to <code>true</code>.
*
* @return true if all tests were executed and passed.
*/
def run(args: Array[String]): Boolean = {
// println("FOR DEBUGGING, THESE ARE THE ARGS PASSED TO run(): " + args.mkString(" "))
val originalThreadName = Thread.currentThread.getName
try {
Thread.currentThread.setName("ScalaTest-run")
runOptionallyWithPassFailReporter(args, true)
}
finally Thread.currentThread.setName(originalThreadName)
}
private def runOptionallyWithPassFailReporter(args: Array[String], runWithPassFailReporter: Boolean): Boolean = {
checkArgsForValidity(args) match {
case Some(s) => {
println(s)
System.exit(1) // TODO: Shouldn't this be returning false?
}
case None =>
}
val ParsedArgs(
runpathArgs,
reporterArgs,
suiteArgs,
againArgs,
junitArgs,
propertiesArgs,
tagsToIncludeArgs,
tagsToExcludeArgs,
concurrentArgs,
membersOnlyArgs,
wildcardArgs,
testNGArgs,
suffixes,
chosenStyles,
spanScaleFactors,
testSortingReporterTimeouts,
slowpokeArgs,
seedArgs
) = parseArgs(args)
val fullReporterConfigurations: ReporterConfigurations =
if (reporterArgs.isEmpty)
// If no reporters specified, just give them a graphic reporter
new ReporterConfigurations(Some(GraphicReporterConfiguration(Set())), Nil, Nil, Nil, /*Nil, Nil, */None, None, Nil, Nil, Nil, Nil)
else
parseReporterArgsIntoConfigurations(reporterArgs)
val (suitesList: List[SuiteParam], testSpecs: List[TestSpec]) =
parseSuiteArgs(suiteArgs)
val agains: List[String] = parseAgainArgs(againArgs)
val junitsList: List[String] = parseSuiteArgsIntoNameStrings(junitArgs, "-j")
val runpathList: List[String] = parseRunpathArgIntoList(runpathArgs)
val propertiesMap: ConfigMap = parsePropertiesArgsIntoMap(propertiesArgs)
val tagsToInclude: Set[String] = parseCompoundArgIntoSet(tagsToIncludeArgs, "-n")
val tagsToExclude: Set[String] = parseCompoundArgIntoSet(tagsToExcludeArgs, "-l")
val concurrent: Boolean = !concurrentArgs.isEmpty
val concurrentConfig: ConcurrentConfig = parseConcurrentConfig(concurrentArgs)
val membersOnlyList: List[String] = parseSuiteArgsIntoNameStrings(membersOnlyArgs, "-m")
val wildcardList: List[String] = parseSuiteArgsIntoNameStrings(wildcardArgs, "-w")
val testNGList: List[String] = parseSuiteArgsIntoNameStrings(testNGArgs, "-b")
val chosenStyleSet: Set[String] = parseChosenStylesIntoChosenStyleSet(chosenStyles, "-y")
val slowpokeConfig: Option[SlowpokeConfig] = parseSlowpokeConfig(slowpokeArgs)
val seedList: Option[Long] = parseLongArgument(seedArgs, "-S")
spanScaleFactor = parseDoubleArgument(spanScaleFactors, "-F", 1.0)
val testSortingReporterTimeout = Span(parseDoubleArgument(testSortingReporterTimeouts, "-T", Suite.defaultTestSortingReporterTimeoutInSeconds), Seconds)
seedList match {
case Some(seed) => Seed.configuredRef.getAndSet(Some(seed))
case None => // do nothing
}
// If there's a graphic reporter, we need to leave it out of
// reporterSpecs, because we want to pass all reporterSpecs except
// the graphic reporter's to the RunnerJFrame (because RunnerJFrame *is*
// the graphic reporter).
val reporterConfigs: ReporterConfigurations =
fullReporterConfigurations.graphicReporterConfiguration match {
case None => fullReporterConfigurations
case Some(grs) => {
new ReporterConfigurations(
None,
fullReporterConfigurations.fileReporterConfigurationList,
fullReporterConfigurations.memoryReporterConfigurationList,
fullReporterConfigurations.junitXmlReporterConfigurationList,
//fullReporterConfigurations.dashboardReporterConfigurationList,
//fullReporterConfigurations.xmlReporterConfigurationList,
fullReporterConfigurations.standardOutReporterConfiguration,
fullReporterConfigurations.standardErrReporterConfiguration,
fullReporterConfigurations.htmlReporterConfigurationList,
fullReporterConfigurations.customReporterConfigurationList,
fullReporterConfigurations.xmlSocketReporterConfigurationList,
fullReporterConfigurations.socketReporterConfigurationList
)
}
}
val passFailReporter = if (runWithPassFailReporter) Some(new PassFailReporter) else None
if (propertiesMap.isDefinedAt(CHOSEN_STYLES))
throw new IllegalArgumentException("Property name '" + CHOSEN_STYLES + "' is used by ScalaTest, please choose other property name.")
val configMap: ConfigMap =
if (chosenStyleSet.isEmpty)
propertiesMap
else
propertiesMap + (CHOSEN_STYLES -> chosenStyleSet)
if (chosenStyleSet.nonEmpty)
println(Resources.deprecatedChosenStyleWarning)
val (detectSlowpokes: Boolean, slowpokeDetectionDelay: Long, slowpokeDetectionPeriod: Long) =
slowpokeConfig match {
case Some(SlowpokeConfig(delayInMillis, periodInMillis)) => (true, delayInMillis, periodInMillis)
case _ => (false, 60000L, 60000L)
}
fullReporterConfigurations.graphicReporterConfiguration match {
case Some(GraphicReporterConfiguration(configSet)) => {
val graphicEventsToPresent: Set[EventToPresent] = EventToPresent.allEventsToPresent filter
(if (configSet.contains(FilterTestStarting)) {_ != PresentTestStarting} else etp => true) filter
(if (configSet.contains(FilterTestSucceeded)) {_ != PresentTestSucceeded} else etp => true) filter
(if (configSet.contains(FilterTestIgnored)) {_ != PresentTestIgnored} else etp => true) filter
(if (configSet.contains(FilterTestPending)) {_ != PresentTestPending} else etp => true) filter
(if (configSet.contains(FilterScopeOpened)) {_ != PresentScopeOpened} else etp => true) filter
(if (configSet.contains(FilterScopeClosed)) {_ != PresentScopeClosed} else etp => true) filter
(if (configSet.contains(FilterScopePending)) {_ != PresentScopePending} else etp => true) filter
(if (configSet.contains(FilterSuiteStarting)) {_ != PresentSuiteStarting} else etp => true) filter
(if (configSet.contains(FilterSuiteCompleted)) {_ != PresentSuiteCompleted} else etp => true) filter
(if (configSet.contains(FilterInfoProvided)) {_ != PresentInfoProvided} else etp => true) filter
(if (configSet.contains(FilterMarkupProvided)) {_ != PresentMarkupProvided} else etp => true)
val abq = new ArrayBlockingQueue[RunnerJFrame](1)
usingEventDispatchThread {
val rjf = new RunnerJFrame(
graphicEventsToPresent,
reporterConfigs,
suitesList,
agains,
testSpecs,
junitsList,
runpathList,
tagsToInclude,
tagsToExclude,
configMap,
concurrent,
membersOnlyList,
wildcardList,
testNGList,
passFailReporter,
concurrentConfig,
suffixes,
chosenStyleSet,
detectSlowpokes,
slowpokeDetectionDelay,
slowpokeDetectionPeriod,
testSortingReporterTimeout
)
rjf.setLocation(RUNNER_JFRAME_START_X, RUNNER_JFRAME_START_Y)
rjf.setVisible(true)
rjf.prepUIForRunning()
rjf.runFromGUI()
abq.put(rjf)
}
// To get the Ant task to work, the main thread needs to block until
// The GUI window exits.
val rjf = abq.take()
rjf.blockUntilWindowClosed()
}
case None => { // Run the test without a GUI
withClassLoaderAndDispatchReporter(
runpathList,
reporterConfigs,
None,
passFailReporter,
detectSlowpokes,
slowpokeDetectionDelay,
slowpokeDetectionPeriod
) { (loader, dispatchReporter) =>
doRunRunRunDaDoRunRun(
dispatchReporter,
suitesList,
agains,
testSpecs,
junitsList,
Stopper.default,
tagsToInclude,
tagsToExclude,
configMap,
concurrent,
membersOnlyList,
wildcardList,
testNGList,
runpathList,
loader,
new RunDoneListener {},
1,
concurrentConfig,
suffixes,
chosenStyleSet,
testSortingReporterTimeout
)
}
}
}
passFailReporter match {
case Some(pfr) => pfr.allTestsPassed
case None => false