forked from VowpalWabbit/vowpal_wabbit
/
RunTests
executable file
·2190 lines (1814 loc) · 88.2 KB
/
RunTests
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env perl
#
# Test suite for vw:
#
# You may add arbitrary (train/test/varying-options) tests
# by adding data files and their expected reference STDOUT and STDERR
#
# See __DATA__ below for how to add more tests
#
require 5.008;
use warnings;
use Getopt::Std;
use File::Basename;
use vars qw($opt_d $opt_D $opt_c $opt_e $opt_f
$opt_E $opt_o $opt_w $opt_y $opt_t
$opt_v $opt_V $opt_F $opt_O);
my $Epsilon = 1e-4;
my $VW;
# External utilities we use. See init() for Windows specific actions.
my $Diff = 'diff';
my $Cat = 'cat';
$ENV{'PATH'} = '../build/vowpalwabbit:test:../vowpalwabbit:vowpalwabbit:.:' . $ENV{'PATH'};
# -V prefixes valgrind like this, we should adjust the default
# options over time to what looks most useful.
my $Valgrind = 'valgrind --quiet --error-exitcode=100 --track-origins=yes --leak-check=full';
# -- timeout is part of GNU coreutils, some systems may not have it
my $TimeOut = '';
my $TimeOutSec = 300; # max allowed time for single vw command run
# By default, we run all tests in the list
my $FullRun = 1;
my $ErrorCount = 0;
# These --side-by-side diff opts are used to make the
# fuzzy-compare easier: just split on '|' and compare numeric values
# word by word:
# NOTE: -W 160 is sufficient for most outputs.
# --bfgs prints widest (134 chars-per-line)
# 160 used to sufficient but there are new tests with VERY wide outputs - so it needs to be huge...
my $DiffOpts = '-N --minimal --suppress-common-lines --ignore-all-space --strip-trailing-cr --side-by-side -W 1500';
$WordSplit = "[ \t:,@]+";
# These diff options are used for the diff we want to show the user
# The intent is to make them easier to parse (and compare values) by a human
my $DisplayDiffOpts = '-u --minimal';
my @PathAdd = qw(. .. ../vowpalwabbit);
my @ToTest = ();
# __DATA__ test counter
my $TestNo = 0;
sub v($;@) {
my $verbose_level = shift @_;
return unless ($opt_v >= $verbose_level);
if (@_ == 1) {
print STDERR @_;
} else {
printf STDERR @_;
}
}
sub usage(@) {
print STDERR @_, "\n" if (@_);
die "Usage: $0 [options] [testno...] [vw-executable]
By default will run against the 1st 'vw' executable found in:
@PathAdd \$PATH
Options:
-c print test-suite commands before running them
-d print diff output on significant diff failure
-D print diff output even if it is not significant
-e exit with non-zero status on first error
-w Ignore white-space differences (diff --ignore-space-change)
-f Ignore small (< $Epsilon) floating-point differences (fuzzy compare)
-E<e> Tolerance epsilon <e> for fuzzy compares (default $Epsilon)
-o Overwrite reference file with new/different result
-y On error, copy bad files to (eg stderr.test21) for later comparison
-v<L> Verbosity <L> (small integer) is verbosity level
-V apply valgrind to vw commands
-F include flatbuffer tests
-t<T> Apply timeout <T> (default $TimeOutSec) secs to individual tests
(will only work where GNU coreutils 'timeout' is present)
-O<O> Add <O> option(s) to all vw commands
[testno...] Optional integer args: explicit test numbers (skip others)
";
}
sub mysystem {
my $cmd = shift;
v(1, "%s\n", $cmd);
system($cmd);
}
sub command_failed($) {
# Deal with cases where vw crashes, exits prematurely etc.
# print a message to distinguish between all cases
# return non-zero status if anything is bad
my $cmd = shift;
my $exitcode = 0;
if ($?) {
$exitcode = $? >> 8;
my $signal = $? & 127;
my $core = ''; if ($? & 128) { $core = ' (core dumped)'; }
if ($signal) {
printf STDERR
"$0: test $TestNo: '%s' died from signal $signal$core\n", $cmd;
$exitcode = 1;
} elsif ($exitcode == 124) {
printf STDERR
"$0: test $TestNo: '%s' timed-out (exitcode=$exitcode)\n" .
"$0: test $TestNo: you may increase the imposed time-out: \$TimeOutSec=%d\n",
$cmd, $TimeOutSec;
} elsif ($exitcode) {
printf STDERR
"$0: test $TestNo: '%s' failed (exitcode=$exitcode)\n", $cmd;
}
}
# This is non-zero only if $cmd failed
$exitcode;
}
sub valgrind_errfile($) {
my $testno = shift;
"Test-$testno.valgrind-err";
}
#
# which vw executable to test against
#
sub which_vw() {
if (@ARGV > 0) {
my $exe = $ARGV[0];
if (-f $exe && -x $exe) {
printf STDERR "Testing vw: %s\n", $exe;
return $exe;
} else {
usage("$0: argument $exe: not an executable file");
}
} elsif (@ARGV == 0) {
foreach my $dir (@PathAdd, split(':', $ENV{PATH})) {
my $exe = "$dir/vw";
if (-x $exe) {
printf STDERR "Testing vw: %s\n", $exe;
return $exe;
}
}
}
usage("can't find a 'vw' executable to test on");
}
sub init() {
$0 =~ s{.*/}{};
getopts('wcdDefyE:ov:VFt:O:') || usage();
$opt_v = 0 unless (defined $opt_v and $opt_v);
if (defined $opt_O) {
$opt_O = " $opt_O";
} else {
$opt_O = '';
}
my $hostname = `hostname`; chomp($hostname);
printf STDERR "Testing on: hostname=%s OS=%s\n", $hostname, $^O;
if ($^O =~ /MSWin/i) {
v(1, "OS is $^O\n");
# On MS Windows we need to change paths to external executables
# Assumes cygwin is installed
$ENV{'PATH'} .= ':/cygdrive/c/cygwin/bin';
# And just to be safe (probably not needed):
$Diff = 'c:\cygwin\bin\diff.exe';
$Cat = 'c:\cygwin\bin\cat.exe';
}
elsif ($^O =~ /cygwin/i){
v(1,"OS is $^O\n");
# On MS Windows we need to change paths to external executables
# Assumes cygwin is installed
$ENV{'PATH'} .= ':/cygdrive/c/cygwin/bin';
# And just to be safe (probably not needed):
# $Diff = 'c:/cygwin/bin/diff.exe';
# $Cat = 'c:/cygwin/bin/cat.exe';
}
$Epsilon = $opt_E if ($opt_E);
$Diff .= ' --ignore-space-change' if ($opt_w);
my @num_args = ();
my @exe_args = ();
foreach my $arg (@ARGV) {
if ($arg =~ /^\d+$/) { # a test number
push(@num_args, $arg);
next;
}
push(@exe_args, $arg);
}
if (@num_args) {
@ToTest = sort { $a <=> $b } @num_args;
# add dummy element so we don't become empty on last test
push(@ToTest, -1);
$FullRun = 0;
}
@ARGV = @exe_args;
$VW = which_vw();
my $timeout = `which timeout 2>/dev/null`;
if ($timeout =~ /timeout$/) {
chomp($timeout);
$TimeOut = $timeout;
v(1,"timeout is: %s\n", $TimeOut);
}
if ($opt_t) {
if ($opt_t =~ /^\d+$/) {
$TimeOutSec = $opt_t;
} else {
usage("-t $opt_t: -t can only accept integer seconds");
}
warn "-t passed but this env doesn't have timeout installed\n"
unless ($TimeOut);
}
}
sub copy_file {
my ($src_file, $dst_file) = @_;
use File::Copy;
print STDERR "\t\t-> copying output to $dst_file\n";
copy($src_file, $dst_file);
}
sub trim_spaces($) {
my $str = shift;
$str =~ s/^\s+//s;
$str =~ s/\s+$//s;
$str =~ s/\n+$//s;
$str;
}
#
# ref_file($default_name)
# Reference file existence: if we're on Windows, AND
# an alternate reference-file exists, give precedence
# to the alternate file (file with a '-mswin' suffix.)
#
sub ref_file($) {
my $file = shift;
if ($^O =~ /MSWin/i or $^O =~ /cygwin/i) {
my $win_reffile = "$file-mswin";
if (-e $win_reffile) {
return $win_reffile;
}
}
$file;
}
sub next_paragraph {
my $paragraph = '';
my $testid = '<unknown id>';
while ($line = <DATA>) {
if ($line =~ /^\s*#/) { # skip comment lines
if ($line =~ /^# Test (\d+)/) {
$testid = $1;
}
next;
}
if ($line =~ /\\$/) { # support line continuation
$line =~ s/\\\n/ /;
}
$paragraph .= $line if $line =~ /\w/;
if ($paragraph and ($line =~ /^\s*$/ || eof(DATA))) {
# end of paragraph
chomp $paragraph;
$paragraph = trim_spaces($paragraph);
return ($testid, $paragraph);
}
}
return;
}
sub next_test() {
my ($cmd, $out_ref, $err_ref, @other_ref);
my $paragraph = '';
($TestNo, $paragraph) = next_paragraph();
return (undef, undef, undef, undef) if !defined $paragraph;
my @lines = split("\n", $paragraph);
# The command line must be first
$cmd = shift @lines;
foreach my $line (@lines) {
if ($line =~ m/\.stdout\b/) {
$out_ref = ref_file(trim_spaces($line));
next;
}
if ($line =~ /\.stderr\b/) {
$err_ref = ref_file(trim_spaces($line));
next;
}
# any other reference file
$line = ref_file(trim_spaces($line));
if (-e $line) {
push(@other_ref, $line);
} else {
unless ($opt_y) {
printf STDERR "__DATA__: line $.: " .
"non-existent reference file: %s\n", $line;
}
next;
}
}
if (eof(DATA) && !defined $cmd) {
return (undef, undef, undef, undef);
}
if ($cmd =~ /{VW}/) {
$cmd = trim_spaces($cmd);
$cmd =~ s/{VW}/$VW$opt_O/g;
}
if ($cmd =~ /--flatbuffer/) {
if (!$opt_F)
{
printf STDERR "Skipping test %s. Run with -F to include\n", $TestNo;
next;
}
}
unless (defined $cmd) {
die "$0: test $TestNo: command is undefined\n";
}
unless (defined $err_ref) {
v(2, "%s: test %s: stderr ref: undefined\n", $0, $TestNo);
$err_ref = '/dev/null';
}
# print STDERR "next_test: (\$cmd, $out_ref, $err_ref, $pred_ref, $pred)\n";
if ($opt_V) {
$cmd = sprintf("%s --log-file='%s' %s",
$Valgrind, valgrind_errfile($TestNo), $cmd);
} elsif ($TimeOut) {
$cmd = sprintf("%s %u %s", $TimeOut, $TimeOutSec, $cmd);
}
($cmd, $out_ref, $err_ref, @other_ref);
}
#
# If the difference is small (least significant digits of numbers)
# treat it as ok. It may be a result of 32 vs 64 bit calculations.
#
use Scalar::Util qw(looks_like_number);
sub lenient_array_compare($$) {
my ($w1_ref, $w2_ref) = @_;
my (@w1) = @$w1_ref;
my (@w2) = @$w2_ref;
# print STDERR "lenient_array_compare: (@w1) (@w2)\n";
if ($#w1 != $#w2) { # arrays not of same size
if ($opt_v > 3) {
v(4, "#-of-words in two arrays are different: %d != %d\n", scalar(@w1), scalar(@w2));
v(4, "line1: "); for (my $i=0; $i <= $#w1; $i++) { v(4, " word[%d]='%s'", $i, $w1[$i]) }; v(4, "\n");
v(4, "line2: "); for (my $i=0; $i <= $#w2; $i++) { v(4, " word[%d]='%s'", $i, $w2[$i]) }; v(4, "\n");
}
return 1;
}
my $nelem = scalar @w1;
for (my $i = 0; $i < $nelem; $i++) {
my ($word1, $word2) = ($w1[$i], $w2[$i]);
# print STDERR "\t$word1 == $word2 ?\n";
next if ($word1 eq $word2);
# Some output contains '...', remove this for comparison.
$word1 =~ s/\.\.\.//;
$word2 =~ s/\.\.\.//;
# There's some difference, is it significant?
unless (looks_like_number($word1)) {
v(4, "$word1 vs $word2: word1=$word1 is not a number!\n");
return 1;
}
unless (looks_like_number($word2)) {
v(4, "$word1 vs $word2: word2=$word2 is not a number!\n");
return 1;
}
my $delta = abs($word1 - $word2);
if ($delta > $Epsilon) {
# We have a 'big enough' difference, but this difference
# may still not be meaningful in all contexts:
# Big numbers should be compared by ratio rather than
# by difference
# Must ensure we can divide (avoid div-by-0)
if (abs($word2) <= 1.0) {
# If numbers are so small (close to zero),
# ($delta > $Epsilon) suffices for deciding that
# the numbers are meaningfully different
v(4, "$word1 vs $word2: delta=$delta > Epsilon=$Epsilon\n");
return 1;
}
# Now we can safely divide (since abs($word2) > 0)
# and determine the ratio difference from 1.0
my $ratio_delta = abs($word1/$word2 - 1.0);
if ($ratio_delta > $Epsilon) {
v(4, "$word1 vs $word2: ratio_delta=$ratio_delta > Epsilon=$Epsilon\n");
return 1;
}
}
}
# print STDERR "lenient_array_compare: no meaningful difference\n";
return 0; # no meaningful difference
}
sub diff_lenient_float($$) {
my ($reffile, $outfile) = @_;
my $status = 0;
my $tmpf = 'lenient-diff.tmp';
mysystem("$Diff $DiffOpts $reffile $outfile >$tmpf");
$status = $? >> 8;
v(2, "diff produced $tmpf: status=$status\n");
if (-s $tmpf) {
# The diff has something in it.
my $fuzzy_status = 0; # assume innocent till proven guilty
open(my $sdiff, $tmpf) || die "$0: diff_lenient_float: $tmpf: $!\n";
while (<$sdiff>) {
chomp;
my ($line1, $line2) = split(/\s*\|\s*/, $_);
unless (defined($line1) && defined($line2)) {
my $save_diff_file = "test-$TestNo.lenient-diff";
warn "$0: test $TestNo: $tmpf: line $.: fuzzy-match missing data on one of the sides. Can't compare\n$_\n";
warn "$0: test $TestNo: saving lenient diff in '$save_diff_file' for later inspection\n";
close $sdiff;
rename($tmpf, $save_diff_file);
return 1;
}
# strip leading spaces if any (happens with --bfgs)
$line1 =~ s/^\s+//;
$line2 =~ s/^\s+//;
v(3, "line1: %s\n", $line1);
v(3, "line2: %s\n", $line2);
# Break lines into tokens/words
my (@w1) = split(/$WordSplit/o, $line1);
my (@w2) = split(/$WordSplit/o, $line2);
if (lenient_array_compare(\@w1, \@w2) != 0) {
$fuzzy_status = 1;
last;
}
}
close $sdiff;
$status = $fuzzy_status;
}
unlink($tmpf) if ($status == 0);
$status;
}
#
# perl internal way to emulate 'touch'
#
sub touch(@) {
my $now = time;
utime $now, $now, @_;
}
sub display_diff($$) {
my ($reference_file, $actual_file) = @_;
my $diff_cmd = "$Diff $DisplayDiffOpts $reference_file $actual_file";
printf STDERR "--- %s\n", $diff_cmd;
mysystem($diff_cmd);
}
sub diff($$) {
my ($reffile, $outfile) = @_;
my $status = 0;
$reffile = '' unless (defined $reffile);
# Special case, empty file w/o reference is not considered a failure.
# This is a most common case with stdout.
unless (-e $reffile) {
if (-s $outfile > 0) {
warn "$0: test $TestNo: stdout ref: $reffile: $!\n";
exit 1 if ($opt_e);
return 2 unless ($opt_o);
} else {
# Empty output without a ref is not considered a failure
v(1, "$0: test $TestNo: empty output w/o reference: ignored.\n");
return 0;
}
}
# Actually run the diff
my $diff_cmd = "$Diff $DiffOpts $reffile $outfile";
my $diftmp = 'diff.tmp';
mysystem("$diff_cmd >$diftmp");
$status = $? >> 8;
v(2, "$diff_cmd >$diftmp: status=$status\n");
if (-s "$diftmp") {
# There's some difference
v(2, "$diftmp has something in it. Is it meaningful?\n");
if ($opt_f && -e $reffile && -e $outfile &&
diff_lenient_float($reffile, $outfile) == 0) {
print STDERR "$0: test $TestNo: minor (<$Epsilon) precision differences ignored\n";
$status = 0;
}
if ($opt_D or ($opt_d && $status)) {
# Print the diff only iff:
# 1) -D is in effect OR
# 2) -d is in effect and diff is significant
display_diff($reffile, $outfile);
}
if ($opt_o) {
print STDERR "-o: overwriting reference:\n";
if (-e $reffile) {
print STDERR "\t$reffile -> $reffile.prev\n";
rename($reffile, "$reffile.prev") ||
die "FATAL: rename($reffile, $reffile.prev): $!\n";
}
print STDERR "\t$outfile -> $reffile\n";
rename($outfile, $reffile) ||
die "FATAL: rename($outfile, $reffile): $!\n";
unless ($opt_e) {
$status = 0;
}
}
}
unlink($diftmp) if ($status == 0);
$status;
}
#
# check_for_time_regression()
# Compare last overall time to run to current to catch
# performance regressions
#
my $LastTimeFile = 'RunTests.last.times';
sub write_times($@) {
my ($file, @times) = @_;
open(my $fh, ">$file") || die "$0: can't open(>$file): $!\n";
print $fh join(' ', @times), "\n";
close $fh;
}
sub read_times($) {
my ($file) = @_;
open(my $fh, $file) || die "$0: can't open($file): $!\n";
my $line = <$fh>; chomp $line;
close $fh;
return (split(' ', $line));
}
sub check_for_time_regression() {
my $tolerate_regress = 1.02;
my $pct_change = 0.0;
my ($overall_time0, $overall_time1);
my ($user0, $system0, $cuser0, $csystem0);
my ($user1, $system1, $cuser1, $csystem1) = times;
$overall_time1 = $cuser1 + $csystem1;
if (-e $LastTimeFile) {
($user0, $system0, $cuser0, $csystem0) = read_times($LastTimeFile);
if (!(defined $csystem0) or !(defined $cuser0)) {
die "$0: undefined times in saved times file: $LastTimeFile," .
" try removing it\n"
}
$overall_time0 = $cuser0 + $csystem0;
$pct_change = 100 * ($overall_time1 - $overall_time0) / (1e-4+$overall_time0);
if ($overall_time0 == 0) {
die "$0: Bad times in saved times file: $LastTimeFile," .
" try removing it\n"
} elsif ($overall_time1/$overall_time0 > $tolerate_regress) {
printf STDERR "$0: RUNTIME REGRESSION: " .
"%.2f sec vs last time %.2f sec. (%.2f%% worse)\n",
$overall_time1, $overall_time0, $pct_change;
}
}
write_times($LastTimeFile, $user1, $system1, $cuser1, $csystem1);
printf STDERR
"$0 runtime: user %g, system %g, total %g sec (%+.2f%% vs. last)\n",
$cuser1, $csystem1, $overall_time1, $pct_change;
}
# only unlink relative path, plain files
# e.g. avoids trying to unlink /dev/null when running as root
sub safe_unlink($) {
my $file = shift;
return 0 if ($file =~ m{^/});
return 0 unless (-f $file);
unlink($file);
}
sub run_tests() {
print STDERR "$0: '-D' to see any diff output\n"
unless ($opt_D);
print STDERR "$0: '-d' to see only significant diff output\n"
unless ($opt_d);
print STDERR "$0: '-o' to force overwrite references\n"
unless ($opt_o);
print STDERR "$0: '-e' to abort/exit on first failure\n"
unless ($opt_e);
my ($cmd, $out_ref, $err_ref, $pred_ref, $cmp_ref);
my ($outf, $errf, $predf, $cmpf);
mkdir('models', 0755) unless (-d 'models');
unlink(glob('*.tmp'));
unlink(glob('*.cache'));
unlink(glob('*/*.cache'));
while (($cmd, $out_ref, $err_ref, @more_refs) = next_test()) {
last unless (defined $cmd);
if (@ToTest) {
if ($ToTest[0] != $TestNo) {
# warn "$0: test $TestNo: skipped\n";
next;
} else {
shift(@ToTest);
}
}
$outf = (defined($out_ref) && -f $out_ref)
? basename($out_ref)
: '/dev/null';
$errf = (defined($err_ref) && -f $err_ref)
? basename($err_ref)
: '/dev/null';
# Run the test
print STDERR "Test $TestNo: ($cmd) >$outf 2>$errf\n" if ($opt_c);
mysystem("($cmd) >$outf 2>$errf");
my $full_status = $?;
my $status = $full_status >> 8;
unless ($opt_V) {
if (my $failure = command_failed($cmd)) {
print STDERR `$Cat $errf`
unless ($failure == 124);
if ($opt_e) {
printf STDERR "$0: exiting with status=$failure\n";
exit $failure;
}
next;
}
}
if ($status) {
$ErrorCount++;
if ($opt_V && $status == 100) {
# If the failing test was a script or Python file, ignore the failure because Valgrind
# doesn't really work there.
if (index($cmd, ".sh") != -1 || index($cmd, ".py") != -1) {
print "$0: test $TestNo: Valgrind failure in script ignored\n";
# Undo the error count change, as this failure is okay.
$ErrorCount--;
next;
}
else {
my $errfile = valgrind_errfile($TestNo);
warn "$0: test $TestNo: FAILED: valgrind errors in $errfile\n";
}
} elsif ($TimeOut && $status == 124) {
warn "$0: test $TestNo: FAILED: timeout $TimeOutSec exceeded\n";
} else {
warn "$0: test $TestNo: '$cmd' failed: status=$status\n";
}
if ($opt_e) {
printf STDERR "exiting with full status $full_status\n";
exit 1;
}
next;
}
# command succeded
# -- compare stdout
$status = diff($out_ref, $outf);
if ($status) {
$ErrorCount++;
printf STDERR "%s: test %d: FAILED: ref(%s) != stdout(%s)\n\tcmd: $cmd\n",
$0, $TestNo, $out_ref, $outf;
copy_file($outf, "$outf.test$TestNo") if ($opt_y);
exit $status if ($opt_e);
} else {
if (defined $out_ref) {
print STDERR "$0: test $TestNo: stdout OK\n";
safe_unlink($outf);
} else {
v(1, "$0: test $TestNo: stdout OK (no reference)\n");
}
}
# -- compare stderr
if (! -e $err_ref and ! $opt_o) {
$ErrorCount++;
print STDERR "$0: test $TestNo: FAILED: stderr ref: $err_ref: $!\n\tcmd: $cmd\n";
exit 1 if ($opt_e);
next;
}
$status = diff($err_ref, $errf);
if ($status) {
$ErrorCount++;
printf STDERR "%s: test %d: FAILED: ref(%s) != stderr(%s)\n\tcmd: $cmd\n",
$0, $TestNo, $err_ref, $errf;
copy_file($errf, "$errf.test$TestNo") if ($opt_y);
exit $status if ($opt_e);
} else {
print STDERR "$0: test $TestNo: stderr OK\n";
safe_unlink($errf);
}
# -- compare all other reference files
if (@more_refs) {
foreach my $ref_path (@more_refs) {
my $ref_base = basename($ref_path);
# Verify that it exists on the shell line
unless ($cmd =~ /$ref_base/) {
printf STDERR "%s: test %d: FAILED: " .
"no match for '%s' in command: '%s'\n" .
"Unable to compare output to reference file\n",
$0, $TestNo, $ref_base, $cmd;
$ErrorCount++;
exit $status if ($opt_e);
next;
}
$status = diff($ref_path, $ref_base);
if ($status) {
$ErrorCount++;
printf STDERR "%s: test %d: FAILED: ref(%s) != (%s)\n\tcmd: $cmd\n",
$0, $TestNo, $ref_path, $ref_base;
copy_file($ref_base, $ref_path) if ($opt_y);
exit $status if ($opt_e);
} else {
print STDERR "$0: test $TestNo: $ref_base OK\n";
unlink($ref_base);
}
}
}
}
if ($FullRun == 0) {
v(1, "Partial run: not recording overall time\n");
} elsif ($ErrorCount > 0) {
v(1, "Errors found: not recording overall time\n");
} elsif ($opt_V) {
v(1, "valgrind run: not recording overall time\n");
} else {
check_for_time_regression();
}
}
# --- main
init();
run_tests();
exit $ErrorCount;
#
# Add tests below the __DATA__ line
#
# Each test is a sequence of non-blank lines, terminated
# by an empty line (or EOF), essentially a paragraph.
#
# Each paragraph/test should look like:
#
# 1st line: shell command to run.
# 2nd-to-Nth line: one-or-more reference files to compare outputs to.
# ONE reference file per line
# (Note: we indent these lines just for readability.)
#
# You may break very long lines using \ at EOL.
#
# # -------------------------------------
# # Test <test_id>: ...
# shell command which may include {VW} ...
# reference/file1
# reference/file2
# ...more reference files...
#
# # -------------------------------------
#
# shell-command can be anything accepted by bash, including pipes,
# redirections, etc., even a sequence of shell-commands separated by ';'
#
# Inside any shell command, all (optional) appearances of {VW}
# will be substituted by the vw executable under test.
#
# By default, 'vw' under our parent dir (../vowpalwabbit/vw) is tested.
# To run against a different reference executable, pass the
# wanted executable as an argument to RunTests
#
# The output line-items are reference files to compare outputs to:
# - *.stdout: expected (reference file) standard output
# - *.stderr: expected (reference file) standard error
# - Any other relative path, pointing to a reference file to compare
# to, this allows adding references to any explicitly named file
# appearing on the shell-line, the only requirement is that the
# _basename_ (path stripped of directory) of the reference file
# would exactly match its respective file in the shell-command.
#
# For example:
#
# #-------------------------------------------------------
# # Test 237: readable_model
# {VW} ... -p test75.predict --readable_model test75.rmodel
# test/train-sets/ref/test75.stderr
# test/pred-sets/ref/test75.predict
# test/whatever/ref/test75.rmodel
#
# #-------------------------------------------------------
#
# All reference filenames are relative to this (test) directory
#
# Only the STDOUT and STDERR streams in the shell command
# are implicit (so only their reference files need to be specified):
# The implicit names would be matched only by their extension
# as opposed to the full basename of the file.
#
# The two implicit names are:
# TestXXX.stdout
# TestXXX.stderr
#
# Windows note:
#
# Due to differences in Random-Number-Generators in Windows,
# floating-point outputs may differ in some tests (not all).
#
# To minimize the need for changes (leverage existing tests and
# reference files as much as possible), on Windows we check for
# existence of files with '-mswin' suffix:
# *.stderr-mswin
# *.stdout-mswin
# and if any of them exists, we use it instead.
#
__DATA__
# Test 1:
{VW} -k -l 20 --initial_t 128000 --power_t 1 -d train-sets/0001.dat \
-f models/0001_1.model -c --passes 8 --invariant \
--ngram 3 --skips 1 --holdout_off
train-sets/ref/0001.stderr
# Test 2: checking predictions as well
{VW} -k -t -d train-sets/0001.dat -i models/0001_1.model -p 0001.predict --invariant
test-sets/ref/0001.stderr
pred-sets/ref/0001.predict
# Test 3: without -d, training only
{VW} -k -d train-sets/0002.dat -f models/0002.model --invariant
train-sets/ref/0002.stderr
# Test 4: same, with -d
{VW} -k -d train-sets/0002.dat -f models/0002.model --invariant
train-sets/ref/0002.stdout
train-sets/ref/0002.stderr
# Test 5: add -q .., adaptive, and more (same input, different outputs)
{VW} -k --initial_t 1 --adaptive --invariant -q Tf -q ff -f models/0002a.model -d train-sets/0002.dat
train-sets/ref/0002a.stderr
# Test 6: run predictions on Test 4 model
# Pretending the labels aren't there
{VW} -k -t -i models/0002.model -d train-sets/0002.dat -p 0002b.predict
test-sets/ref/0002b.stderr
pred-sets/ref/0002b.predict
# Test 7: using normalized adaptive updates and a low --power_t
{VW} -k --power_t 0.45 -f models/0002c.model -d train-sets/0002.dat
train-sets/ref/0002c.stderr
# Test 8: predicts on test 7 model
{VW} -k -t -i models/0002c.model -d train-sets/0002.dat -p 0002c.predict
test-sets/ref/0002c.stderr
pred-sets/ref/0002c.predict
# Test 9: label-dependent features with csoaa_ldf
{VW} -k -c -d train-sets/cs_test.ldf -p cs_test.ldf.csoaa.predict --passes 10 --invariant --csoaa_ldf multiline --holdout_off --noconstant
train-sets/ref/cs_test.ldf.csoaa.stderr
train-sets/ref/cs_test.ldf.csoaa.predict
# Test 10: label-dependent features with wap_ldf
{VW} -k -c -d train-sets/cs_test.ldf -p cs_test.ldf.wap.predict --passes 10 --invariant --wap_ldf multiline --holdout_off --noconstant
train-sets/ref/cs_test.ldf.wap.stderr
train-sets/ref/cs_test.ldf.wap.predict
# Test 11: one-against-all
{VW} -k --oaa 10 -c --passes 10 -d train-sets/multiclass --holdout_off
train-sets/ref/oaa.stderr
# Test 12: Error Correcting Tournament
{VW} -k --ect 10 --error 3 -c --passes 10 --invariant -d train-sets/multiclass --holdout_off
train-sets/ref/multiclass.stderr
# Test 13: LBFGS on zero derivative input
{VW} -k -c -d train-sets/zero.dat --loss_function=squared -b 20 --bfgs --mem 7 --passes 5 --l2 1.0 --holdout_off
train-sets/ref/zero.stdout
train-sets/ref/zero.stderr
# Test 14: LBFGS early termination
{VW} -k -c -d train-sets/rcv1_small.dat --loss_function=logistic --bfgs --mem 7 --passes 20 --termination 0.001 --l2 1.0 --holdout_off
train-sets/ref/rcv1_small.stdout
train-sets/ref/rcv1_small.stderr
# Test 15: Run LDA with 100 topics on 1000 Wikipedia articles
{VW} -k --lda 100 --lda_alpha 0.01 --lda_rho 0.01 --lda_D 1000 -l 1 -b 13 --minibatch 128 -d train-sets/wiki256.dat
train-sets/ref/wiki1K.stderr
# Test 16: neural network 3-parity with 2 hidden units
{VW} -k -c -d train-sets/3parity --hash all --passes 3000 -b 16 --nn 2 -l 10 --invariant -f models/0021.model --random_seed 19 --quiet --holdout_off
train-sets/ref/3parity.stderr
# Test 17: neural network 3-parity with 2 hidden units (predict)
{VW} -d train-sets/3parity -t -i models/0021.model -p 0022.predict
pred-sets/ref/0022.stderr
pred-sets/ref/0022.predict
# Test 18: cubic features -- on a parity test case
{VW} -k -c -f models/xxor.model -d train-sets/xxor.dat --cubic abc --passes 100 --holdout_off --progress 1.33333
train-sets/ref/xxor.stderr
# Test 19: matrix factorization -- training
{VW} -k -d train-sets/ml100k_small_train -b 16 -q ui --rank 10 \
--l2 2e-6 --learning_rate 0.05 --passes 2 \
--decay_learning_rate 0.97 --power_t 0 -f models/movielens.reg \
-c --loss_function classic --holdout_off
train-sets/ref/ml100k_small.stdout
train-sets/ref/ml100k_small.stderr
# Test 20: matrix factorization -- testing
{VW} -i models/movielens.reg -t -d test-sets/ml100k_small_test
test-sets/ref/ml100k_small.stdout
test-sets/ref/ml100k_small.stderr
# Test 21: active-learning -- training
{VW} -k --active --simulation --mellowness 0.000001 -d train-sets/rcv1_small.dat -l 10 --initial_t 10 --random_seed 3
train-sets/ref/active-simulation.t24.stderr
# Test 22: bagging -- training regressor
{VW} -k -d train-sets/0002.dat -f models/bs.reg.model --bootstrap 4 -p bs.reg.predict
train-sets/ref/bs.reg.stderr
train-sets/ref/bs.reg.predict
# Test 23: bagging -- predicting with bagged regressor
{VW} -d train-sets/0002.dat -i models/bs.reg.model -p bs.prreg.predict -t
train-sets/ref/bs.prreg.stderr
train-sets/ref/bs.prreg.predict
# Test 24: bagging -- binary classifiers
{VW} -d train-sets/0001.dat -f models/bs.vote.model --bootstrap 4 --bs_type vote -p bs.vote.predict
train-sets/ref/bs.vote.stderr
train-sets/ref/bs.vote.predict
# Test 25: bagging -- predict with bagged classifier
{VW} -d train-sets/0001.dat -i models/bs.vote.model -p bs.prvote.predict -t
train-sets/ref/bs.prvote.stderr
train-sets/ref/bs.prvote.predict
# Test 26: affix features
{VW} -d train-sets/affix_test.dat -k -c --passes 10 --holdout_off --affix -2
train-sets/ref/affix_test.stderr
# Test 27: train --l1 regularized model
{VW} -d train-sets/0001.dat -f models/mask.model --invert_hash mask.predict --l1 0.01
train-sets/ref/mask.stderr
# Test 28: train model using --feature_mask
{VW} -d train-sets/0001.dat --invert_hash remask.predict --feature_mask models/mask.model -f models/remask.model
train-sets/ref/remask.stderr
# Test 29: train model using --feature_mask and --initial_regressor
{VW} -d train-sets/0001.dat --feature_mask models/mask.model -i models/remask.model
train-sets/ref/remask.final.stderr
# Test 30: train model for topk recommender
{VW} -d train-sets/topk.vw -f topk.model -q MF --passes 100 --cache_file topk-train.cache -k --holdout_off
train-sets/ref/topk-train.stderr
# Test 31: train model for topk recommender
{VW} -P 1 -d train-sets/topk.vw -i topk.model --top 2 -p topk-rec.predict
train-sets/ref/topk-rec.stderr