forked from regehr/calc-compiler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_tests.pl
executable file
·140 lines (133 loc) · 3.55 KB
/
run_tests.pl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
#!/usr/bin/perl -w
use strict;
my $CALCC = $ENV{"CALCC"};
if (!defined($CALCC)) {
$CALCC = "../build/calcc";
}
die "cannot find compiler!" unless
-e $CALCC;
sub runit ($) {
(my $cmd) = @_;
system "$cmd";
return $? >> 8;
}
sub test($) {
(my $f) = @_;
system "rm -f a.out";
print "================================================\n";
print "running test '$f'\n";
open INF, "<$f" or die "OOPS: cannot open '$f'";
my $result;
my $args;
my $check;
while (my $line = <INF>) {
chomp $line;
if ($line =~ /RESULT (.*)$/) {
die "BUGGY TESTCASE: more than one RESULT line"
if defined $result;
$result = $1;
}
if ($line =~ /CHECK$/) {
die "BUGGY TESTCASE: more than one CHECK line"
if defined $check;
$check = 1;
}
if ($line =~ /ARGS\s*([0-9\ \-]*)\s*$/) {
die "BUGGY TESTCASE: more than one ARG line"
if defined $args;
$args = $1;
}
}
close INF;
die "BUGGY TESTCASE: cannot find RESULT line in '$f'" unless
defined $result;
die "BUGGY TESTCASE: cannot find ARGS line in '$f'" unless
defined $args;
my @arglist = split / /, $args;
die "BUGGY TESTCASE: too many args in '$f'" if
scalar(@arglist) > 6;
die "BUGGY TESTCASE: unexpected RESULT '$result'" unless
($result eq "ERROR" ||
$result =~ /^-?[0-9]+$/ ||
$result =~ /^OVERFLOW [0-9]+$/);
my $res;
if ($check){
$res = runit("$CALCC -check < $f > /dev/null 2> out.ll");
} else {
$res = runit("$CALCC < $f > /dev/null 2> out.ll");
}
if ($result eq "ERROR") {
if ($res == 1) {
print "compiler correctly detected erroneous input\n";
goto done;
}
print "COMPILER BUG: failed to correctly give exit code 1\n";
return 0;
}
if ($res != 0) {
print "COMPILER BUG: failed to correctly give exit code 0\n";
return 0;
}
runit("clang out.ll ../driver.c");
if (!(-f "a.out")) {
print "COMPILER BUG: executable could not be generated\n";
return 0;
}
my $argstr = "";
foreach my $arg (@arglist) {
$argstr .= "$arg ";
}
$res = runit("./a.out $argstr > output.txt");
if ($res != 0 && !$check) {
print "COMPILER BUG: executable did not run successfully\n";
return 0;
}
open INF, "<output.txt" or
die "OOPS: cannot read output generated by executable";
my $prog_result;
while (my $line = <INF>) {
chomp $line;
if ($line =~ "integer error at position ([0-9]+)") {
$prog_result = "OVERFLOW $1";
}
if ($line =~ /result = (-?[0-9]+)$/) {
$prog_result = $1;
}
}
close INF;
if (!defined($prog_result)) {
print "COMPILER BUG: output from executable does not contain 'result = ...'\n";
return 0;
}
print "expected output = $result\n";
print "actual output = $prog_result\n";
if ($result ne $prog_result) {
print "COMPILER BUG: output disagreement\n";
return 0;
}
done:
print "TEST SUCCESSFUL\n";
return 1;
}
my $pass = 0;
my $fail = 0;
my @fails;
foreach my $f (@ARGV) {
if (test($f)) {
$pass++;
} else {
$fail++;
push @fails, $f;
}
print "\n";
}
print "\n";
print "================================================\n";
print "SUMMARY: $pass tests passed, $fail tests failed\n";
if ($fail > 0) {
print "failing tests:\n";
foreach my $t (@fails) {
print " $t\n";
}
}
print "================================================\n";