Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100755 235 lines (212 sloc) 7.871 kB
df63750 @andrewrk test.py
authored
1 #!/usr/bin/python
2
3 import os, sys
4 import subprocess
5 import optparse
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
6 import tempfile
7
8 def which(executable, path=None):
9 """Try to find 'executable' in the directories listed in 'path' (a
10 string listing directories separated by 'os.pathsep'; defaults to
11 os.environ['PATH']). Returns the complete filename or None if not
12 found
13 """
14 if path is None:
15 path = os.environ['PATH']
16 paths = path.split(os.pathsep)
17 extlist = ['']
18 if os.name == 'os2':
19 (base, ext) = os.path.splitext(executable)
20 # executable files on OS/2 can have an arbitrary extension, but
21 # .exe is automatically appended if no dot is present in the name
22 if not ext:
23 executable = executable + ".exe"
24 elif sys.platform == 'win32':
25 pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
26 (base, ext) = os.path.splitext(executable)
27 if ext.lower() not in pathext:
28 extlist = pathext
29 for ext in extlist:
30 execname = executable + ext
31 if os.path.isfile(execname):
32 return execname
33 else:
34 for p in paths:
35 f = os.path.join(p, execname)
36 if os.path.isfile(f):
37 return f
38 else:
39 return None
40
df63750 @andrewrk test.py
authored
41
42 def superwalk(folder):
43 for dirpath, _dirnames, filenames in os.walk(folder):
44 for filename in filenames:
45 yield os.path.join(dirpath, filename)
46
47 def absolute(relative_path):
48 return os.path.abspath(os.path.join(os.path.dirname(__file__), relative_path))
49
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
50 def execute_spim_code(asm_code):
51 "execute asm_code and return stdout"
52 exe = which('spim')
53
54 # save asm_code to a file
55 handle = tempfile.NamedTemporaryFile(suffix=".asm", delete=False)
56 handle.write(asm_code)
57 handle.close()
58 spim = subprocess.Popen([exe, '-file', handle.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
59 stdout, stderr = spim.communicate()
60 try:
61 os.remove(handle.name)
62 except:
63 pass
64
65 # chop off the 5 useless lines
66 clean_out = "\n".join(stdout.split('\n')[5:])
67
68 return clean_out
df63750 @andrewrk test.py
authored
69
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
70 def main():
71 parser = optparse.OptionParser()
d8c1a53 @thejoshwolfe added ./test.py -b
thejoshwolfe authored
72 parser.add_option('-f', '--failfast', help="Stop on first failed test", action="store_true")
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
73 parser.add_option("-q", "--quiet", help="only print dots and summary", action="store_true")
d8c1a53 @thejoshwolfe added ./test.py -b
thejoshwolfe authored
74 parser.add_option("-b", "--backwards", help="run tests in reverse order", action="store_true")
9665097 @thejoshwolfe added -v option for test.py
thejoshwolfe authored
75 parser.add_option("-v", "--verbose", action="store_true", default=False)
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
76 options, args = parser.parse_args()
df63750 @andrewrk test.py
authored
77
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
78 if not options.quiet:
79 print("Loading test suite...")
80
81 tests = {}
c7bd85f @thejoshwolfe added documentation in parser.h and added custom tests to test.py
thejoshwolfe authored
82 for f in superwalk("tests/"):
d87ccbb @thejoshwolfe fixed _ ignore in test.py
thejoshwolfe authored
83 if f.find("/_") != -1:
c7bd85f @thejoshwolfe added documentation in parser.h and added custom tests to test.py
thejoshwolfe authored
84 continue
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
85 if f.endswith('.p.errors'):
86 test_name = f[:-len('.p.errors')]
87 ext = '.p.errors'
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
88 elif f.endswith('.p'):
89 test_name = f[:-len('.p')]
90 ext = '.p'
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
91 elif f.endswith('.p.out'):
92 test_name = f[:-len('.p.out')]
93 ext = '.p.out'
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
94 else:
95 continue
96
97 if not tests.has_key(test_name):
98 tests[test_name] = {}
99
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
100 if ext == '.p.errors':
101 compiler_output = open(absolute(f), 'r').read()
102 tests[test_name]['errors'] = compiler_output.replace('Errors detected. Exiting.\n', '')
103 elif ext == '.p.out':
844155c @andrewrk ignore "errors detected, exiting" in tests
authored
104 expected_output = open(absolute(f), 'r').read()
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
105 tests[test_name]['out'] = expected_output
106 else: # ext == '.p'
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
107 tests[test_name]['source'] = open(absolute(f), 'r').read()
108
109 fails = []
110 compiler_exe = absolute('opc')
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
111 interpret_command = execute_spim_code
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
112 passed = 0
d8c1a53 @thejoshwolfe added ./test.py -b
thejoshwolfe authored
113 test_list = sorted(tests.iteritems())
114 if options.backwards:
115 test_list.reverse()
2f106c9 @andrewrk fix some bugs and test cases.
authored
116
117 # complain about missing files before showing progress
d8c1a53 @thejoshwolfe added ./test.py -b
thejoshwolfe authored
118 for test_name, test in test_list:
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
119 if not test.has_key('source'):
120 print("%s missing source" % test_name)
121 continue
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
122 if not test.has_key('errors'):
123 print("%s missing .errors file for expected compiler output" % test_name)
124 continue
125 if not test.has_key('out'):
126 print("%s missing .out file for what the program should output" % test_name)
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
127 continue
128
2f106c9 @andrewrk fix some bugs and test cases.
authored
129 for test_name, test in test_list:
130 if not test.has_key('source'):
131 continue
132 if not test.has_key('errors'):
133 continue
134 if not test.has_key('out'):
135 continue
136
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
137 # try compiling the test file
9665097 @thejoshwolfe added -v option for test.py
thejoshwolfe authored
138 if options.verbose:
139 sys.stdout.write(test_name + "...")
140 sys.stdout.flush()
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
141 compiler = subprocess.Popen([compiler_exe], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
142 stdout, stderr = compiler.communicate(test['source'])
4134762 @andrewrk test framework catches crashes
authored
143 if compiler.returncode not in [0, 1]:
9665097 @thejoshwolfe added -v option for test.py
thejoshwolfe authored
144 if options.verbose:
145 sys.stdout.write("crash\n")
146 else:
147 sys.stdout.write('E')
4134762 @andrewrk test framework catches crashes
authored
148 fails.append({
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
149 'errors': test['errors'],
4134762 @andrewrk test framework catches crashes
authored
150 'output': stderr,
151 'stdout': stdout,
152 'name': test_name,
153 'crash': True,
154 })
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
155 if options.failfast:
156 break
157 elif stderr != test['errors']:
9665097 @thejoshwolfe added -v option for test.py
thejoshwolfe authored
158 if options.verbose:
159 sys.stdout.write("fail\n")
160 else:
161 sys.stdout.write('F')
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
162 fails.append({
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
163 'errors': test['errors'],
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
164 'output': stderr,
23913e0 @andrewrk check types for assignments. we pass the first test
authored
165 'stdout': stdout,
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
166 'name': test_name,
4134762 @andrewrk test framework catches crashes
authored
167 'crash': False,
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
168 })
169 if options.failfast:
170 break
9ebe01f @andrewrk correct some test output and the test suite itself
authored
171 elif compiler.returncode != 1:
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
172 # compiler output correct, now test the generated code output
173 asm_output = interpret_command(stdout)
174 if asm_output != test['out']:
9665097 @thejoshwolfe added -v option for test.py
thejoshwolfe authored
175 if options.verbose:
176 sys.stdout.write("wrong\n")
177 else:
178 sys.stdout.write('W')
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
179 fails.append({
180 'expected_runout': test['out'],
181 'runout': asm_output,
182 'name': test_name,
183 'crash': False,
184 })
185 if options.failfast:
186 break
187 else:
9665097 @thejoshwolfe added -v option for test.py
thejoshwolfe authored
188 if options.verbose:
189 sys.stdout.write("pass\n")
190 else:
191 sys.stdout.write('.')
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
192 passed += 1
9665097 @thejoshwolfe added -v option for test.py
thejoshwolfe authored
193 else:
194 if options.verbose:
195 sys.stdout.write("pass\n")
196 else:
197 sys.stdout.write('.')
198 passed += 1
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
199
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
200 sys.stdout.flush()
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
201
202 if len(fails) > 0:
203 if not options.quiet:
204 print("\n=========================================")
205 for fail in fails:
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
206 print("Test name: %(name)s" % fail)
207 if 'expected_runout' in fail:
208 print("""\
209 ---- Program Output: ----
210 %(runout)s\
211 ---- Expected Program Output: ----
212 %(expected_runout)s\
213 --------""" % fail)
214 else:
215 print("""\
216 ---- Compiler Output: ----
217 %(output)s\
218 ---- Expected Compiler output: ----
219 %(errors)s\
23913e0 @andrewrk check types for assignments. we pass the first test
authored
220 --------""" % fail)
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
221 if fail['crash']:
222 print("The program crashed.")
223 if fail['stdout'].strip() != '':
224 print("---- Compiler stdout ----")
225 print(fail['stdout'])
226 print("----------------")
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
227 print("=========================================")
228 print("\n%i tests passed, %i tests failed." % (passed, len(fails)))
229 else:
cddc1ed @andrewrk revamp test system, but still need to correct the .out files
authored
230 print("\nAll %i tests passed." % passed)
b3e9f16 @thejoshwolfe formatted test.py and added -q
thejoshwolfe authored
231
232 if __name__ == "__main__":
233 main()
234
Something went wrong with that request. Please try again.