-
Notifications
You must be signed in to change notification settings - Fork 0
/
test.py
91 lines (78 loc) · 2.89 KB
/
test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#!/usr/bin/python
import os, os.path, time, datetime, signal, platform, subprocess
import pprint
import sexp
import translator
programdir = './programs/'
testroot = './tests/'
hiddentests = testroot + 'hidden_tests/'
opentests = testroot + 'open_tests/'
class TimeoutError(Exception):
pass
def stripComments(bmFile):
noComments = '('
for line in bmFile:
line = line.split(';', 1)[0]
noComments += line
return noComments + ')'
def run_command(cmd, timeout=60):
is_linux = platform.system() == 'Linux'
p = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=False,
preexec_fn=os.setsid if is_linux else None)
t_beginning = datetime.datetime.now()
seconds_passed = 0
while True:
timepassed = datetime.datetime.now() - t_beginning
rtimepassed = timepassed.seconds + timepassed.microseconds / 1000000.0
if p.poll() is not None:
break
if timeout and rtimepassed > timeout:
if is_linux:
os.killpg(p.pid, signal.SIGTERM)
else:
print 'out of time'
p.terminate()
raise TimeoutError(cmd, timeout)
time.sleep(0.01)
return p.stdout.read(), rtimepassed
def my_test(cmd, outputfile, testname, timeout=300):
outputfile.write('\t%s:' % (testname))
print cmd
try:
result, rtime = run_command(cmd, timeout)
except TimeoutError:
outputfile.write('timeout after %i \n' % (timeout))
else:
print result
benchmarkFile = open(testname)
bm = stripComments(benchmarkFile)
bmExpr = sexp.sexp.parseString(bm, parseAll=True).asList()[0] # Parse string to python list
checker = translator.ReadQuery(bmExpr)
try:
checkresult = checker.check(result)
except:
# outputfile.write('Wrong Answer: Invalid check result %s(%f)\n' %(result,rtime))
outputfile.write('Invalid format(%f)\n' % (rtime))
else:
if (checkresult == None):
outputfile.write('Passed(%f)\n' % (rtime))
else:
# outputfile.write('Wrong Answer: get %s(%f)\n' %(result,rtime))
outputfile.write('Wrong Answer(%f)\n' % (rtime))
if __name__ == '__main__':
timeout = 60
testresultfile = 'testresult.txt'
outfile = open(testresultfile, 'w')
i = 0
for studentname in os.listdir(programdir):
if studentname != 'CEGIS':
continue
toexe = programdir + studentname + '/main.py '
outfile.write(studentname + ': \n')
cmd = ('python3.5 ' if '3.5' in studentname else 'python ')
i = i + 1
print i
for testgroup in [hiddentests, opentests]:
for test in os.listdir(testgroup):
arg = testgroup + test
my_test(cmd + toexe + arg, outfile, arg, timeout)