/
runTests.py
executable file
·186 lines (150 loc) · 7.84 KB
/
runTests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
#!/usr/bin/env python3
from __future__ import print_function, division, unicode_literals, absolute_import
from helpers.automation_tools import directory, get_changed_packages
import subprocess, argparse, shutil, sys, os
import webbrowser
'''
Script for running the test suite.
see pyGSTi/doc/repotools/test.md, or try running ./runTests.py -h
'''
def run_mpi_coverage_tests(coverage_cmd, nproc=4):
shutil.copy('mpi/setup.cfg.mpi', 'setup.cfg')
#OLD: worked with python2.7, but not with 3 (where .coverage files turned to JSON)
#mpicommands = ('time mpiexec -np %s python%s mpi/runtests.py -v ' % (str(nproc), '' if version is None else version)+
# '--with-coverage --cover-package=pygsti --cover-erase mpi/testmpi*.py ' +
# '> ../output/coverage_tests_mpi.out 2>&1')
mpicommands = ('time mpiexec -np %s %s run -p ' % (str(nproc), coverage_cmd) +
'--source=pygsti mpi/runtests.py -v mpi/testmpi*.py ' +
'> ../output/coverage_tests_mpi.out 2>&1')
with open('../output/mpi_output.txt', 'w') as output:
returned = subprocess.call(mpicommands, shell=True, stdout=output, stderr=output)
with open('../output/mpi_output.txt', 'r') as output:
print(output.read())
os.remove('setup.cfg')
return returned
def create_html(dirname, coverage_cmd):
subprocess.call([coverage_cmd, 'html', '--directory=%s' % dirname])
default = ['tools', 'iotest', 'objects', 'construction', 'drivers', 'report', 'reportb', 'algorithms', 'algorithmsb', 'optimize', 'extras', 'mpi']
slowtests = ['report', 'drivers']
def run_tests(testnames, version=None, fast=False, changed=False, coverage=True,
parallel=False, failed=False, cores=None, coverdir='../output/coverage', html=False,
threshold=90, outputfile=None, package='pygsti'):
with directory('test_packages'):
# Don't run report or drivers
if fast:
for slowtest in slowtests:
testnames.remove(slowtest)
# Specify the versions of your test :)
if version is None:
# The version this file was run/imported with
pythoncommands = ['python%s.%s' % (sys.version_info[0], sys.version_info[1])]
else:
# The version specified
pythoncommands = ['python%s' % version]
# Always use nose
pythoncommands += ['-m', 'nose', '-v']
# Since last commit to current branch
if changed:
testnames = [name for name in testnames if name in get_changed_packages()]
if len(testnames) == 0:
print('No tests to run')
sys.exit(0)
# testnames should be final at this point
print('Running tests:\n%s' % ('\n'.join(testnames)))
# Version-specific coverage cmds only sometimes have a dash
# e.g.: coverage, coverage2, coverage3, coverage-2.7, coverage-3.5
if version is None: coverage_cmd = "coverage"
elif "." in version: coverage_cmd = 'coverage-%s' % version
else: coverage_cmd = 'coverage%s' % version
# Run mpi coverage tests differently
covermpi = ('mpi' in testnames) and coverage
if covermpi:
testnames.remove('mpi')
postcommands = []
# Use parallelism native to nose
if parallel:
if cores is None:
pythoncommands.append('--processes=-1')
# (-1) will use all cores
else:
pythoncommands.append('--processes=%s' % cores)
# Some tests take up to an hour
pythoncommands.append('--process-timeout=14400') # Four hours
else:
# Use the failure monitoring native to nose
postcommands = ['--with-id']
if failed:
postcommands = ['--failed']# ~implies --with-id
if coverage:
# html coverage is prettiest
pythoncommands += ['--with-coverage',
'--cover-erase',
'--cover-package={}'.format(package),
'--cover-min-percentage={}'.format(threshold)]
returned = 0
if len(testnames) > 0:
commands = pythoncommands + testnames + postcommands
print(' '.join(commands))
if outputfile is None:
returned = subprocess.call(commands)
else:
with open(outputfile, 'w') as testoutput:
returned = subprocess.call(commands, stdout=testoutput, stderr=testoutput)
with open(outputfile, 'r') as testoutput:
print(testoutput.read())
if parallel:
#Only combine when run in parallel mode, since this
# causes nose tests to create .coverage.<processid>
# files instead of just a single .coverage file, which
# "coverage combine" will overwrite with no-data (eek!).
subprocess.call([coverage_cmd, 'combine'])
if covermpi:
print('Running mpi with coverage')
# Combine serial/parallel coverage
serial_coverage_exists = bool(len(testnames) > 0)
if serial_coverage_exists:
#In this case, nose tests have erased old coverage files
shutil.copy2('.coverage', '../output/temp_coverage')
else:
#If no serial tests have run, then we need to erase old files
subprocess.call([coverage_cmd, 'erase'])
run_mpi_coverage_tests(coverage_cmd) #creates .coverage.xxx files
if serial_coverage_exists:
shutil.copy2('../output/temp_coverage', '.coverage.serial')
subprocess.call([coverage_cmd, 'combine']) #combine everything
if html:
create_html(coverdir, coverage_cmd)
webbrowser.open(coverdir + '/index.html')
sys.exit(returned)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Run tests for pygsti')
parser.add_argument('tests', nargs='*', default=default, type=str,
help='list of packages to run tests for')
parser.add_argument('--package', type=str, default='pygsti',
help='package to test coverage for')
parser.add_argument('--version', '-v', type=str,
help='version of python to run the tests under')
parser.add_argument('--changed', '-c', action='store_true', help='run only the changed packages')
parser.add_argument('--fast', '-f', action='store_true',
help='run only the faster packages')
parser.add_argument('--failed', action='store_true',
help='run last failed tests only')
parser.add_argument('--html', action='store_true',
help='generate html')
parser.add_argument('--parallel', '-p', action='store_true',
help='run tests in parallel')
parser.add_argument('--cover', action='store_true',
help='generate coverage')
parser.add_argument('--cores', type=int, default=None,
help='run tests with n cores')
parser.add_argument('--coverdir', type=str, default='../output/coverage',
help='put html coverage report here')
parser.add_argument('--threshold', type=int, default=90,
help='coverage percentage to beat')
parser.add_argument('--output', type=str, default=None,
help='outputfile')
parsed = parser.parse_args(sys.argv[1:])
# With this many arguments, maybe this function should be refactored?
run_tests(parsed.tests, parsed.version, parsed.fast, parsed.changed, parsed.cover,
parsed.parallel, parsed.failed, parsed.cores, parsed.coverdir,
parsed.html, parsed.threshold, parsed.output, parsed.package)