forked from django/djangobench
-
Notifications
You must be signed in to change notification settings - Fork 2
/
main.py
executable file
·301 lines (268 loc) · 10.5 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
#!/usr/bin/env python
"""
Run us some Django benchmarks.
"""
import subprocess
import argparse
import email
import simplejson
import sys
from djangobench import perf
from unipath import DIRS, FSPath as Path
__version__ = '0.9'
DEFAULT_BENCMARK_DIR = Path(__file__).parent.child('benchmarks').absolute()
def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, vcs=None, record_dir=None, profile_dir=None):
if benchmarks:
print "Running benchmarks: %s" % " ".join(benchmarks)
else:
print "Running all benchmarks"
if record_dir:
record_dir = Path(record_dir).expand().absolute()
if not record_dir.exists():
raise ValueError('Recording directory "%s" does not exist' % record_dir)
print "Recording data to '%s'" % record_dir
control_label = get_django_version(control, vcs=vcs)
experiment_label = get_django_version(experiment, vcs=vcs)
branch_info = "%s branch " % vcs if vcs else ""
print "Control: Django %s (in %s%s)" % (control_label, branch_info, control)
print "Experiment: Django %s (in %s%s)" % (experiment_label, branch_info, experiment)
print
# Calculate the subshell envs that we'll use to execute the
# benchmarks in.
if vcs:
control_env = {
'PYTHONPATH': '%s:%s' % (Path.cwd().absolute(), Path(benchmark_dir)),
}
experiment_env = control_env.copy()
else:
control_env = {'PYTHONPATH': '%s:%s' % (Path(control).absolute(), Path(benchmark_dir))}
experiment_env = {'PYTHONPATH': '%s:%s' % (Path(experiment).absolute(), Path(benchmark_dir))}
for benchmark in discover_benchmarks(benchmark_dir):
if not benchmarks or benchmark.name in benchmarks:
print "Running '%s' benchmark ..." % benchmark.name
settings_mod = '%s.settings' % benchmark.name
control_env['DJANGO_SETTINGS_MODULE'] = settings_mod
experiment_env['DJANGO_SETTINGS_MODULE'] = settings_mod
if profile_dir is not None:
control_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "con-%s" % benchmark.name)
experiment_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "exp-%s" % benchmark.name)
try:
if vcs: switch_to_branch(vcs, control)
control_data = run_benchmark(benchmark, trials, control_env)
if vcs: switch_to_branch(vcs, experiment)
experiment_data = run_benchmark(benchmark, trials, experiment_env)
except SkipBenchmark, reason:
print "Skipped: %s\n" % reason
continue
options = argparse.Namespace(
track_memory = False,
diff_instrumentation = False,
benchmark_name = benchmark.name,
disable_timelines = True,
control_label = control_label,
experiment_label = experiment_label,
)
result = perf.CompareBenchmarkData(control_data, experiment_data, options)
if record_dir:
record_benchmark_results(
dest = record_dir.child('%s.json' % benchmark.name),
name = benchmark.name,
result = result,
control = control_label,
experiment = experiment_label,
control_data = control_data,
experiment_data = experiment_data,
)
print format_benchmark_result(result, len(control_data.runtimes))
print
def discover_benchmarks(benchmark_dir):
for app in Path(benchmark_dir).listdir(filter=DIRS):
if app.child('benchmark.py').exists() and app.child('settings.py').exists():
yield app
class SkipBenchmark(Exception):
pass
def run_benchmark(benchmark, trials, env):
"""
Similar to perf.MeasureGeneric, but modified a bit for our purposes.
"""
# Remove Pycs, then call the command once to prime the pump and
# re-generate fresh ones. This makes sure we're measuring as little of
# Python's startup time as possible.
perf.RemovePycs()
command = [sys.executable, '%s/benchmark.py' % benchmark]
out, _, _ = perf.CallAndCaptureOutput(command + ['-t', 1], env, track_memory=False, inherit_env=[])
if out.startswith('SKIP:'):
raise SkipBenchmark(out.replace('SKIP:', '').strip())
# Now do the actual mesurements.
output = perf.CallAndCaptureOutput(command + ['-t', str(trials)], env, track_memory=False, inherit_env=[])
stdout, stderr, mem_usage = output
message = email.message_from_string(stdout)
data_points = [float(line) for line in message.get_payload().splitlines()]
return perf.RawData(data_points, mem_usage, inst_output=stderr)
def record_benchmark_results(dest, **kwargs):
kwargs['version'] = __version__
simplejson.dump(kwargs, open(dest, 'w'), default=json_encode_custom)
def json_encode_custom(obj):
if isinstance(obj, perf.RawData):
return obj.runtimes
if isinstance(obj, perf.BenchmarkResult):
return {
'min_base' : obj.min_base,
'min_changed' : obj.min_changed,
'delta_min' : obj.delta_min,
'avg_base' : obj.avg_base,
'avg_changed' : obj.avg_changed,
'delta_avg' : obj.delta_avg,
't_msg' : obj.t_msg,
'std_base' : obj.std_base,
'std_changed' : obj.std_changed,
'delta_std' : obj.delta_std,
}
if isinstance(obj, perf.SimpleBenchmarkResult):
return {
'base_time' : obj.base_time,
'changed_time' : obj.changed_time,
'time_delta' : obj.time_delta,
}
raise TypeError("%r is not JSON serializable" % obj)
def supports_color():
return sys.platform != 'win32' and hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
class colorize(object):
GOOD = INSIGNIFICANT = SIGNIFICANT = BAD = ENDC = ''
if supports_color():
GOOD = '\033[92m'
INSIGNIFICANT = '\033[94m'
SIGNIFICANT = '\033[93m'
BAD = '\033[91m'
ENDC = '\033[0m'
@classmethod
def colorize(cls, color, text):
return "%s%s%s" % (color, text, cls.ENDC)
@classmethod
def good(cls, text):
return cls.colorize(cls.GOOD, text)
@classmethod
def significant(cls, text):
return cls.colorize(cls.SIGNIFICANT, text)
@classmethod
def insignificant(cls, text):
return cls.colorize(cls.INSIGNIFICANT, text)
@classmethod
def bad(cls, text):
return cls.colorize(cls.BAD, text)
def format_benchmark_result(result, num_points):
if isinstance(result, perf.BenchmarkResult):
output = ''
delta_min = result.delta_min
if 'faster' in delta_min:
delta_min = colorize.good(delta_min)
elif 'slower' in result.delta_min:
delta_min = colorize.bad(delta_min)
output += "Min: %f -> %f: %s\n" % (result.min_base, result.min_changed, delta_min)
delta_avg = result.delta_avg
if 'faster' in delta_avg:
delta_avg = colorize.good(delta_avg)
elif 'slower' in delta_avg:
delta_avg = colorize.bad(delta_avg)
output += "Avg: %f -> %f: %s\n" % (result.avg_base, result.avg_changed, delta_avg)
t_msg = result.t_msg
if 'Not significant' in t_msg:
t_msg = colorize.insignificant(t_msg)
elif 'Significant' in result.t_msg:
t_msg = colorize.significant(t_msg)
output += t_msg
delta_std = result.delta_std
if 'larger' in delta_std:
delta_std = colorize.bad(delta_std)
elif 'smaller' in delta_std:
delta_std = colorize.good(delta_std)
output += "Stddev: %.5f -> %.5f: %s" %(result.std_base, result.std_changed, delta_std)
output += " (N = %s)" % num_points
output += result.get_timeline()
return output
else:
return str(result)
def get_django_version(loc, vcs=None):
if vcs:
switch_to_branch(vcs, loc)
pythonpath = Path.cwd()
else:
pythonpath = Path(loc).absolute()
out, err, _ = perf.CallAndCaptureOutput(
[sys.executable, '-c' 'import django; print django.get_version()'],
env = {'PYTHONPATH': pythonpath}
)
return out.strip()
def switch_to_branch(vcs, branchname):
if vcs == 'git':
cmd = ['git', 'checkout', branchname]
else:
raise ValueError("Sorry, %s isn't supported (yet?)" % vcs)
subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--control',
metavar = 'PATH',
default = 'django-control',
help = "Path to the Django code tree to use as control."
)
parser.add_argument(
'--experiment',
metavar = 'PATH',
default = 'django-experiment',
help = "Path to the Django version to use as experiment."
)
parser.add_argument(
'--vcs',
choices = ['git'],
help = 'Use a VCS for control/experiment. Makes --control/--experiment specify branches, not paths.'
)
parser.add_argument(
'-t', '--trials',
type = int,
default = 50,
help = 'Number of times to run each benchmark.'
)
parser.add_argument(
'-r', '--record',
default = None,
metavar = 'PATH',
help = 'Directory to record detailed output as a series of JSON files.',
)
parser.add_argument(
'--benchmark-dir',
dest = 'benchmark_dir',
metavar = 'PATH',
default = DEFAULT_BENCMARK_DIR,
help = ('Directory to inspect for benchmarks. Defaults to the '
'benchmarks included with djangobench.'),
)
parser.add_argument(
'benchmarks',
metavar = 'name',
default = None,
help = "Benchmarks to be run. Defaults to all.",
nargs = '*'
)
parser.add_argument(
'-p',
'--profile-dir',
dest = 'profile_dir',
default = None,
metavar = 'PATH',
help = 'Directory to record profiling statistics for the control and experimental run of each benchmark'
)
args = parser.parse_args()
run_benchmarks(
control = args.control,
experiment = args.experiment,
benchmark_dir = args.benchmark_dir,
benchmarks = args.benchmarks,
trials = args.trials,
vcs = args.vcs,
record_dir = args.record,
profile_dir = args.profile_dir
)
if __name__ == '__main__':
main()