Permalink
Browse files

Added ability to profile each benchmark and store results in a specif…

…ied directory
  • Loading branch information...
1 parent ec13b1a commit f2f6084236242682b91137d98f99a21fe40ee552 @sebleier sebleier committed Oct 2, 2010
Showing with 54 additions and 32 deletions.
  1. +21 −9 djangobench/main.py
  2. +33 −23 djangobench/utils.py
View
@@ -16,7 +16,7 @@
DEFAULT_BENCMARK_DIR = Path(__file__).parent.child('benchmarks').absolute()
-def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, vcs=None, record_dir=None):
+def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, vcs=None, record_dir=None, profile_dir=None):
if benchmarks:
print "Running benchmarks: %s" % " ".join(benchmarks)
else:
@@ -27,7 +27,7 @@ def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, vcs=N
if not record_dir.exists():
raise ValueError('Recording directory "%s" does not exist' % record_dir)
print "Recording data to '%s'" % record_dir
-
+
control_label = get_django_version(control, vcs=vcs)
experiment_label = get_django_version(experiment, vcs=vcs)
branch_info = "%s branch " % vcs if vcs else ""
@@ -51,7 +51,9 @@ def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, vcs=N
settings_mod = '%s.settings' % benchmark.name
control_env['DJANGO_SETTINGS_MODULE'] = settings_mod
experiment_env['DJANGO_SETTINGS_MODULE'] = settings_mod
-
+ if profile_dir is not None:
+ control_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "con-%s" % benchmark.name)
+ experiment_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "exp-%s" % benchmark.name)
try:
if vcs: switch_to_branch(vcs, control)
control_data = run_benchmark(benchmark, trials, control_env)
@@ -208,7 +210,7 @@ def format_benchmark_result(result, num_points):
return str(result)
def get_django_version(loc, vcs=None):
- if vcs:
+ if vcs:
switch_to_branch(vcs, loc)
pythonpath = Path.cwd()
else:
@@ -225,7 +227,7 @@ def switch_to_branch(vcs, branchname):
else:
raise ValueError("Sorry, %s isn't supported (yet?)" % vcs)
subprocess.check_call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
+
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -257,6 +259,7 @@ def main():
metavar = 'PATH',
help = 'Directory to record detailed output as a series of JSON files.',
)
+
parser.add_argument(
'--benchmark-dir',
dest = 'benchmark_dir',
@@ -272,16 +275,25 @@ def main():
help = "Benchmarks to be run. Defaults to all.",
nargs = '*'
)
-
+ parser.add_argument(
+ '-p',
+ '--profile-dir',
+ dest = 'profile_dir',
+ default = None,
+ metavar = 'PATH',
+ help = 'Directory to record profiling statistics for the control and experimental run of each benchmark'
+ )
+
args = parser.parse_args()
run_benchmarks(
- control = args.control,
+ control = args.control,
experiment = args.experiment,
benchmark_dir = args.benchmark_dir,
- benchmarks = args.benchmarks,
- trials = args.trials,
+ benchmarks = args.benchmarks,
+ trials = args.trials,
vcs = args.vcs,
record_dir = args.record,
+ profile_dir = args.profile_dir
)
if __name__ == '__main__':
View
@@ -1,60 +1,70 @@
import argparse
import inspect
+import os
import time
+try:
+ import cProfile as profile
+except ImportError:
+ import profile
benchmark_parser = argparse.ArgumentParser()
benchmark_parser.add_argument('-t', '--trials', type=int, default=100)
def run_benchmark(benchmark, syncdb=True, setup=None, trials=None, handle_argv=True, meta={}):
"""
Run a benchmark a few times and report the results.
-
+
Arguments:
-
+
benchmark
The benchmark callable. ``run_benchmark`` will time
the executation of this function and report those times
back to the harness. However, if ``benchmark`` returns
a value, that result will reported instead of the
raw timing.
-
+
syncdb
If True, a syncdb will be performed before running
the benchmark.
-
+
setup
A function to be called before running the benchmark
function(s).
-
+
trials
The number of times to run the benchmark function. If not given
and if ``handle_argv`` is ``True`` this'll be automatically
determined from the ``--trials`` flag.
-
+
handle_argv
``True`` if the script should handle ``sys.argv`` and set
the number of trials accordingly.
-
+
meta
Key/value pairs to be returned as part of the benchmark results.
"""
if handle_argv:
args = benchmark_parser.parse_args()
trials = trials or args.trials
-
+
print_benchmark_header(benchmark, meta)
-
+
if syncdb:
from django.core.management import call_command
call_command("syncdb", verbosity=0)
-
+
if setup:
setup()
-
+
for x in xrange(trials):
start = time.time()
- benchmark_result = benchmark()
-
+ profile_file = os.environ.get('DJANGOBENCH_PROFILE_FILE', None)
+ if profile_file is not None:
+ loc = locals().copy()
+ profile.runctx('benchmark_result = benchmark()', globals(), loc, profile_file)
+ benchmark_result = loc['benchmark_result']
+ else:
+ benchmark_result = benchmark()
if benchmark_result is not None:
print benchmark_result
else:
@@ -63,44 +73,44 @@ def run_benchmark(benchmark, syncdb=True, setup=None, trials=None, handle_argv=T
def run_comparison_benchmark(benchmark_a, benchmark_b, syncdb=True, setup=None, trials=None, handle_argv=True, meta={}):
"""
Benchmark the difference between two functions.
-
+
Arguments are as for ``run_benchmark``, except that this takes 2
benchmark functions, an A and a B, and reports the difference between
them.
-
+
For example, you could use this to test the overhead of an ORM query
versus a raw SQL query -- pass the ORM query as ``benchmark_a`` and the
raw query as ``benchmark_b`` and this function will report the
difference in time between them.
-
+
For best results, the A function should be the more expensive one
(otherwise djangobench will report results like "-1.2x slower", which
- is just confusing).
+ is just confusing).
"""
if handle_argv:
args = benchmark_parser.parse_args()
trials = trials or args.trials
-
+
print_benchmark_header(benchmark_a, meta)
-
+
if syncdb:
from django.core.management import call_command
call_command("syncdb", verbosity=0)
if setup:
setup()
-
+
for x in xrange(trials):
start_a = time.time()
result_a = benchmark_a()
result_a = result_a or time.time() - start_a
-
+
start_b = time.time()
result_b = benchmark_b()
result_b = result_b or time.time() - start_b
-
+
print result_a - result_b
-
+
def print_benchmark_header(benchmark, meta):
if 'title' not in map(str.lower, meta.keys()):
meta['title'] = inspect.getmodule(benchmark).__name__

0 comments on commit f2f6084

Please sign in to comment.