Permalink
Browse files

Initial refactoring into a proper package.

Doesn't actually work yet, but the deck chairs have been moved.
  • Loading branch information...
jacobian committed Jun 22, 2010
1 parent 59cbca3 commit 048fe9e6de9694e6228c25d559faf2eb67b3f1f6
Showing with 101 additions and 43 deletions.
  1. +30 −41 README.rst
  2. +39 −0 TODO
  3. 0 { → djangobench}/__init__.py
  4. 0 { → djangobench}/base_settings.py
  5. 0 { → djangobench}/benchmarks/default_middleware/__init__.py
  6. 0 { → djangobench}/benchmarks/default_middleware/benchmark.py
  7. 0 { → djangobench}/benchmarks/default_middleware/models.py
  8. 0 { → djangobench}/benchmarks/default_middleware/settings.py
  9. 0 { → djangobench}/benchmarks/default_middleware/urls.py
  10. 0 { → djangobench}/benchmarks/default_middleware/views.py
  11. 0 { → djangobench}/benchmarks/form_clean/__init__.py
  12. 0 { → djangobench}/benchmarks/form_clean/benchmark.py
  13. 0 { → djangobench}/benchmarks/form_clean/settings.py
  14. 0 { → djangobench}/benchmarks/form_create/__init__.py
  15. 0 { → djangobench}/benchmarks/form_create/benchmark.py
  16. 0 { → djangobench}/benchmarks/form_create/settings.py
  17. 0 { → djangobench}/benchmarks/model_creation/__init__.py
  18. 0 { → djangobench}/benchmarks/model_creation/benchmark.py
  19. 0 { → djangobench}/benchmarks/model_creation/models.py
  20. 0 { → djangobench}/benchmarks/model_creation/settings.py
  21. 0 { → djangobench}/benchmarks/multi_value_dict/__init__.py
  22. 0 { → djangobench}/benchmarks/multi_value_dict/benchmark.py
  23. 0 { → djangobench}/benchmarks/multi_value_dict/settings.py
  24. 0 { → djangobench}/benchmarks/qs_filter_chaining/__init__.py
  25. 0 { → djangobench}/benchmarks/qs_filter_chaining/benchmark.py
  26. 0 { → djangobench}/benchmarks/qs_filter_chaining/models.py
  27. 0 { → djangobench}/benchmarks/qs_filter_chaining/settings.py
  28. 0 { → djangobench}/benchmarks/query_aggregate/__init__.py
  29. 0 { → djangobench}/benchmarks/query_aggregate/benchmark.py
  30. 0 { → djangobench}/benchmarks/query_aggregate/fixtures/initial_data.json
  31. 0 { → djangobench}/benchmarks/query_aggregate/models.py
  32. 0 { → djangobench}/benchmarks/query_aggregate/settings.py
  33. 0 { → djangobench}/benchmarks/query_annotate/__init__.py
  34. 0 { → djangobench}/benchmarks/query_annotate/benchmark.py
  35. 0 { → djangobench}/benchmarks/query_annotate/fixtures/initial_data.json
  36. 0 { → djangobench}/benchmarks/query_annotate/models.py
  37. 0 { → djangobench}/benchmarks/query_annotate/settings.py
  38. 0 { → djangobench}/benchmarks/query_complex_filter/__init__.py
  39. 0 { → djangobench}/benchmarks/query_complex_filter/benchmark.py
  40. 0 { → djangobench}/benchmarks/query_complex_filter/fixtures/initial_data.json
  41. 0 { → djangobench}/benchmarks/query_complex_filter/models.py
  42. 0 { → djangobench}/benchmarks/query_complex_filter/settings.py
  43. 0 { → djangobench}/benchmarks/query_count/__init__.py
  44. 0 { → djangobench}/benchmarks/query_count/benchmark.py
  45. 0 { → djangobench}/benchmarks/query_count/fixtures/initial_data.json
  46. 0 { → djangobench}/benchmarks/query_count/models.py
  47. 0 { → djangobench}/benchmarks/query_count/settings.py
  48. 0 { → djangobench}/benchmarks/query_dates/__init__.py
  49. 0 { → djangobench}/benchmarks/query_dates/benchmark.py
  50. 0 { → djangobench}/benchmarks/query_dates/fixtures/initial_data.json
  51. 0 { → djangobench}/benchmarks/query_dates/models.py
  52. 0 { → djangobench}/benchmarks/query_dates/settings.py
  53. 0 { → djangobench}/benchmarks/query_delete/__init__.py
  54. 0 { → djangobench}/benchmarks/query_delete/benchmark.py
  55. 0 { → djangobench}/benchmarks/query_delete/models.py
  56. 0 { → djangobench}/benchmarks/query_delete/settings.py
  57. 0 { → djangobench}/benchmarks/query_distinct/__init__.py
  58. 0 { → djangobench}/benchmarks/query_distinct/benchmark.py
  59. 0 { → djangobench}/benchmarks/query_distinct/fixtures/initial_data.json
  60. 0 { → djangobench}/benchmarks/query_distinct/models.py
  61. 0 { → djangobench}/benchmarks/query_distinct/settings.py
  62. 0 { → djangobench}/benchmarks/query_exclude/__init__.py
  63. 0 { → djangobench}/benchmarks/query_exclude/benchmark.py
  64. 0 { → djangobench}/benchmarks/query_exclude/fixtures/initial_data.json
  65. 0 { → djangobench}/benchmarks/query_exclude/models.py
  66. 0 { → djangobench}/benchmarks/query_exclude/settings.py
  67. 0 { → djangobench}/benchmarks/query_exists/__init__.py
  68. 0 { → djangobench}/benchmarks/query_exists/benchmark.py
  69. 0 { → djangobench}/benchmarks/query_exists/fixtures/initial_data.json
  70. 0 { → djangobench}/benchmarks/query_exists/models.py
  71. 0 { → djangobench}/benchmarks/query_exists/settings.py
  72. 0 { → djangobench}/benchmarks/query_filter/__init__.py
  73. 0 { → djangobench}/benchmarks/query_filter/benchmark.py
  74. 0 { → djangobench}/benchmarks/query_filter/fixtures/initial_data.json
  75. 0 { → djangobench}/benchmarks/query_filter/models.py
  76. 0 { → djangobench}/benchmarks/query_filter/settings.py
  77. 0 { → djangobench}/benchmarks/query_get/__init__.py
  78. 0 { → djangobench}/benchmarks/query_get/benchmark.py
  79. 0 { → djangobench}/benchmarks/query_get/fixtures/initial_data.json
  80. 0 { → djangobench}/benchmarks/query_get/models.py
  81. 0 { → djangobench}/benchmarks/query_get/settings.py
  82. 0 { → djangobench}/benchmarks/query_get_or_create/__init__.py
  83. 0 { → djangobench}/benchmarks/query_get_or_create/benchmark.py
  84. 0 { → djangobench}/benchmarks/query_get_or_create/models.py
  85. 0 { → djangobench}/benchmarks/query_get_or_create/settings.py
  86. 0 { → djangobench}/benchmarks/query_in_bulk/__init__.py
  87. 0 { → djangobench}/benchmarks/query_in_bulk/benchmark.py
  88. 0 { → djangobench}/benchmarks/query_in_bulk/fixtures/initial_data.json
  89. 0 { → djangobench}/benchmarks/query_in_bulk/models.py
  90. 0 { → djangobench}/benchmarks/query_in_bulk/settings.py
  91. 0 { → djangobench}/benchmarks/query_iterator/__init__.py
  92. 0 { → djangobench}/benchmarks/query_iterator/benchmark.py
  93. 0 { → djangobench}/benchmarks/query_iterator/fixtures/initial_data.json
  94. 0 { → djangobench}/benchmarks/query_iterator/models.py
  95. 0 { → djangobench}/benchmarks/query_iterator/settings.py
  96. 0 { → djangobench}/benchmarks/query_latest/__init__.py
  97. 0 { → djangobench}/benchmarks/query_latest/benchmark.py
  98. 0 { → djangobench}/benchmarks/query_latest/fixtures/initial_data.json
  99. 0 { → djangobench}/benchmarks/query_latest/models.py
  100. 0 { → djangobench}/benchmarks/query_latest/settings.py
  101. 0 { → djangobench}/benchmarks/query_none/__init__.py
  102. 0 { → djangobench}/benchmarks/query_none/benchmark.py
  103. 0 { → djangobench}/benchmarks/query_none/models.py
  104. 0 { → djangobench}/benchmarks/query_none/settings.py
  105. 0 { → djangobench}/benchmarks/query_order_by/__init__.py
  106. 0 { → djangobench}/benchmarks/query_order_by/benchmark.py
  107. 0 { → djangobench}/benchmarks/query_order_by/fixtures/initial_data.json
  108. 0 { → djangobench}/benchmarks/query_order_by/models.py
  109. 0 { → djangobench}/benchmarks/query_order_by/settings.py
  110. 0 { → djangobench}/benchmarks/query_select_related/__init__.py
  111. 0 { → djangobench}/benchmarks/query_select_related/benchmark.py
  112. 0 { → djangobench}/benchmarks/query_select_related/fixtures/initial_data.json
  113. 0 { → djangobench}/benchmarks/query_select_related/models.py
  114. 0 { → djangobench}/benchmarks/query_select_related/settings.py
  115. 0 { → djangobench}/benchmarks/query_update/__init__.py
  116. 0 { → djangobench}/benchmarks/query_update/benchmark.py
  117. 0 { → djangobench}/benchmarks/query_update/fixtures/initial_data.json
  118. 0 { → djangobench}/benchmarks/query_update/models.py
  119. 0 { → djangobench}/benchmarks/query_update/settings.py
  120. 0 { → djangobench}/benchmarks/query_values/__init__.py
  121. 0 { → djangobench}/benchmarks/query_values/benchmark.py
  122. 0 { → djangobench}/benchmarks/query_values/fixtures/initial_data.json
  123. 0 { → djangobench}/benchmarks/query_values/models.py
  124. 0 { → djangobench}/benchmarks/query_values/settings.py
  125. 0 { → djangobench}/benchmarks/query_values_list/__init__.py
  126. 0 { → djangobench}/benchmarks/query_values_list/benchmark.py
  127. 0 { → djangobench}/benchmarks/query_values_list/fixtures/initial_data.json
  128. 0 { → djangobench}/benchmarks/query_values_list/models.py
  129. 0 { → djangobench}/benchmarks/query_values_list/settings.py
  130. 0 { → djangobench}/benchmarks/startup/__init__.py
  131. 0 { → djangobench}/benchmarks/startup/benchmark.py
  132. 0 { → djangobench}/benchmarks/startup/models.py
  133. 0 { → djangobench}/benchmarks/startup/settings.py
  134. 0 { → djangobench}/benchmarks/template_compilation/__init__.py
  135. 0 { → djangobench}/benchmarks/template_compilation/benchmark.py
  136. 0 { → djangobench}/benchmarks/template_compilation/settings.py
  137. 0 { → djangobench}/benchmarks/template_render/__init__.py
  138. 0 { → djangobench}/benchmarks/template_render/benchmark.py
  139. 0 { → djangobench}/benchmarks/template_render/settings.py
  140. 0 { → djangobench}/benchmarks/template_render/templates/base.html
  141. 0 { → djangobench}/benchmarks/template_render/templates/confirmation_message.html
  142. 0 { → djangobench}/benchmarks/template_render/templates/footer.html
  143. 0 { → djangobench}/benchmarks/template_render/templates/for_loop.html
  144. 0 { → djangobench}/benchmarks/template_render/templates/permalink.html
  145. 0 { → djangobench}/benchmarks/template_render/templates/sidebar.html
  146. 0 { → djangobench}/benchmarks/template_render/templates/somefile.js
  147. 0 { → djangobench}/benchmarks/template_render/urls.py
  148. 0 { → djangobench}/benchmarks/template_render/views.py
  149. 0 { → djangobench}/benchmarks/url_resolve/__init__.py
  150. 0 { → djangobench}/benchmarks/url_resolve/benchmark.py
  151. 0 { → djangobench}/benchmarks/url_resolve/settings.py
  152. 0 { → djangobench}/benchmarks/url_resolve/urlconf.py
  153. 0 { → djangobench}/benchmarks/url_resolve/views.py
  154. 0 { → djangobench}/benchmarks/url_reverse/__init__.py
  155. 0 { → djangobench}/benchmarks/url_reverse/benchmark.py
  156. 0 { → djangobench}/benchmarks/url_reverse/settings.py
  157. 0 djangobench.py → djangobench/main.py
  158. 0 { → djangobench}/perf.py
  159. 0 { → djangobench}/utils.py
  160. +0 −2 requirements.txt
  161. +32 −0 setup.py
View
@@ -1,28 +1,37 @@
Djangobench
===========
-This is a harness for a (to-be-written) set of benchmarks for measuring
-Django's performance over time.
+A harness and a set of benchmarks for measuring Django's performance over
+time.
Running the benchmarks
----------------------
-This doesn't test a single Django version in isolation -- that wouldn't be
-very useful. Instead, it benchmarks an "experiment" Django against a
-"control", reporting on the difference between the two and measuring for
-statistical significance.
-
-So to run this, you'll need two complete Django source trees. By default
-``djangobench.py`` looks for directories named ``django-control`` and
-``django-experiment`` here in this directory, but you can change that
-by using the ``--control`` or ``--experiment`` options.
-
-So, for example, to benchmark Django 1.2 against trunk::
+Here's the short version::
+ mkvirtualenv --no-site-packages djangobench
+ pip install djangobench
svn co http://code.djangoproject.com/svn/django/tags/releases/1.2/ django-control
svn co http://code.djangoproject.com/svn/django/trunk django-experiment
- ./djangobench.py
+ djangobench
+Okay, so what the heck's going on here?
+
+First, ``djangobench`` doesn't test a single Django version in isolation --
+that wouldn't be very useful. Instead, it benchmarks an "experiment" Django
+against a "control", reporting on the difference between the two and
+measuring for statistical significance.
+
+So to run this, you'll need two complete Django source trees. By default
+``djangobench`` looks for directories named ``django-control`` and
+``django-experiment`` in the current working directory, but you can change
+that by using the ``--control`` or ``--experiment`` options.
+
+Now, because you need two Django source trees, you can't exactly install
+them: ``djangobench`` works its magic by mucking with ``PYTHONPATH``.
+However, the benchmarks themselves need access to the ``djangobench``
+module, so you'll need to install it.
+
At the time of this writing Django's trunk hasn't significantly diverged
from Django 1.2, so you should expect to see not-statistically-significant
results::
@@ -38,33 +47,13 @@ Writing new benchmarks
Benchmarks are very simple: they're a Django app, along with a settings
file, and an executable ``benchmarks.py`` that gets run by the harness.
-This file should print timing data to stdout.
-
-See the ``startup`` directory benchmark for an example.
-
-Please write new benchmarks and send me pull requests on Github!
+The benchmark file should print data points to stdout; those points will be compared
+between the two runs.
-TODO
-----
+There's a few utility functions in ``djangobench.utils`` that you can use
+to automate some of the steps of running benchmarks.
-* Right now each benchmark gets run multiple times by the harness,
- incurring startup overhead. The startup benchmark shows this is a non-trivial
- amount of time, so there really needs to be a way for individual benchmarks
- to run n-trials in-process to avoid that overhead and warmup time. Unladen's
- ``perf.py`` supports this; the harness code needs to, also.
-
-* The number of trials is hard-coded. This should be an --option, or, better
- yet, it could be automatically determined by running trials until the results
- reach a particular confidence internal or some large ceiling is hit.
-
-* Lots and lots and lots more benchmarks. Some ideas:
+The existing benchmarks should be pretty easy to read for inspiration. The
+``query_delete`` benchmark is probably a good place to start.
- * template rendering (something useful, not the unladen one)
- * ORM queries
- * ORM overhead compared to cursor.execute()
- * signal/dispatch
- * datastructures (specifically MultiDict)
- * url resolving and reversing
- * form/model validation
- * holistic request/response round-trip time
- * middleware (piecemeal and standard "stacks" together)
+**Please write new benchmarks and send me pull requests on Github!**
View
39 TODO
@@ -0,0 +1,39 @@
+* Lots and lots and lots more benchmarks. Some remaining ideas
+
+ * ORM overhead compared to cursor.execute()
+ * signal dispatch
+ * more datastructures
+ * more intensive url resolving and reversing
+ * model validation
+ * holistic request/response round-trip time (probably requires spawning
+ a running server and testing it).
+
+* The number of trials is hard-coded. This should be an --option, or, better
+ yet, it could be automatically determined by running trials until the results
+ reach a particular confidence internal or some large ceiling is hit.
+
+* Along similar lines, the number of trials is weirdly split between
+ djangobench's trials and each individual benchmarks. This ideally should
+ be centralized so that it's not an (N*M) situation.
+
+ Probably the best way to do this is to pass the number of trials as a
+ command-line arg to the benchmark script. However, this doesn't exactly
+ work for the startup benchmark, or any other benchmark that can't
+ loop for some reason. Those benchmarks could further spawn processes,
+ I s'pose, but it might be nice to make the harness smart about that.
+
+* Add benchmark metadata: benchmarks should provide short descriptions,
+ explanations of what the data points are, etc. I'd picture the benchmark
+ output being something like::
+
+ Title: form clean
+ Description: Speed of a Form.clean() call
+ Units: seconds
+
+ <data points>
+
+* Parameterized benchmark output: it'd be awesome if there could be, say, a
+ single middleware benchmark that tested the overhead of each individual
+ middleware component. This could be done with multiple benchmarks, but
+ that starts getting into serious boilerplate. If a single benchmark could
+ report multiple result sets, that'd rock.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
View
@@ -1,2 +0,0 @@
-argparse == 1.1
-Unipath == 0.2.1
View
@@ -0,0 +1,32 @@
+from setuptools import setup, find_packages
+
+def read(fname):
+ return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+setup(
+ name = "djangobench",
+ version = "1.0",
+ description = "A harness and a set of benchmarks for measuring Django's performance over time.",
+ long_description = read('README.rst'),
+ url = 'http://github.com/jacobian/djangobench',
+ license = 'BSD',
+ author = 'Jacob Kaplan-Moss',
+ author_email = 'jacob@jacobian.org',
+ packages = find_packages(),
+ classifiers = [
+ 'Development Status :: 4 - Beta',
+ 'Environment :: Console',
+ 'Framework :: Django',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: BSD License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python',
+ 'Topic :: Software Development :: Testing',
+ 'Topic :: System :: Benchmark'
+ ],
+ install_requires = ['argparse==1.1', 'Unipath==0.2.1'],
+
+ entry_points = {
+ 'console_scripts': ['djangobench = djangobench.main:main']
+ }
+)

0 comments on commit 048fe9e

Please sign in to comment.