Initial import of keas.profile: a WSGI profiler middleware.

CHANGES.txt

@@ -0,0 +1,9 @@
+=======
+CHANGES
+=======
+
+
+0.1.0 (2008-12-??)
+------------------
+
+- Initial release

README.txt

@@ -0,0 +1,26 @@
+===================
+Profiler middleware
+===================
+
+This package provides middleware for profiling of the application.  It's based
+on `paste.debug.profile <http://pythonpaste.org/modules/debug.profile.html>`_,
+but uses cProfile instead of hotshot.
+
+If you use PasteScript, enabling the profiler is as simple as adding ::
+
+  [filter-app:profile]
+  use = egg:keas.profile#profiler
+  next = main
+
+to your paster configuration file and passing ``--app-name=profile`` to
+``paster``.  When you access your web application, every page will have the
+profiler output appended to the end of the document body.
+
+
+Viewing profiles with KCacheGrind
+---------------------------------
+
+KCacheGrind is a GUI application for digging through the profile data and
+visualizing call trees.  keas.profile uses pyprof2calltree to convert the
+profiler data into KCacheGrind format for your convenience.  To view it,
+open the log file (``profile.log.tmp.kgrind`` by default) in KCacheGrind.

bootstrap.py

@@ -0,0 +1,52 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Bootstrap a buildout-based project
+
+Simply run this script in a directory containing a buildout.cfg.
+The script accepts buildout command-line options, so you can
+use the -c option to specify an alternate configuration file.
+
+$Id$
+"""
+
+import os, shutil, sys, tempfile, urllib2
+
+tmpeggs = tempfile.mkdtemp()
+
+ez = {}
+exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
+                     ).read() in ez
+ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+
+import pkg_resources
+
+cmd = 'from setuptools.command.easy_install import main; main()'
+if sys.platform == 'win32':
+    cmd = '"%s"' % cmd # work around spawn lamosity on windows
+
+ws = pkg_resources.working_set
+assert os.spawnle(
+    os.P_WAIT, sys.executable, sys.executable,
+    '-c', cmd, '-mqNxd', tmpeggs, 'zc.buildout',
+    dict(os.environ,
+         PYTHONPATH=
+         ws.find(pkg_resources.Requirement.parse('setuptools')).location
+         ),
+    ) == 0
+
+ws.add_entry(tmpeggs)
+ws.require('zc.buildout')
+import zc.buildout.buildout
+zc.buildout.buildout.main(sys.argv[1:] + ['bootstrap'])
+shutil.rmtree(tmpeggs)

buildout.cfg

@@ -0,0 +1,30 @@
+[buildout]
+develop = .
+parts = test coverage-test coverage-report python ctags
+
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = keas.profile [test]
+
+
+[coverage-test]
+recipe = zc.recipe.testrunner
+eggs = keas.profile [test]
+defaults = ['--coverage', '../../coverage']
+
+
+[coverage-report]
+recipe = zc.recipe.egg
+eggs = z3c.coverage
+scripts = coverage=coverage-report
+arguments = ('coverage', 'coverage/report')
+
+[python]
+recipe = zc.recipe.egg
+eggs = keas.profile
+interpreter = python
+
+[ctags]
+recipe = z3c.recipe.tag:tags
+eggs = keas.profile

setup.py

@@ -0,0 +1,57 @@
+###############################################################################
+#
+# Copyright 2008 by Keas, Inc., San Francisco, CA
+#
+###############################################################################
+"""Package setup.
+
+$Id$
+"""
+import os
+from setuptools import setup, find_packages
+
+def read(*rnames):
+    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+
+setup(
+    name='keas.profile',
+    version = '0.2.0dev',
+    author='Marius Gedminas and the Zope Community.',
+    author_email="zope-dev@zope.org",
+    description='WSGI Profiler for Python Paste',
+    long_description=(
+        read('README.txt')
+        + '\n\n' +
+        read('CHANGES.txt')
+        ),
+    license="ZPL 2.1",
+    keywords="zope3 profile paste wsgi",
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Environment :: Web Environment',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Zope Public License',
+        'Programming Language :: Python',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Topic :: Internet :: WWW/HTTP',
+        'Framework :: Zope3'],
+    url = 'http://pypi.python.org/pypi/keas.profile',
+    packages=find_packages('src'),
+    package_dir={'': 'src'},
+    namespace_packages=['keas'],
+    extras_require=dict(
+        test=['zope.testing',],
+        ),
+    install_requires=[
+        'setuptools',
+        'paste',
+        'pyprof2calltree',
+        ],
+    include_package_data=True,
+    zip_safe=False,
+    entry_points = """
+      [paste.filter_app_factory]
+      profiler = keas.profile.profiler:make_profiler
+      """
+    )

src/keas/__init__.py

@@ -0,0 +1,7 @@
+try:
+    # Declare this a namespace package if pkg_resources is available.
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    pass
+

src/keas/profile/README.txt

@@ -0,0 +1,34 @@
+Keas.profile
+------------
+
+Middleware that profiles the request and displays profiling information at the
+bottom of each page.
+
+  >>> from email import utils
+  >>> from keas.profile import profiler
+
+Lets start with a simple "hello world" app to profile
+
+  >>> def simple_app(environ, start_response):
+  ...     start_response('200 OK', [('content-type', 'text/html')])
+  ...     return "hello world!"
+  ...
+  >>> def start_response(status, headers, exc_info=None):
+  ...   pass
+
+we can now generate a middleware profiler for the app
+
+  >>> profiled_app = profiler.make_profiler(simple_app, global_conf=None)
+
+and call the app to profile it
+
+  >>> environ = {'HTTP_DATE': utils.formatdate(),
+  ...            'PATH_INFO': '/' ,
+  ...            'REQUEST_METHOD': 'GET'}
+
+  >>> profiled_app(environ, start_response)
+  ['hello world!... function calls in ... CPU seconds...
+  ...Ordered by: cumulative time, call count...]
+
+The profiler output is appended to the end of the response body if it returns
+HTML.  (Yes, this violates the HTML standard, but seems to work in practice.)

src/keas/profile/__init__.py

@@ -0,0 +1 @@
+# make a package

src/keas/profile/profiler.py

@@ -0,0 +1,117 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+# Portions copyright (c) 2005 Ian Bicking and contributors; written for Paste
+# (http://pythonpaste.org) and licensed under the MIT license:
+# http://www.opensource.org/licenses/mit-license.php
+#
+###############################################################################
+"""WSGI Profiler Middleware.
+
+$Id$
+"""
+
+import cgi
+import threading
+import cProfile
+import pstats
+from cStringIO import StringIO
+
+from paste import response
+
+
+class ProfileMiddleware(object):
+    """Middleware that profiles all requests.
+
+    This is a fork of paste.debug.profile.ProfileMiddleware.  It uses
+    cProfile instead of hotshot (which is buggy).  It doesn't cause the
+    truncate the profiler output to be truncated in the browser.  It sorts
+    the stats by cumulative time rather than internal time.
+
+    If the following bugs were fixed upstream, we could switch to paste.debug
+    again:
+
+        http://trac.pythonpaste.org/pythonpaste/ticket/204
+        http://trac.pythonpaste.org/pythonpaste/ticket/311
+        http://trac.pythonpaste.org/pythonpaste/ticket/312
+
+    However upstream says at leas one of those won't be fixed, and suggests
+    we look into better-maintained WSGI profiler middleware products such as
+    repoze.profile or Dozer.
+    """
+
+    style = ('clear: both; background-color: #ff9; color: #000; '
+             'border: 2px solid #000; padding: 5px;')
+
+    def __init__(self, app, global_conf=None,
+                 log_filename='profile.log.tmp',
+                 limit=40):
+        self.app = app
+        self.lock = threading.Lock()
+        self.log_filename = log_filename
+        self.limit = limit
+
+    def __call__(self, environ, start_response):
+        catch_response = []
+        body = []
+        def replace_start_response(status, headers, exc_info=None):
+            catch_response.extend([status, headers])
+            start_response(status, headers, exc_info)
+            return body.append
+        def run_app():
+            app_iter = self.app(environ, replace_start_response)
+            try:
+                body.extend(app_iter)
+            finally:
+                if hasattr(app_iter, 'close'):
+                    app_iter.close()
+        self.lock.acquire()
+        try:
+            profiler = cProfile.Profile()
+            profiler.runctx("run_app()", globals(), locals())
+            body = ''.join(body)
+            headers = catch_response[1]
+            content_type = response.header_value(headers, 'content-type')
+            if content_type is None or not content_type.startswith('text/html'):
+                # We can't add info to non-HTML output
+                return [body]
+            stream = StringIO()
+            stats = pstats.Stats(profiler, stream=stream)
+            stats.strip_dirs()
+            stats.sort_stats('cumulative', 'calls')
+            stats.print_stats(self.limit)
+            output = stream.getvalue()
+            stream.reset()
+            stream.truncate()
+            stats.print_callers(self.limit)
+            output_callers = stream.getvalue()
+            stream.close()
+            body += '<pre style="%s">%s\n%s</pre>' % (
+                self.style, cgi.escape(output), cgi.escape(output_callers))
+            response.replace_header(headers, 'Content-Length', str(len(body)))
+            try:
+                import pyprof2calltree
+            except ImportError:
+                pass
+            else:
+                # Use kcachegrind to view the profile interactively.
+                pyprof2calltree.convert(profiler.getstats(),
+                                        self.log_filename + '.kgrind')
+            return [body]
+        finally:
+            self.lock.release()
+
+
+def make_profiler(app, global_conf, **local_conf):
+    """Create the Profiler app."""
+    return ProfileMiddleware(app, global_conf)
+

src/keas/profile/tests.py

@@ -0,0 +1,27 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+###############################################################################
+"""Test Setup.
+
+$Id$
+"""
+import unittest
+from zope.testing import doctestunit, doctest
+
+def test_suite():
+    return unittest.TestSuite((
+        doctestunit.DocFileSuite(
+            'README.txt',
+            optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS,
+            ),
+        ))