Permalink
Browse files

Initial checkin

  • Loading branch information...
0 parents commit d43016ddee3b5f13920696cafa36fe60ab54fee2 Ben Darnell committed Jul 23, 2010
Showing with 224 additions and 0 deletions.
  1. +2 −0 .gitignore
  2. +67 −0 demo/main.py
  3. 0 tornado_tracing/__init__.py
  4. +155 −0 tornado_tracing/recording.py
@@ -0,0 +1,2 @@
+*.pyc
+*~
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+#
+# export PYTHONPATH=.:../tornado:/usr/local/google_appengine:/usr/local/google_appengine/lib/webob
+# $PWD/demo/main.py --enable_appstats
+
+from tornado.httpserver import HTTPServer
+from tornado.ioloop import IOLoop
+from tornado.options import define, options, parse_command_line
+from tornado.web import Application, asynchronous
+from tornado_tracing import recording
+
+import time
+
+define('port', type=int, default=8888)
+define('memcache', default='localhost:11211')
+
+class DelayHandler(recording.RecordingRequestHandler):
+ @asynchronous
+ def get(self):
+ IOLoop.instance().add_timeout(
+ time.time() + int(self.get_argument('ms')) / 1000.0,
+ self.handle_timeout)
+
+ def handle_timeout(self):
+ self.finish("ok")
+
+class RootHandler(recording.RecordingRequestHandler):
+ @asynchronous
+ def get(self):
+ self.client = recording.AsyncHTTPClient()
+ self.client.fetch('http://localhost:%d/delay?ms=100' % options.port,
+ self.step2)
+
+ def handle_step2_fetch(self, response):
+ assert response.body == 'ok'
+ self.fetches_remaining -= 1
+ if self.fetches_remaining == 0:
+ self.step3()
+
+ def step2(self, response):
+ assert response.body == 'ok'
+ self.fetches_remaining = 3
+ self.client.fetch('http://localhost:%d/delay?ms=50' % options.port,
+ self.handle_step2_fetch)
+ self.client.fetch('http://localhost:%d/delay?ms=20' % options.port,
+ self.handle_step2_fetch)
+ self.client.fetch('http://localhost:%d/delay?ms=30' % options.port,
+ self.handle_step2_fetch)
+
+ def step3(self):
+ self.finish('all done')
+
+def main():
+ parse_command_line()
+ recording.setup_memcache([options.memcache])
+
+ app = Application([
+ ('/', RootHandler),
+ ('/delay', DelayHandler),
+ recording.get_urlspec('/appstats/.*'),
+ ], debug=True)
+ server = HTTPServer(app)
+ server.listen(options.port)
+ IOLoop.instance().start()
+
+if __name__ == '__main__':
+ main()
No changes.
@@ -0,0 +1,155 @@
+'''RPC Tracing support.
+
+Records timing information about rpcs and other operations for performance
+profiling. Currently just a wrapper around the Google App Engine appstats
+module.
+'''
+
+import contextlib
+import functools
+import logging
+import memcache
+import tornado.httpclient
+import tornado.web
+import tornado.wsgi
+
+from google.appengine.api import memcache as appengine_memcache
+from google.appengine.api import lib_config
+from google.appengine.ext import webapp
+from google.appengine.ext.appstats import recording
+from google.appengine.ext.appstats.recording import (
+ start_recording, end_recording, pre_call_hook, post_call_hook)
+from tornado.httpclient import AsyncHTTPClient
+from tornado.options import define, options
+from tornado.stack_context import StackContext
+from tornado.web import RequestHandler
+
+define('enable_appstats', type=bool, default=False)
+
+class RecordingRequestHandler(RequestHandler):
+ def __init__(self, *args, **kwargs):
+ super(RecordingRequestHandler, self).__init__(*args, **kwargs)
+ self.__recorder = None
+
+ def _execute(self, transforms, *args, **kwargs):
+ if options.enable_appstats:
+ start_recording(tornado.wsgi.WSGIContainer.environ(self.request))
+ recorder = recording.recorder
+ @contextlib.contextmanager
+ def transfer_recorder():
+ recording.recorder = recorder
+ yield
+ with StackContext(transfer_recorder):
+ super(RecordingRequestHandler, self)._execute(transforms,
+ *args, **kwargs)
+ else:
+ super(RecordingRequestHandler, self)._execute(transforms,
+ *args, **kwargs)
+
+ def finish(self, chunk=None):
+ super(RecordingRequestHandler, self).finish(chunk)
+ if options.enable_appstats:
+ end_recording(self._status_code)
+
+class RecordingFallbackHandler(tornado.web.FallbackHandler):
+ def prepare(self):
+ if options.enable_appstats:
+ recording.start_recording(
+ tornado.wsgi.WSGIContainer.environ(self.request))
+ recorder = recording.recorder
+ @contextlib.contextmanager
+ def transfer_recorder():
+ recording.recorder = recorder
+ yield
+ with StackContext(transfer_recorder):
+ super(RecordingFallbackHandler, self).prepare()
+ recording.end_recording(self._status_code)
+ else:
+ super(RecordingFallbackHandler, self).prepare()
+
+def setup_memcache(*args, **kwargs):
+ '''Configures the app engine memcache interface with a set of regular
+ memcache servers. All arguments are passed to the memcache.Client
+ constructor.
+ '''
+ client = memcache.Client(*args, **kwargs)
+ # The appengine memcache interface has some methods that aren't
+ # currently available in the regular memcache module (at least
+ # in version 1.4.4). Fortunately appstats doesn't use them, but
+ # the setup_client function expects them to be there.
+ client.add_multi = None
+ client.replace_multi = None
+ client.offset_multi = None
+ # Appengine adds a 'namespace' parameter to many methods. Since
+ # appstats.recording uses both namespace and key_prefix, just drop
+ # the namespace. (This list of methods is not exhaustive, it's just
+ # the ones appstats uses)
+ for method in ('set_multi', 'set', 'add', 'delete', 'get', 'get_multi'):
+ def wrapper(old_method, *args, **kwargs):
+ # appstats.recording always passes namespace by keyword
+ if 'namespace' in kwargs:
+ del kwargs['namespace']
+ return old_method(*args, **kwargs)
+ setattr(client, method,
+ functools.partial(wrapper, getattr(client, method)))
+ appengine_memcache.setup_client(client)
+
+def get_urlspec(prefix):
+ '''Returns a tornado.web.URLSpec for the appstats UI.
+ Should be mapped to a url prefix ending with 'stats/'.
+
+ Example:
+ app = tornado.web.Application([
+ ...
+ recording.get_urlspec(r'/_stats/.*'),
+ ])
+ '''
+ # This import can't happen at the top level because things get horribly
+ # confused if it happens before django settings are initialized.
+ from google.appengine.ext.appstats import ui
+ wsgi_app = tornado.wsgi.WSGIContainer(webapp.WSGIApplication(ui.URLMAP))
+ return tornado.web.url(prefix,
+ tornado.web.FallbackHandler,
+ dict(fallback=wsgi_app))
+
+def save():
+ return recording.recorder
+
+def restore(recorder):
+ recording.recorder = recorder
+
+def _recording_request(request):
+ if isinstance(request, tornado.httpclient.HTTPRequest):
+ return request.url
+ else:
+ return request
+
+class HTTPClient(tornado.httpclient.HTTPClient):
+ def fetch(self, request, *args, **kwargs):
+ recording_request = _recording_request(request)
+ recording.pre_call_hook('http', 'sync', recording_request, None)
+ response = super(HTTPClient, self).fetch(request, *args, **kwargs)
+ recording.post_call_hook('http', 'sync', recording_request, None)
+ return response
+
+class AsyncHTTPClient(AsyncHTTPClient):
+ def fetch(self, request, callback, *args, **kwargs):
+ recording_request = _recording_request(request)
+ recording.pre_call_hook('http', 'async', recording_request, None)
+ def wrapper(request, callback, response, *args):
+ recording.post_call_hook('http', 'async', recording_request, None)
+ callback(response)
+ super(AsyncHTTPClient, self).fetch(
+ request,
+ functools.partial(wrapper, request, callback),
+ *args, **kwargs)
+
+def config(**kwargs):
+ '''Sets configuration options for appstats. See
+ /usr/local/google_appengine/ext/appstats/recording.py for possible keys.
+
+ Example:
+ recording.config(RECORD_FRACTION=0.1,
+ KEY_PREFIX='__appstats_myapp__')
+ '''
+ lib_config.register('appstats', kwargs)

0 comments on commit d43016d

Please sign in to comment.