Permalink
Browse files

major rework of async_yield to use a with statement to encapsulate ca…

…pturing the iterator for each function call. This solves an issue of having multiple functions on a Handler each doing a yield. The previous method stored the iterator at the Handler level and so got confused and errored out when resuming a nonstopped function
  • Loading branch information...
1 parent ab8aad3 commit e344a4cc228a5899c97b2377c8044545f43d772f Jeremy Kelley committed Feb 25, 2011
Showing with 98 additions and 116 deletions.
  1. +6 −2 tests/test_async_yield.py
  2. +8 −2 tests/test_cushion_mixin.py
  3. +66 −78 tornado_addons/async_yield.py
  4. +18 −34 tornado_addons/cushion.py
@@ -7,7 +7,7 @@
from tornado.testing import AsyncTestCase
-class AYHandler(tornado.web.RequestHandler, AsyncYieldMixin):
+class AYHandler(AsyncYieldMixin, tornado.web.RequestHandler):
"""
very basic handler for writing async yield tests on a RequestHandler
"""
@@ -18,6 +18,9 @@ def __init__(self):
"""
self.application = tornado.web.Application([], {})
+ def prepare(self):
+ super(AYHandler, self).prepare()
+
def async_assign(self, newdata, callback):
"""
totally contrived async function
@@ -27,7 +30,7 @@ def async_assign(self, newdata, callback):
@async_yield
def some_async_func(self, ioloop, val, callback):
self.test_ioloop = ioloop # we have to fake this for tests
- results = yield self.async_assign(val, self.yield_cb)
+ results = yield self.async_assign(val, self.mycb)
callback(results)
@@ -36,6 +39,7 @@ class AYHandlerTests(AsyncTestCase):
def setUp(self):
AsyncTestCase.setUp(self)
self.handler = AYHandler()
+ self.handler.prepare()
def tearDown(self):
del self.handler
@@ -26,11 +26,16 @@
from tornado.testing import AsyncTestCase
-class CushionHandler(CushionDBMixin, AsyncYieldMixin):
+class CushionHandler(CushionDBMixin, AsyncYieldMixin, tornado.web.RequestHandler):
"""
very basic handler for writing async yield tests on a RequestHandler
"""
+ def __init__(self):
+ pass
+
+ def prepare(self):
+ super(CushionHandler,self).prepare()
@skipIf(no_trombi, "not testing Cushion, trombi failed to import")
@@ -41,6 +46,7 @@ def setUp(self):
dbname = 'test_db' + str(randint(100, 100000))
print "WORKING ON", dbname
self.handler = CushionHandler()
+ self.handler.prepare()
# typically, this would be called in the Handler.prepare()
self.handler.db_setup(
dbname, baseurl,
@@ -58,7 +64,7 @@ def tearDown(self):
del self.handler
def test_db_one(self):
- self.handler.db_one(self.record['_id'], callback=self.stop)
+ self.handler.db_one(self.record['_id'], self.stop)
rec = self.wait()
self.assertTrue(self.record['fake'] == rec['fake'])
@@ -1,84 +1,16 @@
from types import GeneratorType
+import inspect
import tornado.web
-def async_yield(f):
- """
- Decorates request handler methods on an AppBaseRequestHandler object such
- that we can use the yield keyword for a bit of code cleanup to make it
- easier to write and use asynchronous calls w/o having to drop into the
- callback function to continue execution. This makes execution much easier
- to handle as we don't have to attach all method state to self and also
- following the code execution now stays in the same method.
-
- USAGE
- =====
-
- class MyHandler(RequestHandler, AsyncYieldMixin):
- @async_yield
- def get(self):
- ... stuff ...
- x = yield http_fetch(
- 'http://blah',
- callback=self.yield_cb # always use this
- )
- print "stuff returned is", x
-
- TECH NOTES
- ==========
-
- This is hard coded to be used with AppBaseRequestHandler objects and
- actually just turns the get/post/etc methods into generators and we then
- take advantage of this nature and begin execution with a call to
- self._yield_iter.next() which actually begins execution of the method.
- Then, when yield is encountered with an asynchronous call, we halt
- execution of the method until the our generic callback, self._yield_cb, is
- reached where we save off the results in self._yielded and
- self._yielded_kwargs and then call self._yield_iter.next() again to pick up
- right after the asynchronous call.
-
- BUT WAIT
- ========
-
- Q: Doesn't the yield essentially halt (or pause) the execution of the
- handler until the callback returns?
- A: YES. And, I'm ok with that. This implementation is much more preferable
- than the series of hops we were doing and accomplishes the same thing in
- a much more elegant way. And, if we need to do true async in multiple
- paths, there's always the original way.
-
- Q: What if I mix this yield stuff with the tornado async way of doing
- things like in the docs?
- A: Go for it.
- """
-
- f = tornado.web.asynchronous(f)
- def yielding_asynchronously(self, *a, **ka):
- self._yield_iter = f(self, *a, **ka)
- if type(self._yield_iter) is GeneratorType:
- try: self._yield_iter.next()
- except StopIteration: pass
- else: return self._yield_iter
- return yielding_asynchronously
-
-
-
-class AsyncYieldMixin(object):
- """
- Any RequestHandler wishing to use the async_yield wrapper on it's methods
- must extend this mixin.
- """
-
- def ignored_cb(self, *a, **ka):
- """
- Placeholder callback for when we just don't care what happens to it.
- """
- pass
+class TwasBrillig(object):
+ def __init__(self, func, *a, **ka):
+ self.func = func
+ self.a = a
+ self.ka = ka
+ self.yielding = None
def _yield_continue(self, response=None):
- # by sending the response to the generator, we can treat it as a
- # yield expression and do stuff like x= yield async_fun(..)
- # This takes the place of a .next() on the generator.
- try: self._yield_iter.send(response)
+ try: self.yielding.send(response)
except StopIteration: pass
def yield_cb(self, *args, **ka):
@@ -100,10 +32,13 @@ def yield_cb(self, *args, **ka):
If there are both args and kwargs, retval = (args, kwargs). If none,
retval is None.
+ It's a little gross but works for a large majority of the cases.
+
"""
+
if args and ka:
self._yield_continue((args, ka))
- if ka and not args:
+ elif ka and not args:
self._yield_continue(ka)
elif args and not ka:
if len(args) == 1:
@@ -114,5 +49,58 @@ def yield_cb(self, *args, **ka):
else:
self._yield_continue()
- ycb = yield_cb # because typing is lame
+ def __enter__(self):
+ # munge this instance's yield_cb to map to THIS instance of a context
+ self.yielding = self.func(*self.a, **self.ka)
+ if type(self.yielding) is GeneratorType:
+ # the first member of self.a is going to be the instance the
+ # function belongs to. attach our yield_cb to that
+ self.a[0].add_func_callback(self.func.func_name, self.yield_cb)
+ return self.yielding
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if exc_type: print exc_type
+ self.a[0].rm_func_callback(self.func.func_name)
+
+
+def async_yield(f):
+ f = tornado.web.asynchronous(f)
+ def yielding_(*a, **ka):
+ with TwasBrillig(f, *a, **ka) as f_:
+ if type(f_) is not GeneratorType:
+ return f_
+ else:
+ try:
+ f_.next() # kickstart it
+ except StopIteration:
+ print "STOPPED ITERATION"
+ pass
+ return yielding_
+
+
+class AsyncYieldMixin(object):
+
+ def prepare(self):
+ self._yield_callbacks = {}
+ super(AsyncYieldMixin, self).prepare()
+
+ def add_func_callback(self, _id, cb):
+ self._yield_callbacks[_id] = cb
+
+ def rm_func_callback(self, _id):
+ del self._yield_callbacks[_id]
+
+ @property
+ def mycb(self):
+ """
+ make a callback
+ """
+ # return inspect.stack()[1][3] # <-- calling functions name
+ # technically, this just looks up the callback, but eh. whatev
+ key = inspect.stack()[1][3]
+ cb = self._yield_callbacks[key]
+ print "\n....... key",key," cb",cb
+
+ return cb
+
View
@@ -162,8 +162,11 @@ def delete(self, db, data, callback=None):
class CushionDBMixin(object):
- db_default = ''
- cushion = None
+
+ def prepare(self):
+ super(CushionDBMixin, self).prepare()
+ self.db_default = ''
+ self.cushion = None
def db_ignored_cb(self, *a, **ka):
"""
@@ -181,22 +184,15 @@ def db_setup(self, dbname, uri, callback, **kwa):
callback=callback,
create=kwa.get('create') )
- def _db_cb_get(self, callback, ignore_cb):
+ def _db_cb_get(self, callback=None, ignore_cb=False):
# we should never have a callback AND ignore_cb
- # and we should have at least one
- assert(not callback and ignore_cb)
- assert(callback or not ignore_cb)
- if ignore_cb: callback_ = self.db_ignored_cb
- elif not callback:
- if not hasattr(self, 'yield_cb'):
- raise Exception(
- "default callbacks must extend AsyncYieldMixin" )
- callback_ = self.yield_cb
- else: callback_ = callback
- return callback_
-
-
- def db_save(self, data, db=None, callback=None, ignore_cb=False):
+ assert(bool(not callback) ^ bool(ignore_cb)) # logical xor
+
+ if ignore_cb: callback = self.db_ignored_cb
+ return callback
+
+
+ def db_save(self, data, callback=None, db=None, ignore_cb=False):
# default to the account database
if not db: db = self.db_default
@@ -211,7 +207,7 @@ def db_save(self, data, db=None, callback=None, ignore_cb=False):
cush.save(db, data, callback)
- def db_delete(self, obj, db=None, callback=None, ignore_cb=False):
+ def db_delete(self, obj, callback, db=None, ignore_cb=False):
if not db: db = self.db_default
callback = self._db_cb_get(callback, ignore_cb)
@@ -230,14 +226,11 @@ def db_delete(self, obj, db=None, callback=None, ignore_cb=False):
else: cush.delete(db, obj, callback)
- def db_one(self, key, db=None, callback=None, **kwargs):
+ def db_one(self, key, callback, db=None, **kwargs):
"""
Retrieve a particular document from couchdb.
- If no callback is specified, this will assume self.yield_cb. This
- means, for convenience, this can be called like the following:
-
- x = yield self.db_one(dbname, key)
+ x = yield self.db_one(key, cb, dbname)
Parameters:
db <- name of the db to hit. If this db isn't in our cushion, we'll
@@ -252,11 +245,6 @@ def db_one(self, key, db=None, callback=None, **kwargs):
# default to the account db
if not db: db = self.db_default
- if not callback:
- # for convenience, if no callback is passed in, we'll assume an
- # async_yield situation.
- callback = self.yield_cb
-
cush = self.cushion
# if the db's not open, we're going to open the db with the callback
# being the same way we were called
@@ -268,7 +256,7 @@ def db_one(self, key, db=None, callback=None, **kwargs):
else: cush.one(db, key, callback, **kwargs)
- def db_view(self, resource, db=None, callback=None, **kwargs):
+ def db_view(self, resource, callback, db=None, **kwargs):
"""
see comments for db_one
"""
@@ -277,10 +265,6 @@ def db_view(self, resource, db=None, callback=None, **kwargs):
# default to the account db
if not db: db = self.db_default
- if not callback:
- # for convenience, if no callback is passed in, we'll assume an
- # async_yield situation.
- callback = self.yield_cb
cush = self.cushion
# if the db's not open, we're going to open the db with the callback
# being the same way we were called
@@ -289,7 +273,7 @@ def db_view(self, resource, db=None, callback=None, **kwargs):
cush.open(
db,
lambda *a: self.db_view(
- resource, db, callback=callback, **kwargs )
+ resource, db, callback, **kwargs )
)
else:
cush.view(db, resource, callback, **kwargs)

0 comments on commit e344a4c

Please sign in to comment.