Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 350 lines (285 sloc) 12.125 kb
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
1 """``tornado.gen`` is a generator-based interface to make it easier to
2 work in an asynchronous environment. Code using the ``gen`` module
3 is technically asynchronous, but it is written as a single generator
4 instead of a collection of separate functions.
5
6 For example, the following asynchronous handler::
7
8 class AsyncHandler(RequestHandler):
9 @asynchronous
10 def get(self):
11 http_client = AsyncHTTPClient()
12 http_client.fetch("http://example.com",
13 callback=self.on_fetch)
14
15 def on_fetch(self, response):
16 do_something_with_response(response)
17 self.render("template.html")
18
19 could be written with ``gen`` as::
20
21 class GenAsyncHandler(RequestHandler):
22 @asynchronous
23 @gen.engine
24 def get(self):
25 http_client = AsyncHTTPClient()
93a0dc8 @bdarnell Fix error in docs
bdarnell authored
26 response = yield gen.Task(http_client.fetch, "http://example.com")
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
27 do_something_with_response(response)
28 self.render("template.html")
29
5872db2 @bdarnell Add support for callbacks that take more than a single positional argume...
bdarnell authored
30 `Task` works with any function that takes a ``callback`` keyword
31 argument. You can also yield a list of ``Tasks``, which will be
32 started at the same time and run in parallel; a list of results will
33 be returned when they are all finished::
94b483e @bdarnell Add support for lists of YieldPoints in the gen framework.
bdarnell authored
34
35 def get(self):
36 http_client = AsyncHTTPClient()
37 response1, response2 = yield [gen.Task(http_client.fetch, url1),
38 gen.Task(http_client.fetch, url2)]
39
40 For more complicated interfaces, `Task` can be split into two parts:
41 `Callback` and `Wait`::
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
42
43 class GenAsyncHandler2(RequestHandler):
44 @asynchronous
45 @gen.engine
46 def get(self):
47 http_client = AsyncHTTPClient()
48 http_client.fetch("http://example.com",
49 callback=(yield gen.Callback("key"))
50 response = yield gen.Wait("key")
51 do_something_with_response(response)
52 self.render("template.html")
53
54 The ``key`` argument to `Callback` and `Wait` allows for multiple
94b483e @bdarnell Add support for lists of YieldPoints in the gen framework.
bdarnell authored
55 asynchronous operations to be started at different times and proceed
56 in parallel: yield several callbacks with different keys, then wait
57 for them once all the async operations have started.
5872db2 @bdarnell Add support for callbacks that take more than a single positional argume...
bdarnell authored
58
59 The result of a `Wait` or `Task` yield expression depends on how the callback
60 was run. If it was called with no arguments, the result is ``None``. If
61 it was called with one argument, the result is that argument. If it was
62 called with more than one argument or any keyword arguments, the result
63 is an `Arguments` object, which is a named tuple ``(args, kwargs)``.
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
64 """
65
66 import functools
5872db2 @bdarnell Add support for callbacks that take more than a single positional argume...
bdarnell authored
67 import operator
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
68 import sys
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
69 import types
70
71 class KeyReuseError(Exception): pass
72 class UnknownKeyError(Exception): pass
73 class LeakedCallbackError(Exception): pass
74 class BadYieldError(Exception): pass
75
76 def engine(func):
77 """Decorator for asynchronous generators.
78
79 Any generator that yields objects from this module must be wrapped
80 in this decorator. The decorator only works on functions that are
81 already asynchronous. For `~tornado.web.RequestHandler`
82 ``get``/``post``/etc methods, this means that both the `tornado.gen.engine`
83 and `tornado.web.asynchronous` decorators must be used (in either order).
84 In most other cases, it means that it doesn't make sense to use
85 ``gen.engine`` on functions that don't already take a callback argument.
86 """
87 @functools.wraps(func)
88 def wrapper(*args, **kwargs):
89 gen = func(*args, **kwargs)
90 if isinstance(gen, types.GeneratorType):
91 Runner(gen).run()
92 return
93 assert gen is None, gen
94 # no yield, so we're done
95 return wrapper
96
97 class YieldPoint(object):
98 """Base class for objects that may be yielded from the generator."""
99 def start(self, runner):
100 """Called by the runner after the generator has yielded.
101
102 No other methods will be called on this object before ``start``.
103 """
104 raise NotImplementedError()
105
106 def is_ready(self):
107 """Called by the runner to determine whether to resume the generator.
108
2f91460 @bdarnell Allow any sequence in gen.WaitAll, not just lists.
bdarnell authored
109 Returns a boolean; may be called more than once.
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
110 """
111 raise NotImplementedError()
112
113 def get_result(self):
114 """Returns the value to use as the result of the yield expression.
115
116 This method will only be called once, and only after `is_ready`
117 has returned true.
118 """
119 raise NotImplementedError()
120
121 class Callback(YieldPoint):
122 """Returns a callable object that will allow a matching `Wait` to proceed.
123
124 The key may be any value suitable for use as a dictionary key, and is
125 used to match ``Callbacks`` to their corresponding ``Waits``. The key
126 must be unique among outstanding callbacks within a single run of the
127 generator function, but may be reused across different runs of the same
128 function (so constants generally work fine).
129
130 The callback may be called with zero or one arguments; if an argument
131 is given it will be returned by `Wait`.
132 """
133 def __init__(self, key):
134 self.key = key
135
136 def start(self, runner):
137 self.runner = runner
138 runner.register_callback(self.key)
139
140 def is_ready(self):
141 return True
142
143 def get_result(self):
5872db2 @bdarnell Add support for callbacks that take more than a single positional argume...
bdarnell authored
144 return self.runner.result_callback(self.key)
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
145
146 class Wait(YieldPoint):
147 """Returns the argument passed to the result of a previous `Callback`."""
148 def __init__(self, key):
149 self.key = key
150
151 def start(self, runner):
152 self.runner = runner
153
154 def is_ready(self):
155 return self.runner.is_ready(self.key)
156
157 def get_result(self):
158 return self.runner.pop_result(self.key)
159
0965952 @bdarnell Add gen.WaitAll
bdarnell authored
160 class WaitAll(YieldPoint):
161 """Returns the results of multiple previous `Callbacks`.
162
163 The argument is a sequence of `Callback` keys, and the result is
164 a list of results in the same order.
94b483e @bdarnell Add support for lists of YieldPoints in the gen framework.
bdarnell authored
165
166 `WaitAll` is equivalent to yielding a list of `Wait` objects.
0965952 @bdarnell Add gen.WaitAll
bdarnell authored
167 """
168 def __init__(self, keys):
169 self.keys = keys
170
171 def start(self, runner):
172 self.runner = runner
173
174 def is_ready(self):
175 return all(self.runner.is_ready(key) for key in self.keys)
176
177 def get_result(self):
178 return [self.runner.pop_result(key) for key in self.keys]
179
180
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
181 class Task(YieldPoint):
182 """Runs a single asynchronous operation.
183
184 Takes a function (and optional additional arguments) and runs it with
185 those arguments plus a ``callback`` keyword argument. The argument passed
186 to the callback is returned as the result of the yield expression.
187
188 A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique
189 key generated automatically)::
190
191 result = yield gen.Task(func, args)
192
193 func(args, callback=(yield gen.Callback(key)))
194 result = yield gen.Wait(key)
195 """
196 def __init__(self, func, *args, **kwargs):
197 assert "callback" not in kwargs
5872db2 @bdarnell Add support for callbacks that take more than a single positional argume...
bdarnell authored
198 self.args = args
199 self.kwargs = kwargs
200 self.func = func
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
201
202 def start(self, runner):
203 self.runner = runner
204 self.key = object()
205 runner.register_callback(self.key)
5872db2 @bdarnell Add support for callbacks that take more than a single positional argume...
bdarnell authored
206 self.kwargs["callback"] = runner.result_callback(self.key)
207 self.func(*self.args, **self.kwargs)
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
208
209 def is_ready(self):
210 return self.runner.is_ready(self.key)
211
212 def get_result(self):
213 return self.runner.pop_result(self.key)
214
94b483e @bdarnell Add support for lists of YieldPoints in the gen framework.
bdarnell authored
215 class Multi(YieldPoint):
216 """Runs multiple asynchronous operations in parallel.
217
218 Takes a list of ``Tasks`` or other ``YieldPoints`` and returns a list of
219 their responses. It is not necessary to call `Multi` explicitly,
220 since the engine will do so automatically when the generator yields
221 a list of ``YieldPoints``.
222 """
223 def __init__(self, children):
224 assert all(isinstance(i, YieldPoint) for i in children)
225 self.children = children
226
227 def start(self, runner):
228 for i in self.children:
229 i.start(runner)
230
231 def is_ready(self):
232 return all(i.is_ready() for i in self.children)
233
234 def get_result(self):
235 return [i.get_result() for i in self.children]
236
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
237 class _NullYieldPoint(YieldPoint):
238 def start(self, runner):
239 pass
240 def is_ready(self):
241 return True
242 def get_result(self):
243 return None
244
245 class Runner(object):
246 """Internal implementation of `tornado.gen.engine`.
247
248 Maintains information about pending callbacks and their results.
249 """
250 def __init__(self, gen):
251 self.gen = gen
252 self.yield_point = _NullYieldPoint()
253 self.pending_callbacks = set()
254 self.results = {}
255 self.running = False
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
256 self.finished = False
257 self.exc_info = None
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
258
259 def register_callback(self, key):
260 """Adds ``key`` to the list of callbacks."""
261 if key in self.pending_callbacks:
262 raise KeyReuseError("key %r is already pending" % key)
263 self.pending_callbacks.add(key)
264
265 def is_ready(self, key):
266 """Returns true if a result is available for ``key``."""
267 if key not in self.pending_callbacks:
268 raise UnknownKeyError("key %r is not pending" % key)
269 return key in self.results
270
271 def set_result(self, key, result):
272 """Sets the result for ``key`` and attempts to resume the generator."""
273 self.results[key] = result
274 self.run()
275
276 def pop_result(self, key):
277 """Returns the result for ``key`` and unregisters it."""
278 self.pending_callbacks.remove(key)
279 return self.results.pop(key)
280
281 def run(self):
282 """Starts or resumes the generator, running until it reaches a
283 yield point that is not ready.
284 """
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
285 if self.running or self.finished:
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
286 return
287 try:
288 self.running = True
289 while True:
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
290 if self.exc_info is None:
291 try:
292 if not self.yield_point.is_ready():
293 return
294 next = self.yield_point.get_result()
295 except Exception:
296 self.exc_info = sys.exc_info()
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
297 try:
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
298 if self.exc_info is not None:
299 exc_info = self.exc_info
300 self.exc_info = None
301 yielded = self.gen.throw(*exc_info)
302 else:
303 yielded = self.gen.send(next)
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
304 except StopIteration:
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
305 self.finished = True
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
306 if self.pending_callbacks:
307 raise LeakedCallbackError(
308 "finished without waiting for callbacks %r" %
309 self.pending_callbacks)
310 return
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
311 except Exception:
312 self.finished = True
313 raise
94b483e @bdarnell Add support for lists of YieldPoints in the gen framework.
bdarnell authored
314 if isinstance(yielded, list):
315 yielded = Multi(yielded)
5997f41 @bdarnell Improve exception handling for gen module.
bdarnell authored
316 if isinstance(yielded, YieldPoint):
317 self.yield_point = yielded
318 self.yield_point.start(self)
319 else:
320 self.exc_info = (BadYieldError("yielded unknown object %r" % yielded),)
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
321 finally:
322 self.running = False
323
5872db2 @bdarnell Add support for callbacks that take more than a single positional argume...
bdarnell authored
324 def result_callback(self, key):
325 def inner(*args, **kwargs):
326 if kwargs or len(args) > 1:
327 result = Arguments(args, kwargs)
328 elif args:
329 result = args[0]
330 else:
331 result = None
332 self.set_result(key, result)
333 return inner
334
335 # in python 2.6+ this could be a collections.namedtuple
336 class Arguments(tuple):
337 """The result of a yield expression whose callback had more than one
338 argument (or keyword arguments).
339
340 The `Arguments` object can be used as a tuple ``(args, kwargs)``
341 or an object with attributes ``args`` and ``kwargs``.
342 """
343 __slots__ = ()
344
345 def __new__(cls, args, kwargs):
346 return tuple.__new__(cls, (args, kwargs))
347
348 args = property(operator.itemgetter(0))
349 kwargs = property(operator.itemgetter(1))
Something went wrong with that request. Please try again.