Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Newer
Older
100644 270 lines (216 sloc) 9.062 kb
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
1 """``tornado.gen`` is a generator-based interface to make it easier to
2 work in an asynchronous environment. Code using the ``gen`` module
3 is technically asynchronous, but it is written as a single generator
4 instead of a collection of separate functions.
5
6 For example, the following asynchronous handler::
7
8 class AsyncHandler(RequestHandler):
9 @asynchronous
10 def get(self):
11 http_client = AsyncHTTPClient()
12 http_client.fetch("http://example.com",
13 callback=self.on_fetch)
14
15 def on_fetch(self, response):
16 do_something_with_response(response)
17 self.render("template.html")
18
19 could be written with ``gen`` as::
20
21 class GenAsyncHandler(RequestHandler):
22 @asynchronous
23 @gen.engine
24 def get(self):
25 http_client = AsyncHTTPClient()
93a0dc8 @bdarnell Fix error in docs
bdarnell authored
26 response = yield gen.Task(http_client.fetch, "http://example.com")
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
27 do_something_with_response(response)
28 self.render("template.html")
29
30 `Task` works with any function that takes a ``callback`` keyword argument
31 (and runs that callback with zero or one arguments). For more complicated
32 interfaces, `Task` can be split into two parts: `Callback` and `Wait`::
33
34 class GenAsyncHandler2(RequestHandler):
35 @asynchronous
36 @gen.engine
37 def get(self):
38 http_client = AsyncHTTPClient()
39 http_client.fetch("http://example.com",
40 callback=(yield gen.Callback("key"))
41 response = yield gen.Wait("key")
42 do_something_with_response(response)
43 self.render("template.html")
44
45 The ``key`` argument to `Callback` and `Wait` allows for multiple
46 asynchronous operations to proceed in parallel: yield several
47 callbacks with different keys, then wait for them once all the async
48 operations have started.
49 """
50
51 import functools
52 import types
53
54 class KeyReuseError(Exception): pass
55 class UnknownKeyError(Exception): pass
56 class LeakedCallbackError(Exception): pass
57 class BadYieldError(Exception): pass
58
59 def engine(func):
60 """Decorator for asynchronous generators.
61
62 Any generator that yields objects from this module must be wrapped
63 in this decorator. The decorator only works on functions that are
64 already asynchronous. For `~tornado.web.RequestHandler`
65 ``get``/``post``/etc methods, this means that both the `tornado.gen.engine`
66 and `tornado.web.asynchronous` decorators must be used (in either order).
67 In most other cases, it means that it doesn't make sense to use
68 ``gen.engine`` on functions that don't already take a callback argument.
69 """
70 @functools.wraps(func)
71 def wrapper(*args, **kwargs):
72 gen = func(*args, **kwargs)
73 if isinstance(gen, types.GeneratorType):
74 Runner(gen).run()
75 return
76 assert gen is None, gen
77 # no yield, so we're done
78 return wrapper
79
80 class YieldPoint(object):
81 """Base class for objects that may be yielded from the generator."""
82 def start(self, runner):
83 """Called by the runner after the generator has yielded.
84
85 No other methods will be called on this object before ``start``.
86 """
87 raise NotImplementedError()
88
89 def is_ready(self):
90 """Called by the runner to determine whether to resume the generator.
91
2f91460 @bdarnell Allow any sequence in gen.WaitAll, not just lists.
bdarnell authored
92 Returns a boolean; may be called more than once.
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
93 """
94 raise NotImplementedError()
95
96 def get_result(self):
97 """Returns the value to use as the result of the yield expression.
98
99 This method will only be called once, and only after `is_ready`
100 has returned true.
101 """
102 raise NotImplementedError()
103
104 class Callback(YieldPoint):
105 """Returns a callable object that will allow a matching `Wait` to proceed.
106
107 The key may be any value suitable for use as a dictionary key, and is
108 used to match ``Callbacks`` to their corresponding ``Waits``. The key
109 must be unique among outstanding callbacks within a single run of the
110 generator function, but may be reused across different runs of the same
111 function (so constants generally work fine).
112
113 The callback may be called with zero or one arguments; if an argument
114 is given it will be returned by `Wait`.
115 """
116 def __init__(self, key):
117 self.key = key
118
119 def start(self, runner):
120 self.runner = runner
121 runner.register_callback(self.key)
122
123 def is_ready(self):
124 return True
125
126 def get_result(self):
127 return self.callback
128
129 def callback(self, arg=None):
130 self.runner.set_result(self.key, arg)
131
132 class Wait(YieldPoint):
133 """Returns the argument passed to the result of a previous `Callback`."""
134 def __init__(self, key):
135 self.key = key
136
137 def start(self, runner):
138 self.runner = runner
139
140 def is_ready(self):
141 return self.runner.is_ready(self.key)
142
143 def get_result(self):
144 return self.runner.pop_result(self.key)
145
0965952 @bdarnell Add gen.WaitAll
bdarnell authored
146 class WaitAll(YieldPoint):
147 """Returns the results of multiple previous `Callbacks`.
148
149 The argument is a sequence of `Callback` keys, and the result is
150 a list of results in the same order.
151 """
152 def __init__(self, keys):
153 self.keys = keys
154
155 def start(self, runner):
156 self.runner = runner
157
158 def is_ready(self):
159 return all(self.runner.is_ready(key) for key in self.keys)
160
161 def get_result(self):
162 return [self.runner.pop_result(key) for key in self.keys]
163
164
e4ead59 @bdarnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
165 class Task(YieldPoint):
166 """Runs a single asynchronous operation.
167
168 Takes a function (and optional additional arguments) and runs it with
169 those arguments plus a ``callback`` keyword argument. The argument passed
170 to the callback is returned as the result of the yield expression.
171
172 A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique
173 key generated automatically)::
174
175 result = yield gen.Task(func, args)
176
177 func(args, callback=(yield gen.Callback(key)))
178 result = yield gen.Wait(key)
179 """
180 def __init__(self, func, *args, **kwargs):
181 assert "callback" not in kwargs
182 kwargs["callback"] = self.callback
183 self.func = functools.partial(func, *args, **kwargs)
184
185 def start(self, runner):
186 self.runner = runner
187 self.key = object()
188 runner.register_callback(self.key)
189 self.func()
190
191 def is_ready(self):
192 return self.runner.is_ready(self.key)
193
194 def get_result(self):
195 return self.runner.pop_result(self.key)
196
197 def callback(self, arg=None):
198 self.runner.set_result(self.key, arg)
199
200 class _NullYieldPoint(YieldPoint):
201 def start(self, runner):
202 pass
203 def is_ready(self):
204 return True
205 def get_result(self):
206 return None
207
208 class Runner(object):
209 """Internal implementation of `tornado.gen.engine`.
210
211 Maintains information about pending callbacks and their results.
212 """
213 def __init__(self, gen):
214 self.gen = gen
215 self.yield_point = _NullYieldPoint()
216 self.pending_callbacks = set()
217 self.results = {}
218 self.waiting = None
219 self.running = False
220
221 def register_callback(self, key):
222 """Adds ``key`` to the list of callbacks."""
223 if key in self.pending_callbacks:
224 raise KeyReuseError("key %r is already pending" % key)
225 self.pending_callbacks.add(key)
226
227 def is_ready(self, key):
228 """Returns true if a result is available for ``key``."""
229 if key not in self.pending_callbacks:
230 raise UnknownKeyError("key %r is not pending" % key)
231 return key in self.results
232
233 def set_result(self, key, result):
234 """Sets the result for ``key`` and attempts to resume the generator."""
235 self.results[key] = result
236 self.run()
237
238 def pop_result(self, key):
239 """Returns the result for ``key`` and unregisters it."""
240 self.pending_callbacks.remove(key)
241 return self.results.pop(key)
242
243 def run(self):
244 """Starts or resumes the generator, running until it reaches a
245 yield point that is not ready.
246 """
247 if self.running:
248 return
249 try:
250 self.running = True
251 while True:
252 if not self.yield_point.is_ready():
253 return
254 next = self.yield_point.get_result()
255 try:
256 yielded = self.gen.send(next)
257 except StopIteration:
258 if self.pending_callbacks:
259 raise LeakedCallbackError(
260 "finished without waiting for callbacks %r" %
261 self.pending_callbacks)
262 return
263 if not isinstance(yielded, YieldPoint):
264 raise BadYieldError("yielded unknown object %r" % yielded)
265 self.yield_point = yielded
266 self.yield_point.start(self)
267 finally:
268 self.running = False
269
Something went wrong with that request. Please try again.