Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 271 lines (217 sloc) 9.099 kb
e4ead59 Ben Darnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
1 """``tornado.gen`` is a generator-based interface to make it easier to
2 work in an asynchronous environment. Code using the ``gen`` module
3 is technically asynchronous, but it is written as a single generator
4 instead of a collection of separate functions.
5
6 For example, the following asynchronous handler::
7
8 class AsyncHandler(RequestHandler):
9 @asynchronous
10 def get(self):
11 http_client = AsyncHTTPClient()
12 http_client.fetch("http://example.com",
13 callback=self.on_fetch)
14
15 def on_fetch(self, response):
16 do_something_with_response(response)
17 self.render("template.html")
18
19 could be written with ``gen`` as::
20
21 class GenAsyncHandler(RequestHandler):
22 @asynchronous
23 @gen.engine
24 def get(self):
25 http_client = AsyncHTTPClient()
26 response = yield gen.Task(http_client.fetch("http://example.com"))
27 do_something_with_response(response)
28 self.render("template.html")
29
30 `Task` works with any function that takes a ``callback`` keyword argument
31 (and runs that callback with zero or one arguments). For more complicated
32 interfaces, `Task` can be split into two parts: `Callback` and `Wait`::
33
34 class GenAsyncHandler2(RequestHandler):
35 @asynchronous
36 @gen.engine
37 def get(self):
38 http_client = AsyncHTTPClient()
39 http_client.fetch("http://example.com",
40 callback=(yield gen.Callback("key"))
41 response = yield gen.Wait("key")
42 do_something_with_response(response)
43 self.render("template.html")
44
45 The ``key`` argument to `Callback` and `Wait` allows for multiple
46 asynchronous operations to proceed in parallel: yield several
47 callbacks with different keys, then wait for them once all the async
48 operations have started.
49 """
50
51 import functools
52 import types
53
54 class KeyReuseError(Exception): pass
55 class UnknownKeyError(Exception): pass
56 class LeakedCallbackError(Exception): pass
57 class BadYieldError(Exception): pass
58
59 def engine(func):
60 """Decorator for asynchronous generators.
61
62 Any generator that yields objects from this module must be wrapped
63 in this decorator. The decorator only works on functions that are
64 already asynchronous. For `~tornado.web.RequestHandler`
65 ``get``/``post``/etc methods, this means that both the `tornado.gen.engine`
66 and `tornado.web.asynchronous` decorators must be used (in either order).
67 In most other cases, it means that it doesn't make sense to use
68 ``gen.engine`` on functions that don't already take a callback argument.
69 """
70 @functools.wraps(func)
71 def wrapper(*args, **kwargs):
72 gen = func(*args, **kwargs)
73 if isinstance(gen, types.GeneratorType):
74 Runner(gen).run()
75 return
76 assert gen is None, gen
77 # no yield, so we're done
78 return wrapper
79
80 class YieldPoint(object):
81 """Base class for objects that may be yielded from the generator."""
82 def start(self, runner):
83 """Called by the runner after the generator has yielded.
84
85 No other methods will be called on this object before ``start``.
86 """
87 raise NotImplementedError()
88
89 def is_ready(self):
90 """Called by the runner to determine whether to resume the generator.
91
92 May be called repeatedly until it returns True.
93 """
94 raise NotImplementedError()
95
96 def get_result(self):
97 """Returns the value to use as the result of the yield expression.
98
99 This method will only be called once, and only after `is_ready`
100 has returned true.
101 """
102 raise NotImplementedError()
103
104 class Callback(YieldPoint):
105 """Returns a callable object that will allow a matching `Wait` to proceed.
106
107 The key may be any value suitable for use as a dictionary key, and is
108 used to match ``Callbacks`` to their corresponding ``Waits``. The key
109 must be unique among outstanding callbacks within a single run of the
110 generator function, but may be reused across different runs of the same
111 function (so constants generally work fine).
112
113 The callback may be called with zero or one arguments; if an argument
114 is given it will be returned by `Wait`.
115 """
116 def __init__(self, key):
117 self.key = key
118
119 def start(self, runner):
120 self.runner = runner
121 runner.register_callback(self.key)
122
123 def is_ready(self):
124 return True
125
126 def get_result(self):
127 return self.callback
128
129 def callback(self, arg=None):
130 self.runner.set_result(self.key, arg)
131
132 class Wait(YieldPoint):
133 """Returns the argument passed to the result of a previous `Callback`."""
134 def __init__(self, key):
135 self.key = key
136
137 def start(self, runner):
138 self.runner = runner
139
140 def is_ready(self):
141 return self.runner.is_ready(self.key)
142
143 def get_result(self):
144 return self.runner.pop_result(self.key)
145
0965952 Ben Darnell Add gen.WaitAll
bdarnell authored
146 class WaitAll(YieldPoint):
147 """Returns the results of multiple previous `Callbacks`.
148
149 The argument is a sequence of `Callback` keys, and the result is
150 a list of results in the same order.
151 """
152 def __init__(self, keys):
153 assert isinstance(keys, list)
154 self.keys = keys
155
156 def start(self, runner):
157 self.runner = runner
158
159 def is_ready(self):
160 return all(self.runner.is_ready(key) for key in self.keys)
161
162 def get_result(self):
163 return [self.runner.pop_result(key) for key in self.keys]
164
165
e4ead59 Ben Darnell Add tornado.gen module for simpler generator-based async code.
bdarnell authored
166 class Task(YieldPoint):
167 """Runs a single asynchronous operation.
168
169 Takes a function (and optional additional arguments) and runs it with
170 those arguments plus a ``callback`` keyword argument. The argument passed
171 to the callback is returned as the result of the yield expression.
172
173 A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique
174 key generated automatically)::
175
176 result = yield gen.Task(func, args)
177
178 func(args, callback=(yield gen.Callback(key)))
179 result = yield gen.Wait(key)
180 """
181 def __init__(self, func, *args, **kwargs):
182 assert "callback" not in kwargs
183 kwargs["callback"] = self.callback
184 self.func = functools.partial(func, *args, **kwargs)
185
186 def start(self, runner):
187 self.runner = runner
188 self.key = object()
189 runner.register_callback(self.key)
190 self.func()
191
192 def is_ready(self):
193 return self.runner.is_ready(self.key)
194
195 def get_result(self):
196 return self.runner.pop_result(self.key)
197
198 def callback(self, arg=None):
199 self.runner.set_result(self.key, arg)
200
201 class _NullYieldPoint(YieldPoint):
202 def start(self, runner):
203 pass
204 def is_ready(self):
205 return True
206 def get_result(self):
207 return None
208
209 class Runner(object):
210 """Internal implementation of `tornado.gen.engine`.
211
212 Maintains information about pending callbacks and their results.
213 """
214 def __init__(self, gen):
215 self.gen = gen
216 self.yield_point = _NullYieldPoint()
217 self.pending_callbacks = set()
218 self.results = {}
219 self.waiting = None
220 self.running = False
221
222 def register_callback(self, key):
223 """Adds ``key`` to the list of callbacks."""
224 if key in self.pending_callbacks:
225 raise KeyReuseError("key %r is already pending" % key)
226 self.pending_callbacks.add(key)
227
228 def is_ready(self, key):
229 """Returns true if a result is available for ``key``."""
230 if key not in self.pending_callbacks:
231 raise UnknownKeyError("key %r is not pending" % key)
232 return key in self.results
233
234 def set_result(self, key, result):
235 """Sets the result for ``key`` and attempts to resume the generator."""
236 self.results[key] = result
237 self.run()
238
239 def pop_result(self, key):
240 """Returns the result for ``key`` and unregisters it."""
241 self.pending_callbacks.remove(key)
242 return self.results.pop(key)
243
244 def run(self):
245 """Starts or resumes the generator, running until it reaches a
246 yield point that is not ready.
247 """
248 if self.running:
249 return
250 try:
251 self.running = True
252 while True:
253 if not self.yield_point.is_ready():
254 return
255 next = self.yield_point.get_result()
256 try:
257 yielded = self.gen.send(next)
258 except StopIteration:
259 if self.pending_callbacks:
260 raise LeakedCallbackError(
261 "finished without waiting for callbacks %r" %
262 self.pending_callbacks)
263 return
264 if not isinstance(yielded, YieldPoint):
265 raise BadYieldError("yielded unknown object %r" % yielded)
266 self.yield_point = yielded
267 self.yield_point.start(self)
268 finally:
269 self.running = False
270
Something went wrong with that request. Please try again.