Permalink
Browse files

Support coroutines compiled with cython

On Python 3.5, this means supporting awaitables that are not
iterables. On older versions of python, this includes
* In `@gen.coroutine`, recognizing cython's coroutines
  via the backports_abc module.
* At various points in the gen module, recognize cython's
  use of `StopIteration.args[0]` in place of `StopIteration.value`.
* Implementing Future.__await__ and _wrap_awaitable for
  pre-3.3 versions of python.
  • Loading branch information...
bdarnell committed Sep 14, 2015
1 parent f302e7e commit 3463036d750bcccf0977321f73c9a07cecfe2e7e
@@ -0,0 +1,3 @@
.eggs
cythonapp.egg-info
dist
@@ -0,0 +1 @@
include cythonapp.pyx
@@ -0,0 +1,15 @@
from tornado import gen
import pythonmodule
async def native_coroutine():
x = await pythonmodule.hello()
if x != "hello":
raise ValueError("expected hello, got %r" % x)
return "goodbye"
@gen.coroutine
def decorated_coroutine():
x = yield pythonmodule.hello()
if x != "hello":
raise ValueError("expected hello, got %r" % x)
return "goodbye"
@@ -0,0 +1,22 @@
try:
import backports_abc
except ImportError:
raise
else:
backports_abc.patch()
from tornado.testing import AsyncTestCase, gen_test
import cythonapp
class CythonCoroutineTest(AsyncTestCase):
@gen_test
def test_native_coroutine(self):
x = yield cythonapp.native_coroutine()
self.assertEqual(x, "goodbye")
@gen_test
def test_decorated_coroutine(self):
x = yield cythonapp.decorated_coroutine()
self.assertEqual(x, "goodbye")
@@ -0,0 +1,6 @@
from tornado import gen
@gen.coroutine
def hello():
yield gen.sleep(0.001)
raise gen.Return("hello")
View
@@ -0,0 +1,18 @@
from setuptools import setup
try:
import Cython.Build
except:
Cython = None
if Cython is None:
ext_modules = None
else:
ext_modules=Cython.Build.cythonize('cythonapp.pyx')
setup(
name='cythonapp',
py_modules=['cythonapp_test', 'pythonmodule'],
ext_modules=ext_modules,
setup_requires='Cython>=0.23.1',
)
View
@@ -0,0 +1,19 @@
[tox]
# This currently segfaults on pypy.
envlist = py27,py32,py33,py34,py35
[testenv]
deps =
../../..
Cython>= 0.23.1
backports_abc
singledispatch
commands = python -m unittest cythonapp_test
# Most of these are defaults, but if you specify any you can't fall back
# defaults for the others.
basepython =
py27: python2.7
py32: python3.2
py33: python3.3
py34: python3.4
py35: python3.5
View
@@ -177,6 +177,16 @@ def __init__(self):
def __await__(self):
return (yield self)
"""))
else:
# Py2-compatible version for use with cython.
# Late import of gen.Return to avoid cycles.

This comment has been minimized.

Show comment
Hide comment
@scoder

scoder Sep 15, 2015

stale comment?

@scoder

scoder Sep 15, 2015

stale comment?

This comment has been minimized.

Show comment
Hide comment
@bdarnell

bdarnell Sep 15, 2015

Member

Yup, thanks.

@bdarnell

bdarnell Sep 15, 2015

Member

Yup, thanks.

def __await__(self):
result = yield self
# StopIteration doesn't take args before py33,
# but Cython recognizes the args tuple.
e = StopIteration()
e.args = (result,)
raise e
def cancel(self):
"""Cancel the operation, if possible.
View
@@ -101,7 +101,10 @@ def get(self):
try:
from collections.abc import Generator as GeneratorType # py35+
except ImportError:
from types import GeneratorType
try:
from collections import Generator as GeneratorType # py2 with backports_abc
except ImportError:
from types import GeneratorType
try:
from inspect import isawaitable # py35+
@@ -139,6 +142,21 @@ class TimeoutError(Exception):
"""Exception raised by ``with_timeout``."""
def _value_from_stopiteration(e):
try:
# StopIteration has a value attribute beginning in py33.
# So does our Return class.
return e.value
except AttributeError:
pass
try:
# Cython backports coroutine functionality by putting the value in
# e.args[0].
return e.args[0]
except (AttributeError, IndexError):
return None
def engine(func):
"""Callback-oriented decorator for asynchronous generators.
@@ -236,7 +254,7 @@ def wrapper(*args, **kwargs):
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
result = _value_from_stopiteration(e)
except Exception:
future.set_exc_info(sys.exc_info())
return future
@@ -257,7 +275,7 @@ def wrapper(*args, **kwargs):
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)'))
except (StopIteration, Return) as e:
future.set_result(getattr(e, 'value', None))
future.set_result(_value_from_stopiteration(e))
except Exception:
future.set_exc_info(sys.exc_info())
else:
@@ -302,6 +320,8 @@ def fetch_json(url):
def __init__(self, value=None):
super(Return, self).__init__()
self.value = value
# Cython recognizes subclasses of StopIteration with a .args tuple.
self.args = (value,)
class WaitIterator(object):
@@ -946,7 +966,7 @@ def run(self):
raise LeakedCallbackError(
"finished without waiting for callbacks %r" %
self.pending_callbacks)
self.result_future.set_result(getattr(e, 'value', None))
self.result_future.set_result(_value_from_stopiteration(e))
self.result_future = None
self._deactivate_stack_context()
return
@@ -1057,11 +1077,57 @@ def wrapper(*args, **kwargs):
exec(textwrap.dedent("""
@coroutine
def _wrap_awaitable(x):
if hasattr(x, '__await__'):
x = x.__await__()
return (yield from x)
"""))
else:
# Py2-compatible version for use with Cython.
# Copied from PEP 380.
@coroutine
def _wrap_awaitable(x):
raise NotImplementedError()
if hasattr(x, '__await__'):
_i = x.__await__()
else:
_i = iter(x)
try:
_y = next(_i)
except StopIteration as _e:
_r = _value_from_stopiteration(_e)
else:
while 1:
try:
_s = yield _y
except GeneratorExit as _e:
try:
_m = _i.close
except AttributeError:
pass
else:
_m()
raise _e
except BaseException as _e:
_x = sys.exc_info()
try:
_m = _i.throw
except AttributeError:
raise _e
else:
try:
_y = _m(*_x)
except StopIteration as _e:
_r = _value_from_stopiteration(_e)
break
else:
try:
if _s is None:
_y = next(_i)
else:
_y = _i.send(_s)
except StopIteration as _e:
_r = _value_from_stopiteration(_e)
break
raise Return(_r)
def convert_yielded(yielded):

0 comments on commit 3463036

Please sign in to comment.