From 44b2a6032d6edb82a79daeccec6994daa8a601e8 Mon Sep 17 00:00:00 2001 From: jab Date: Fri, 24 Aug 2018 05:08:55 +0000 Subject: [PATCH 01/39] bpo-31861: Add operator.aiter and operator.anext --- Lib/operator.py | 67 +++++++++++++++-- Lib/test/test_asyncgen.py | 75 +++++++++++++++++++ .../2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst | 1 + 3 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst diff --git a/Lib/operator.py b/Lib/operator.py index fb58851fa6ef67..ed4b3719e67c36 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -10,16 +10,18 @@ This is the pure Python implementation of the module. """ -__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', - 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', - 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', - 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', - 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', - 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', - 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'truediv', 'truth', 'xor'] +__all__ = [ + 'abs', 'add', 'aiter', 'anext', 'and_', 'attrgetter', 'concat', 'contains', + 'countOf', 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', + 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', + 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', 'is_', + 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', + 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', + 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor', +] from builtins import abs as _abs +from collections.abc import AsyncIterable, AsyncIterator # Comparison Operations *******************************************************# @@ -404,6 +406,55 @@ def ixor(a, b): return a +# Asynchronous Iterator Operations ********************************************# + +async def aiter(*args): + """aiter(async_iterable) -> async_iterator + aiter(async_callable, sentinel) -> async_iterator + + An async version of the iter() builtin. + """ + lenargs = len(args) + if lenargs != 1 and lenargs != 2: + raise TypeError(f'aiter expected 1 or 2 arguments, got {lenargs}') + if lenargs == 1: + obj, = args + if not isinstance(obj, AsyncIterable): + raise TypeError(f'aiter expected an AsyncIterable, got {type(obj)}') + async for i in obj.__aiter__(): + yield i + return + # lenargs == 2 + async_callable, sentinel = args + while True: + value = await async_callable() + if value == sentinel: + break + yield value + + +async def anext(*args): + """anext(async_iterator[, default]) + + Return the next item from the async iterator. + If default is given and the iterator is exhausted, + it is returned instead of raising StopAsyncIteration. + """ + lenargs = len(args) + if lenargs != 1 and lenargs != 2: + raise TypeError(f'anext expected 1 or 2 arguments, got {lenargs}') + ait = args[0] + if not isinstance(ait, AsyncIterator): + raise TypeError(f'anext expected an AsyncIterable, got {type(ait)}') + anxt = ait.__anext__ + try: + return await anxt() + except StopAsyncIteration: + if lenargs == 1: + raise + return args[1] # default + + try: from _operator import * except ImportError: diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 1f7e05b42be99c..41d243094e6a41 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1,4 +1,5 @@ import inspect +import operator import types import unittest @@ -372,6 +373,80 @@ def tearDown(self): self.loop = None asyncio.set_event_loop_policy(None) + def test_async_gen_operator_anext(self): + async def gen(): + yield 1 + yield 2 + g = gen() + async def consume(): + results = [] + results.append(await operator.anext(g)) + results.append(await operator.anext(g)) + results.append(await operator.anext(g, 'buckle my shoe')) + return results + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2, 'buckle my shoe']) + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(consume()) + + def test_async_gen_operator_aiter(self): + async def gen(): + yield 1 + yield 2 + g = gen() + async def consume(): + return [i async for i in operator.aiter(g)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + + def test_async_gen_operator_aiter_class(self): + loop = self.loop + class Gen: + async def __aiter__(self): + yield 1 + await asyncio.sleep(0.01, loop=loop) + yield 2 + g = Gen() + async def consume(): + return [i async for i in operator.aiter(g)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + + def test_async_gen_operator_aiter_2_arg(self): + async def gen(): + yield 1 + yield 2 + yield None + g = gen() + async def foo(): + return await operator.anext(g) + async def consume(): + return [i async for i in operator.aiter(foo, None)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + + def test_operator_anext_bad_args(self): + self._test_bad_args(operator.anext) + + def test_operator_aiter_bad_args(self): + self._test_bad_args(operator.aiter) + + def _test_bad_args(self, afn): + async def gen(): + yield 1 + async def call_with_no_args(): + await afn() + async def call_with_3_args(): + await afn(gen(), 1, 2) + async def call_with_bad_args(): + await afn(1, gen()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_no_args()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_3_args()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_bad_args()) + async def to_list(self, gen): res = [] async for i in gen: diff --git a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst new file mode 100644 index 00000000000000..f218a1aa2af71e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst @@ -0,0 +1 @@ +Add the operator.aiter and operator.anext functions. Patch by Josh Bronson. From 142523bf8bb41990b9dc0970664eb45833985849 Mon Sep 17 00:00:00 2001 From: jab Date: Fri, 7 Sep 2018 12:24:20 -0400 Subject: [PATCH 02/39] address comments from first review --- Lib/operator.py | 59 +++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/Lib/operator.py b/Lib/operator.py index ed4b3719e67c36..9a07c50af99dab 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -408,51 +408,52 @@ def ixor(a, b): # Asynchronous Iterator Operations ********************************************# -async def aiter(*args): + +_NOT_PROVIDED = object() # sentinel object to detect when a kwarg was not given + + +def aiter(obj, sentinel=_NOT_PROVIDED): """aiter(async_iterable) -> async_iterator aiter(async_callable, sentinel) -> async_iterator - An async version of the iter() builtin. + Like the iter() builtin but for async iterables and callables. """ - lenargs = len(args) - if lenargs != 1 and lenargs != 2: - raise TypeError(f'aiter expected 1 or 2 arguments, got {lenargs}') - if lenargs == 1: - obj, = args + if sentinel is _NOT_PROVIDED: if not isinstance(obj, AsyncIterable): raise TypeError(f'aiter expected an AsyncIterable, got {type(obj)}') - async for i in obj.__aiter__(): - yield i - return - # lenargs == 2 - async_callable, sentinel = args - while True: - value = await async_callable() - if value == sentinel: - break - yield value - - -async def anext(*args): + if isinstance(obj, AsyncIterator): + return obj + return (i async for i in obj) + + if not callable(obj): + raise TypeError(f'aiter expected an async callable, got {type(obj)}') + + async def ait(): + while True: + value = await obj() + if value == sentinel: + break + yield value + + return ait() + + +async def anext(async_iterator, default=_NOT_PROVIDED): """anext(async_iterator[, default]) Return the next item from the async iterator. If default is given and the iterator is exhausted, it is returned instead of raising StopAsyncIteration. """ - lenargs = len(args) - if lenargs != 1 and lenargs != 2: - raise TypeError(f'anext expected 1 or 2 arguments, got {lenargs}') - ait = args[0] - if not isinstance(ait, AsyncIterator): - raise TypeError(f'anext expected an AsyncIterable, got {type(ait)}') - anxt = ait.__anext__ + if not isinstance(async_iterator, AsyncIterator): + raise TypeError(f'anext expected an AsyncIterator, got {type(async_iterator)}') + anxt = async_iterator.__anext__ try: return await anxt() except StopAsyncIteration: - if lenargs == 1: + if default is _NOT_PROVIDED: raise - return args[1] # default + return default try: From 3ebce05681c0cd6095cc8efb45f0ec024a05a48b Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 30 Nov 2020 19:02:36 +0000 Subject: [PATCH 03/39] Address new review comments + rebase ...on top of latest master. Also drop now-removed `loop` kwarg from asyncio.sleep call. Ref: https://bugs.python.org/issue42392 --- Lib/operator.py | 14 ++++++++------ Lib/test/test_asyncgen.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/operator.py b/Lib/operator.py index 9a07c50af99dab..f18cc8630963b1 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -21,7 +21,6 @@ ] from builtins import abs as _abs -from collections.abc import AsyncIterable, AsyncIterator # Comparison Operations *******************************************************# @@ -418,12 +417,14 @@ def aiter(obj, sentinel=_NOT_PROVIDED): Like the iter() builtin but for async iterables and callables. """ + from collections.abc import AsyncIterable, AsyncIterator if sentinel is _NOT_PROVIDED: if not isinstance(obj, AsyncIterable): raise TypeError(f'aiter expected an AsyncIterable, got {type(obj)}') - if isinstance(obj, AsyncIterator): - return obj - return (i async for i in obj) + ait = type(obj).__aiter__(obj) + if not isinstance(ait, AsyncIterator): + raise TypeError(f'obj.__aiter__() returned non-AsyncIterator: {type(ait)}') + return ait if not callable(obj): raise TypeError(f'aiter expected an async callable, got {type(obj)}') @@ -445,11 +446,12 @@ async def anext(async_iterator, default=_NOT_PROVIDED): If default is given and the iterator is exhausted, it is returned instead of raising StopAsyncIteration. """ + from collections.abc import AsyncIterator if not isinstance(async_iterator, AsyncIterator): raise TypeError(f'anext expected an AsyncIterator, got {type(async_iterator)}') - anxt = async_iterator.__anext__ + anxt = type(async_iterator).__anext__ try: - return await anxt() + return await anxt(async_iterator) except StopAsyncIteration: if default is _NOT_PROVIDED: raise diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 41d243094e6a41..77da57b39117d5 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -404,7 +404,7 @@ def test_async_gen_operator_aiter_class(self): class Gen: async def __aiter__(self): yield 1 - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) yield 2 g = Gen() async def consume(): From 2f8df9945e698d8cb3d315671ef991209af91f8f Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Fri, 4 Dec 2020 16:21:04 +0000 Subject: [PATCH 04/39] improve tests --- Lib/test/test_asyncgen.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 77da57b39117d5..5c2dad836ccb19 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -400,6 +400,7 @@ async def consume(): self.assertEqual(res, [1, 2]) def test_async_gen_operator_aiter_class(self): + results = [] loop = self.loop class Gen: async def __aiter__(self): @@ -408,9 +409,14 @@ async def __aiter__(self): yield 2 g = Gen() async def consume(): - return [i async for i in operator.aiter(g)] - res = self.loop.run_until_complete(consume()) - self.assertEqual(res, [1, 2]) + ait = operator.aiter(g) + while True: + try: + results.append(await operator.anext(ait)) + except StopAsyncIteration: + break + self.loop.run_until_complete(consume()) + self.assertEqual(results, [1, 2]) def test_async_gen_operator_aiter_2_arg(self): async def gen(): From ad12116258c227da4e1711a68683be3bfe0a2b7d Mon Sep 17 00:00:00 2001 From: Daniel Pope Date: Fri, 4 Dec 2020 18:02:18 +0000 Subject: [PATCH 05/39] Sketch out async iterator in C --- Objects/iterobject.c | 141 +++++++++++++++++++++++++++++++++++++++++++ Python/bltinmodule.c | 37 ++++++++++++ 2 files changed, 178 insertions(+) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 6cac41ad539db1..cfcd3c82fa7631 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -288,3 +288,144 @@ PyTypeObject PyCallIter_Type = { }; +/* -------------------------------------- */ + +typedef struct { + PyObject_HEAD + PyObject *it_callable; /* Set to NULL when iterator is exhausted */ + PyObject *it_sentinel; /* Set to NULL when iterator is exhausted */ +} asynccalliterobject; + +typedef struct { + PyObject_HEAD + PyObject *result; /* The return value of the callable, to be awaited */ + PyObject *it; /* The iterator object, in order to clear it when done. */ +} asynccallawaitableobject; + +PyObject * +PyCallIter_New(PyObject *callable, PyObject *sentinel) +{ + asynccalliterobject *it; + it = PyObject_GC_New(asynccalliterobject, &PyCallIter_Type); + if (it == NULL) + return NULL; + Py_INCREF(callable); + it->it_callable = callable; + Py_INCREF(sentinel); + it->it_sentinel = sentinel; + _PyObject_GC_TRACK(it); + return (PyObject *)it; +} +static void +asynccalliter_dealloc(asynccalliterobject *it) +{ + _PyObject_GC_UNTRACK(it); + Py_XDECREF(it->it_callable); + Py_XDECREF(it->it_sentinel); + PyObject_GC_Del(it); +} + +static int +asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) +{ + Py_VISIT(it->it_callable); + Py_VISIT(it->it_sentinel); + return 0; +} + +static PyObject * +asynccalliter_iternext(asynccalliterobject *it) +{ + PyObject *result; + + if (it->it_callable == NULL) { + return NULL; + } + + result = _PyObject_CallNoArg(it->it_callable); + + Py_INCREF(it); + return Awaitable(result, it); +} + + +static PyObject * +awaitable_await(PyObject *self) { + + PyObject *iterator, *result; + + Py_INCREF(self->result); + iterator = (*Py_TYPE(self->result)->tp_aiter.am_await)(self->result); + Py_DECREF(self->result); + + result = PyIter_Next(iterator); + Py_DECREF(iterator); + if (result != NULL) { + int ok; + + ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); + if (ok == 0) { + return result; /* Common case, fast path */ + } + + Py_DECREF(result); + if (ok > 0) { + Py_CLEAR(it->it_callable); + Py_CLEAR(it->it_sentinel); + } + } + else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + Py_CLEAR(self->it->it_callable); + Py_CLEAR(self->it->it_sentinel); + } + return NULL; +} + +static PyObject * +asynccalliter_reduce(asynccalliterobject *it, PyObject *Py_UNUSED(ignored)) +{ + if (it->it_callable != NULL && it->it_sentinel != NULL) + return Py_BuildValue("N(OO)", _PyEval_GetBuiltinId(&PyId_iter), + it->it_callable, it->it_sentinel); + else + return Py_BuildValue("N(())", _PyEval_GetBuiltinId(&PyId_iter)); +} + +static PyMethodDef asynccalliter_methods[] = { + {"__reduce__", (PyCFunction)asynccalliter_reduce, METH_NOARGS, reduce_doc}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyCallIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "callable_iterator", /* tp_name */ + sizeof(asynccalliterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)asynccalliter_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + 0, /* tp_doc */ + (traverseproc)asynccalliter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)asynccalliter_iternext, /* tp_iternext */ + asynccalliter_methods, /* tp_methods */ +}; + diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 1ce55b6ec5a1ce..26ae63686a8df2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1535,6 +1535,42 @@ supply its own iterator, or be a sequence.\n\ In the second form, the callable is called until it returns the sentinel."); +/*[clinic input] +aiter as builtin_aiter + + aiterable: object + sentinel: object = NULL + / + +Return an async iterator for an async iterable object. +[clinic start generated code]*/ +static PyObject * +builtin_aiter(PyObject *self, PyObject *aiterable, PyObject *sentinel) +{ + PyObject *v; + + if (sentinel == NULL) { + return PyObject_GetIter(v); + } + + if (!PyCallable_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "iter(v, w): v must be callable"); + return NULL; + } + PyObject *sentinel = args[1]; + return PyCallIter_New(v, sentinel); +} + +PyDoc_STRVAR(iter_doc, +"iter(iterable) -> iterator\n\ +iter(callable, sentinel) -> iterator\n\ +\n\ +Get an iterator from an object. In the first form, the argument must\n\ +supply its own iterator, or be a sequence.\n\ +In the second form, the callable is called until it returns the sentinel."); + + /*[clinic input] len as builtin_len @@ -2810,6 +2846,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF {"iter", (PyCFunction)(void(*)(void))builtin_iter, METH_FASTCALL, iter_doc}, + {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, aiter_doc}, BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", (PyCFunction)(void(*)(void))builtin_max, METH_VARARGS | METH_KEYWORDS, max_doc}, From ce35092a898be5da4fd6ff2d1c92b4e020ccf934 Mon Sep 17 00:00:00 2001 From: Daniel Pope Date: Mon, 7 Dec 2020 11:34:41 +0000 Subject: [PATCH 06/39] Implement aiter() and anext() using sync methods only --- Lib/operator.py | 78 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/Lib/operator.py b/Lib/operator.py index f18cc8630963b1..ff2d067608bcb0 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -429,17 +429,46 @@ def aiter(obj, sentinel=_NOT_PROVIDED): if not callable(obj): raise TypeError(f'aiter expected an async callable, got {type(obj)}') - async def ait(): - while True: - value = await obj() - if value == sentinel: - break - yield value + return _aiter_callable(obj, sentinel) - return ait() +class _aiter_callable: + __slots__ = ('acallable', 'sentinel') -async def anext(async_iterator, default=_NOT_PROVIDED): + def __init__(self, acallable, sentinel): + self.acallable = acallable + self.sentinel = sentinel + + def __aiter__(self): + return self + + def __anext__(self): + return _aiter_anext(self.acallable().__await__(), self.sentinel) + + +class _aiter_anext: + __slots__ = ('iterator', 'sentinel') + + def __init__(self, iterator, sentinel): + self.iterator = iterator + self.sentinel = sentinel + + def __await__(self): + return self + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.iterator) + except StopIteration as end: + if end.value == self.sentinel: + raise StopAsyncIteration(end.value) from None + raise + + +def anext(async_iterator, default=_NOT_PROVIDED): """anext(async_iterator[, default]) Return the next item from the async iterator. @@ -449,13 +478,32 @@ async def anext(async_iterator, default=_NOT_PROVIDED): from collections.abc import AsyncIterator if not isinstance(async_iterator, AsyncIterator): raise TypeError(f'anext expected an AsyncIterator, got {type(async_iterator)}') - anxt = type(async_iterator).__anext__ - try: - return await anxt(async_iterator) - except StopAsyncIteration: - if default is _NOT_PROVIDED: - raise - return default + anxt = type(async_iterator).__anext__(async_iterator) + + if default is _NOT_PROVIDED: + return anxt + + return _anext_default(anxt, default) + + +class _anext_default: + __slots__ = ('iterator', 'default') + + def __init__(self, iterator, default): + self.iterator = iterator + self.default = default + + def __await__(self): + return self + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self.iterator) + except StopAsyncIteration: + raise StopIteration(self.default) from None try: From f9dc183ce3a6cfd50b4df8b4ac065e014a25c9db Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Mon, 7 Dec 2020 00:20:05 -0500 Subject: [PATCH 07/39] Add basic aiter built-in [WIP] --- Include/abstract.h | 3 +++ Include/iterobject.h | 2 ++ Objects/abstract.c | 20 ++++++++++++++++ Objects/iterobject.c | 45 ++++++++++++++++++++++++----------- Python/bltinmodule.c | 17 ++++++------- Python/clinic/bltinmodule.c.h | 36 +++++++++++++++++++++++++++- 6 files changed, 100 insertions(+), 23 deletions(-) diff --git a/Include/abstract.h b/Include/abstract.h index 0bd1ca936846fe..c0382a4546bc5e 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -324,6 +324,9 @@ PyAPI_FUNC(PyObject *) PyObject_Format(PyObject *obj, returns itself. */ PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *); +// TODO +PyAPI_FUNC(PyObject *) PyObject_GetAiter(PyObject *); + /* Returns 1 if the object 'obj' provides iterator protocols, and 0 otherwise. This function always succeeds. */ diff --git a/Include/iterobject.h b/Include/iterobject.h index 51139bf1874088..98650f4ef761e5 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -7,6 +7,7 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; +PyAPI_DATA(PyTypeObject) PyCallAiter_Type; #define PySeqIter_Check(op) Py_IS_TYPE(op, &PySeqIter_Type) @@ -16,6 +17,7 @@ PyAPI_FUNC(PyObject *) PySeqIter_New(PyObject *); #define PyCallIter_Check(op) Py_IS_TYPE(op, &PyCallIter_Type) PyAPI_FUNC(PyObject *) PyCallIter_New(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyCallAiter_New(PyObject *, PyObject *); #ifdef __cplusplus } diff --git a/Objects/abstract.c b/Objects/abstract.c index 44ed5b3932bf21..8ea848701a889a 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2638,6 +2638,26 @@ PyObject_GetIter(PyObject *o) } } + +PyObject * +PyObject_GetAiter(PyObject *o) { + PyTypeObject *t = Py_TYPE(o); + unaryfunc f; + + f = t->tp_as_async->am_aiter; + if (f == NULL) { + // TODO: is this check relevant? + // if (PySequence_Check(o)) + // return PySeqIter_New(o); + return type_error("'%.200s' object is not iterable", o); + } + else { + PyObject *it = (*f)(o); + // TODO: deleted a check here, do we need an async version? + return it; + } +} + #undef PyIter_Check int PyIter_Check(PyObject *obj) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index cfcd3c82fa7631..26f02066f347c8 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -299,14 +299,14 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *result; /* The return value of the callable, to be awaited */ - PyObject *it; /* The iterator object, in order to clear it when done. */ + asynccalliterobject *it; /* The iterator object, in order to clear it when done. */ } asynccallawaitableobject; PyObject * -PyCallIter_New(PyObject *callable, PyObject *sentinel) +PyCallAiter_New(PyObject *callable, PyObject *sentinel) { asynccalliterobject *it; - it = PyObject_GC_New(asynccalliterobject, &PyCallIter_Type); + it = PyObject_GC_New(asynccalliterobject, &PyCallAiter_Type); if (it == NULL) return NULL; Py_INCREF(callable); @@ -316,6 +316,7 @@ PyCallIter_New(PyObject *callable, PyObject *sentinel) _PyObject_GC_TRACK(it); return (PyObject *)it; } + static void asynccalliter_dealloc(asynccalliterobject *it) { @@ -336,30 +337,39 @@ asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) static PyObject * asynccalliter_iternext(asynccalliterobject *it) { - PyObject *result; + PyObject *obj; if (it->it_callable == NULL) { return NULL; } - result = _PyObject_CallNoArg(it->it_callable); + obj = _PyObject_CallNoArg(it->it_callable); + int ok = PyObject_RichCompareBool(it->it_sentinel, obj, Py_EQ); + if (ok == 0) { + PyObject *result = (*Py_TYPE(obj)->tp_as_async->am_await)(obj); + return result; + } else { + Py_CLEAR(it->it_callable); + Py_CLEAR(it->it_sentinel); + } Py_INCREF(it); - return Awaitable(result, it); + return NULL; } static PyObject * -awaitable_await(PyObject *self) { +awaitable_await(asynccallawaitableobject *self) { - PyObject *iterator, *result; + asynccalliterobject *it; + PyObject *result; Py_INCREF(self->result); - iterator = (*Py_TYPE(self->result)->tp_aiter.am_await)(self->result); + it = (asynccalliterobject*)(*Py_TYPE(self->result)->tp_as_async->am_await)(self->result); Py_DECREF(self->result); - result = PyIter_Next(iterator); - Py_DECREF(iterator); + result = PyIter_Next(result); + Py_DECREF(it); if (result != NULL) { int ok; @@ -397,9 +407,16 @@ static PyMethodDef asynccalliter_methods[] = { {NULL, NULL} /* sentinel */ }; -PyTypeObject PyCallIter_Type = { +static PyAsyncMethods async_iter_as_async = { + PyObject_SelfIter, /* am_await */ /* am_await */ + PyObject_SelfIter, /* am_aiter */ + (unaryfunc)asynccalliter_iternext, /* am_anext */ + 0, /* am_send */ +}; + +PyTypeObject PyCallAiter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "callable_iterator", /* tp_name */ + "callable_async_iterator", /* tp_name */ sizeof(asynccalliterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ @@ -407,7 +424,7 @@ PyTypeObject PyCallIter_Type = { 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ - 0, /* tp_as_async */ + &async_iter_as_async, /* tp_as_async */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 26ae63686a8df2..e778c01a6fe73d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1544,25 +1544,26 @@ aiter as builtin_aiter Return an async iterator for an async iterable object. [clinic start generated code]*/ + static PyObject * -builtin_aiter(PyObject *self, PyObject *aiterable, PyObject *sentinel) +builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) +/*[clinic end generated code: output=ea120e90169d9f32 input=a12efceda1863b3c]*/ { - PyObject *v; - if (sentinel == NULL) { - return PyObject_GetIter(v); + return PyObject_GetAiter(aiterable); } - if (!PyCallable_Check(v)) { + if (!PyCallable_Check(aiterable)) { PyErr_SetString(PyExc_TypeError, "iter(v, w): v must be callable"); return NULL; } - PyObject *sentinel = args[1]; - return PyCallIter_New(v, sentinel); + // PyObject *sentinel = args[1]; + return PyCallAiter_New(aiterable, sentinel); } -PyDoc_STRVAR(iter_doc, +// TODO +PyDoc_STRVAR(aiter_doc, "iter(iterable) -> iterator\n\ iter(callable, sentinel) -> iterator\n\ \n\ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index bc3b518792811d..d3b9ef9c226801 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -530,6 +530,40 @@ PyDoc_STRVAR(builtin_hex__doc__, #define BUILTIN_HEX_METHODDEF \ {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, +PyDoc_STRVAR(builtin_aiter__doc__, +"aiter($module, aiterable, sentinel=, /)\n" +"--\n" +"\n" +"Return an async iterator for an async iterable object."); + +#define BUILTIN_AITER_METHODDEF \ + {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, builtin_aiter__doc__}, + +static PyObject * +builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel); + +static PyObject * +builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *aiterable; + PyObject *sentinel = NULL; + + if (!_PyArg_CheckPositional("aiter", nargs, 1, 2)) { + goto exit; + } + aiterable = args[0]; + if (nargs < 2) { + goto skip_optional; + } + sentinel = args[1]; +skip_optional: + return_value = builtin_aiter_impl(module, aiterable, sentinel); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_len__doc__, "len($module, obj, /)\n" "--\n" @@ -830,4 +864,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e2fcf0201790367c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=64b83ddea2a46bc5 input=a9049054013a1b77]*/ From 086bb798f09b5e29ada42fbcb42199115b565c1c Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Wed, 9 Dec 2020 12:01:08 -0500 Subject: [PATCH 08/39] Start aiter implementation --- Objects/iterobject.c | 162 +++++++++++++++++++++++++++++++------------ 1 file changed, 119 insertions(+), 43 deletions(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 26f02066f347c8..095d42586d5c6f 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -298,7 +298,12 @@ typedef struct { typedef struct { PyObject_HEAD - PyObject *result; /* The return value of the callable, to be awaited */ + PyObject *wrapped_iter; /* + The iterator returned by the callable, unwrapped via __await__. + + If NULL this means that the iterator is already exhausted and when + iterated it should raise StopAsyncIteration; + */ asynccalliterobject *it; /* The iterator object, in order to clear it when done. */ } asynccallawaitableobject; @@ -326,6 +331,15 @@ asynccalliter_dealloc(asynccalliterobject *it) PyObject_GC_Del(it); } +static void +asynccallawaitable_dealloc(asynccallawaitableobject *obj) +{ + _PyObject_GC_UNTRACK(obj); + Py_XDECREF(obj->wrapped_awaitable); + Py_XDECREF(obj->it); + PyObject_GC_Del(obj); +} + static int asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) { @@ -334,61 +348,80 @@ asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) return 0; } +static int +asynccallawaitable_traverse(asynccallawaitableobject *obj, visitproc visit, void *arg) +{ + Py_VISIT(obj->wrapped_awaitable); + PyVISIT(obj->it); + return 0; +} + static PyObject * -asynccalliter_iternext(asynccalliterobject *it) +asynccalliter_anext(asynccalliterobject *it) { - PyObject *obj; + PyObject *obj = NULL, *wrapped_iter = NULL; + asynccallawaitableobject *awaitable; - if (it->it_callable == NULL) { - return NULL; + if (it->it_callable != NULL) { + obj = _PyObject_CallNoArg(it->it_callable); + if (!obj) { + return NULL; + } } + + wrapped_iter = (*Py_TYPE(obj)->tp_as_async->am_await)(obj); - obj = _PyObject_CallNoArg(it->it_callable); - int ok = PyObject_RichCompareBool(it->it_sentinel, obj, Py_EQ); - if (ok == 0) { - PyObject *result = (*Py_TYPE(obj)->tp_as_async->am_await)(obj); - return result; - } else { - Py_CLEAR(it->it_callable); - Py_CLEAR(it->it_sentinel); + awaitable = PyObject_GC_New( + asynccallawaitableobject, + &PyAsyncCallAwaitable_Type); + if (!awaitable) { + return NULL; } Py_INCREF(it); + awaitable->it = it; + awaitable->wrapped_iter = wrapped_iter; + return NULL; } - static PyObject * -awaitable_await(asynccallawaitableobject *self) { - - asynccalliterobject *it; +asynccallawaitable_iternext(asynccallawaitableobject *obj) +{ PyObject *result; + PyObject *type, *value, *traceback; - Py_INCREF(self->result); - it = (asynccalliterobject*)(*Py_TYPE(self->result)->tp_as_async->am_await)(self->result); - Py_DECREF(self->result); + result = PyIter_Next(obj->wrapped_iter); - result = PyIter_Next(result); - Py_DECREF(it); - if (result != NULL) { - int ok; + if (!PyErr_ExceptionMatches(&PyExc_StopIteration)) { + return result; + } + result = NULL; + + PyErr_Fetch(&type, &value, &traceback); + result = PyObject_GetAttrString(value, "value"); + Py_DECREF(type); + Py_DECREF(traceback); + Py_DECREF(value); + if (!result) { + PyErr_SetString(&PyExc_TypeError, "Coroutine iterator raised StopIteration without value"); + return NULL; + } - ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); - if (ok == 0) { - return result; /* Common case, fast path */ - } + int ok = PyObject_RichCompareBool(obj->it->it_sentinel, result, Py_EQ); + if (ok == 0) { Py_DECREF(result); - if (ok > 0) { - Py_CLEAR(it->it_callable); - Py_CLEAR(it->it_sentinel); - } - } - else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Clear(); - Py_CLEAR(self->it->it_callable); - Py_CLEAR(self->it->it_sentinel); + return NULL; } + + + Py_CLEAR(obj->it->it_callable); + Py_CLEAR(obj->it->it_sentinel); + + // value = ... + PyErr_SetObject(&PyExc_StopAsyncIteration, value) + return NULL; } @@ -396,10 +429,14 @@ static PyObject * asynccalliter_reduce(asynccalliterobject *it, PyObject *Py_UNUSED(ignored)) { if (it->it_callable != NULL && it->it_sentinel != NULL) - return Py_BuildValue("N(OO)", _PyEval_GetBuiltinId(&PyId_iter), - it->it_callable, it->it_sentinel); + return Py_BuildValue("N(OO)", + _PyEval_GetBuiltinId(&PyId_iter), // TODO: not this + it->it_callable, it->it_sentinel); else - return Py_BuildValue("N(())", _PyEval_GetBuiltinId(&PyId_iter)); + return Py_BuildValue( + "N(())", + _PyEval_GetBuiltinId(&PyId_iter) // TODO: not this + ); } static PyMethodDef asynccalliter_methods[] = { @@ -408,9 +445,9 @@ static PyMethodDef asynccalliter_methods[] = { }; static PyAsyncMethods async_iter_as_async = { - PyObject_SelfIter, /* am_await */ /* am_await */ + PyObject_SelfIter, /* am_await */ PyObject_SelfIter, /* am_aiter */ - (unaryfunc)asynccalliter_iternext, /* am_anext */ + (unaryfunc)asynccalliter_anext, /* am_anext */ 0, /* am_send */ }; @@ -442,7 +479,46 @@ PyTypeObject PyCallAiter_Type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)asynccalliter_iternext, /* tp_iternext */ + 0, /* tp_iternext */ asynccalliter_methods, /* tp_methods */ }; +static PyAsyncMethods async_awaitable_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0, /* am_anext */ + 0, /* am_send */ +}; + +PyTypeObject PyAsyncCallAwaitable_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "callable_async_awaitable", /* tp_name */ + sizeof(asynccallawaitableobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)asynccallawaitable_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &async_awaitable_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)asynccallawaitable_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + 0, /* tp_iternext */ + wrapped_awaitable_methods, /* tp_methods */ +}; + From 29ef712b0238e33678579da733c6a9d7a0668522 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Wed, 9 Dec 2020 19:44:08 -0500 Subject: [PATCH 09/39] Get test_asyncgen tests passing --- Include/iterobject.h | 1 + Lib/test/test_asyncgen.py | 15 +++++- Objects/iterobject.c | 100 ++++++++++++++++++++++++++++---------- 3 files changed, 89 insertions(+), 27 deletions(-) diff --git a/Include/iterobject.h b/Include/iterobject.h index 98650f4ef761e5..76052ec2fff56f 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -8,6 +8,7 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; PyAPI_DATA(PyTypeObject) PyCallAiter_Type; +PyAPI_DATA(PyTypeObject) PyAsyncCallAwaitable_Type; #define PySeqIter_Check(op) Py_IS_TYPE(op, &PySeqIter_Type) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 5c2dad836ccb19..cf5d318e6accf3 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -422,12 +422,23 @@ def test_async_gen_operator_aiter_2_arg(self): async def gen(): yield 1 yield 2 - yield None + yield 3 g = gen() async def foo(): return await operator.anext(g) async def consume(): - return [i async for i in operator.aiter(foo, None)] + return [i async for i in aiter(foo, 3)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + + def test_async_aiter_callable(self): + v = 0 + async def foo(): + nonlocal v + v += 1 + return v + async def consume(): + return [i async for i in aiter(foo, 3)] res = self.loop.run_until_complete(consume()) self.assertEqual(res, [1, 2]) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 095d42586d5c6f..6d941a12415c2e 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -335,7 +335,7 @@ static void asynccallawaitable_dealloc(asynccallawaitableobject *obj) { _PyObject_GC_UNTRACK(obj); - Py_XDECREF(obj->wrapped_awaitable); + Py_XDECREF(obj->wrapped_iter); Py_XDECREF(obj->it); PyObject_GC_Del(obj); } @@ -351,8 +351,8 @@ asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) static int asynccallawaitable_traverse(asynccallawaitableobject *obj, visitproc visit, void *arg) { - Py_VISIT(obj->wrapped_awaitable); - PyVISIT(obj->it); + Py_VISIT(obj->wrapped_iter); + Py_VISIT(obj->it); return 0; } @@ -360,21 +360,37 @@ static PyObject * asynccalliter_anext(asynccalliterobject *it) { PyObject *obj = NULL, *wrapped_iter = NULL; + PyTypeObject *t = NULL; asynccallawaitableobject *awaitable; - if (it->it_callable != NULL) { - obj = _PyObject_CallNoArg(it->it_callable); - if (!obj) { + if (it->it_callable == NULL) { + /* Can we raise this at this point, or do we need to return an awaitable + * that raises it? + */ + PyObject *value = _PyObject_New((PyTypeObject *) PyExc_StopAsyncIteration); + if (value == NULL) { return NULL; } + PyErr_SetObject(PyExc_StopAsyncIteration, value); + return NULL; + } + + obj = _PyObject_CallNoArg(it->it_callable); + if (obj == NULL) { + return NULL; + } + + t = Py_TYPE(obj); + if (t->tp_as_async == NULL || t->tp_as_async->am_await == NULL) { + return PyErr_Format(PyExc_TypeError, "'%.200s' object is not awaitable", obj); } - wrapped_iter = (*Py_TYPE(obj)->tp_as_async->am_await)(obj); + wrapped_iter = (*t->tp_as_async->am_await)(obj); awaitable = PyObject_GC_New( asynccallawaitableobject, &PyAsyncCallAwaitable_Type); - if (!awaitable) { + if (awaitable == NULL) { return NULL; } @@ -382,7 +398,8 @@ asynccalliter_anext(asynccalliterobject *it) awaitable->it = it; awaitable->wrapped_iter = wrapped_iter; - return NULL; + _PyObject_GC_TRACK(awaitable); + return (PyObject *) awaitable; } static PyObject * @@ -391,37 +408,64 @@ asynccallawaitable_iternext(asynccallawaitableobject *obj) PyObject *result; PyObject *type, *value, *traceback; - result = PyIter_Next(obj->wrapped_iter); + // result = PyIter_Next(obj->wrapped_iter); + result = (*Py_TYPE(obj->wrapped_iter)->tp_iternext)(obj->wrapped_iter); - if (!PyErr_ExceptionMatches(&PyExc_StopIteration)) { + if (result != NULL) { + PyObject_Print(result, stdout, 0); + printf("result was not null!\n"); + return result; + } + + if (PyErr_Occurred() == NULL) { + PyErr_SetString(PyExc_AssertionError, "No exception set"); + return NULL; + // PyErr_SetObject(PyExc_StopAsyncIteration, obj->it->it_sentinel); + // return NULL; + } else if (PyErr_ExceptionMatches(PyExc_StopIteration) == 0) { + PyObject_Print(value, stdout, 0); + printf("Was not Stop\n"); + Py_DECREF(type); + Py_DECREF(traceback); + Py_DECREF(value); return result; } - result = NULL; PyErr_Fetch(&type, &value, &traceback); - result = PyObject_GetAttrString(value, "value"); - Py_DECREF(type); - Py_DECREF(traceback); - Py_DECREF(value); - if (!result) { - PyErr_SetString(&PyExc_TypeError, "Coroutine iterator raised StopIteration without value"); + + // result = PyObject_GetAttrString(value, "value"); + // Py_DECREF(type); + // Py_DECREF(traceback); + // Py_DECREF(value); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Coroutine iterator raised StopIteration without value"); return NULL; } + if (obj->it->it_sentinel == NULL) { + return PyErr_Format(PyExc_TypeError, "'%.200s' object is already exhausted", obj->it); + } - int ok = PyObject_RichCompareBool(obj->it->it_sentinel, result, Py_EQ); + int ok = PyObject_RichCompareBool(obj->it->it_sentinel, value, Py_EQ); if (ok == 0) { - Py_DECREF(result); + Py_DECREF(value); + PyErr_SetObject(PyExc_StopIteration, value); return NULL; } - Py_CLEAR(obj->it->it_callable); Py_CLEAR(obj->it->it_sentinel); - // value = ... - PyErr_SetObject(&PyExc_StopAsyncIteration, value) + // value = _PyObject_New((PyTypeObject *) PyExc_StopAsyncIteration); + // if (value == NULL) { + // return NULL; + // } + /* + if (-1 == PyObject_SetAttrString(value, "value", result)) { + return NULL; + } */ + PyErr_SetObject(PyExc_StopAsyncIteration, value); return NULL; } @@ -490,6 +534,12 @@ static PyAsyncMethods async_awaitable_as_async = { 0, /* am_send */ }; +static PyMethodDef async_awaitable_methods[] = { + // TODO... + {"__reduce__", (PyCFunction)asynccalliter_reduce, METH_NOARGS, reduce_doc}, + {NULL, NULL} /* sentinel */ +}; + PyTypeObject PyAsyncCallAwaitable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "callable_async_awaitable", /* tp_name */ @@ -518,7 +568,7 @@ PyTypeObject PyAsyncCallAwaitable_Type = { 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - 0, /* tp_iternext */ - wrapped_awaitable_methods, /* tp_methods */ + (unaryfunc)asynccallawaitable_iternext, /* tp_iternext */ + async_awaitable_methods, /* tp_methods */ }; From 331e80e290fa7f25ff55cbabc167a5e9b8a7c79d Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Thu, 10 Dec 2020 12:24:57 -0500 Subject: [PATCH 10/39] Fix awaitable iternext --- Objects/iterobject.c | 51 +++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 6d941a12415c2e..202f8789b51980 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -407,65 +407,52 @@ asynccallawaitable_iternext(asynccallawaitableobject *obj) { PyObject *result; PyObject *type, *value, *traceback; + PyObject *stop_value = NULL; + + if (obj->it->it_sentinel == NULL) { + return PyErr_Format(PyExc_TypeError, "'%.200s' object is already exhausted", obj->it); + } - // result = PyIter_Next(obj->wrapped_iter); result = (*Py_TYPE(obj->wrapped_iter)->tp_iternext)(obj->wrapped_iter); if (result != NULL) { - PyObject_Print(result, stdout, 0); - printf("result was not null!\n"); return result; } if (PyErr_Occurred() == NULL) { PyErr_SetString(PyExc_AssertionError, "No exception set"); return NULL; - // PyErr_SetObject(PyExc_StopAsyncIteration, obj->it->it_sentinel); - // return NULL; } else if (PyErr_ExceptionMatches(PyExc_StopIteration) == 0) { - PyObject_Print(value, stdout, 0); - printf("Was not Stop\n"); - Py_DECREF(type); - Py_DECREF(traceback); - Py_DECREF(value); return result; } PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); - // result = PyObject_GetAttrString(value, "value"); - // Py_DECREF(type); - // Py_DECREF(traceback); - // Py_DECREF(value); - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Coroutine iterator raised StopIteration without value"); - return NULL; + if (value != NULL) { + stop_value = PyObject_GetAttrString(value, "value"); } - if (obj->it->it_sentinel == NULL) { - return PyErr_Format(PyExc_TypeError, "'%.200s' object is already exhausted", obj->it); + if (stop_value == NULL) { + PyErr_SetString(PyExc_TypeError, "Coroutine iterator raised StopIteration without value"); + goto raise; } - int ok = PyObject_RichCompareBool(obj->it->it_sentinel, value, Py_EQ); + int ok = PyObject_RichCompareBool(obj->it->it_sentinel, stop_value, Py_EQ); + Py_DECREF(stop_value); if (ok == 0) { - Py_DECREF(value); - PyErr_SetObject(PyExc_StopIteration, value); + PyErr_Restore(type, value, traceback); return NULL; } Py_CLEAR(obj->it->it_callable); Py_CLEAR(obj->it->it_sentinel); + PyErr_SetNone(PyExc_StopAsyncIteration); - // value = _PyObject_New((PyTypeObject *) PyExc_StopAsyncIteration); - // if (value == NULL) { - // return NULL; - // } - - /* - if (-1 == PyObject_SetAttrString(value, "value", result)) { - return NULL; - } */ - PyErr_SetObject(PyExc_StopAsyncIteration, value); +raise: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(traceback); return NULL; } From 95879d87d75ed5a1e098442b554ae210a6634373 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Fri, 11 Dec 2020 00:26:16 -0500 Subject: [PATCH 11/39] Add anext builtin --- Include/iterobject.h | 2 + Lib/operator.py | 2 +- Lib/test/test_asyncgen.py | 22 ++++----- Objects/iterobject.c | 89 +++++++++++++++++++++++++++++++++++ Python/bltinmodule.c | 43 +++++++++++++++++ Python/clinic/bltinmodule.c.h | 37 ++++++++++++++- 6 files changed, 182 insertions(+), 13 deletions(-) diff --git a/Include/iterobject.h b/Include/iterobject.h index 76052ec2fff56f..8ff9a8eee40635 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -8,6 +8,7 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; PyAPI_DATA(PyTypeObject) PyCallAiter_Type; +PyAPI_DATA(PyTypeObject) PyAnext_Type; PyAPI_DATA(PyTypeObject) PyAsyncCallAwaitable_Type; #define PySeqIter_Check(op) Py_IS_TYPE(op, &PySeqIter_Type) @@ -19,6 +20,7 @@ PyAPI_FUNC(PyObject *) PySeqIter_New(PyObject *); PyAPI_FUNC(PyObject *) PyCallIter_New(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyCallAiter_New(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyCallAnext_New(PyObject *, PyObject *); #ifdef __cplusplus } diff --git a/Lib/operator.py b/Lib/operator.py index ff2d067608bcb0..c0e94ce4bb3960 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -483,7 +483,7 @@ def anext(async_iterator, default=_NOT_PROVIDED): if default is _NOT_PROVIDED: return anxt - return _anext_default(anxt, default) + return _anext_default(anxt.__await__(), default) class _anext_default: diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index cf5d318e6accf3..dd06b0b59e73fc 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -373,16 +373,16 @@ def tearDown(self): self.loop = None asyncio.set_event_loop_policy(None) - def test_async_gen_operator_anext(self): + def test_async_gen_anext(self): async def gen(): yield 1 yield 2 g = gen() async def consume(): results = [] - results.append(await operator.anext(g)) - results.append(await operator.anext(g)) - results.append(await operator.anext(g, 'buckle my shoe')) + results.append(await anext(g)) + results.append(await anext(g)) + results.append(await anext(g, 'buckle my shoe')) return results res = self.loop.run_until_complete(consume()) self.assertEqual(res, [1, 2, 'buckle my shoe']) @@ -395,7 +395,7 @@ async def gen(): yield 2 g = gen() async def consume(): - return [i async for i in operator.aiter(g)] + return [i async for i in aiter(g)] res = self.loop.run_until_complete(consume()) self.assertEqual(res, [1, 2]) @@ -409,10 +409,10 @@ async def __aiter__(self): yield 2 g = Gen() async def consume(): - ait = operator.aiter(g) + ait = aiter(g) while True: try: - results.append(await operator.anext(ait)) + results.append(await anext(ait)) except StopAsyncIteration: break self.loop.run_until_complete(consume()) @@ -425,7 +425,7 @@ async def gen(): yield 3 g = gen() async def foo(): - return await operator.anext(g) + return await anext(g) async def consume(): return [i async for i in aiter(foo, 3)] res = self.loop.run_until_complete(consume()) @@ -442,11 +442,11 @@ async def consume(): res = self.loop.run_until_complete(consume()) self.assertEqual(res, [1, 2]) - def test_operator_anext_bad_args(self): - self._test_bad_args(operator.anext) + def test_anext_bad_args(self): + self._test_bad_args(anext) def test_operator_aiter_bad_args(self): - self._test_bad_args(operator.aiter) + self._test_bad_args(aiter) def _test_bad_args(self, afn): async def gen(): diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 202f8789b51980..ef118ad2d8a37b 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -307,6 +307,12 @@ typedef struct { asynccalliterobject *it; /* The iterator object, in order to clear it when done. */ } asynccallawaitableobject; +typedef struct { + PyObject_HEAD + PyObject *wrapped; + PyObject *default_value; +} anextobject; + PyObject * PyCallAiter_New(PyObject *callable, PyObject *sentinel) { @@ -322,6 +328,16 @@ PyCallAiter_New(PyObject *callable, PyObject *sentinel) return (PyObject *)it; } +PyObject * +PyCallAnext_New(PyObject *awaitable, PyObject *default_value) +{ + anextobject *anext = PyObject_GC_New(anextobject, &PyAnext_Type); + anext->wrapped = awaitable; + anext->default_value = default_value; + _PyObject_GC_TRACK(anext); + return (PyObject *)anext; +} + static void asynccalliter_dealloc(asynccalliterobject *it) { @@ -340,6 +356,15 @@ asynccallawaitable_dealloc(asynccallawaitableobject *obj) PyObject_GC_Del(obj); } +static void +anext_dealloc(anextobject *obj) +{ + _PyObject_GC_UNTRACK(obj); + Py_XDECREF(obj->wrapped); + Py_XDECREF(obj->default_value); + PyObject_GC_Del(obj); +} + static int asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) { @@ -356,6 +381,14 @@ asynccallawaitable_traverse(asynccallawaitableobject *obj, visitproc visit, void return 0; } +static int +anext_traverse(anextobject *obj, visitproc visit, void *arg) +{ + Py_VISIT(obj->wrapped); + Py_VISIT(obj->default_value); + return 0; +} + static PyObject * asynccalliter_anext(asynccalliterobject *it) { @@ -456,6 +489,24 @@ asynccallawaitable_iternext(asynccallawaitableobject *obj) return NULL; } +static PyObject * +anext_iternext(anextobject *obj) +{ + // PyTypeObject *t; + // t = Py_TYPE(obj->wrapped); + // PyObject *result = (*t->tp_as_async->am_anext)(obj->wrapped); + PyObject *result = PyIter_Next(obj->wrapped); + if (result == NULL) { + if (obj->default_value != NULL) { + Py_INCREF(obj->default_value); + PyErr_SetObject(PyExc_StopIteration, obj->default_value); + } else { + PyErr_SetNone(PyExc_StopIteration); + } + } + return result; +} + static PyObject * asynccalliter_reduce(asynccalliterobject *it, PyObject *Py_UNUSED(ignored)) { @@ -559,3 +610,41 @@ PyTypeObject PyAsyncCallAwaitable_Type = { async_awaitable_methods, /* tp_methods */ }; +static PyAsyncMethods anext_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0, /* am_anext */ + 0, /* am_send */ +}; + +PyTypeObject PyAnext_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "anext", /* tp_name */ + sizeof(anextobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)anext_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + &anext_as_async, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)anext_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (unaryfunc)anext_iternext, /* tp_iternext */ + 0, /* tp_methods */ +}; \ No newline at end of file diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e778c01a6fe73d..da861fb943827b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1571,6 +1571,48 @@ Get an iterator from an object. In the first form, the argument must\n\ supply its own iterator, or be a sequence.\n\ In the second form, the callable is called until it returns the sentinel."); +/*[clinic input] +anext as builtin_anext + + aiterator: object + default: object = NULL + / + +Return the next item from the async iterator. +[clinic start generated code]*/ + +static PyObject * +builtin_anext_impl(PyObject *module, PyObject *aiterator, + PyObject *default_value) +/*[clinic end generated code: output=f02c060c163a81fa input=699d11f4e38eca24]*/ +{ + PyTypeObject *t; + PyObject *awaitable; + + t = Py_TYPE(aiterator); + if (t->tp_as_async == NULL || t->tp_as_async->am_anext == NULL) { + PyErr_Format(PyExc_TypeError, + "'%.200s' object is not an async iterator", + t->tp_name); + return NULL; + } + + awaitable = (*t->tp_as_async->am_anext)(aiterator); + if (default_value == NULL) { + // do we need to account for _NOT_PROVIDED i.e. if the default is None? + return awaitable; + } + + return PyCallAnext_New(awaitable, default_value); +} + +// TODO +PyDoc_STRVAR(anext_doc, +"anext(iterator[, default])\n\ +\n\ +Return the next item from the iterator. If default is given and the iterator\n\ +is exhausted, it is returned instead of raising StopIteration."); + /*[clinic input] len as builtin_len @@ -2853,6 +2895,7 @@ static PyMethodDef builtin_methods[] = { {"max", (PyCFunction)(void(*)(void))builtin_max, METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", (PyCFunction)(void(*)(void))builtin_min, METH_VARARGS | METH_KEYWORDS, min_doc}, {"next", (PyCFunction)(void(*)(void))builtin_next, METH_FASTCALL, next_doc}, + {"anext", (PyCFunction)(void(*)(void))builtin_anext, METH_FASTCALL, anext_doc}, BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index d3b9ef9c226801..822496ca82a4af 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -564,6 +564,41 @@ builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(builtin_anext__doc__, +"anext($module, aiterator, default=, /)\n" +"--\n" +"\n" +"Return the next item from the async iterator."); + +#define BUILTIN_ANEXT_METHODDEF \ + {"anext", (PyCFunction)(void(*)(void))builtin_anext, METH_FASTCALL, builtin_anext__doc__}, + +static PyObject * +builtin_anext_impl(PyObject *module, PyObject *aiterator, + PyObject *default_value); + +static PyObject * +builtin_anext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *aiterator; + PyObject *default_value = NULL; + + if (!_PyArg_CheckPositional("anext", nargs, 1, 2)) { + goto exit; + } + aiterator = args[0]; + if (nargs < 2) { + goto skip_optional; + } + default_value = args[1]; +skip_optional: + return_value = builtin_anext_impl(module, aiterator, default_value); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_len__doc__, "len($module, obj, /)\n" "--\n" @@ -864,4 +899,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=64b83ddea2a46bc5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b5248d88ee495198 input=a9049054013a1b77]*/ From 5b6458939ad7ba116b63aa7e721fbc2fb78f1eff Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Mon, 14 Dec 2020 15:04:16 -0500 Subject: [PATCH 12/39] Use stop iter functions for anext --- Objects/iterobject.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index ef118ad2d8a37b..31b14229c6f694 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -332,7 +332,9 @@ PyObject * PyCallAnext_New(PyObject *awaitable, PyObject *default_value) { anextobject *anext = PyObject_GC_New(anextobject, &PyAnext_Type); + Py_INCREF(awaitable); anext->wrapped = awaitable; + Py_INCREF(default_value); anext->default_value = default_value; _PyObject_GC_TRACK(anext); return (PyObject *)anext; @@ -492,19 +494,14 @@ asynccallawaitable_iternext(asynccallawaitableobject *obj) static PyObject * anext_iternext(anextobject *obj) { - // PyTypeObject *t; - // t = Py_TYPE(obj->wrapped); - // PyObject *result = (*t->tp_as_async->am_anext)(obj->wrapped); PyObject *result = PyIter_Next(obj->wrapped); - if (result == NULL) { - if (obj->default_value != NULL) { - Py_INCREF(obj->default_value); - PyErr_SetObject(PyExc_StopIteration, obj->default_value); - } else { - PyErr_SetNone(PyExc_StopIteration); - } + if (result != NULL) { + return result; } - return result; + if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + _PyGen_SetStopIterationValue(obj->default_value); + } + return NULL; } static PyObject * From 6e145db1e0b03a3b4e69f2b95e8a8ce3719570a4 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Mon, 14 Dec 2020 15:12:29 -0500 Subject: [PATCH 13/39] Use stop iter functions in aiter --- Objects/iterobject.c | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 31b14229c6f694..142fc0160674db 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -441,8 +441,8 @@ static PyObject * asynccallawaitable_iternext(asynccallawaitableobject *obj) { PyObject *result; - PyObject *type, *value, *traceback; PyObject *stop_value = NULL; + int ok; if (obj->it->it_sentinel == NULL) { return PyErr_Format(PyExc_TypeError, "'%.200s' object is already exhausted", obj->it); @@ -454,40 +454,28 @@ asynccallawaitable_iternext(asynccallawaitableobject *obj) return result; } - if (PyErr_Occurred() == NULL) { - PyErr_SetString(PyExc_AssertionError, "No exception set"); + ok = _PyGen_FetchStopIterationValue(&stop_value); + if (ok == -1) { + // wasn't StopIteration (TODO: should this error message be better?) + PyErr_SetString(PyExc_AssertionError, "Unexpected exception"); return NULL; - } else if (PyErr_ExceptionMatches(PyExc_StopIteration) == 0) { - return result; - } - - PyErr_Fetch(&type, &value, &traceback); - PyErr_NormalizeException(&type, &value, &traceback); - - if (value != NULL) { - stop_value = PyObject_GetAttrString(value, "value"); } if (stop_value == NULL) { PyErr_SetString(PyExc_TypeError, "Coroutine iterator raised StopIteration without value"); - goto raise; + return NULL; } - int ok = PyObject_RichCompareBool(obj->it->it_sentinel, stop_value, Py_EQ); + ok = PyObject_RichCompareBool(obj->it->it_sentinel, stop_value, Py_EQ); Py_DECREF(stop_value); if (ok == 0) { - PyErr_Restore(type, value, traceback); + _PyGen_SetStopIterationValue(stop_value); return NULL; } Py_CLEAR(obj->it->it_callable); Py_CLEAR(obj->it->it_sentinel); PyErr_SetNone(PyExc_StopAsyncIteration); - -raise: - Py_XDECREF(value); - Py_XDECREF(type); - Py_XDECREF(traceback); return NULL; } From 2fd4be61217f7110261950a16c053fbfd06367fb Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Wed, 16 Dec 2020 09:06:43 -0500 Subject: [PATCH 14/39] Note about implementing __reduce__ for aiter --- Objects/iterobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 142fc0160674db..a3dd51e1443696 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -501,7 +501,7 @@ asynccalliter_reduce(asynccalliterobject *it, PyObject *Py_UNUSED(ignored)) it->it_callable, it->it_sentinel); else return Py_BuildValue( - "N(())", + "N(...)", // FIXME: need to construct with an empty *async* iterable _PyEval_GetBuiltinId(&PyId_iter) // TODO: not this ); } From 430dd59a329fe958497c40e0776d6f6fc5faee25 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Wed, 16 Dec 2020 09:40:38 -0500 Subject: [PATCH 15/39] Refactor aiter and anext type names --- Include/iterobject.h | 8 +- Objects/iterobject.c | 196 ++++++++++++++++++++----------------------- Python/bltinmodule.c | 4 +- 3 files changed, 96 insertions(+), 112 deletions(-) diff --git a/Include/iterobject.h b/Include/iterobject.h index 8ff9a8eee40635..40920fd6f15878 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -7,9 +7,9 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; -PyAPI_DATA(PyTypeObject) PyCallAiter_Type; -PyAPI_DATA(PyTypeObject) PyAnext_Type; +PyAPI_DATA(PyTypeObject) PyCallAsyncIter_Type; PyAPI_DATA(PyTypeObject) PyAsyncCallAwaitable_Type; +PyAPI_DATA(PyTypeObject) PyAnextAwaitable_Type; #define PySeqIter_Check(op) Py_IS_TYPE(op, &PySeqIter_Type) @@ -19,8 +19,8 @@ PyAPI_FUNC(PyObject *) PySeqIter_New(PyObject *); #define PyCallIter_Check(op) Py_IS_TYPE(op, &PyCallIter_Type) PyAPI_FUNC(PyObject *) PyCallIter_New(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyCallAiter_New(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyCallAnext_New(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyCallAsyncIter_New(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyAnextAwaitable_New(PyObject *, PyObject *); #ifdef __cplusplus } diff --git a/Objects/iterobject.c b/Objects/iterobject.c index a3dd51e1443696..d47afc544f59f3 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -294,7 +294,7 @@ typedef struct { PyObject_HEAD PyObject *it_callable; /* Set to NULL when iterator is exhausted */ PyObject *it_sentinel; /* Set to NULL when iterator is exhausted */ -} asynccalliterobject; +} callasynciterobject; typedef struct { PyObject_HEAD @@ -304,20 +304,15 @@ typedef struct { If NULL this means that the iterator is already exhausted and when iterated it should raise StopAsyncIteration; */ - asynccalliterobject *it; /* The iterator object, in order to clear it when done. */ -} asynccallawaitableobject; + callasynciterobject *it; /* The iterator object, in order to clear it when done. */ +} callasynciterawaitableobject; -typedef struct { - PyObject_HEAD - PyObject *wrapped; - PyObject *default_value; -} anextobject; PyObject * -PyCallAiter_New(PyObject *callable, PyObject *sentinel) +PyCallAsyncIter_New(PyObject *callable, PyObject *sentinel) { - asynccalliterobject *it; - it = PyObject_GC_New(asynccalliterobject, &PyCallAiter_Type); + callasynciterobject *it; + it = PyObject_GC_New(callasynciterobject, &PyCallAsyncIter_Type); if (it == NULL) return NULL; Py_INCREF(callable); @@ -328,47 +323,26 @@ PyCallAiter_New(PyObject *callable, PyObject *sentinel) return (PyObject *)it; } -PyObject * -PyCallAnext_New(PyObject *awaitable, PyObject *default_value) -{ - anextobject *anext = PyObject_GC_New(anextobject, &PyAnext_Type); - Py_INCREF(awaitable); - anext->wrapped = awaitable; - Py_INCREF(default_value); - anext->default_value = default_value; - _PyObject_GC_TRACK(anext); - return (PyObject *)anext; -} - static void -asynccalliter_dealloc(asynccalliterobject *it) -{ - _PyObject_GC_UNTRACK(it); - Py_XDECREF(it->it_callable); - Py_XDECREF(it->it_sentinel); - PyObject_GC_Del(it); -} - -static void -asynccallawaitable_dealloc(asynccallawaitableobject *obj) +callasynciter_dealloc(callasynciterobject *obj) { _PyObject_GC_UNTRACK(obj); - Py_XDECREF(obj->wrapped_iter); - Py_XDECREF(obj->it); + Py_XDECREF(obj->it_callable); + Py_XDECREF(obj->it_sentinel); PyObject_GC_Del(obj); } static void -anext_dealloc(anextobject *obj) +callasynciterawaitable_dealloc(callasynciterawaitableobject *obj) { _PyObject_GC_UNTRACK(obj); - Py_XDECREF(obj->wrapped); - Py_XDECREF(obj->default_value); + Py_XDECREF(obj->wrapped_iter); + Py_XDECREF(obj->it); PyObject_GC_Del(obj); } static int -asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) +callasynciter_traverse(callasynciterobject *it, visitproc visit, void *arg) { Py_VISIT(it->it_callable); Py_VISIT(it->it_sentinel); @@ -376,27 +350,20 @@ asynccalliter_traverse(asynccalliterobject *it, visitproc visit, void *arg) } static int -asynccallawaitable_traverse(asynccallawaitableobject *obj, visitproc visit, void *arg) +callasyncawaitable_traverse(callasynciterawaitableobject *obj, visitproc visit, void *arg) { Py_VISIT(obj->wrapped_iter); Py_VISIT(obj->it); return 0; } -static int -anext_traverse(anextobject *obj, visitproc visit, void *arg) -{ - Py_VISIT(obj->wrapped); - Py_VISIT(obj->default_value); - return 0; -} static PyObject * -asynccalliter_anext(asynccalliterobject *it) +callasynciter_anext(callasynciterobject *it) { PyObject *obj = NULL, *wrapped_iter = NULL; PyTypeObject *t = NULL; - asynccallawaitableobject *awaitable; + callasynciterawaitableobject *awaitable; if (it->it_callable == NULL) { /* Can we raise this at this point, or do we need to return an awaitable @@ -423,7 +390,7 @@ asynccalliter_anext(asynccalliterobject *it) wrapped_iter = (*t->tp_as_async->am_await)(obj); awaitable = PyObject_GC_New( - asynccallawaitableobject, + callasynciterawaitableobject, &PyAsyncCallAwaitable_Type); if (awaitable == NULL) { return NULL; @@ -438,7 +405,7 @@ asynccalliter_anext(asynccalliterobject *it) } static PyObject * -asynccallawaitable_iternext(asynccallawaitableobject *obj) +asynccallawaitable_iternext(callasynciterawaitableobject *obj) { PyObject *result; PyObject *stop_value = NULL; @@ -479,52 +446,22 @@ asynccallawaitable_iternext(asynccallawaitableobject *obj) return NULL; } -static PyObject * -anext_iternext(anextobject *obj) -{ - PyObject *result = PyIter_Next(obj->wrapped); - if (result != NULL) { - return result; - } - if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { - _PyGen_SetStopIterationValue(obj->default_value); - } - return NULL; -} - -static PyObject * -asynccalliter_reduce(asynccalliterobject *it, PyObject *Py_UNUSED(ignored)) -{ - if (it->it_callable != NULL && it->it_sentinel != NULL) - return Py_BuildValue("N(OO)", - _PyEval_GetBuiltinId(&PyId_iter), // TODO: not this - it->it_callable, it->it_sentinel); - else - return Py_BuildValue( - "N(...)", // FIXME: need to construct with an empty *async* iterable - _PyEval_GetBuiltinId(&PyId_iter) // TODO: not this - ); -} -static PyMethodDef asynccalliter_methods[] = { - {"__reduce__", (PyCFunction)asynccalliter_reduce, METH_NOARGS, reduce_doc}, - {NULL, NULL} /* sentinel */ -}; static PyAsyncMethods async_iter_as_async = { PyObject_SelfIter, /* am_await */ PyObject_SelfIter, /* am_aiter */ - (unaryfunc)asynccalliter_anext, /* am_anext */ + (unaryfunc)callasynciter_anext, /* am_anext */ 0, /* am_send */ }; -PyTypeObject PyCallAiter_Type = { +PyTypeObject PyCallAsyncIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "callable_async_iterator", /* tp_name */ - sizeof(asynccalliterobject), /* tp_basicsize */ + sizeof(callasynciterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)asynccalliter_dealloc, /* tp_dealloc */ + (destructor)callasynciter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -541,15 +478,16 @@ PyTypeObject PyCallAiter_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ 0, /* tp_doc */ - (traverseproc)asynccalliter_traverse, /* tp_traverse */ + (traverseproc)callasynciter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ 0, /* tp_iternext */ - asynccalliter_methods, /* tp_methods */ + 0, /* tp_methods */ }; + static PyAsyncMethods async_awaitable_as_async = { PyObject_SelfIter, /* am_await */ 0, /* am_aiter */ @@ -557,19 +495,13 @@ static PyAsyncMethods async_awaitable_as_async = { 0, /* am_send */ }; -static PyMethodDef async_awaitable_methods[] = { - // TODO... - {"__reduce__", (PyCFunction)asynccalliter_reduce, METH_NOARGS, reduce_doc}, - {NULL, NULL} /* sentinel */ -}; - PyTypeObject PyAsyncCallAwaitable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "callable_async_awaitable", /* tp_name */ - sizeof(asynccallawaitableobject), /* tp_basicsize */ + "callable_async_iterator_awaitable", /* tp_name */ + sizeof(callasynciterawaitableobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)asynccallawaitable_dealloc, /* tp_dealloc */ + (destructor)callasynciterawaitable_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -586,33 +518,85 @@ PyTypeObject PyAsyncCallAwaitable_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)asynccallawaitable_traverse, /* tp_traverse */ + (traverseproc)callasyncawaitable_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (unaryfunc)asynccallawaitable_iternext, /* tp_iternext */ - async_awaitable_methods, /* tp_methods */ + 0, /* tp_methods */ }; -static PyAsyncMethods anext_as_async = { +/* -------------------------------------- */ + +typedef struct { + PyObject_HEAD + PyObject *wrapped; + PyObject *default_value; +} anextawaitableobject; + + +PyObject * +PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) +{ + anextawaitableobject *anext = PyObject_GC_New(anextawaitableobject, &PyAnextAwaitable_Type); + Py_INCREF(awaitable); + anext->wrapped = awaitable; + Py_INCREF(default_value); + anext->default_value = default_value; + _PyObject_GC_TRACK(anext); + return (PyObject *)anext; +} + +static void +anextawaitable_dealloc(anextawaitableobject *obj) +{ + _PyObject_GC_UNTRACK(obj); + Py_XDECREF(obj->wrapped); + Py_XDECREF(obj->default_value); + PyObject_GC_Del(obj); +} + +static int +anextawaitable_traverse(anextawaitableobject *obj, visitproc visit, void *arg) +{ + Py_VISIT(obj->wrapped); + Py_VISIT(obj->default_value); + return 0; +} + +static PyObject * +anextawaitable_iternext(anextawaitableobject *obj) +{ + PyObject *result = PyIter_Next(obj->wrapped); + if (result != NULL) { + return result; + } + if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + _PyGen_SetStopIterationValue(obj->default_value); + } + return NULL; +} + + +static PyAsyncMethods anextawaitable_as_async = { PyObject_SelfIter, /* am_await */ 0, /* am_aiter */ 0, /* am_anext */ 0, /* am_send */ }; -PyTypeObject PyAnext_Type = { +PyTypeObject PyAnextAwaitable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "anext", /* tp_name */ - sizeof(anextobject), /* tp_basicsize */ + "anext_awaitable", /* tp_name */ + sizeof(anextawaitableobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)anext_dealloc, /* tp_dealloc */ + (destructor)anextawaitable_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ - &anext_as_async, /* tp_as_async */ + &anextawaitable_as_async, /* tp_as_async */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ @@ -625,11 +609,11 @@ PyTypeObject PyAnext_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)anext_traverse, /* tp_traverse */ + (traverseproc)anextawaitable_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (unaryfunc)anext_iternext, /* tp_iternext */ + (unaryfunc)anextawaitable_iternext, /* tp_iternext */ 0, /* tp_methods */ }; \ No newline at end of file diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index da861fb943827b..f37692f30f0101 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1559,7 +1559,7 @@ builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) return NULL; } // PyObject *sentinel = args[1]; - return PyCallAiter_New(aiterable, sentinel); + return PyCallAsyncIter_New(aiterable, sentinel); } // TODO @@ -1603,7 +1603,7 @@ builtin_anext_impl(PyObject *module, PyObject *aiterator, return awaitable; } - return PyCallAnext_New(awaitable, default_value); + return PyAnextAwaitable_New(awaitable, default_value); } // TODO From 4266e65e45a539d180a71e1dc1d0b991bdade903 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Wed, 16 Dec 2020 10:01:58 -0500 Subject: [PATCH 16/39] Add documentation for aiter() --- Doc/library/functions.rst | 69 ++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a8a4ca42007c63..93dfc90de77506 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -10,33 +10,33 @@ are always available. They are listed here in alphabetical order. +---------------------------------------------------------------------------------------------------+ | Built-in Functions | +=========================+=======================+=======================+=========================+ -| | **A** | | **E** | | **L** | | **R** | -| | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | -| | :func:`all` | | :func:`eval` | | |func-list|_ | | :func:`repr` | -| | :func:`any` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`ascii` | | | | | | :func:`round` | -| | | | **F** | | **M** | | | -| | **B** | | :func:`filter` | | :func:`map` | | **S** | -| | :func:`bin` | | :func:`float` | | :func:`max` | | |func-set|_ | -| | :func:`bool` | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`breakpoint` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | |func-bytearray|_ | | | | | | :func:`sorted` | -| | |func-bytes|_ | | **G** | | **N** | | :func:`staticmethod` | -| | | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | **C** | | :func:`globals` | | | | :func:`sum` | -| | :func:`callable` | | | | **O** | | :func:`super` | -| | :func:`chr` | | **H** | | :func:`object` | | | -| | :func:`classmethod` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`compile` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`complex` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | | | :func:`hex` | | | | | -| | **D** | | | | **P** | | **V** | -| | :func:`delattr` | | **I** | | :func:`pow` | | :func:`vars` | -| | |func-dict|_ | | :func:`id` | | :func:`print` | | | -| | :func:`dir` | | :func:`input` | | :func:`property` | | **Z** | -| | :func:`divmod` | | :func:`int` | | | | :func:`zip` | -| | | | :func:`isinstance` | | | | | -| | | | :func:`issubclass` | | | | **_** | +| | **A** | | **E** | | **L** | | **R** | +| | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | +| | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | +| | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | +| | :func:`any` | | | | | | :func:`round` | +| | :func:`anext` | | **F** | | **M** | | | +| | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | +| | | | :func:`float` | | :func:`max` | | |func-set|_ | +| | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | +| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | +| | :func:`bool` | | | | | | :func:`sorted` | +| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | +| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | +| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | +| | | | | | **O** | | :func:`super` | +| | **C** | | **H** | | :func:`object` | | | +| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | +| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | +| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | +| | :func:`compile` | | :func:`hex` | | | | | +| | :func:`complex` | | | | **P** | | **V** | +| | | | **I** | | :func:`pow` | | :func:`vars` | +| | **D** | | :func:`id` | | :func:`print` | | | +| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | +| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | +| | :func:`dir` | | :func:`isinstance` | | | | | +| | :func:`divmod` | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ @@ -61,6 +61,21 @@ are always available. They are listed here in alphabetical order. If the argument is a complex number, its magnitude is returned. +.. function:: aiter(object, [sentinel]) + + Return an async iterator object. This is the async variant of the + :func:`iter()` builtin, and behaves similarly. + + If sentinel is omitted, then *object* must be an async iterable object, and + ``aiter()`` returns an iterator for it. + + Otherwise, *object* must be an async callable (a callable which returns an + awaitable). When the returned async iterator is async iterated, the callable + is called and awaited and the values it returns becomes the values produced + by the iterator. When the awaited return value is equal to sentinel, the + async iterator terminates with :cls:`StopAsyncIteration`. + + .. function:: all(iterable) Return ``True`` if all elements of the *iterable* are true (or if the iterable From ecbedc7dc4afadf3c7ade0a9bc037550739b8fdd Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Fri, 18 Dec 2020 14:59:12 -0500 Subject: [PATCH 17/39] Update documentation for aiter and anext --- Doc/glossary.rst | 5 +++++ Doc/library/functions.rst | 33 ++++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 0661c8283290ce..274166443647f5 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -88,6 +88,11 @@ Glossary :ref:`the difference between arguments and parameters `, and :pep:`362`. + asynchronous callable + Any callable that returns an :term:`awaitable`. Examples include + :term:`coroutine functions ` and the built-in + :func:`anext` function. + asynchronous context manager An object which controls the environment seen in an :keyword:`async with` statement by defining :meth:`__aenter__` and diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 93dfc90de77506..44503d57835c69 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -63,17 +63,18 @@ are always available. They are listed here in alphabetical order. .. function:: aiter(object, [sentinel]) - Return an async iterator object. This is the async variant of the - :func:`iter()` builtin, and behaves similarly. + Return an :term:`asynchronous iterator` object. This is the async variant + of the :func:`iter()` builtin, and behaves similarly. - If sentinel is omitted, then *object* must be an async iterable object, and - ``aiter()`` returns an iterator for it. + If sentinel is omitted, then *object* must be an + :term:`asynchronous iterable` object, and ``aiter()`` returns an iterator + for it. - Otherwise, *object* must be an async callable (a callable which returns an - awaitable). When the returned async iterator is async iterated, the callable - is called and awaited and the values it returns becomes the values produced - by the iterator. When the awaited return value is equal to sentinel, the - async iterator terminates with :cls:`StopAsyncIteration`. + Otherwise, *object* must be an :term:`asynchronous callable`. When the + resulting async iterator is async iterated, the passed callable is + called and awaited and the values returned become the values produced + by the iterator. When the awaited return value is equal to sentinel, + the async iterator terminates with :exc:`StopAsyncIteration`. .. function:: all(iterable) @@ -88,6 +89,20 @@ are always available. They are listed here in alphabetical order. return True +.. awaitablefunction:: anext(async_iterator[, default]) + + When awaited, return the next item from the given :term:`asynchronous + iterator`, or *default* if given and the iterator is exhausted. + + This is the async variant of the :func:`next()` builtin, and behaves + similarly. + + Immediately, call the :meth:`~object.__anext__` method of *async_iterator*, + then return an :term:`awaitable`. Awaiting this returns the next value of the + iterator. If *default* is given, it is returned if the iterator is exhausted, + otherwise :exc:`StopAsyncIteration` is raised. + + .. function:: any(iterable) Return ``True`` if any element of the *iterable* is true. If the iterable From 5fa3812808d8573caf313dbda5e4fd5926607361 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Fri, 18 Dec 2020 15:43:15 -0500 Subject: [PATCH 18/39] Clean up docs and formatting --- Doc/library/functions.rst | 48 +++++++++---------- Include/abstract.h | 4 +- Include/cpython/abstract.h | 7 +++ .../2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst | 3 +- Objects/abstract.c | 14 ++++-- Objects/iterobject.c | 29 ++++++----- Python/bltinmodule.c | 21 ++++---- 7 files changed, 69 insertions(+), 57 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 44503d57835c69..4fb03f83f92054 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -10,33 +10,33 @@ are always available. They are listed here in alphabetical order. +---------------------------------------------------------------------------------------------------+ | Built-in Functions | +=========================+=======================+=======================+=========================+ -| | **A** | | **E** | | **L** | | **R** | -| | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | -| | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | -| | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`any` | | | | | | :func:`round` | -| | :func:`anext` | | **F** | | **M** | | | -| | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | +| | **A** | | **E** | | **L** | | **R** | +| | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | +| | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | +| | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | +| | :func:`any` | | | | | | :func:`round` | +| | :func:`anext` | | **F** | | **M** | | | +| | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | -| | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | +| | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | +| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | +| | :func:`bool` | | | | | | :func:`sorted` | +| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | +| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | +| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | | | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | -| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | :func:`compile` | | :func:`hex` | | | | | -| | :func:`complex` | | | | **P** | | **V** | +| | **C** | | **H** | | :func:`object` | | | +| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | +| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | +| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | +| | :func:`compile` | | :func:`hex` | | | | | +| | :func:`complex` | | | | **P** | | **V** | | | | | **I** | | :func:`pow` | | :func:`vars` | -| | **D** | | :func:`id` | | :func:`print` | | | -| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | -| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | -| | :func:`dir` | | :func:`isinstance` | | | | | -| | :func:`divmod` | | :func:`issubclass` | | | | **_** | +| | **D** | | :func:`id` | | :func:`print` | | | +| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | +| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | +| | :func:`dir` | | :func:`isinstance` | | | | | +| | :func:`divmod` | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ diff --git a/Include/abstract.h b/Include/abstract.h index c0382a4546bc5e..0a0eabb0263517 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -324,7 +324,9 @@ PyAPI_FUNC(PyObject *) PyObject_Format(PyObject *obj, returns itself. */ PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *); -// TODO +/* Takes an object and returns an async iterator for it. + This is typically a new iterator but if the argument is an async iterator, + this returns itself. */ PyAPI_FUNC(PyObject *) PyObject_GetAiter(PyObject *); /* Returns 1 if the object 'obj' provides iterator protocols, and 0 otherwise. diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index b5b6e4819788c5..141c3dbfec54d8 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -331,6 +331,13 @@ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); (Py_TYPE(obj)->tp_iternext != NULL && \ Py_TYPE(obj)->tp_iternext != &_PyObject_NextNotImplemented) +#define PyAiter_Check(obj) \ + (Py_TYPE(obj)->tp_as_async != NULL && \ + Py_TYPE(obj)->tp_as_async->am_aiter != NULL && \ + Py_TYPE(obj)->tp_as_async->am_aiter != &_PyObject_NextNotImplemented && \ + Py_TYPE(obj)->tp_as_async->am_anext != NULL && \ + Py_TYPE(obj)->tp_as_async->am_anext != &_PyObject_NextNotImplemented) + /* === Sequence protocol ================================================ */ /* Assume tp_as_sequence and sq_item exist and that 'i' does not diff --git a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst index f218a1aa2af71e..d4c7efda2c0790 100644 --- a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst +++ b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst @@ -1 +1,2 @@ -Add the operator.aiter and operator.anext functions. Patch by Josh Bronson. +Add builtins.aiter and builtins.anext. +Patch by Daniel Pope, Joshua Bronson, and Justin Wang. diff --git a/Objects/abstract.c b/Objects/abstract.c index 8ea848701a889a..6161ee6a920b65 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2646,14 +2646,18 @@ PyObject_GetAiter(PyObject *o) { f = t->tp_as_async->am_aiter; if (f == NULL) { - // TODO: is this check relevant? - // if (PySequence_Check(o)) - // return PySeqIter_New(o); - return type_error("'%.200s' object is not iterable", o); + return type_error("'%.200s' object is not async iterable", o); } else { PyObject *it = (*f)(o); - // TODO: deleted a check here, do we need an async version? + if (it != NULL && !PyAiter_Check(it)) { + PyErr_Format(PyExc_TypeError, + "aiter() returned non-async-iterator " + "of type '%.100s'", + Py_TYPE(it)->tp_name); + Py_DECREF(it); + it = NULL; + } return it; } } diff --git a/Objects/iterobject.c b/Objects/iterobject.c index d47afc544f59f3..986784d2177d0c 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -157,7 +157,7 @@ PyTypeObject PySeqIter_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ (traverseproc)iter_traverse, /* tp_traverse */ 0, /* tp_clear */ @@ -276,7 +276,7 @@ PyTypeObject PyCallIter_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ (traverseproc)calliter_traverse, /* tp_traverse */ 0, /* tp_clear */ @@ -423,8 +423,7 @@ asynccallawaitable_iternext(callasynciterawaitableobject *obj) ok = _PyGen_FetchStopIterationValue(&stop_value); if (ok == -1) { - // wasn't StopIteration (TODO: should this error message be better?) - PyErr_SetString(PyExc_AssertionError, "Unexpected exception"); + PyErr_SetString(PyExc_AssertionError, "StopIteration not raised"); return NULL; } @@ -451,17 +450,17 @@ asynccallawaitable_iternext(callasynciterawaitableobject *obj) static PyAsyncMethods async_iter_as_async = { PyObject_SelfIter, /* am_await */ PyObject_SelfIter, /* am_aiter */ - (unaryfunc)callasynciter_anext, /* am_anext */ + (unaryfunc)callasynciter_anext, /* am_anext */ 0, /* am_send */ }; PyTypeObject PyCallAsyncIter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "callable_async_iterator", /* tp_name */ - sizeof(callasynciterobject), /* tp_basicsize */ + "callable_async_iterator", /* tp_name */ + sizeof(callasynciterobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)callasynciter_dealloc, /* tp_dealloc */ + (destructor)callasynciter_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -476,9 +475,9 @@ PyTypeObject PyCallAsyncIter_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)callasynciter_traverse, /* tp_traverse */ + (traverseproc)callasynciter_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -498,7 +497,7 @@ static PyAsyncMethods async_awaitable_as_async = { PyTypeObject PyAsyncCallAwaitable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "callable_async_iterator_awaitable", /* tp_name */ - sizeof(callasynciterawaitableobject), /* tp_basicsize */ + sizeof(callasynciterawaitableobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)callasynciterawaitable_dealloc, /* tp_dealloc */ @@ -589,10 +588,10 @@ static PyAsyncMethods anextawaitable_as_async = { PyTypeObject PyAnextAwaitable_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "anext_awaitable", /* tp_name */ - sizeof(anextawaitableobject), /* tp_basicsize */ + sizeof(anextawaitableobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)anextawaitable_dealloc, /* tp_dealloc */ + (destructor)anextawaitable_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -609,11 +608,11 @@ PyTypeObject PyAnextAwaitable_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)anextawaitable_traverse, /* tp_traverse */ + (traverseproc)anextawaitable_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (unaryfunc)anextawaitable_iternext, /* tp_iternext */ + (unaryfunc)anextawaitable_iternext, /* tp_iternext */ 0, /* tp_methods */ }; \ No newline at end of file diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f37692f30f0101..d4ff94243aac91 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1562,14 +1562,14 @@ builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) return PyCallAsyncIter_New(aiterable, sentinel); } -// TODO PyDoc_STRVAR(aiter_doc, -"iter(iterable) -> iterator\n\ -iter(callable, sentinel) -> iterator\n\ +"aiter(iterable) -> iterator\n\ +aiter(acallable, sentinel) -> iterator\n\ \n\ -Get an iterator from an object. In the first form, the argument must\n\ -supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the sentinel."); +Get an async iterator from an object. In the first form, the\n\ +argument must supply its own async iterator.\n\ +In the second form, the async callable is called and awaited\n\ +until it returns the sentinel."); /*[clinic input] anext as builtin_anext @@ -1599,19 +1599,18 @@ builtin_anext_impl(PyObject *module, PyObject *aiterator, awaitable = (*t->tp_as_async->am_anext)(aiterator); if (default_value == NULL) { - // do we need to account for _NOT_PROVIDED i.e. if the default is None? return awaitable; } return PyAnextAwaitable_New(awaitable, default_value); } -// TODO PyDoc_STRVAR(anext_doc, -"anext(iterator[, default])\n\ +"anext(aiterator[, default])\n\ \n\ -Return the next item from the iterator. If default is given and the iterator\n\ -is exhausted, it is returned instead of raising StopIteration."); +When awaited, return the next item from the given async iterator.\n\ +If the iterator is exhausted and *default* is given, return *default*.\n\ +Otherwise, raise StopAsyncIteration."); /*[clinic input] From 0f9c814f77d9fb25b26a4ef13edaba240fa3784a Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Fri, 18 Dec 2020 15:49:02 -0500 Subject: [PATCH 19/39] Remove async iterator code from operator.py --- Lib/operator.py | 118 ++++-------------------------------------------- 1 file changed, 8 insertions(+), 110 deletions(-) diff --git a/Lib/operator.py b/Lib/operator.py index c0e94ce4bb3960..fb58851fa6ef67 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -10,15 +10,14 @@ This is the pure Python implementation of the module. """ -__all__ = [ - 'abs', 'add', 'aiter', 'anext', 'and_', 'attrgetter', 'concat', 'contains', - 'countOf', 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', - 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', - 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', 'is_', - 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', - 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', - 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor', -] +__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', + 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', + 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', + 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', + 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', + 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', + 'setitem', 'sub', 'truediv', 'truth', 'xor'] from builtins import abs as _abs @@ -405,107 +404,6 @@ def ixor(a, b): return a -# Asynchronous Iterator Operations ********************************************# - - -_NOT_PROVIDED = object() # sentinel object to detect when a kwarg was not given - - -def aiter(obj, sentinel=_NOT_PROVIDED): - """aiter(async_iterable) -> async_iterator - aiter(async_callable, sentinel) -> async_iterator - - Like the iter() builtin but for async iterables and callables. - """ - from collections.abc import AsyncIterable, AsyncIterator - if sentinel is _NOT_PROVIDED: - if not isinstance(obj, AsyncIterable): - raise TypeError(f'aiter expected an AsyncIterable, got {type(obj)}') - ait = type(obj).__aiter__(obj) - if not isinstance(ait, AsyncIterator): - raise TypeError(f'obj.__aiter__() returned non-AsyncIterator: {type(ait)}') - return ait - - if not callable(obj): - raise TypeError(f'aiter expected an async callable, got {type(obj)}') - - return _aiter_callable(obj, sentinel) - - -class _aiter_callable: - __slots__ = ('acallable', 'sentinel') - - def __init__(self, acallable, sentinel): - self.acallable = acallable - self.sentinel = sentinel - - def __aiter__(self): - return self - - def __anext__(self): - return _aiter_anext(self.acallable().__await__(), self.sentinel) - - -class _aiter_anext: - __slots__ = ('iterator', 'sentinel') - - def __init__(self, iterator, sentinel): - self.iterator = iterator - self.sentinel = sentinel - - def __await__(self): - return self - - def __iter__(self): - return self - - def __next__(self): - try: - return next(self.iterator) - except StopIteration as end: - if end.value == self.sentinel: - raise StopAsyncIteration(end.value) from None - raise - - -def anext(async_iterator, default=_NOT_PROVIDED): - """anext(async_iterator[, default]) - - Return the next item from the async iterator. - If default is given and the iterator is exhausted, - it is returned instead of raising StopAsyncIteration. - """ - from collections.abc import AsyncIterator - if not isinstance(async_iterator, AsyncIterator): - raise TypeError(f'anext expected an AsyncIterator, got {type(async_iterator)}') - anxt = type(async_iterator).__anext__(async_iterator) - - if default is _NOT_PROVIDED: - return anxt - - return _anext_default(anxt.__await__(), default) - - -class _anext_default: - __slots__ = ('iterator', 'default') - - def __init__(self, iterator, default): - self.iterator = iterator - self.default = default - - def __await__(self): - return self - - def __iter__(self): - return self - - def __next__(self): - try: - return next(self.iterator) - except StopAsyncIteration: - raise StopIteration(self.default) from None - - try: from _operator import * except ImportError: From 8160c82f9e45c47111bbaa7a1d87589595308077 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Fri, 18 Dec 2020 15:55:28 -0500 Subject: [PATCH 20/39] Cleanup formatting --- Python/bltinmodule.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d4ff94243aac91..32440f0d15cee2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1558,7 +1558,6 @@ builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) "iter(v, w): v must be callable"); return NULL; } - // PyObject *sentinel = args[1]; return PyCallAsyncIter_New(aiterable, sentinel); } @@ -2888,13 +2887,13 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF {"iter", (PyCFunction)(void(*)(void))builtin_iter, METH_FASTCALL, iter_doc}, - {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, aiter_doc}, + {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, aiter_doc}, BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", (PyCFunction)(void(*)(void))builtin_max, METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", (PyCFunction)(void(*)(void))builtin_min, METH_VARARGS | METH_KEYWORDS, min_doc}, {"next", (PyCFunction)(void(*)(void))builtin_next, METH_FASTCALL, next_doc}, - {"anext", (PyCFunction)(void(*)(void))builtin_anext, METH_FASTCALL, anext_doc}, + {"anext", (PyCFunction)(void(*)(void))builtin_anext, METH_FASTCALL, anext_doc}, BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF From fa8b12aebee8bf37db667e765ffab64992184172 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Fri, 18 Dec 2020 22:04:24 +0000 Subject: [PATCH 21/39] Fix test_builtins_have_signatures + misc. cleanups --- Lib/test/test_asyncgen.py | 9 ++++----- Lib/test/test_inspect.py | 2 +- .../Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst | 2 +- Python/bltinmodule.c | 10 +++++----- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index dd06b0b59e73fc..ab98138d7c5a88 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1,5 +1,4 @@ import inspect -import operator import types import unittest @@ -389,7 +388,7 @@ async def consume(): with self.assertRaises(StopAsyncIteration): self.loop.run_until_complete(consume()) - def test_async_gen_operator_aiter(self): + def test_async_gen_aiter(self): async def gen(): yield 1 yield 2 @@ -399,7 +398,7 @@ async def consume(): res = self.loop.run_until_complete(consume()) self.assertEqual(res, [1, 2]) - def test_async_gen_operator_aiter_class(self): + def test_async_gen_aiter_class(self): results = [] loop = self.loop class Gen: @@ -418,7 +417,7 @@ async def consume(): self.loop.run_until_complete(consume()) self.assertEqual(results, [1, 2]) - def test_async_gen_operator_aiter_2_arg(self): + def test_async_gen_aiter_2_arg(self): async def gen(): yield 1 yield 2 @@ -445,7 +444,7 @@ async def consume(): def test_anext_bad_args(self): self._test_bad_args(anext) - def test_operator_aiter_bad_args(self): + def test_aiter_bad_args(self): self._test_bad_args(aiter) def _test_bad_args(self, afn): diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 71c4f27d27b982..5337c564420eaa 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3812,7 +3812,7 @@ def test_builtins_have_signatures(self): no_signature = set() # These need PEP 457 groups needs_groups = {"range", "slice", "dir", "getattr", - "next", "iter", "vars"} + "next", "iter", "anext", "aiter", "vars"} no_signature |= needs_groups # These need PEP 457 groups or a signature change to accept None needs_semantic_update = {"round"} diff --git a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst index d4c7efda2c0790..4f1259ba4112c5 100644 --- a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst +++ b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst @@ -1,2 +1,2 @@ Add builtins.aiter and builtins.anext. -Patch by Daniel Pope, Joshua Bronson, and Justin Wang. +Patch by Daniel Pope (@lordmauve), Joshua Bronson (@jab), and Justin Wang (@justin39). diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 32440f0d15cee2..21a375b7264c03 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1562,8 +1562,8 @@ builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) } PyDoc_STRVAR(aiter_doc, -"aiter(iterable) -> iterator\n\ -aiter(acallable, sentinel) -> iterator\n\ +"aiter(async_iterable) -> async_iterator\n\ +aiter(async_callable, sentinel) -> async_iterator\n\ \n\ Get an async iterator from an object. In the first form, the\n\ argument must supply its own async iterator.\n\ @@ -1605,7 +1605,7 @@ builtin_anext_impl(PyObject *module, PyObject *aiterator, } PyDoc_STRVAR(anext_doc, -"anext(aiterator[, default])\n\ +"anext(async_iterator[, default])\n\ \n\ When awaited, return the next item from the given async iterator.\n\ If the iterator is exhausted and *default* is given, return *default*.\n\ @@ -2887,13 +2887,13 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF {"iter", (PyCFunction)(void(*)(void))builtin_iter, METH_FASTCALL, iter_doc}, - {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, aiter_doc}, + {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, aiter_doc}, BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", (PyCFunction)(void(*)(void))builtin_max, METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", (PyCFunction)(void(*)(void))builtin_min, METH_VARARGS | METH_KEYWORDS, min_doc}, {"next", (PyCFunction)(void(*)(void))builtin_next, METH_FASTCALL, next_doc}, - {"anext", (PyCFunction)(void(*)(void))builtin_anext, METH_FASTCALL, anext_doc}, + {"anext", (PyCFunction)(void(*)(void))builtin_anext, METH_FASTCALL, anext_doc}, BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF From a2242d936827018b40fbbc3f2741f75b62f2984d Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Fri, 18 Dec 2020 22:30:38 +0000 Subject: [PATCH 22/39] Use PyErr_SetNone now that we can --- Objects/iterobject.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 986784d2177d0c..732e57cb3b6984 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -367,13 +367,8 @@ callasynciter_anext(callasynciterobject *it) if (it->it_callable == NULL) { /* Can we raise this at this point, or do we need to return an awaitable - * that raises it? - */ - PyObject *value = _PyObject_New((PyTypeObject *) PyExc_StopAsyncIteration); - if (value == NULL) { - return NULL; - } - PyErr_SetObject(PyExc_StopAsyncIteration, value); + * that raises it? */ + PyErr_SetNone(PyExc_StopAsyncIteration); return NULL; } @@ -615,4 +610,4 @@ PyTypeObject PyAnextAwaitable_Type = { PyObject_SelfIter, /* tp_iter */ (unaryfunc)anextawaitable_iternext, /* tp_iternext */ 0, /* tp_methods */ -}; \ No newline at end of file +}; From 3ce5675a17c7d5580cac3b269544e5b33392859b Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Fri, 18 Dec 2020 22:45:18 +0000 Subject: [PATCH 23/39] whitespace --- Doc/glossary.rst | 2 +- Doc/library/functions.rst | 2 +- Lib/test/test_posix.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 274166443647f5..2144384f6f1846 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -90,7 +90,7 @@ Glossary asynchronous callable Any callable that returns an :term:`awaitable`. Examples include - :term:`coroutine functions ` and the built-in + :term:`coroutine functions ` and the built-in :func:`anext` function. asynchronous context manager diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 4fb03f83f92054..e9160371548aa4 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -93,7 +93,7 @@ are always available. They are listed here in alphabetical order. When awaited, return the next item from the given :term:`asynchronous iterator`, or *default* if given and the iterator is exhausted. - + This is the async variant of the :func:`next()` builtin, and behaves similarly. diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index a522717751ac17..e053fdc567179f 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1929,7 +1929,7 @@ def test_posix_spawnp(self): class TestPosixWeaklinking(unittest.TestCase): # These test cases verify that weak linking support on macOS works # as expected. These cases only test new behaviour introduced by weak linking, - # regular behaviour is tested by the normal test cases. + # regular behaviour is tested by the normal test cases. # # See the section on Weak Linking in Mac/README.txt for more information. def setUp(self): From 0038cde18805b683ecb12e9872d24aa97690edae Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Sat, 19 Dec 2020 02:19:20 +0000 Subject: [PATCH 24/39] cosmetic fixes --- Doc/library/functions.rst | 6 +++--- Objects/iterobject.c | 5 +---- Python/bltinmodule.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e9160371548aa4..5d5879b9b8a7e7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -64,10 +64,10 @@ are always available. They are listed here in alphabetical order. .. function:: aiter(object, [sentinel]) Return an :term:`asynchronous iterator` object. This is the async variant - of the :func:`iter()` builtin, and behaves similarly. + of the :func:`iter` builtin, and behaves similarly. If sentinel is omitted, then *object* must be an - :term:`asynchronous iterable` object, and ``aiter()`` returns an iterator + :term:`asynchronous iterable` object, and :func:`aiter` returns an iterator for it. Otherwise, *object* must be an :term:`asynchronous callable`. When the @@ -94,7 +94,7 @@ are always available. They are listed here in alphabetical order. When awaited, return the next item from the given :term:`asynchronous iterator`, or *default* if given and the iterator is exhausted. - This is the async variant of the :func:`next()` builtin, and behaves + This is the async variant of the :func:`next` builtin, and behaves similarly. Immediately, call the :meth:`~object.__anext__` method of *async_iterator*, diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 732e57cb3b6984..850a156ed268a4 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -302,7 +302,7 @@ typedef struct { The iterator returned by the callable, unwrapped via __await__. If NULL this means that the iterator is already exhausted and when - iterated it should raise StopAsyncIteration; + iterated it should raise StopAsyncIteration. */ callasynciterobject *it; /* The iterator object, in order to clear it when done. */ } callasynciterawaitableobject; @@ -357,7 +357,6 @@ callasyncawaitable_traverse(callasynciterawaitableobject *obj, visitproc visit, return 0; } - static PyObject * callasynciter_anext(callasynciterobject *it) { @@ -440,8 +439,6 @@ asynccallawaitable_iternext(callasynciterawaitableobject *obj) return NULL; } - - static PyAsyncMethods async_iter_as_async = { PyObject_SelfIter, /* am_await */ PyObject_SelfIter, /* am_aiter */ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 21a375b7264c03..740c39e2136982 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1555,7 +1555,7 @@ builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) if (!PyCallable_Check(aiterable)) { PyErr_SetString(PyExc_TypeError, - "iter(v, w): v must be callable"); + "aiter(v, w): v must be async callable"); return NULL; } return PyCallAsyncIter_New(aiterable, sentinel); From 06019ea05ebcd477f1545f2fccc8b089204ade42 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Sat, 19 Dec 2020 15:43:00 -0500 Subject: [PATCH 25/39] Add null check and use StopAsyncIteration when aiter is exhausted --- Objects/abstract.c | 4 ++-- Objects/iterobject.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 6161ee6a920b65..ec541a69ccbf6e 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2644,11 +2644,11 @@ PyObject_GetAiter(PyObject *o) { PyTypeObject *t = Py_TYPE(o); unaryfunc f; - f = t->tp_as_async->am_aiter; - if (f == NULL) { + if (t->tp_as_async == NULL || t->tp_as_async->am_aiter == NULL) { return type_error("'%.200s' object is not async iterable", o); } else { + f = t->tp_as_async->am_aiter; PyObject *it = (*f)(o); if (it != NULL && !PyAiter_Check(it)) { PyErr_Format(PyExc_TypeError, diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 850a156ed268a4..d765c1d958d731 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -406,7 +406,8 @@ asynccallawaitable_iternext(callasynciterawaitableobject *obj) int ok; if (obj->it->it_sentinel == NULL) { - return PyErr_Format(PyExc_TypeError, "'%.200s' object is already exhausted", obj->it); + PyErr_SetNone(PyExc_StopAsyncIteration); + return NULL; } result = (*Py_TYPE(obj->wrapped_iter)->tp_iternext)(obj->wrapped_iter); From 0cc00f20b579594ed96f71b39922bcdb84464e78 Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Mon, 21 Dec 2020 11:08:07 -0500 Subject: [PATCH 26/39] Fix AC definition and resulting signature --- Lib/test/test_inspect.py | 2 +- Python/bltinmodule.c | 27 ++++++--------------------- Python/clinic/bltinmodule.c.h | 10 +++++----- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 5337c564420eaa..71c4f27d27b982 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3812,7 +3812,7 @@ def test_builtins_have_signatures(self): no_signature = set() # These need PEP 457 groups needs_groups = {"range", "slice", "dir", "getattr", - "next", "iter", "anext", "aiter", "vars"} + "next", "iter", "vars"} no_signature |= needs_groups # These need PEP 457 groups or a signature change to accept None needs_semantic_update = {"round"} diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 740c39e2136982..5fd5c1817bc45e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1539,7 +1539,7 @@ In the second form, the callable is called until it returns the sentinel."); aiter as builtin_aiter aiterable: object - sentinel: object = NULL + sentinel: object = None / Return an async iterator for an async iterable object. @@ -1547,7 +1547,7 @@ Return an async iterator for an async iterable object. static PyObject * builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) -/*[clinic end generated code: output=ea120e90169d9f32 input=a12efceda1863b3c]*/ +/*[clinic end generated code: output=ea120e90169d9f32 input=586f672fb18a94a5]*/ { if (sentinel == NULL) { return PyObject_GetAiter(aiterable); @@ -1561,20 +1561,12 @@ builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) return PyCallAsyncIter_New(aiterable, sentinel); } -PyDoc_STRVAR(aiter_doc, -"aiter(async_iterable) -> async_iterator\n\ -aiter(async_callable, sentinel) -> async_iterator\n\ -\n\ -Get an async iterator from an object. In the first form, the\n\ -argument must supply its own async iterator.\n\ -In the second form, the async callable is called and awaited\n\ -until it returns the sentinel."); /*[clinic input] anext as builtin_anext aiterator: object - default: object = NULL + default: object = None / Return the next item from the async iterator. @@ -1583,7 +1575,7 @@ Return the next item from the async iterator. static PyObject * builtin_anext_impl(PyObject *module, PyObject *aiterator, PyObject *default_value) -/*[clinic end generated code: output=f02c060c163a81fa input=699d11f4e38eca24]*/ +/*[clinic end generated code: output=f02c060c163a81fa input=6051f80000c06306]*/ { PyTypeObject *t; PyObject *awaitable; @@ -1604,13 +1596,6 @@ builtin_anext_impl(PyObject *module, PyObject *aiterator, return PyAnextAwaitable_New(awaitable, default_value); } -PyDoc_STRVAR(anext_doc, -"anext(async_iterator[, default])\n\ -\n\ -When awaited, return the next item from the given async iterator.\n\ -If the iterator is exhausted and *default* is given, return *default*.\n\ -Otherwise, raise StopAsyncIteration."); - /*[clinic input] len as builtin_len @@ -2887,13 +2872,13 @@ static PyMethodDef builtin_methods[] = { BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF {"iter", (PyCFunction)(void(*)(void))builtin_iter, METH_FASTCALL, iter_doc}, - {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, aiter_doc}, + BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", (PyCFunction)(void(*)(void))builtin_max, METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", (PyCFunction)(void(*)(void))builtin_min, METH_VARARGS | METH_KEYWORDS, min_doc}, {"next", (PyCFunction)(void(*)(void))builtin_next, METH_FASTCALL, next_doc}, - {"anext", (PyCFunction)(void(*)(void))builtin_anext, METH_FASTCALL, anext_doc}, + BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 822496ca82a4af..4482e0d9c33db9 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -531,7 +531,7 @@ PyDoc_STRVAR(builtin_hex__doc__, {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, PyDoc_STRVAR(builtin_aiter__doc__, -"aiter($module, aiterable, sentinel=, /)\n" +"aiter($module, aiterable, sentinel=None, /)\n" "--\n" "\n" "Return an async iterator for an async iterable object."); @@ -547,7 +547,7 @@ builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *aiterable; - PyObject *sentinel = NULL; + PyObject *sentinel = Py_None; if (!_PyArg_CheckPositional("aiter", nargs, 1, 2)) { goto exit; @@ -565,7 +565,7 @@ builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_anext__doc__, -"anext($module, aiterator, default=, /)\n" +"anext($module, aiterator, default=None, /)\n" "--\n" "\n" "Return the next item from the async iterator."); @@ -582,7 +582,7 @@ builtin_anext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *aiterator; - PyObject *default_value = NULL; + PyObject *default_value = Py_None; if (!_PyArg_CheckPositional("anext", nargs, 1, 2)) { goto exit; @@ -899,4 +899,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=b5248d88ee495198 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=02b80d529a5c972b input=a9049054013a1b77]*/ From 8b8a689128ccb3d0fd1ae8e49b86ad26d7ce9b9c Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Mon, 21 Dec 2020 11:46:01 -0500 Subject: [PATCH 27/39] Fix comparison to NULL instead of Py_None --- Python/bltinmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 5fd5c1817bc45e..470982b2861a5b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1549,7 +1549,7 @@ static PyObject * builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) /*[clinic end generated code: output=ea120e90169d9f32 input=586f672fb18a94a5]*/ { - if (sentinel == NULL) { + if (sentinel == Py_None) { return PyObject_GetAiter(aiterable); } @@ -1589,7 +1589,7 @@ builtin_anext_impl(PyObject *module, PyObject *aiterator, } awaitable = (*t->tp_as_async->am_anext)(aiterator); - if (default_value == NULL) { + if (default_value == Py_None) { return awaitable; } From 894600d8e272944b7e13cf3b7014c7325982827e Mon Sep 17 00:00:00 2001 From: Justin Wang Date: Mon, 21 Dec 2020 13:13:35 -0500 Subject: [PATCH 28/39] Revert None deafult for aiter/anext and add whatsnew entry --- Doc/whatsnew/3.10.rst | 5 +++++ Lib/test/test_inspect.py | 3 +++ Python/bltinmodule.c | 12 ++++++------ Python/clinic/bltinmodule.c.h | 10 +++++----- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f96a3bcbca95f3..6e655f8be71ee6 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -157,6 +157,11 @@ Other Language Changes * Assignment expressions can now be used unparenthesized within set literals and set comprehensions, as well as in sequence indexes (but not slices). +* Two new builtin functions -- :func:`aiter` and :func:`anext` have been added + to provide asynchronous counterparts to :func:`iter` and :func:`next`, + respectively. + (Contributed by Joshua Bronson, Justin Wang and Daniel Pope in :issue:`31861`) + New Modules =========== diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 71c4f27d27b982..9ccdfcabd4ef04 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3814,6 +3814,9 @@ def test_builtins_have_signatures(self): needs_groups = {"range", "slice", "dir", "getattr", "next", "iter", "vars"} no_signature |= needs_groups + # These have unpresentable parameter default values of NULL + needs_null = {"aiter", "anext"} + no_signature |= needs_null # These need PEP 457 groups or a signature change to accept None needs_semantic_update = {"round"} no_signature |= needs_semantic_update diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 470982b2861a5b..e532c2a488d5e8 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1539,7 +1539,7 @@ In the second form, the callable is called until it returns the sentinel."); aiter as builtin_aiter aiterable: object - sentinel: object = None + sentinel: object = NULL / Return an async iterator for an async iterable object. @@ -1547,9 +1547,9 @@ Return an async iterator for an async iterable object. static PyObject * builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) -/*[clinic end generated code: output=ea120e90169d9f32 input=586f672fb18a94a5]*/ +/*[clinic end generated code: output=ea120e90169d9f32 input=a12efceda1863b3c]*/ { - if (sentinel == Py_None) { + if (sentinel == NULL) { return PyObject_GetAiter(aiterable); } @@ -1566,7 +1566,7 @@ builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) anext as builtin_anext aiterator: object - default: object = None + default: object = NULL / Return the next item from the async iterator. @@ -1575,7 +1575,7 @@ Return the next item from the async iterator. static PyObject * builtin_anext_impl(PyObject *module, PyObject *aiterator, PyObject *default_value) -/*[clinic end generated code: output=f02c060c163a81fa input=6051f80000c06306]*/ +/*[clinic end generated code: output=f02c060c163a81fa input=699d11f4e38eca24]*/ { PyTypeObject *t; PyObject *awaitable; @@ -1589,7 +1589,7 @@ builtin_anext_impl(PyObject *module, PyObject *aiterator, } awaitable = (*t->tp_as_async->am_anext)(aiterator); - if (default_value == Py_None) { + if (default_value == NULL) { return awaitable; } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 4482e0d9c33db9..822496ca82a4af 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -531,7 +531,7 @@ PyDoc_STRVAR(builtin_hex__doc__, {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, PyDoc_STRVAR(builtin_aiter__doc__, -"aiter($module, aiterable, sentinel=None, /)\n" +"aiter($module, aiterable, sentinel=, /)\n" "--\n" "\n" "Return an async iterator for an async iterable object."); @@ -547,7 +547,7 @@ builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *aiterable; - PyObject *sentinel = Py_None; + PyObject *sentinel = NULL; if (!_PyArg_CheckPositional("aiter", nargs, 1, 2)) { goto exit; @@ -565,7 +565,7 @@ builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_anext__doc__, -"anext($module, aiterator, default=None, /)\n" +"anext($module, aiterator, default=, /)\n" "--\n" "\n" "Return the next item from the async iterator."); @@ -582,7 +582,7 @@ builtin_anext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *aiterator; - PyObject *default_value = Py_None; + PyObject *default_value = NULL; if (!_PyArg_CheckPositional("anext", nargs, 1, 2)) { goto exit; @@ -899,4 +899,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=02b80d529a5c972b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b5248d88ee495198 input=a9049054013a1b77]*/ From 259af97639e97491385c878b724e0f803d4e1fc4 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Sat, 20 Mar 2021 20:00:16 +0000 Subject: [PATCH 29/39] Delint tests + docs (fixes CI), alphabetize names. --- Doc/whatsnew/3.10.rst | 4 ++-- Lib/test/test_asyncgen.py | 2 -- .../next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index fa5a1d4718cbab..9a73961bd96707 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -279,7 +279,7 @@ Other Language Changes * Assignment expressions can now be used unparenthesized within set literals and set comprehensions, as well as in sequence indexes (but not slices). - + * Functions have a new ``__builtins__`` attribute which is used to look for builtin symbols when a function is executed, instead of looking into ``__globals__['__builtins__']``. The attribute is initialized from @@ -289,7 +289,7 @@ Other Language Changes * Two new builtin functions -- :func:`aiter` and :func:`anext` have been added to provide asynchronous counterparts to :func:`iter` and :func:`next`, respectively. - (Contributed by Joshua Bronson, Justin Wang and Daniel Pope in :issue:`31861`) + (Contributed by Josh Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) New Modules diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index ab98138d7c5a88..31cfb8e8dd5967 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -400,11 +400,9 @@ async def consume(): def test_async_gen_aiter_class(self): results = [] - loop = self.loop class Gen: async def __aiter__(self): yield 1 - await asyncio.sleep(0.01) yield 2 g = Gen() async def consume(): diff --git a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst index 4f1259ba4112c5..790f682294270c 100644 --- a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst +++ b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst @@ -1,2 +1,2 @@ Add builtins.aiter and builtins.anext. -Patch by Daniel Pope (@lordmauve), Joshua Bronson (@jab), and Justin Wang (@justin39). +Patch by Josh Bronson (@jab), Daniel Pope (@lordmauve), and Justin Wang (@justin39). From dd7c02d27047aeac1c00394e73eda3380db15b99 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Sat, 20 Mar 2021 20:11:40 +0000 Subject: [PATCH 30/39] Fix code style and comments. --- Lib/test/test_inspect.py | 2 +- Objects/abstract.c | 23 ++++++++++------------- Objects/iterobject.c | 6 ++---- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 32da0c6c5c8018..ff069e481d28f1 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3860,7 +3860,7 @@ def test_builtins_have_signatures(self): needs_groups = {"range", "slice", "dir", "getattr", "next", "iter", "vars"} no_signature |= needs_groups - # These have unpresentable parameter default values of NULL + # These have unrepresentable parameter default values of NULL needs_null = {"aiter", "anext"} no_signature |= needs_null # These need PEP 457 groups or a signature change to accept None diff --git a/Objects/abstract.c b/Objects/abstract.c index d5d94c227bd6d6..e237379cc72115 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2744,21 +2744,18 @@ PyObject_GetAiter(PyObject *o) { unaryfunc f; if (t->tp_as_async == NULL || t->tp_as_async->am_aiter == NULL) { - return type_error("'%.200s' object is not async iterable", o); + return type_error("'%.200s' object is not async-iterable", o); } - else { - f = t->tp_as_async->am_aiter; - PyObject *it = (*f)(o); - if (it != NULL && !PyAiter_Check(it)) { - PyErr_Format(PyExc_TypeError, - "aiter() returned non-async-iterator " - "of type '%.100s'", - Py_TYPE(it)->tp_name); - Py_DECREF(it); - it = NULL; - } - return it; + f = t->tp_as_async->am_aiter; + PyObject *it = (*f)(o); + if (it != NULL && !PyAiter_Check(it)) { + PyErr_Format(PyExc_TypeError, + "aiter() returned non-async-iterator of type '%.100s'", + Py_TYPE(it)->tp_name); + Py_DECREF(it); + it = NULL; } + return it; } int diff --git a/Objects/iterobject.c b/Objects/iterobject.c index d765c1d958d731..251e574f4845b9 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -301,8 +301,8 @@ typedef struct { PyObject *wrapped_iter; /* The iterator returned by the callable, unwrapped via __await__. - If NULL this means that the iterator is already exhausted and when - iterated it should raise StopAsyncIteration. + If NULL, this means that the iterator is already exhausted, + and when iterated, it should raise StopAsyncIteration. */ callasynciterobject *it; /* The iterator object, in order to clear it when done. */ } callasynciterawaitableobject; @@ -365,8 +365,6 @@ callasynciter_anext(callasynciterobject *it) callasynciterawaitableobject *awaitable; if (it->it_callable == NULL) { - /* Can we raise this at this point, or do we need to return an awaitable - * that raises it? */ PyErr_SetNone(PyExc_StopAsyncIteration); return NULL; } From 2fbdd5bb82aeaf4936e03f7753f0306fb169198b Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 22 Mar 2021 13:58:23 +0000 Subject: [PATCH 31/39] Remove 2-arg variant of aiter. --- Doc/library/functions.rst | 19 +- Doc/whatsnew/3.10.rst | 2 +- Include/abstract.h | 6 +- Include/iterobject.h | 2 - Lib/test/test_asyncgen.py | 58 ++--- .../2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst | 2 +- Objects/abstract.c | 4 +- Objects/iterobject.c | 229 ------------------ Python/bltinmodule.c | 18 +- Python/clinic/bltinmodule.c.h | 33 +-- 10 files changed, 46 insertions(+), 327 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 1ff87cdd03203c..4f3e0cb46145c9 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -61,20 +61,19 @@ are always available. They are listed here in alphabetical order. If the argument is a complex number, its magnitude is returned. -.. function:: aiter(object, [sentinel]) +.. function:: aiter(async_iterable) - Return an :term:`asynchronous iterator` object. This is the async variant + Return an :term:`asynchronous iterator`. This is the async variant of the :func:`iter` builtin, and behaves similarly. - If sentinel is omitted, then *object* must be an - :term:`asynchronous iterable` object, and :func:`aiter` returns an iterator - for it. + *async_iterable* must be an :term:`asynchronous iterable`, + and :func:`aiter` returns an asynchronous iterator for it. - Otherwise, *object* must be an :term:`asynchronous callable`. When the - resulting async iterator is async iterated, the passed callable is - called and awaited and the values returned become the values produced - by the iterator. When the awaited return value is equal to sentinel, - the async iterator terminates with :exc:`StopAsyncIteration`. + Unlike the :func:`iter` builtin, :func:`aiter` has no 2-argument variant. + Often, this variant can be replaced with assignment expressions:: + + while chunk := await sock.read(CHUNK_SIZE): + ... .. function:: all(iterable) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 03a59f5f6a849d..a45e0fe6acf8e3 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -591,7 +591,7 @@ Other Language Changes * Two new builtin functions -- :func:`aiter` and :func:`anext` have been added to provide asynchronous counterparts to :func:`iter` and :func:`next`, respectively. - (Contributed by Josh Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) + (Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) New Modules diff --git a/Include/abstract.h b/Include/abstract.h index 54a4fa09d43212..1af1487deec916 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -324,8 +324,8 @@ PyAPI_FUNC(PyObject *) PyObject_Format(PyObject *obj, returns itself. */ PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *); -/* Takes an object and returns an async iterator for it. - This is typically a new iterator but if the argument is an async iterator, +/* Takes an AsyncIterable object and returns an AsyncIterator for it. + This is typically a new iterator but if the argument is an AsyncIterator, this returns itself. */ PyAPI_FUNC(PyObject *) PyObject_GetAiter(PyObject *); @@ -334,7 +334,7 @@ PyAPI_FUNC(PyObject *) PyObject_GetAiter(PyObject *); This function always succeeds. */ PyAPI_FUNC(int) PyIter_Check(PyObject *); -/* Returns non-zero if the object 'obj' provides async iterator protocols, and 0 otherwise. +/* Returns non-zero if the object 'obj' provides AsyncIterator protocols, and 0 otherwise. This function always succeeds. */ PyAPI_FUNC(int) PyAiter_Check(PyObject *); diff --git a/Include/iterobject.h b/Include/iterobject.h index 40920fd6f15878..8251a7dba49f46 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -7,7 +7,6 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; -PyAPI_DATA(PyTypeObject) PyCallAsyncIter_Type; PyAPI_DATA(PyTypeObject) PyAsyncCallAwaitable_Type; PyAPI_DATA(PyTypeObject) PyAnextAwaitable_Type; @@ -19,7 +18,6 @@ PyAPI_FUNC(PyObject *) PySeqIter_New(PyObject *); #define PyCallIter_Check(op) Py_IS_TYPE(op, &PyCallIter_Type) PyAPI_FUNC(PyObject *) PyCallIter_New(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyCallAsyncIter_New(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyAnextAwaitable_New(PyObject *, PyObject *); #ifdef __cplusplus diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 31cfb8e8dd5967..6a1df7ca3989f9 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -415,51 +415,37 @@ async def consume(): self.loop.run_until_complete(consume()) self.assertEqual(results, [1, 2]) - def test_async_gen_aiter_2_arg(self): + def test_anext_bad_args(self): async def gen(): yield 1 - yield 2 - yield 3 - g = gen() - async def foo(): - return await anext(g) - async def consume(): - return [i async for i in aiter(foo, 3)] - res = self.loop.run_until_complete(consume()) - self.assertEqual(res, [1, 2]) - - def test_async_aiter_callable(self): - v = 0 - async def foo(): - nonlocal v - v += 1 - return v - async def consume(): - return [i async for i in aiter(foo, 3)] - res = self.loop.run_until_complete(consume()) - self.assertEqual(res, [1, 2]) - - def test_anext_bad_args(self): - self._test_bad_args(anext) + async def call_with_too_few_args(): + await anext() + async def call_with_too_many_args(): + await anext(gen(), 1, 3) + async def call_with_wrong_type_args(): + await anext(1, gen()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_too_few_args()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_too_many_args()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_wrong_type_args()) def test_aiter_bad_args(self): - self._test_bad_args(aiter) - - def _test_bad_args(self, afn): async def gen(): yield 1 - async def call_with_no_args(): - await afn() - async def call_with_3_args(): - await afn(gen(), 1, 2) - async def call_with_bad_args(): - await afn(1, gen()) + async def call_with_too_few_args(): + await aiter() + async def call_with_too_many_args(): + await aiter(gen(), 1) + async def call_with_wrong_type_arg(): + await aiter(1) with self.assertRaises(TypeError): - self.loop.run_until_complete(call_with_no_args()) + self.loop.run_until_complete(call_with_too_few_args()) with self.assertRaises(TypeError): - self.loop.run_until_complete(call_with_3_args()) + self.loop.run_until_complete(call_with_too_many_args()) with self.assertRaises(TypeError): - self.loop.run_until_complete(call_with_bad_args()) + self.loop.run_until_complete(call_with_wrong_type_arg()) async def to_list(self, gen): res = [] diff --git a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst index 790f682294270c..1526deb995c3dc 100644 --- a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst +++ b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst @@ -1,2 +1,2 @@ Add builtins.aiter and builtins.anext. -Patch by Josh Bronson (@jab), Daniel Pope (@lordmauve), and Justin Wang (@justin39). +Patch by Joshua Bronson (@jab), Daniel Pope (@lordmauve), and Justin Wang (@justin39). diff --git a/Objects/abstract.c b/Objects/abstract.c index e237379cc72115..1e7c83108fed84 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2744,13 +2744,13 @@ PyObject_GetAiter(PyObject *o) { unaryfunc f; if (t->tp_as_async == NULL || t->tp_as_async->am_aiter == NULL) { - return type_error("'%.200s' object is not async-iterable", o); + return type_error("'%.200s' object is not an AsyncIterable", o); } f = t->tp_as_async->am_aiter; PyObject *it = (*f)(o); if (it != NULL && !PyAiter_Check(it)) { PyErr_Format(PyExc_TypeError, - "aiter() returned non-async-iterator of type '%.100s'", + "aiter() returned non-AsyncIterator of type '%.100s'", Py_TYPE(it)->tp_name); Py_DECREF(it); it = NULL; diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 251e574f4845b9..3b42edcabf91bd 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -288,235 +288,6 @@ PyTypeObject PyCallIter_Type = { }; -/* -------------------------------------- */ - -typedef struct { - PyObject_HEAD - PyObject *it_callable; /* Set to NULL when iterator is exhausted */ - PyObject *it_sentinel; /* Set to NULL when iterator is exhausted */ -} callasynciterobject; - -typedef struct { - PyObject_HEAD - PyObject *wrapped_iter; /* - The iterator returned by the callable, unwrapped via __await__. - - If NULL, this means that the iterator is already exhausted, - and when iterated, it should raise StopAsyncIteration. - */ - callasynciterobject *it; /* The iterator object, in order to clear it when done. */ -} callasynciterawaitableobject; - - -PyObject * -PyCallAsyncIter_New(PyObject *callable, PyObject *sentinel) -{ - callasynciterobject *it; - it = PyObject_GC_New(callasynciterobject, &PyCallAsyncIter_Type); - if (it == NULL) - return NULL; - Py_INCREF(callable); - it->it_callable = callable; - Py_INCREF(sentinel); - it->it_sentinel = sentinel; - _PyObject_GC_TRACK(it); - return (PyObject *)it; -} - -static void -callasynciter_dealloc(callasynciterobject *obj) -{ - _PyObject_GC_UNTRACK(obj); - Py_XDECREF(obj->it_callable); - Py_XDECREF(obj->it_sentinel); - PyObject_GC_Del(obj); -} - -static void -callasynciterawaitable_dealloc(callasynciterawaitableobject *obj) -{ - _PyObject_GC_UNTRACK(obj); - Py_XDECREF(obj->wrapped_iter); - Py_XDECREF(obj->it); - PyObject_GC_Del(obj); -} - -static int -callasynciter_traverse(callasynciterobject *it, visitproc visit, void *arg) -{ - Py_VISIT(it->it_callable); - Py_VISIT(it->it_sentinel); - return 0; -} - -static int -callasyncawaitable_traverse(callasynciterawaitableobject *obj, visitproc visit, void *arg) -{ - Py_VISIT(obj->wrapped_iter); - Py_VISIT(obj->it); - return 0; -} - -static PyObject * -callasynciter_anext(callasynciterobject *it) -{ - PyObject *obj = NULL, *wrapped_iter = NULL; - PyTypeObject *t = NULL; - callasynciterawaitableobject *awaitable; - - if (it->it_callable == NULL) { - PyErr_SetNone(PyExc_StopAsyncIteration); - return NULL; - } - - obj = _PyObject_CallNoArg(it->it_callable); - if (obj == NULL) { - return NULL; - } - - t = Py_TYPE(obj); - if (t->tp_as_async == NULL || t->tp_as_async->am_await == NULL) { - return PyErr_Format(PyExc_TypeError, "'%.200s' object is not awaitable", obj); - } - - wrapped_iter = (*t->tp_as_async->am_await)(obj); - - awaitable = PyObject_GC_New( - callasynciterawaitableobject, - &PyAsyncCallAwaitable_Type); - if (awaitable == NULL) { - return NULL; - } - - Py_INCREF(it); - awaitable->it = it; - awaitable->wrapped_iter = wrapped_iter; - - _PyObject_GC_TRACK(awaitable); - return (PyObject *) awaitable; -} - -static PyObject * -asynccallawaitable_iternext(callasynciterawaitableobject *obj) -{ - PyObject *result; - PyObject *stop_value = NULL; - int ok; - - if (obj->it->it_sentinel == NULL) { - PyErr_SetNone(PyExc_StopAsyncIteration); - return NULL; - } - - result = (*Py_TYPE(obj->wrapped_iter)->tp_iternext)(obj->wrapped_iter); - - if (result != NULL) { - return result; - } - - ok = _PyGen_FetchStopIterationValue(&stop_value); - if (ok == -1) { - PyErr_SetString(PyExc_AssertionError, "StopIteration not raised"); - return NULL; - } - - if (stop_value == NULL) { - PyErr_SetString(PyExc_TypeError, "Coroutine iterator raised StopIteration without value"); - return NULL; - } - - ok = PyObject_RichCompareBool(obj->it->it_sentinel, stop_value, Py_EQ); - Py_DECREF(stop_value); - if (ok == 0) { - _PyGen_SetStopIterationValue(stop_value); - return NULL; - } - - Py_CLEAR(obj->it->it_callable); - Py_CLEAR(obj->it->it_sentinel); - PyErr_SetNone(PyExc_StopAsyncIteration); - return NULL; -} - -static PyAsyncMethods async_iter_as_async = { - PyObject_SelfIter, /* am_await */ - PyObject_SelfIter, /* am_aiter */ - (unaryfunc)callasynciter_anext, /* am_anext */ - 0, /* am_send */ -}; - -PyTypeObject PyCallAsyncIter_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "callable_async_iterator", /* tp_name */ - sizeof(callasynciterobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)callasynciter_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - &async_iter_as_async, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - 0, /* tp_doc */ - (traverseproc)callasynciter_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ -}; - - -static PyAsyncMethods async_awaitable_as_async = { - PyObject_SelfIter, /* am_await */ - 0, /* am_aiter */ - 0, /* am_anext */ - 0, /* am_send */ -}; - -PyTypeObject PyAsyncCallAwaitable_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "callable_async_iterator_awaitable", /* tp_name */ - sizeof(callasynciterawaitableobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)callasynciterawaitable_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - &async_awaitable_as_async, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - 0, /* tp_doc */ - (traverseproc)callasyncawaitable_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (unaryfunc)asynccallawaitable_iternext, /* tp_iternext */ - 0, /* tp_methods */ -}; - /* -------------------------------------- */ typedef struct { diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 1d81dee2c6529c..98689a1a3a98d0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1574,26 +1574,16 @@ In the second form, the callable is called until it returns the sentinel."); aiter as builtin_aiter aiterable: object - sentinel: object = NULL / -Return an async iterator for an async iterable object. +Return an async iterator for an AsyncIterable object. [clinic start generated code]*/ static PyObject * -builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel) -/*[clinic end generated code: output=ea120e90169d9f32 input=a12efceda1863b3c]*/ +builtin_aiter(PyObject *module, PyObject *aiterable) +/*[clinic end generated code: output=0acd83729a7081e9 input=9311745fdf3f9f1b]*/ { - if (sentinel == NULL) { - return PyObject_GetAiter(aiterable); - } - - if (!PyCallable_Check(aiterable)) { - PyErr_SetString(PyExc_TypeError, - "aiter(v, w): v must be async callable"); - return NULL; - } - return PyCallAsyncIter_New(aiterable, sentinel); + return PyObject_GetAiter(aiterable); } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 822496ca82a4af..0142338e799df6 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -531,38 +531,13 @@ PyDoc_STRVAR(builtin_hex__doc__, {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, PyDoc_STRVAR(builtin_aiter__doc__, -"aiter($module, aiterable, sentinel=, /)\n" +"aiter($module, aiterable, /)\n" "--\n" "\n" -"Return an async iterator for an async iterable object."); +"Return an AsyncIterator for an AsyncIterable object."); #define BUILTIN_AITER_METHODDEF \ - {"aiter", (PyCFunction)(void(*)(void))builtin_aiter, METH_FASTCALL, builtin_aiter__doc__}, - -static PyObject * -builtin_aiter_impl(PyObject *module, PyObject *aiterable, PyObject *sentinel); - -static PyObject * -builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *aiterable; - PyObject *sentinel = NULL; - - if (!_PyArg_CheckPositional("aiter", nargs, 1, 2)) { - goto exit; - } - aiterable = args[0]; - if (nargs < 2) { - goto skip_optional; - } - sentinel = args[1]; -skip_optional: - return_value = builtin_aiter_impl(module, aiterable, sentinel); - -exit: - return return_value; -} + {"aiter", (PyCFunction)builtin_aiter, METH_O, builtin_aiter__doc__}, PyDoc_STRVAR(builtin_anext__doc__, "anext($module, aiterator, default=, /)\n" @@ -899,4 +874,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=b5248d88ee495198 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2b8d3a868450f2fe input=a9049054013a1b77]*/ From 009118ce1b36d81307667d735ec274905ce9a322 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 22 Mar 2021 14:37:27 +0000 Subject: [PATCH 32/39] Fix use of clinic. --- Python/bltinmodule.c | 10 +++++----- Python/clinic/bltinmodule.c.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 98689a1a3a98d0..d55544092bbd16 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1573,17 +1573,17 @@ In the second form, the callable is called until it returns the sentinel."); /*[clinic input] aiter as builtin_aiter - aiterable: object + async_iterable: object / -Return an async iterator for an AsyncIterable object. +Return an AsyncIterator for an AsyncIterable object. [clinic start generated code]*/ static PyObject * -builtin_aiter(PyObject *module, PyObject *aiterable) -/*[clinic end generated code: output=0acd83729a7081e9 input=9311745fdf3f9f1b]*/ +builtin_aiter(PyObject *module, PyObject *async_iterable) +/*[clinic end generated code: output=1bae108d86f7960e input=473993d0cacc7d23]*/ { - return PyObject_GetAiter(aiterable); + return PyObject_GetAiter(async_iterable); } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 0142338e799df6..545f5b53f6e549 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -531,7 +531,7 @@ PyDoc_STRVAR(builtin_hex__doc__, {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, PyDoc_STRVAR(builtin_aiter__doc__, -"aiter($module, aiterable, /)\n" +"aiter($module, async_iterable, /)\n" "--\n" "\n" "Return an AsyncIterator for an AsyncIterable object."); @@ -874,4 +874,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=2b8d3a868450f2fe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=da9ae459e9233259 input=a9049054013a1b77]*/ From 71fede3446707007005886aee553176cc3712246 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 22 Mar 2021 20:23:24 +0000 Subject: [PATCH 33/39] Address some feedback from Guido's review. --- Doc/glossary.rst | 5 ----- Doc/library/functions.rst | 14 ++++++-------- Objects/abstract.c | 8 ++++---- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 2144384f6f1846..0661c8283290ce 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -88,11 +88,6 @@ Glossary :ref:`the difference between arguments and parameters `, and :pep:`362`. - asynchronous callable - Any callable that returns an :term:`awaitable`. Examples include - :term:`coroutine functions ` and the built-in - :func:`anext` function. - asynchronous context manager An object which controls the environment seen in an :keyword:`async with` statement by defining :meth:`__aenter__` and diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 4f3e0cb46145c9..ffdfa6e5b8c146 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -63,17 +63,15 @@ are always available. They are listed here in alphabetical order. .. function:: aiter(async_iterable) - Return an :term:`asynchronous iterator`. This is the async variant - of the :func:`iter` builtin, and behaves similarly. + Equivalent to calling ``x.__aiter__()``. - *async_iterable* must be an :term:`asynchronous iterable`, - and :func:`aiter` returns an asynchronous iterator for it. + ``aiter(aiter(x))`` is the same as ``aiter(x)``. + (``aiter(x)`` itself has an ``__aiter__()`` method that returns ``self``.) - Unlike the :func:`iter` builtin, :func:`aiter` has no 2-argument variant. - Often, this variant can be replaced with assignment expressions:: + Formally, given an :term:`asynchronous iterable`, + return an :term:`asynchronous iterator`. - while chunk := await sock.read(CHUNK_SIZE): - ... + Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. .. function:: all(iterable) diff --git a/Objects/abstract.c b/Objects/abstract.c index 1e7c83108fed84..fcfe2dbe483f4f 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2770,10 +2770,10 @@ int PyAiter_Check(PyObject *obj) { PyTypeObject *tp = Py_TYPE(obj); - return (tp->tp_as_async != NULL && \ - tp->tp_as_async->am_aiter != NULL && \ - tp->tp_as_async->am_aiter != &_PyObject_NextNotImplemented && \ - tp->tp_as_async->am_anext != NULL && \ + return (tp->tp_as_async != NULL && + tp->tp_as_async->am_aiter != NULL && + tp->tp_as_async->am_aiter != &_PyObject_NextNotImplemented && + tp->tp_as_async->am_anext != NULL && tp->tp_as_async->am_anext != &_PyObject_NextNotImplemented); } From 4a51ace740d959e8067a4306c927b028b06740d4 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 22 Mar 2021 20:37:45 +0000 Subject: [PATCH 34/39] No longer need to exclude aiter from test_builtins_have_signatures. --- Lib/test/test_inspect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index ff069e481d28f1..72feaedbe7513a 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -3861,7 +3861,7 @@ def test_builtins_have_signatures(self): "next", "iter", "vars"} no_signature |= needs_groups # These have unrepresentable parameter default values of NULL - needs_null = {"aiter", "anext"} + needs_null = {"anext"} no_signature |= needs_null # These need PEP 457 groups or a signature change to accept None needs_semantic_update = {"round"} From 108e4a3b8e969d1353a9d528e324913b3eda6992 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 22 Mar 2021 21:32:00 +0000 Subject: [PATCH 35/39] Improve aiter() docs. --- Doc/library/functions.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index ffdfa6e5b8c146..0825db62d54b2f 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -63,13 +63,11 @@ are always available. They are listed here in alphabetical order. .. function:: aiter(async_iterable) + Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. - ``aiter(aiter(x))`` is the same as ``aiter(x)``. - (``aiter(x)`` itself has an ``__aiter__()`` method that returns ``self``.) - - Formally, given an :term:`asynchronous iterable`, - return an :term:`asynchronous iterator`. + ``aiter(x)`` itself has an ``__aiter__()`` method that returns ``self``, + so ``aiter(aiter(x))`` is the same as ``aiter(x)``. Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. From 042f596d51e61709525d9b60bbc6f85410692dcb Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 22 Mar 2021 21:33:44 +0000 Subject: [PATCH 36/39] Fix typo. --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 0825db62d54b2f..9b433c4ccfe158 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -66,7 +66,7 @@ are always available. They are listed here in alphabetical order. Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. - ``aiter(x)`` itself has an ``__aiter__()`` method that returns ``self``, + ``aiter(x)`` itself has an ``__aiter__()`` method that returns ``x``, so ``aiter(aiter(x))`` is the same as ``aiter(x)``. Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. From 6ee8824007e2797b0cb460cc78ee40608ab6809d Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Tue, 23 Mar 2021 15:16:12 +0000 Subject: [PATCH 37/39] Add test_aiter_idempotent(). --- Lib/test/test_asyncgen.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 6a1df7ca3989f9..99464e3d0929fd 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -415,6 +415,13 @@ async def consume(): self.loop.run_until_complete(consume()) self.assertEqual(results, [1, 2]) + def test_aiter_idempotent(self): + async def gen(): + yield 1 + applied_once = aiter(gen()) + applied_twice = aiter(applied_once) + self.assertIs(applied_once, applied_twice) + def test_anext_bad_args(self): async def gen(): yield 1 From 6f50ef8946b4b5b8f2f4aba50f3f027ec3b89b75 Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Tue, 23 Mar 2021 19:52:42 +0000 Subject: [PATCH 38/39] Remove public API added for anext. --- Include/iterobject.h | 3 --- Objects/iterobject.c | 26 ++++++++++++-------------- Python/bltinmodule.c | 1 + 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Include/iterobject.h b/Include/iterobject.h index 8251a7dba49f46..51139bf1874088 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -7,8 +7,6 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; -PyAPI_DATA(PyTypeObject) PyAsyncCallAwaitable_Type; -PyAPI_DATA(PyTypeObject) PyAnextAwaitable_Type; #define PySeqIter_Check(op) Py_IS_TYPE(op, &PySeqIter_Type) @@ -18,7 +16,6 @@ PyAPI_FUNC(PyObject *) PySeqIter_New(PyObject *); #define PyCallIter_Check(op) Py_IS_TYPE(op, &PyCallIter_Type) PyAPI_FUNC(PyObject *) PyCallIter_New(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyAnextAwaitable_New(PyObject *, PyObject *); #ifdef __cplusplus } diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 3b42edcabf91bd..06a6da5695f8a6 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -296,19 +296,6 @@ typedef struct { PyObject *default_value; } anextawaitableobject; - -PyObject * -PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) -{ - anextawaitableobject *anext = PyObject_GC_New(anextawaitableobject, &PyAnextAwaitable_Type); - Py_INCREF(awaitable); - anext->wrapped = awaitable; - Py_INCREF(default_value); - anext->default_value = default_value; - _PyObject_GC_TRACK(anext); - return (PyObject *)anext; -} - static void anextawaitable_dealloc(anextawaitableobject *obj) { @@ -339,7 +326,6 @@ anextawaitable_iternext(anextawaitableobject *obj) return NULL; } - static PyAsyncMethods anextawaitable_as_async = { PyObject_SelfIter, /* am_await */ 0, /* am_aiter */ @@ -378,3 +364,15 @@ PyTypeObject PyAnextAwaitable_Type = { (unaryfunc)anextawaitable_iternext, /* tp_iternext */ 0, /* tp_methods */ }; + +PyObject * +PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) +{ + anextawaitableobject *anext = PyObject_GC_New(anextawaitableobject, &PyAnextAwaitable_Type); + Py_INCREF(awaitable); + anext->wrapped = awaitable; + Py_INCREF(default_value); + anext->default_value = default_value; + _PyObject_GC_TRACK(anext); + return (PyObject *)anext; +} diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d55544092bbd16..c80bbaebcdf80c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1586,6 +1586,7 @@ builtin_aiter(PyObject *module, PyObject *async_iterable) return PyObject_GetAiter(async_iterable); } +PyObject *PyAnextAwaitable_New(PyObject *, PyObject *); /*[clinic input] anext as builtin_anext From ef40fb74e1a1c66927429e0bef522b45c411eff2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 23 Mar 2021 15:19:11 -0700 Subject: [PATCH 39/39] Slightly improve wording --- Doc/library/functions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 9b433c4ccfe158..4e2e58ee9dd8b0 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -92,8 +92,8 @@ are always available. They are listed here in alphabetical order. This is the async variant of the :func:`next` builtin, and behaves similarly. - Immediately, call the :meth:`~object.__anext__` method of *async_iterator*, - then return an :term:`awaitable`. Awaiting this returns the next value of the + This calls the :meth:`~object.__anext__` method of *async_iterator*, + returning an :term:`awaitable`. Awaiting this returns the next value of the iterator. If *default* is given, it is returned if the iterator is exhausted, otherwise :exc:`StopAsyncIteration` is raised.