Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bpo-34616: add flags to allow top-level-await #13148

Merged
merged 19 commits into from May 21, 2019
Merged
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -257,6 +257,12 @@ are always available. They are listed here in alphabetical order.
can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on
the :class:`~__future__._Feature` instance in the :mod:`__future__` module.

The optional argument *flags* also controls whether the compiled source is
allowed to contain top-level ``await``, ``async for`` and ``async with``.
When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code
object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively
executed via ``await eval(code_object)``.

The argument *optimize* specifies the optimization level of the compiler; the
default value of ``-1`` selects the optimization level of the interpreter as
given by :option:`-O` options. Explicit levels are ``0`` (no optimization;
@@ -290,6 +296,10 @@ are always available. They are listed here in alphabetical order.
Previously, :exc:`TypeError` was raised when null bytes were encountered
in *source*.

.. versionadded:: 3.8
``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
support for top-level ``await``, ``async for``, and ``async with``.


.. class:: complex([real[, imag]])

@@ -23,6 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
#define PyCF_ONLY_AST 0x0400
#define PyCF_IGNORE_COOKIE 0x0800
#define PyCF_TYPE_COMMENTS 0x1000
#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000

#ifndef Py_LIMITED_API
typedef struct {
@@ -1,6 +1,7 @@
# Python test set -- built-in functions

import ast
import asyncio
import builtins
import collections
import decimal
@@ -18,9 +19,14 @@
import unittest
import warnings
from contextlib import ExitStack
from inspect import CO_COROUTINE
from itertools import product
from textwrap import dedent
from types import AsyncGeneratorType, FunctionType
from operator import neg
from test.support import (
EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink)
EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink,
maybe_get_event_loop_policy)
from test.support.script_helper import assert_python_ok
from unittest.mock import MagicMock, patch
try:
@@ -358,6 +364,71 @@ def f(): """doc"""
rv = ns['f']()
self.assertEqual(rv, tuple(expected))

def test_compile_top_level_await(self):
"""Test whether code some top level await can be compiled.
Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set,
and make sure the generated code object has the CO_COROUTINE flag set in
order to execute it with `await eval(.....)` instead of exec, or via a
FunctionType.
"""

# helper function just to check we can run top=level async-for
async def arange(n):
for i in range(n):
yield i

modes = ('single', 'exec')
code_samples = ['''a = await asyncio.sleep(0, result=1)''',
'''async for i in arange(1):
a = 1''',
'''async with asyncio.Lock() as l:
a = 1''']
policy = maybe_get_event_loop_policy()
try:
for mode, code_sample in product(modes,code_samples):
source = dedent(code_sample)
with self.assertRaises(SyntaxError, msg=f"{source=} {mode=}"):
compile(source, '?' , mode)

co = compile(source,
'?',
mode,
flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)

self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
msg=f"{source=} {mode=}")


# test we can create and advance a function type
globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
async_f = FunctionType(co, globals_)
asyncio.run(async_f())
self.assertEqual(globals_['a'], 1)

# test we can await-eval,
globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange}
asyncio.run(eval(co, globals_))
self.assertEqual(globals_['a'], 1)
finally:
asyncio.set_event_loop_policy(policy)

def test_compile_async_generator(self):
"""
With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to
make sure AsyncGenerators are still properly not marked with CO_COROUTINE
"""
code = dedent("""async def ticker():
for i in range(10):
yield i
await asyncio.sleep(0)""")

co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
glob = {}
exec(co, glob)
self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)


def test_delattr(self):
sys.spam = 1
delattr(sys, 'spam')
@@ -0,0 +1 @@
The ``compile()`` builtin functions now support the ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` flag, which allow to compile sources that contains top-level ``await``, ``async with`` or ``async for``. This is useful to evaluate async-code from with an already async functions; for example in a custom REPL.
@@ -1000,6 +1000,8 @@ def visitModule(self, mod):
self.emit("if (!m) return NULL;", 1)
self.emit("d = PyModule_GetDict(m);", 1)
self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1)
self.emit("return NULL;", 2)
self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
self.emit("return NULL;", 2)
self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1)

Some generated files are not rendered by default. Learn more.

@@ -2609,7 +2609,9 @@ static int
compiler_async_for(struct compiler *c, stmt_ty s)
{
basicblock *start, *except, *end;
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
c->u->u_ste->ste_coroutine = 1;
} else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
return compiler_error(c, "'async for' outside async function");
}

@@ -4564,7 +4566,9 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);

assert(s->kind == AsyncWith_kind);
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
c->u->u_ste->ste_coroutine = 1;

This comment has been minimized.

Copy link
@1st1

1st1 May 21, 2019

Member

Have you checked if this c->u->u_ste->ste_coroutine = 1; line is needed? Does it work without it?

This comment has been minimized.

Copy link
@Carreau

Carreau May 21, 2019

Author Contributor

yes it is necessary, otherwise:

>>> async with acm() as delta:
>>>     count += delta

Traceback (most recent call last):
  File "minirepl.py", line 97, in f
    co = compile(inp, '', 'single', flags=PyCF_ALLOW_TOP_LEVEL_AWAIT)
  File "<string>", line 2
SyntaxError: 'async with' outside async function

This comment has been minimized.

Copy link
@1st1

1st1 May 21, 2019

Member

OK, cool, let's keep it then :)

This comment has been minimized.

Copy link
@Carreau

Carreau May 21, 2019

Author Contributor

And there is be a test in Lib/test/test_builtin.py for that in case you ask.

} else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){
return compiler_error(c, "'async with' outside async function");
}

@@ -4773,12 +4777,16 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
ADDOP(c, YIELD_FROM);
break;
case Await_kind:
if (c->u->u_ste->ste_type != FunctionBlock)
return compiler_error(c, "'await' outside function");
if (!(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){
if (c->u->u_ste->ste_type != FunctionBlock){
return compiler_error(c, "'await' outside function");
}

if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
return compiler_error(c, "'await' outside async function");
if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){
return compiler_error(c, "'await' outside async function");
}
}

VISIT(c, expr, e->v.Await.value);
ADDOP(c, GET_AWAITABLE);
@@ -5712,6 +5720,12 @@ compute_code_flags(struct compiler *c)
/* (Only) inherit compilerflags in PyCF_MASK */
flags |= (c->c_flags->cf_flags & PyCF_MASK);

if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) &&
ste->ste_coroutine &&
!ste->ste_generator) {
flags |= CO_COROUTINE;
This conversation was marked as resolved by Carreau

This comment has been minimized.

Copy link
@Carreau

Carreau May 7, 2019

Author Contributor

This thing has some weird effect and turn async_generators into coroutine; I'm going to change that to !ste->ste_generator

This comment has been minimized.

Copy link
@1st1

1st1 May 21, 2019

Member

Please fix to fit 79 characters

This comment has been minimized.

Copy link
@Carreau

Carreau May 21, 2019

Author Contributor

Done.

}

return flags;
}

ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.