Skip to content

[3.7] Revert "bpo-30406: Make async and await proper keywords" (GH-6143) #6145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 18, 2018
Merged

[3.7] Revert "bpo-30406: Make async and await proper keywords" (GH-6143) #6145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions Lib/lib2to3/Grammar.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef | async_funcdef)
async_funcdef: 'async' funcdef
async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: ((tfpdef ['=' test] ',')*
Expand Down Expand Up @@ -66,7 +66,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]

compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
async_stmt: 'async' (funcdef | with_stmt | for_stmt)
async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
Expand Down Expand Up @@ -105,7 +105,7 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: ['await'] atom trailer* ['**' factor]
power: [AWAIT] atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_gexp] ')' |
'[' [listmaker] ']' |
'{' [dictsetmaker] '}' |
Expand Down Expand Up @@ -142,7 +142,7 @@ argument: ( test [comp_for] |
star_expr )

comp_iter: comp_for | comp_if
comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
comp_if: 'if' old_test [comp_iter]

# As noted above, testlist_safe extends the syntax allowed in list
Expand All @@ -161,7 +161,7 @@ comp_if: 'if' old_test [comp_iter]
#
# See https://bugs.python.org/issue27494
old_comp_iter: old_comp_for | old_comp_if
old_comp_for: ['async'] 'for' exprlist 'in' testlist_safe [old_comp_iter]
old_comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [old_comp_iter]
old_comp_if: 'if' old_test [old_comp_iter]

testlist1: test (',' test)*
Expand Down
6 changes: 4 additions & 2 deletions Lib/lib2to3/pgen2/token.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@
COMMENT = 53
NL = 54
RARROW = 55
ERRORTOKEN = 56
N_TOKENS = 57
AWAIT = 56
ASYNC = 57
ERRORTOKEN = 58
N_TOKENS = 59
NT_OFFSET = 256
#--end constants--

Expand Down
76 changes: 74 additions & 2 deletions Lib/lib2to3/pgen2/tokenize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ def compat(self, token, iterable):
for tok in iterable:
toknum, tokval = tok[:2]

if toknum in (NAME, NUMBER):
if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '

if toknum == INDENT:
Expand Down Expand Up @@ -380,6 +380,12 @@ def generate_tokens(readline):
contline = None
indents = [0]

# 'stashed' and 'async_*' are used for async/await parsing
stashed = None
async_def = False
async_def_indent = 0
async_def_nl = False

while 1: # loop over lines in stream
try:
line = readline()
Expand Down Expand Up @@ -420,6 +426,10 @@ def generate_tokens(readline):
pos = pos + 1
if pos == max: break

if stashed:
yield stashed
stashed = None

if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n')
Expand All @@ -443,8 +453,18 @@ def generate_tokens(readline):
("<tokenize>", lnum, pos, line))
indents = indents[:-1]

if async_def and async_def_indent >= indents[-1]:
async_def = False
async_def_nl = False
async_def_indent = 0

yield (DEDENT, '', (lnum, pos), (lnum, pos), line)

if async_def and async_def_nl and async_def_indent >= indents[-1]:
async_def = False
async_def_nl = False
async_def_indent = 0

else: # continued statement
if not line:
raise TokenError("EOF in multi-line statement", (lnum, 0))
Expand All @@ -464,17 +484,28 @@ def generate_tokens(readline):
newline = NEWLINE
if parenlev > 0:
newline = NL
elif async_def:
async_def_nl = True
if stashed:
yield stashed
stashed = None
yield (newline, token, spos, epos, line)

elif initial == '#':
assert not token.endswith("\n")
if stashed:
yield stashed
stashed = None
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
endmatch = endprog.match(line, pos)
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
if stashed:
yield stashed
stashed = None
yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
Expand All @@ -492,22 +523,63 @@ def generate_tokens(readline):
contline = line
break
else: # ordinary string
if stashed:
yield stashed
stashed = None
yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
yield (NAME, token, spos, epos, line)
if token in ('async', 'await'):
if async_def:
yield (ASYNC if token == 'async' else AWAIT,
token, spos, epos, line)
continue

tok = (NAME, token, spos, epos, line)
if token == 'async' and not stashed:
stashed = tok
continue

if token == 'def':
if (stashed
and stashed[0] == NAME
and stashed[1] == 'async'):

async_def = True
async_def_indent = indents[-1]

yield (ASYNC, stashed[1],
stashed[2], stashed[3],
stashed[4])
stashed = None

if stashed:
yield stashed
stashed = None

yield tok
elif initial == '\\': # continued stmt
# This yield is new; needed for better idempotency:
if stashed:
yield stashed
stashed = None
yield (NL, token, spos, (lnum, pos), line)
continued = 1
else:
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
if stashed:
yield stashed
stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1

if stashed:
yield stashed
stashed = None

for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
Expand Down
22 changes: 11 additions & 11 deletions Lib/lib2to3/tests/test_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,34 +181,34 @@ def foo(): pass
async def foo(): await x
""")

self.validate("await x")
self.validate("""def foo():
await x""")
self.invalid_syntax("await x")
self.invalid_syntax("""def foo():
await x""")

self.validate("""def foo():
self.invalid_syntax("""def foo():
def foo(): pass
async def foo(): pass
await x
""")

def test_async_var(self):
self.invalid_syntax("""async = 1""")
self.invalid_syntax("""await = 1""")
self.invalid_syntax("""def async(): pass""")
self.validate("""async = 1""")
self.validate("""await = 1""")
self.validate("""def async(): pass""")

def test_async_with(self):
self.validate("""async def foo():
async for a in b: pass""")

self.validate("""def foo():
async for a in b: pass""")
self.invalid_syntax("""def foo():
async for a in b: pass""")

def test_async_for(self):
self.validate("""async def foo():
async with a: pass""")

self.validate("""def foo():
async with a: pass""")
self.invalid_syntax("""def foo():
async with a: pass""")


class TestRaiseChanges(GrammarTest):
Expand Down