Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bpo-43224: Implement PEP 646 grammar changes #31018

Merged
merged 17 commits into from Mar 26, 2022
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 8 additions & 1 deletion Grammar/python.gram
Expand Up @@ -308,6 +308,8 @@ slash_with_default[SlashWithDefault*]:
star_etc[StarEtc*]:
| '*' a=param_no_default b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' a=param_no_default_star_annotation b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' ',' b=param_maybe_default+ c=[kwds] {
_PyPegen_star_etc(p, NULL, b, c) }
| a=kwds { _PyPegen_star_etc(p, NULL, NULL, a) }
Expand All @@ -331,14 +333,19 @@ kwds[arg_ty]: '**' a=param_no_default { a }
param_no_default[arg_ty]:
| a=param ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_no_default_star_annotation[arg_ty]:
pablogsal marked this conversation as resolved.
Show resolved Hide resolved
| a=param_star_annotation ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param_star_annotation tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_with_default[NameDefaultPair*]:
| a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param_maybe_default[NameDefaultPair*]:
| a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
annotation[expr_ty]: ':' a=expression { a }
star_annotation[expr_ty]: ':' a=star_expression { a }
default[expr_ty]: '=' a=expression { a }

# If statement
Expand Down Expand Up @@ -780,7 +787,7 @@ primary[expr_ty]:

slices[expr_ty]:
| a=slice !',' { a }
| a[asdl_expr_seq*]=','.slice+ [','] { _PyAST_Tuple(a, Load, EXTRA) }
| a[asdl_expr_seq*]=','.(slice | starred_expression)+ [','] { _PyAST_Tuple(a, Load, EXTRA) }
mrahtz marked this conversation as resolved.
Show resolved Hide resolved

slice[expr_ty]:
| a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _PyAST_Slice(a, b, c, EXTRA) }
Expand Down
25 changes: 24 additions & 1 deletion Lib/test/test_ast.py
Expand Up @@ -13,7 +13,7 @@
from test import support

def to_tuple(t):
if t is None or isinstance(t, (str, int, complex)):
if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis:
return t
elif isinstance(t, list):
return [to_tuple(e) for e in t]
Expand Down Expand Up @@ -46,10 +46,20 @@ def to_tuple(t):
"def f(a=0): pass",
# FunctionDef with varargs
"def f(*args): pass",
# FunctionDef with varargs as TypeVarTuple
"def f(*args: *Ts): pass",
# FunctionDef with varargs as unpacked Tuple
"def f(*args: *tuple[int, ...]): pass",
# FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple
"def f(*args: *tuple[int, *Ts]): pass",
# FunctionDef with kwargs
"def f(**kwargs): pass",
# FunctionDef with all kind of args and docstring
"def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'",
# FunctionDef with type annotation on return involving unpacking
"def f() -> tuple[*Ts]: pass",
"def f() -> tuple[int, *Ts]: pass",
"def f() -> tuple[int, *tuple[int, ...]]: pass",
# ClassDef
"class C:pass",
# ClassDef with docstring
Expand All @@ -65,6 +75,10 @@ def to_tuple(t):
"a,b = c",
"(a,b) = c",
"[a,b] = c",
# AnnAssign with unpacked types
"x: tuple[*Ts]",
"x: tuple[int, *Ts]",
"x: tuple[int, *tuple[str, ...]]",
# AugAssign
"v += 1",
# For
Expand Down Expand Up @@ -2316,8 +2330,14 @@ def main():
('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [])], []),
Expand All @@ -2327,6 +2347,9 @@ def main():
('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []),
('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []),
('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []),
Expand Down
286 changes: 286 additions & 0 deletions Lib/test/test_pep646_syntax.py
@@ -0,0 +1,286 @@
doctests = """

Setup

>>> class AClass:
... def __init__(self):
... self._setitem_name = None
... self._setitem_val = None
... self._delitem_name = None
... def __setitem__(self, name, val):
... self._delitem_name = None
... self._setitem_name = name
... self._setitem_val = val
... def __repr__(self):
... if self._setitem_name is not None:
... return f"A[{self._setitem_name}]={self._setitem_val}"
... elif self._delitem_name is not None:
... return f"delA[{self._delitem_name}]"
... def __getitem__(self, name):
... return ParameterisedA(name)
... def __delitem__(self, name):
... self._setitem_name = None
... self._delitem_name = name
...
>>> class ParameterisedA:
... def __init__(self, name):
... self._name = name
... def __repr__(self):
... return f"A[{self._name}]"
... def __iter__(self):
... for p in self._name:
... yield p
>>> class B:
... def __iter__(self):
... yield StarredB()
... def __repr__(self):
... return "B"
>>> class StarredB:
... def __repr__(self):
... return "StarredB"
>>> A = AClass()
>>> b = B()

Slices that are supposed to work, starring our custom B class

>>> A[*b]
A[(StarredB,)]
>>> A[*b] = 1; A
A[(StarredB,)]=1
>>> del A[*b]; A
delA[(StarredB,)]

>>> A[*b, *b]
A[(StarredB, StarredB)]
>>> A[*b, *b] = 1; A
A[(StarredB, StarredB)]=1
>>> del A[*b, *b]; A
delA[(StarredB, StarredB)]

>>> A[b, *b]
A[(B, StarredB)]
>>> A[b, *b] = 1; A
A[(B, StarredB)]=1
>>> del A[b, *b]; A
delA[(B, StarredB)]

>>> A[*b, b]
A[(StarredB, B)]
>>> A[*b, b] = 1; A
A[(StarredB, B)]=1
>>> del A[*b, b]; A
delA[(StarredB, B)]

>>> A[b, b, *b]
A[(B, B, StarredB)]
>>> A[b, b, *b] = 1; A
A[(B, B, StarredB)]=1
>>> del A[b, b, *b]; A
delA[(B, B, StarredB)]

>>> A[*b, b, b]
A[(StarredB, B, B)]
>>> A[*b, b, b] = 1; A
A[(StarredB, B, B)]=1
>>> del A[*b, b, b]; A
delA[(StarredB, B, B)]

>>> A[b, *b, b]
A[(B, StarredB, B)]
>>> A[b, *b, b] = 1; A
A[(B, StarredB, B)]=1
>>> del A[b, *b, b]; A
delA[(B, StarredB, B)]

>>> A[b, b, *b, b]
A[(B, B, StarredB, B)]
>>> A[b, b, *b, b] = 1; A
A[(B, B, StarredB, B)]=1
>>> del A[b, b, *b, b]; A
delA[(B, B, StarredB, B)]

>>> A[b, *b, b, b]
A[(B, StarredB, B, B)]
>>> A[b, *b, b, b] = 1; A
A[(B, StarredB, B, B)]=1
>>> del A[b, *b, b, b]; A
delA[(B, StarredB, B, B)]

>>> A[A[b, *b, b]]
A[A[(B, StarredB, B)]]
>>> A[A[b, *b, b]] = 1; A
A[A[(B, StarredB, B)]]=1
>>> del A[A[b, *b, b]]; A
delA[A[(B, StarredB, B)]]

>>> A[*A[b, *b, b]]
A[(B, StarredB, B)]
>>> A[*A[b, *b, b]] = 1; A
A[(B, StarredB, B)]=1
>>> del A[*A[b, *b, b]]; A
delA[(B, StarredB, B)]

>>> A[b, ...]
A[(B, Ellipsis)]
>>> A[b, ...] = 1; A
A[(B, Ellipsis)]=1
>>> del A[b, ...]; A
delA[(B, Ellipsis)]

>>> A[*A[b, ...]]
A[(B, Ellipsis)]
>>> A[*A[b, ...]] = 1; A
A[(B, Ellipsis)]=1
>>> del A[*A[b, ...]]; A
delA[(B, Ellipsis)]

Slices that are supposed to work, starring a list

>>> l = [1, 2, 3]

>>> A[*l]
A[(1, 2, 3)]
>>> A[*l] = 1; A
A[(1, 2, 3)]=1
>>> del A[*l]; A
delA[(1, 2, 3)]

>>> A[*l, 4]
A[(1, 2, 3, 4)]
>>> A[*l, 4] = 1; A
A[(1, 2, 3, 4)]=1
>>> del A[*l, 4]; A
delA[(1, 2, 3, 4)]

>>> A[0, *l]
A[(0, 1, 2, 3)]
>>> A[0, *l] = 1; A
A[(0, 1, 2, 3)]=1
>>> del A[0, *l]; A
delA[(0, 1, 2, 3)]

>>> A[1:2, *l]
A[(slice(1, 2, None), 1, 2, 3)]
>>> A[1:2, *l] = 1; A
A[(slice(1, 2, None), 1, 2, 3)]=1
>>> del A[1:2, *l]; A
delA[(slice(1, 2, None), 1, 2, 3)]

>>> repr(A[1:2, *l]) == repr(A[1:2, 1, 2, 3])
True

Slices that are supposed to work, starring a tuple

>>> t = (1, 2, 3)

>>> A[*t]
A[(1, 2, 3)]
>>> A[*t] = 1; A
A[(1, 2, 3)]=1
>>> del A[*t]; A
delA[(1, 2, 3)]

>>> A[*t, 4]
A[(1, 2, 3, 4)]
>>> A[*t, 4] = 1; A
A[(1, 2, 3, 4)]=1
>>> del A[*t, 4]; A
delA[(1, 2, 3, 4)]

>>> A[0, *t]
A[(0, 1, 2, 3)]
>>> A[0, *t] = 1; A
A[(0, 1, 2, 3)]=1
>>> del A[0, *t]; A
delA[(0, 1, 2, 3)]

>>> A[1:2, *t]
A[(slice(1, 2, None), 1, 2, 3)]
>>> A[1:2, *t] = 1; A
A[(slice(1, 2, None), 1, 2, 3)]=1
>>> del A[1:2, *t]; A
delA[(slice(1, 2, None), 1, 2, 3)]

>>> repr(A[1:2, *t]) == repr(A[1:2, 1, 2, 3])
True

Starring an expression (rather than a name) in a slice

>>> def returns_list():
... return [1, 2, 3]

>>> A[returns_list()]
A[[1, 2, 3]]
>>> A[returns_list()] = 1; A
A[[1, 2, 3]]=1
>>> del A[returns_list()]; A
delA[[1, 2, 3]]

>>> A[returns_list(), 4]
A[([1, 2, 3], 4)]
>>> A[returns_list(), 4] = 1; A
A[([1, 2, 3], 4)]=1
>>> del A[returns_list(), 4]; A
delA[([1, 2, 3], 4)]

>>> A[*returns_list()]
A[(1, 2, 3)]
>>> A[*returns_list()] = 1; A
A[(1, 2, 3)]=1
>>> del A[*returns_list()]; A
delA[(1, 2, 3)]

>>> A[*returns_list(), 4]
A[(1, 2, 3, 4)]
>>> A[*returns_list(), 4] = 1; A
A[(1, 2, 3, 4)]=1
>>> del A[*returns_list(), 4]; A
delA[(1, 2, 3, 4)]

>>> A[0, *returns_list()]
A[(0, 1, 2, 3)]
>>> A[0, *returns_list()] = 1; A
A[(0, 1, 2, 3)]=1
>>> del A[0, *returns_list()]; A
delA[(0, 1, 2, 3)]

>>> A[*returns_list(), *returns_list()]
A[(1, 2, 3, 1, 2, 3)]
>>> A[*returns_list(), *returns_list()] = 1; A
A[(1, 2, 3, 1, 2, 3)]=1
>>> del A[*returns_list(), *returns_list()]; A
delA[(1, 2, 3, 1, 2, 3)]

*args annotated as starred expression

>>> def f1(*args: *b): pass
>>> f1.__annotations__
{'args': StarredB}

>>> def f2(*args: *b, arg1): pass
>>> f2.__annotations__
{'args': StarredB}

>>> def f3(*args: *b, arg1: int): pass
>>> f3.__annotations__
{'args': StarredB, 'arg1': <class 'int'>}

>>> def f4(*args: *b, arg1: int = 2): pass
>>> f4.__annotations__
{'args': StarredB, 'arg1': <class 'int'>}

>>> def f5(*args: *b = (1,)): pass
Traceback (most recent call last):
...
SyntaxError: invalid syntax
"""

__test__ = {'doctests' : doctests}

def test_main(verbose=False):
from test import support
from test import test_pep646_syntax
support.run_doctest(test_pep646_syntax, verbose)

if __name__ == "__main__":
test_main(verbose=True)