Skip to content
Permalink
Browse files

bpo-35766: Merge typed_ast back into CPython (GH-11645)

  • Loading branch information...
gvanrossum authored and ambv committed Jan 31, 2019
1 parent d97daeb commit dcfcd146f8e6fc5c2fc16a4c192a0c5f5ca8c53c
@@ -126,16 +126,33 @@ The abstract grammar is currently defined as follows:
Apart from the node classes, the :mod:`ast` module defines these utility functions
and classes for traversing abstract syntax trees:

.. function:: parse(source, filename='<unknown>', mode='exec')
.. function:: parse(source, filename='<unknown>', mode='exec', *, type_comments=False)

Parse the source into an AST node. Equivalent to ``compile(source,
filename, mode, ast.PyCF_ONLY_AST)``.

If ``type_comments=True`` is given, the parser is modified to check
and return type comments as specified by :pep:`484` and :pep:`526`.
This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the
flags passed to :func:`compile()`. This will report syntax errors
for misplaced type comments. Without this flag, type comments will
be ignored, and the ``type_comment`` field on selected AST nodes
will always be ``None``. In addition, the locations of ``# type:
ignore`` comments will be returned as the ``type_ignores``
attribute of :class:`Module` (otherwise it is always an empty list).

In addition, if ``mode`` is ``'func_type'``, the input syntax is
modified to correspond to :pep:`484` "signature type comments",
e.g. ``(str, int) -> List[str]``.

.. warning::
It is possible to crash the Python interpreter with a
sufficiently large/complex string due to stack depth limitations
in Python's AST compiler.

.. versionchanged:: 3.8
Added ``type_comments=True`` and ``mode='func_type'``.


.. function:: literal_eval(node_or_string)

Some generated files are not rendered by default. Learn more.

@@ -69,6 +69,13 @@ the :mod:`tokenize` module.
always be an ``ENCODING`` token.


.. data:: TYPE_COMMENT

Token value indicating that a type comment was recognized. Such
tokens are only produced when :func:`ast.parse()` is invoked with
``type_comments=True``.


.. versionchanged:: 3.5
Added :data:`AWAIT` and :data:`ASYNC` tokens.

@@ -78,3 +85,6 @@ the :mod:`tokenize` module.
.. versionchanged:: 3.7
Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are
now tokenized as :data:`NAME` tokens.

.. versionchanged:: 3.8
Added :data:`TYPE_COMMENT`.
@@ -7,7 +7,9 @@
# single_input is a single interactive statement;
# file_input is a module or sequence of commands read from an input file;
# eval_input is the input for the eval() functions.
# func_type_input is a PEP 484 Python 2 function type comment
# NB: compound_stmt in single_input is followed by extra NEWLINE!
# NB: due to the way TYPE_COMMENT is tokenized it will always be followed by a NEWLINE
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
file_input: (NEWLINE | stmt)* ENDMARKER
eval_input: testlist NEWLINE* ENDMARKER
@@ -17,14 +19,14 @@ decorators: decorator+
decorated: decorators (classdef | funcdef | async_funcdef)

async_funcdef: 'async' funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] func_body_suite

parameters: '(' [typedargslist] ')'
typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
'*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
| '**' tfpdef [',']]]
| '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
| '**' tfpdef [','])
typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [
'*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
| '**' tfpdef [','] [TYPE_COMMENT]]])
| '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
| '**' tfpdef [','] [TYPE_COMMENT])
tfpdef: NAME [':' test]
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
'*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
@@ -39,7 +41,7 @@ simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
('=' (yield_expr|testlist_star_expr))*)
[('=' (yield_expr|testlist_star_expr))+ [TYPE_COMMENT]] )
annassign: ':' test ['=' (yield_expr|testlist)]
testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
@@ -71,13 +73,13 @@ compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef
async_stmt: 'async' (funcdef | with_stmt | for_stmt)
if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite]
try_stmt: ('try' ':' suite
((except_clause ':' suite)+
['else' ':' suite]
['finally' ':' suite] |
'finally' ':' suite))
with_stmt: 'with' with_item (',' with_item)* ':' suite
with_stmt: 'with' with_item (',' with_item)* ':' [TYPE_COMMENT] suite
with_item: test ['as' expr]
# NB compile.c makes sure that the default except clause is last
except_clause: 'except' [test ['as' NAME]]
@@ -150,3 +152,14 @@ encoding_decl: NAME

yield_expr: 'yield' [yield_arg]
yield_arg: 'from' test | testlist_star_expr

# the TYPE_COMMENT in suites is only parsed for funcdefs,
# but can't go elsewhere due to ambiguity
func_body_suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT

func_type_input: func_type NEWLINE* ENDMARKER
func_type: '(' [typelist] ')' '->' test
# typelist is a modified typedargslist (see above)
typelist: (test (',' test)* [','
['*' [test] (',' test)* [',' '**' test] | '**' test]]
| '*' [test] (',' test)* [',' '**' test] | '**' test)
@@ -55,6 +55,8 @@ ELLIPSIS '...'
COLONEQUAL ':='

OP
TYPE_IGNORE
TYPE_COMMENT
ERRORTOKEN

# These aren't used by the C tokenizer but are needed for tokenize.py

Some generated files are not rendered by default. Learn more.

@@ -22,6 +22,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
#define PyCF_DONT_IMPLY_DEDENT 0x0200
#define PyCF_ONLY_AST 0x0400
#define PyCF_IGNORE_COOKIE 0x0800
#define PyCF_TYPE_COMMENTS 0x1000

#ifndef Py_LIMITED_API
typedef struct {
@@ -85,10 +86,10 @@ PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize);

#endif /* !Py_LIMITED_API */

/* These definitions must match corresponding definitions in graminit.h.
There's code in compile.c that checks that they are the same. */
/* These definitions must match corresponding definitions in graminit.h. */
#define Py_single_input 256
#define Py_file_input 257
#define Py_eval_input 258
#define Py_func_type_input 345

#endif /* !Py_COMPILE_H */

Some generated files are not rendered by default. Learn more.

0 comments on commit dcfcd14

Please sign in to comment.
You can’t perform that action at this time.