Transform annotations to use python 3.10 style (#781)

* Fix #780

README.rst

@@ -91,6 +91,8 @@ This is the last major release to support Python 3.7.
 * Make sure the line number coming from ast analysis has precedence over the line of a ``ivar`` field.
 * Ensure that all docutils generated css classes have the ``rst-`` prefix, the base theme have been updated accordingly.
 * Fix compatibility issue with docutils 0.21.x
+* Transform annotations to use python 3.10 style: ``typing.Union[x ,y]`` -> ``x | y``; ``typing.Optional[x]`` -> ``x | None``; ``typing.List[x]`` -> ``list[x]``.
+* Do not output useless parenthesis when colourizing subscripts. 
 
 pydoctor 23.9.1
 ^^^^^^^^^^^^^^^

pydoctor/astbuilder.py

@@ -16,7 +16,7 @@
 from pydoctor import epydoc2stan, model, node2stan, extensions, linker
 from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval
 from pydoctor.astutils import (is_none_literal, is_typing_annotation, is_using_annotations, is_using_typing_final, node2dottedname, node2fullname, 
-                               is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents,
+                               is__name__equals__main__, unstring_annotation, upgrade_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents,
                                get_docstring_node, unparse, NodeVisitor, Parentage, Str)
 
 
@@ -128,9 +128,9 @@ def visit_Assign(self, node: Union[ast.Assign, ast.AnnAssign]) -> None:
                 if self._isTypeAlias(attr) is True:
                     attr.kind = model.DocumentableKind.TYPE_ALIAS
                     # unstring type aliases
-                    attr.value = unstring_annotation(
+                    attr.value = upgrade_annotation(unstring_annotation(
                         # this cast() is safe because _isTypeAlias() return True only if value is not None
-                        cast(ast.expr, attr.value), attr, section='type alias')
+                        cast(ast.expr, attr.value), attr, section='type alias'), attr, section='type alias')
                 elif self._isTypeVariable(attr) is True:
                     # TODO: unstring bound argument of type variables
                     attr.kind = model.DocumentableKind.TYPE_VARIABLE
@@ -752,7 +752,8 @@ def visit_Assign(self, node: ast.Assign) -> None:
         if type_comment is None:
             annotation = None
         else:
-            annotation = unstring_annotation(ast.Constant(type_comment, lineno=lineno), self.builder.current)
+            annotation = upgrade_annotation(unstring_annotation(
+                ast.Constant(type_comment, lineno=lineno), self.builder.current), self.builder.current)
 
         for target in node.targets:
             if isinstance(target, ast.Tuple):
@@ -764,7 +765,8 @@ def visit_Assign(self, node: ast.Assign) -> None:
                 self._handleAssignment(target, annotation, expr, lineno)
 
     def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
-        annotation = unstring_annotation(node.annotation, self.builder.current)
+        annotation = upgrade_annotation(unstring_annotation(
+            node.annotation, self.builder.current), self.builder.current)
         self._handleAssignment(node.target, annotation, node.value, node.lineno)
 
     def visit_AugAssign(self, node:ast.AugAssign) -> None:
@@ -968,7 +970,7 @@ def _handlePropertyDef(self,
             attr.parsed_docstring = pdoc
 
         if node.returns is not None:
-            attr.annotation = unstring_annotation(node.returns, attr)
+            attr.annotation = upgrade_annotation(unstring_annotation(node.returns, attr), attr)
         attr.decorators = node.decorator_list
 
         return attr
@@ -1009,7 +1011,8 @@ def _get_all_ast_annotations() -> Iterator[Tuple[str, Optional[ast.expr]]]:
             # Include parameter names even if they're not annotated, so that
             # we can use the key set to know which parameters exist and warn
             # when non-existing parameters are documented.
-            name: None if value is None else unstring_annotation(value, self.builder.current)
+            name: None if value is None else upgrade_annotation(unstring_annotation(
+                value, self.builder.current), self.builder.current)
             for name, value in _get_all_ast_annotations()
             }
 

pydoctor/astutils.py

@@ -282,6 +282,83 @@ def visit_Constant(self, node: ast.Constant) -> ast.expr:
         def visit_Str(self, node: ast.Str) -> ast.expr:
             return ast.copy_location(self._parse_string(node.s), node)
 
+def upgrade_annotation(node: ast.expr, ctx: model.Documentable, section:str='annotation') -> ast.expr:
+    """
+    Transform the annotation to use python 3.10+ syntax. 
+    """
+    return _UpgradeDeprecatedAnnotations(ctx).visit(node)
+
+class _UpgradeDeprecatedAnnotations(ast.NodeTransformer):
+    if TYPE_CHECKING:
+        def visit(self, node:ast.AST) -> ast.expr:...
+
+    def __init__(self, ctx: model.Documentable) -> None:
+        def _node2fullname(node:ast.expr) -> str | None:
+            return node2fullname(node, ctx)
+        self.node2fullname = _node2fullname
+
+    def _union_args_to_bitor(self, args: list[ast.expr], ctxnode:ast.AST) -> ast.BinOp:
+        assert len(args) > 1
+        *others, right = args
+        if len(others) == 1:
+            rnode = ast.BinOp(left=others[0], right=right, op=ast.BitOr())
+        else:
+            rnode = ast.BinOp(left=self._union_args_to_bitor(others, ctxnode), right=right, op=ast.BitOr())
+
+        return ast.fix_missing_locations(ast.copy_location(rnode, ctxnode))
+
+    def visit_Name(self, node: ast.Name | ast.Attribute) -> Any:
+        fullName = self.node2fullname(node)
+        if fullName in DEPRECATED_TYPING_ALIAS_BUILTINS:
+            return ast.Name(id=DEPRECATED_TYPING_ALIAS_BUILTINS[fullName], ctx=ast.Load())
+        # TODO: Support all deprecated aliases including the ones in the collections.abc module.
+        # In order to support that we need to generate the parsed docstring directly and include 
+        # custom refmap or transform the ast such that missing imports are added.
+        return node
+
+    visit_Attribute = visit_Name
+
+    def visit_Subscript(self, node: ast.Subscript) -> ast.expr:
+        node.value = self.visit(node.value)
+        node.slice = self.visit(node.slice)
+        fullName = self.node2fullname(node.value)
+
+        if fullName == 'typing.Union':
+            # typing.Union can be used with a single type or a 
+            # tuple of types, includea single element tuple, which is the same
+            # as the directly using the type: Union[x] == Union[(x,)] == x
+            slice_ = node.slice
+            if sys.version_info <= (3,9) and isinstance(slice_, ast.Index): # Compat
+                slice_ = slice_.value
+            if isinstance(slice_, ast.Tuple):
+                args = slice_.elts
+                if len(args) > 1:
+                    return self._union_args_to_bitor(args, node)
+                elif len(args) == 1:
+                    return args[0]
+            elif isinstance(slice_, (ast.Attribute, ast.Name, ast.Subscript, ast.BinOp)):
+                return slice_
+
+        elif fullName == 'typing.Optional':
+            # typing.Optional requires a single type, so we don't process when slice is a tuple.
+            slice_ = node.slice
+            if sys.version_info <= (3,9) and isinstance(slice_, ast.Index): # Compat
+                slice_ = slice_.value
+            if isinstance(slice_, (ast.Attribute, ast.Name, ast.Subscript, ast.BinOp)):
+                return self._union_args_to_bitor([slice_, ast.Constant(value=None)], node)
+
+        return node
+
+DEPRECATED_TYPING_ALIAS_BUILTINS = {
+        "typing.Text": 'str',
+        "typing.Dict": 'dict',
+        "typing.Tuple": 'tuple',
+        "typing.Type": 'type',
+        "typing.List": 'list',
+        "typing.Set": 'set',
+        "typing.FrozenSet": 'frozenset',
+}
+
 TYPING_ALIAS = (
         "typing.Hashable",
         "typing.Awaitable",
@@ -302,31 +379,26 @@ def visit_Str(self, node: ast.Str) -> ast.expr:
         "typing.Sequence",
         "typing.MutableSequence",
         "typing.ByteString",
-        "typing.Tuple",
-        "typing.List",
         "typing.Deque",
-        "typing.Set",
-        "typing.FrozenSet",
         "typing.MappingView",
         "typing.KeysView",
         "typing.ItemsView",
         "typing.ValuesView",
         "typing.ContextManager",
         "typing.AsyncContextManager",
-        "typing.Dict",
         "typing.DefaultDict",
         "typing.OrderedDict",
         "typing.Counter",
         "typing.ChainMap",
         "typing.Generator",
         "typing.AsyncGenerator",
-        "typing.Type",
         "typing.Pattern",
         "typing.Match",
         # Special forms
         "typing.Union",
         "typing.Literal",
         "typing.Optional",
+        *DEPRECATED_TYPING_ALIAS_BUILTINS, 
     )
 
 SUBSCRIPTABLE_CLASSES_PEP585 = (
@@ -336,6 +408,12 @@ def visit_Str(self, node: ast.Str) -> ast.expr:
         "set",
         "frozenset",
         "type",
+        "builtins.tuple",
+        "builtins.list",
+        "builtins.dict",
+        "builtins.set",
+        "builtins.frozenset",
+        "builtins.type",
         "collections.deque",
         "collections.defaultdict",
         "collections.OrderedDict",
@@ -645,18 +723,22 @@ class op_util:
     AST nodes to symbols and precedences.
     """
     @classmethod
-    def get_op_symbol(cls, obj:ast.operator|ast.boolop|ast.cmpop|ast.unaryop, 
+    def get_op_symbol(cls, obj:ast.operator|ast.boolop|ast.cmpop|ast.unaryop,
                       fmt:str='%s', 
                       symbol_data:dict[type[ast.AST]|None, str]=_symbol_data, 
                       type:Callable[[object], type[Any]]=type) -> str:
         """Given an AST node object, returns a string containing the symbol.
         """
         return fmt % symbol_data[type(obj)]
     @classmethod
-    def get_op_precedence(cls, obj:ast.operator|ast.boolop|ast.cmpop|ast.unaryop, 
+    def get_op_precedence(cls, obj:ast.AST, 
                           precedence_data:dict[type[ast.AST]|None, int]=_precedence_data, 
                           type:Callable[[object], type[Any]]=type) -> int:
         """Given an AST node object, returns the precedence.
+
+        @raises KeyError: If the node is not explicitely supported by this function. 
+            This is a very legacy piece of code, all calls to L{get_op_precedence} should be
+            guarded in a C{try:... except KeyError:...} statement.
         """
         return precedence_data[type(obj)]
 

pydoctor/epydoc/markup/_pyval_repr.py

@@ -130,7 +130,7 @@ class _OperatorDelimiter:
     """
 
     def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState, 
-                 node: Union[ast.UnaryOp, ast.BinOp, ast.BoolOp],) -> None:
+                 node: ast.expr,) -> None:
 
         self.discard = True
         """No parenthesis by default."""
@@ -148,17 +148,21 @@ def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState,
 
         # avoid needless parenthesis, since we now collect parents for every nodes 
         if isinstance(parent_node, (ast.expr, ast.keyword, ast.comprehension)):
-            precedence = op_util.get_op_precedence(node.op)
-            if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)):
-                parent_precedence = op_util.get_op_precedence(parent_node.op)
-                if isinstance(parent_node.op, ast.Pow) or isinstance(parent_node, ast.BoolOp):
-                    parent_precedence+=1
-            else:
-                parent_precedence = colorizer.explicit_precedence.get(
-                    node, op_util.Precedence.highest)
-
-            if precedence < parent_precedence:
+            try:
+                precedence = op_util.get_op_precedence(getattr(node, 'op', node))
+            except KeyError:
                 self.discard = False
+            else:
+                try:
+                    parent_precedence = op_util.get_op_precedence(getattr(parent_node, 'op', parent_node))
+                    if isinstance(getattr(parent_node, 'op', None), ast.Pow) or isinstance(parent_node, ast.BoolOp):
+                        parent_precedence+=1
+                except KeyError:
+                    parent_precedence = colorizer.explicit_precedence.get(
+                        node, op_util.Precedence.highest)
+
+                if precedence < parent_precedence:
+                    self.discard = False
 
     def __enter__(self) -> '_OperatorDelimiter':
         return self
@@ -613,38 +617,16 @@ def _colorize_ast_unary_op(self, pyval: ast.UnaryOp, state: _ColorizerState) ->
     def _colorize_ast_binary_op(self, pyval: ast.BinOp, state: _ColorizerState) -> None:
         with _OperatorDelimiter(self, state, pyval):
             # Colorize first operand
+            mark = state.mark()
             self._colorize(pyval.left, state)
-
             # Colorize operator
-            if isinstance(pyval.op, ast.Sub):
-                self._output('-', None, state)
-            elif isinstance(pyval.op, ast.Add):
-                self._output('+', None, state)
-            elif isinstance(pyval.op, ast.Mult):
-                self._output('*', None, state)
-            elif isinstance(pyval.op, ast.Div):
-                self._output('/', None, state)
-            elif isinstance(pyval.op, ast.FloorDiv):
-                self._output('//', None, state)
-            elif isinstance(pyval.op, ast.Mod):
-                self._output('%', None, state)
-            elif isinstance(pyval.op, ast.Pow):
-                self._output('**', None, state)
-            elif isinstance(pyval.op, ast.LShift):
-                self._output('<<', None, state)
-            elif isinstance(pyval.op, ast.RShift):
-                self._output('>>', None, state)
-            elif isinstance(pyval.op, ast.BitOr):
-                self._output('|', None, state)
-            elif isinstance(pyval.op, ast.BitXor):
-                self._output('^', None, state)
-            elif isinstance(pyval.op, ast.BitAnd):
-                self._output('&', None, state)
-            elif isinstance(pyval.op, ast.MatMult):
-                self._output('@', None, state)
-            else:
+            try:
+                self._output(op_util.get_op_symbol(pyval.op, ' %s '), None, state)
+            except KeyError:
                 state.warnings.append(f"Unknow binary operator: {pyval}")
+                state.restore(mark)
                 self._colorize_ast_generic(pyval, state)
+                return
 
             # Colorize second operand
             self._colorize(pyval.right, state)
@@ -687,6 +669,8 @@ def _colorize_ast_subscript(self, node: ast.Subscript, state: _ColorizerState) -
             # In Python < 3.9, non-slices are always wrapped in an Index node.
             sub = sub.value
         self._output('[', self.GROUP_TAG, state)
+        self._set_precedence(op_util.Precedence.Subscript, node)
+        self._set_precedence(op_util.Precedence.Index, sub)
         if isinstance(sub, ast.Tuple):
             self._multiline(self._colorize_iter, sub.elts, state)
         else: