Merge pull request #8465 from tk0miya/8460_NewType

Fix #8460: autodoc: Support custom types defined by typing.NewType

CHANGES

@@ -31,6 +31,7 @@ Features added
   value equal to None is set.
 * #8209: autodoc: Add ``:no-value:`` option to :rst:dir:`autoattribute` and
   :rst:dir:`autodata` directive to suppress the default value of the variable
+* #8460: autodoc: Support custom types defined by typing.NewType
 * #6914: Add a new event :event:`warn-missing-reference` to custom warning
   messages when failed to resolve a cross-reference
 * #6914: Emit a detailed warning when failed to resolve a ``:ref:`` reference

sphinx/ext/autodoc/__init__.py

@@ -1677,7 +1677,25 @@ def can_document_member(cls, member: Any, membername: str, isattr: bool, parent:
         return isinstance(member, type) and issubclass(member, BaseException)
 
 
-class DataDocumenter(ModuleLevelDocumenter):
+class NewTypeMixin:
+    """
+    Mixin for DataDocumenter and AttributeDocumenter to provide the feature for
+    supporting NewTypes.
+    """
+
+    def should_suppress_directive_header(self) -> bool:
+        """Check directive header should be suppressed."""
+        return inspect.isNewType(self.object)  # type: ignore
+
+    def update_content(self, more_content: StringList) -> None:
+        """Update docstring for the NewType object."""
+        if inspect.isNewType(self.object):  # type: ignore
+            supertype = restify(self.object.__supertype__)  # type: ignore
+            more_content.append(_('alias of %s') % supertype, '')
+            more_content.append('', '')
+
+
+class DataDocumenter(ModuleLevelDocumenter, NewTypeMixin):
     """
     Specialized Documenter subclass for data items.
     """
@@ -1718,7 +1736,12 @@ def import_object(self, raiseerror: bool = False) -> bool:
     def add_directive_header(self, sig: str) -> None:
         super().add_directive_header(sig)
         sourcename = self.get_sourcename()
-        if not self.options.annotation:
+        if self.options.annotation is SUPPRESS or self.should_suppress_directive_header():
+            pass
+        elif self.options.annotation:
+            self.add_line('   :annotation: %s' % self.options.annotation,
+                          sourcename)
+        else:
             # obtain annotation for this data
             annotations = get_type_hints(self.parent, None, self.config.autodoc_type_aliases)
             if self.objpath[-1] in annotations:
@@ -1738,11 +1761,6 @@ def add_directive_header(self, sig: str) -> None:
                     self.add_line('   :value: ' + objrepr, sourcename)
             except ValueError:
                 pass
-        elif self.options.annotation is SUPPRESS:
-            pass
-        else:
-            self.add_line('   :annotation: %s' % self.options.annotation,
-                          sourcename)
 
     def document_members(self, all_members: bool = False) -> None:
         pass
@@ -1757,6 +1775,10 @@ def add_content(self, more_content: Optional[StringList], no_docstring: bool = F
             # suppress docstring of the value
             super().add_content(more_content, no_docstring=True)
         else:
+            if not more_content:
+                more_content = StringList()
+
+            self.update_content(more_content)
             super().add_content(more_content, no_docstring=no_docstring)
 
 
@@ -1804,6 +1826,24 @@ def add_content(self, more_content: Optional[StringList], no_docstring: bool = F
         super().add_content(content)
 
 
+class NewTypeDataDocumenter(DataDocumenter):
+    """
+    Specialized Documenter subclass for NewTypes.
+
+    Note: This must be invoked before FunctionDocumenter because NewType is a kind of
+    function object.
+    """
+
+    objtype = 'newtypedata'
+    directivetype = 'data'
+    priority = FunctionDocumenter.priority + 1
+
+    @classmethod
+    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
+                            ) -> bool:
+        return inspect.isNewType(member) and isattr
+
+
 class TypeVarDocumenter(DataDocumenter):
     """
     Specialized Documenter subclass for TypeVars.
@@ -2009,7 +2049,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
 
-class AttributeDocumenter(DocstringStripSignatureMixin, ClassLevelDocumenter):  # type: ignore
+class AttributeDocumenter(DocstringStripSignatureMixin, ClassLevelDocumenter, NewTypeMixin):  # type: ignore  # NOQA
     """
     Specialized Documenter subclass for attributes.
     """
@@ -2105,7 +2145,11 @@ def get_real_modname(self) -> str:
     def add_directive_header(self, sig: str) -> None:
         super().add_directive_header(sig)
         sourcename = self.get_sourcename()
-        if not self.options.annotation:
+        if self.options.annotation is SUPPRESS or self.should_suppress_directive_header():
+            pass
+        elif self.options.annotation:
+            self.add_line('   :annotation: %s' % self.options.annotation, sourcename)
+        else:
             # obtain type annotation for this attribute
             annotations = get_type_hints(self.parent, None, self.config.autodoc_type_aliases)
             if self.objpath[-1] in annotations:
@@ -2127,10 +2171,6 @@ def add_directive_header(self, sig: str) -> None:
                         self.add_line('   :value: ' + objrepr, sourcename)
                 except ValueError:
                     pass
-        elif self.options.annotation is SUPPRESS:
-            pass
-        else:
-            self.add_line('   :annotation: %s' % self.options.annotation, sourcename)
 
     def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
         try:
@@ -2149,6 +2189,10 @@ def add_content(self, more_content: Optional[StringList], no_docstring: bool = F
             # if it's not a data descriptor, its docstring is very probably the
             # wrong thing to display
             no_docstring = True
+
+        if more_content is None:
+            more_content = StringList()
+        self.update_content(more_content)
         super().add_content(more_content, no_docstring)
 
 
@@ -2288,6 +2332,24 @@ def get_doc(self, encoding: str = None, ignore: int = None) -> List[List[str]]:
             return []
 
 
+class NewTypeAttributeDocumenter(AttributeDocumenter):
+    """
+    Specialized Documenter subclass for NewTypes.
+
+    Note: This must be invoked before MethodDocumenter because NewType is a kind of
+    function object.
+    """
+
+    objtype = 'newvarattribute'
+    directivetype = 'attribute'
+    priority = MethodDocumenter.priority + 1
+
+    @classmethod
+    def can_document_member(cls, member: Any, membername: str, isattr: bool, parent: Any
+                            ) -> bool:
+        return not isinstance(parent, ModuleDocumenter) and inspect.isNewType(member)
+
+
 def get_documenters(app: Sphinx) -> Dict[str, "Type[Documenter]"]:
     """Returns registered Documenter classes"""
     warnings.warn("get_documenters() is deprecated.", RemovedInSphinx50Warning, stacklevel=2)
@@ -2317,6 +2379,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_autodocumenter(ExceptionDocumenter)
     app.add_autodocumenter(DataDocumenter)
     app.add_autodocumenter(GenericAliasDocumenter)
+    app.add_autodocumenter(NewTypeDataDocumenter)
     app.add_autodocumenter(TypeVarDocumenter)
     app.add_autodocumenter(FunctionDocumenter)
     app.add_autodocumenter(DecoratorDocumenter)
@@ -2325,6 +2388,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_autodocumenter(PropertyDocumenter)
     app.add_autodocumenter(InstanceAttributeDocumenter)
     app.add_autodocumenter(SlotsAttributeDocumenter)
+    app.add_autodocumenter(NewTypeAttributeDocumenter)
 
     app.add_config_value('autoclass_content', 'class', True, ENUM('both', 'class', 'init'))
     app.add_config_value('autodoc_member_order', 'alphabetical', True,

sphinx/ext/autosummary/generate.py

@@ -89,13 +89,15 @@ def setup_documenters(app: Any) -> None:
                                     DecoratorDocumenter, ExceptionDocumenter,
                                     FunctionDocumenter, GenericAliasDocumenter,
                                     InstanceAttributeDocumenter, MethodDocumenter,
-                                    ModuleDocumenter, PropertyDocumenter,
+                                    ModuleDocumenter, NewTypeAttributeDocumenter,
+                                    NewTypeDataDocumenter, PropertyDocumenter,
                                     SingledispatchFunctionDocumenter, SlotsAttributeDocumenter)
     documenters = [
         ModuleDocumenter, ClassDocumenter, ExceptionDocumenter, DataDocumenter,
-        FunctionDocumenter, MethodDocumenter, AttributeDocumenter,
-        InstanceAttributeDocumenter, DecoratorDocumenter, PropertyDocumenter,
-        SlotsAttributeDocumenter, GenericAliasDocumenter, SingledispatchFunctionDocumenter,
+        FunctionDocumenter, MethodDocumenter, NewTypeAttributeDocumenter,
+        NewTypeDataDocumenter, AttributeDocumenter, InstanceAttributeDocumenter,
+        DecoratorDocumenter, PropertyDocumenter, SlotsAttributeDocumenter,
+        GenericAliasDocumenter, SingledispatchFunctionDocumenter,
     ]  # type: List[Type[Documenter]]
     for documenter in documenters:
         app.registry.add_documenter(documenter.objtype, documenter)

sphinx/util/inspect.py

@@ -177,6 +177,16 @@ def getslots(obj: Any) -> Optional[Dict]:
         raise ValueError
 
 
+def isNewType(obj: Any) -> bool:
+    """Check the if object is a kind of NewType."""
+    __module__ = safe_getattr(obj, '__module__', None)
+    __qualname__ = safe_getattr(obj, '__qualname__', None)
+    if __module__ == 'typing' and __qualname__ == 'NewType.<locals>.new_type':
+        return True
+    else:
+        return False
+
+
 def isenumclass(x: Any) -> bool:
     """Check if the object is subclass of enum."""
     return inspect.isclass(x) and issubclass(x, enum.Enum)

sphinx/util/typing.py

@@ -88,10 +88,14 @@ def is_system_TypeVar(typ: Any) -> bool:
 
 def restify(cls: Optional["Type"]) -> str:
     """Convert python class to a reST reference."""
+    from sphinx.util import inspect  # lazy loading
+
     if cls is None or cls is NoneType:
         return ':obj:`None`'
     elif cls is Ellipsis:
         return '...'
+    elif inspect.isNewType(cls):
+        return ':class:`%s`' % cls.__name__
     elif cls.__module__ in ('__builtin__', 'builtins'):
         return ':class:`%s`' % cls.__name__
     else:
@@ -277,6 +281,8 @@ def _restify_py36(cls: Optional["Type"]) -> str:
 
 def stringify(annotation: Any) -> str:
     """Stringify type annotation object."""
+    from sphinx.util import inspect  # lazy loading
+
     if isinstance(annotation, str):
         if annotation.startswith("'") and annotation.endswith("'"):
             # might be a double Forward-ref'ed type.  Go unquoting.
@@ -285,6 +291,9 @@ def stringify(annotation: Any) -> str:
             return annotation
     elif isinstance(annotation, TypeVar):
         return annotation.__name__
+    elif inspect.isNewType(annotation):
+        # Could not get the module where it defiend
+        return annotation.__name__
     elif not annotation:
         return repr(annotation)
     elif annotation is NoneType:

tests/roots/test-ext-autodoc/target/typevar.py

@@ -1,4 +1,4 @@
-from typing import TypeVar
+from typing import NewType, TypeVar
 
 #: T1
 T1 = TypeVar("T1")
@@ -13,3 +13,13 @@
 
 #: T5
 T5 = TypeVar("T5", contravariant=True)
+
+#: T6
+T6 = NewType("T6", int)
+
+
+class Class:
+    # TODO: TypeVar
+
+    #: T6
+    T6 = NewType("T6", int)

tests/test_ext_autodoc.py

@@ -1731,6 +1731,18 @@ def test_autodoc_TypeVar(app):
         '.. py:module:: target.typevar',
         '',
         '',
+        '.. py:class:: Class()',
+        '   :module: target.typevar',
+        '',
+        '',
+        '   .. py:attribute:: Class.T6',
+        '      :module: target.typevar',
+        '',
+        '      T6',
+        '',
+        '      alias of :class:`int`',
+        '',
+        '',
         '.. py:data:: T1',
         '   :module: target.typevar',
         '',
@@ -1758,6 +1770,14 @@ def test_autodoc_TypeVar(app):
         '   T5',
         '',
         "   alias of TypeVar('T5', contravariant=True)",
+        '',
+        '.. py:data:: T6',
+        '   :module: target.typevar',
+        '',
+        '   T6',
+        '',
+        '   alias of :class:`int`',
+        '',
     ]
 
 

tests/test_ext_autodoc_autoattribute.py

@@ -70,3 +70,18 @@ def test_autoattribute_instance_variable(app):
         '   attr4',
         '',
     ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autoattribute_NewType(app):
+    actual = do_autodoc(app, 'attribute', 'target.typevar.Class.T6')
+    assert list(actual) == [
+        '',
+        '.. py:attribute:: Class.T6',
+        '   :module: target.typevar',
+        '',
+        '   T6',
+        '',
+        '   alias of :class:`int`',
+        '',
+    ]

tests/test_ext_autodoc_autodata.py

@@ -73,3 +73,18 @@ def test_autodata_type_comment(app):
         '   attr3',
         '',
     ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_autodata_NewType(app):
+    actual = do_autodoc(app, 'data', 'target.typevar.T6')
+    assert list(actual) == [
+        '',
+        '.. py:data:: T6',
+        '   :module: target.typevar',
+        '',
+        '   T6',
+        '',
+        '   alias of :class:`int`',
+        '',
+    ]

tests/test_util_typing.py

@@ -10,7 +10,8 @@
 
 import sys
 from numbers import Integral
-from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, TypeVar, Union
+from typing import (Any, Callable, Dict, Generator, List, NewType, Optional, Tuple, TypeVar,
+                    Union)
 
 import pytest
 
@@ -26,6 +27,7 @@ class MyClass2(MyClass1):
 
 
 T = TypeVar('T')
+MyInt = NewType('MyInt', int)
 
 
 class MyList(List[T]):
@@ -92,6 +94,7 @@ def test_restify_type_hints_typevars():
     assert restify(T_co) == ":obj:`tests.test_util_typing.T_co`"
     assert restify(T_contra) == ":obj:`tests.test_util_typing.T_contra`"
     assert restify(List[T]) == ":class:`List`\\ [:obj:`tests.test_util_typing.T`]"
+    assert restify(MyInt) == ":class:`MyInt`"
 
 
 def test_restify_type_hints_custom_class():
@@ -179,6 +182,7 @@ def test_stringify_type_hints_typevars():
     assert stringify(T_co) == "T_co"
     assert stringify(T_contra) == "T_contra"
     assert stringify(List[T]) == "List[T]"
+    assert stringify(MyInt) == "MyInt"
 
 
 def test_stringify_type_hints_custom_class():