Honor return type of __new__ (#7188)

This basically follows the approach Jukka laid out in #1020 four years ago:
 * If the return type is Any, ignore that and keep the class type as
   the return type
 * Otherwise respect `__new__`'s return type
 * Produce an error if the return type is not a subtype of the class.

The main motivation for me in implementing this is to support
overloading `__new__` in order to select type variable arguments,
which will be useful for subprocess.Popen.

Fixes #1020.

mypy/checker.py

@@ -803,6 +803,10 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str])
                         self.fail(message_registry.MUST_HAVE_NONE_RETURN_TYPE.format(fdef.name()),
                                   item)
 
+                    # Check validity of __new__ signature
+                    if fdef.info and fdef.name() == '__new__':
+                        self.check___new___signature(fdef, typ)
+
                     self.check_for_missing_annotations(fdef)
                     if self.options.disallow_any_unimported:
                         if fdef.type and isinstance(fdef.type, CallableType):
@@ -1015,6 +1019,27 @@ def is_unannotated_any(t: Type) -> bool:
                 if any(is_unannotated_any(t) for t in fdef.type.arg_types):
                     self.fail(message_registry.ARGUMENT_TYPE_EXPECTED, fdef)
 
+    def check___new___signature(self, fdef: FuncDef, typ: CallableType) -> None:
+        self_type = fill_typevars_with_any(fdef.info)
+        bound_type = bind_self(typ, self_type, is_classmethod=True)
+        # Check that __new__ (after binding cls) returns an instance
+        # type (or any)
+        if not isinstance(bound_type.ret_type, (AnyType, Instance, TupleType)):
+            self.fail(
+                message_registry.NON_INSTANCE_NEW_TYPE.format(
+                    self.msg.format(bound_type.ret_type)),
+                fdef)
+        else:
+            # And that it returns a subtype of the class
+            self.check_subtype(
+                bound_type.ret_type,
+                self_type,
+                fdef,
+                message_registry.INVALID_NEW_TYPE,
+                'returns',
+                'but must return a subtype of'
+            )
+
     def is_trivial_body(self, block: Block) -> bool:
         """Returns 'true' if the given body is "trivial" -- if it contains just a "pass",
         "..." (ellipsis), or "raise NotImplementedError()". A trivial body may also

mypy/checkmember.py

@@ -803,8 +803,10 @@ def type_object_type(info: TypeInfo, builtin_type: Callable[[str], Instance]) ->
     fallback = info.metaclass_type or builtin_type('builtins.type')
     if init_index < new_index:
         method = init_method.node  # type: Union[FuncBase, Decorator]
+        is_new = False
     elif init_index > new_index:
         method = new_method.node
+        is_new = True
     else:
         if init_method.node.info.fullname() == 'builtins.object':
             # Both are defined by object.  But if we've got a bogus
@@ -817,20 +819,21 @@ def type_object_type(info: TypeInfo, builtin_type: Callable[[str], Instance]) ->
                                    arg_names=["_args", "_kwds"],
                                    ret_type=any_type,
                                    fallback=builtin_type('builtins.function'))
-                return class_callable(sig, info, fallback, None)
+                return class_callable(sig, info, fallback, None, is_new=False)
 
         # Otherwise prefer __init__ in a tie. It isn't clear that this
         # is the right thing, but __new__ caused problems with
         # typeshed (#5647).
         method = init_method.node
+        is_new = False
     # Construct callable type based on signature of __init__. Adjust
     # return type and insert type arguments.
     if isinstance(method, FuncBase):
         t = function_type(method, fallback)
     else:
         assert isinstance(method.type, FunctionLike)  # is_valid_constructor() ensures this
         t = method.type
-    return type_object_type_from_function(t, info, method.info, fallback)
+    return type_object_type_from_function(t, info, method.info, fallback, is_new)
 
 
 def is_valid_constructor(n: Optional[SymbolNode]) -> bool:
@@ -849,7 +852,8 @@ def is_valid_constructor(n: Optional[SymbolNode]) -> bool:
 def type_object_type_from_function(signature: FunctionLike,
                                    info: TypeInfo,
                                    def_info: TypeInfo,
-                                   fallback: Instance) -> FunctionLike:
+                                   fallback: Instance,
+                                   is_new: bool) -> FunctionLike:
     # The __init__ method might come from a generic superclass
     # (init_or_new.info) with type variables that do not map
     # identically to the type variables of the class being constructed
@@ -859,7 +863,7 @@ def type_object_type_from_function(signature: FunctionLike,
     #   class B(A[List[T]], Generic[T]): pass
     #
     # We need to first map B's __init__ to the type (List[T]) -> None.
-    signature = bind_self(signature)
+    signature = bind_self(signature, original_type=fill_typevars(info), is_classmethod=is_new)
     signature = cast(FunctionLike,
                      map_type_from_supertype(signature, info, def_info))
     special_sig = None  # type: Optional[str]
@@ -868,25 +872,31 @@ def type_object_type_from_function(signature: FunctionLike,
         special_sig = 'dict'
 
     if isinstance(signature, CallableType):
-        return class_callable(signature, info, fallback, special_sig)
+        return class_callable(signature, info, fallback, special_sig, is_new)
     else:
         # Overloaded __init__/__new__.
         assert isinstance(signature, Overloaded)
         items = []  # type: List[CallableType]
         for item in signature.items():
-            items.append(class_callable(item, info, fallback, special_sig))
+            items.append(class_callable(item, info, fallback, special_sig, is_new))
         return Overloaded(items)
 
 
 def class_callable(init_type: CallableType, info: TypeInfo, type_type: Instance,
-                   special_sig: Optional[str]) -> CallableType:
+                   special_sig: Optional[str],
+                   is_new: bool) -> CallableType:
     """Create a type object type based on the signature of __init__."""
     variables = []  # type: List[TypeVarDef]
     variables.extend(info.defn.type_vars)
     variables.extend(init_type.variables)
 
+    if is_new and isinstance(init_type.ret_type, (Instance, TupleType)):
+        ret_type = init_type.ret_type  # type: Type
+    else:
+        ret_type = fill_typevars(info)
+
     callable_type = init_type.copy_modified(
-        ret_type=fill_typevars(info), fallback=type_type, name=None, variables=variables,
+        ret_type=ret_type, fallback=type_type, name=None, variables=variables,
         special_sig=special_sig)
     c = callable_type.with_name(info.name())
     return c

mypy/interpreted_plugin.py

@@ -22,7 +22,8 @@ class InterpretedPlugin:
     that proxies to this interpreted version.
     """
 
-    def __new__(cls, *args: Any, **kwargs: Any) -> 'mypy.plugin.Plugin':
+    # ... mypy doesn't like these shenanigans so we have to type ignore it!
+    def __new__(cls, *args: Any, **kwargs: Any) -> 'mypy.plugin.Plugin':  # type: ignore
         from mypy.plugin import WrapperPlugin
         plugin = object.__new__(cls)
         plugin.__init__(*args, **kwargs)

mypy/message_registry.py

@@ -51,6 +51,8 @@
 INVALID_SLICE_INDEX = 'Slice index must be an integer or None'  # type: Final
 CANNOT_INFER_LAMBDA_TYPE = 'Cannot infer type of lambda'  # type: Final
 CANNOT_ACCESS_INIT = 'Cannot access "__init__" directly'  # type: Final
+NON_INSTANCE_NEW_TYPE = '"__new__" must return a class instance (got {})'  # type: Final
+INVALID_NEW_TYPE = 'Incompatible return type for "__new__"'  # type: Final
 BAD_CONSTRUCTOR_TYPE = 'Unsupported decorated constructor type'  # type: Final
 CANNOT_ASSIGN_TO_METHOD = 'Cannot assign to a method'  # type: Final
 CANNOT_ASSIGN_TO_TYPE = 'Cannot assign to a type'  # type: Final

test-data/unit/check-class-namedtuple.test

@@ -598,7 +598,7 @@ class XMethBad(NamedTuple):
 class MagicalFields(NamedTuple):
     x: int
     def __slots__(self) -> None: pass  # E: Cannot overwrite NamedTuple attribute "__slots__"
-    def __new__(cls) -> None: pass  # E: Cannot overwrite NamedTuple attribute "__new__"
+    def __new__(cls) -> MagicalFields: pass  # E: Cannot overwrite NamedTuple attribute "__new__"
     def _source(self) -> int: pass  # E: Cannot overwrite NamedTuple attribute "_source"
     __annotations__ = {'x': float}  # E: NamedTuple field name cannot start with an underscore: __annotations__ \
         # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" \

test-data/unit/check-classes.test

@@ -344,12 +344,12 @@ main:6: error: Return type "A" of "f" incompatible with return type "None" in su
 
 [case testOverride__new__WithDifferentSignature]
 class A:
-    def __new__(cls, x: int) -> str:
-        return ''
+    def __new__(cls, x: int) -> A:
+        pass
 
 class B(A):
-    def __new__(cls) -> int:
-        return 1
+    def __new__(cls) -> B:
+        pass
 
 [case testOverride__new__AndCallObject]
 from typing import TypeVar, Generic
@@ -5363,8 +5363,8 @@ class A:
         pass
 
 class B(A):
-    def __new__(cls) -> int:
-        return 10
+    def __new__(cls) -> B:
+        pass
 
 B()
 
@@ -5975,3 +5975,105 @@ class E(C):
         reveal_type(self.x) # N: Revealed type is 'builtins.int'
 
 [targets __main__, __main__, __main__.D.g, __main__.D.f, __main__.C.__init__, __main__.E.g, __main__.E.f]
+
+[case testNewReturnType1]
+class A:
+    def __new__(cls) -> B:
+        pass
+
+class B(A): pass
+
+reveal_type(A())  # N: Revealed type is '__main__.B'
+reveal_type(B())  # N: Revealed type is '__main__.B'
+
+[case testNewReturnType2]
+from typing import Any
+
+# make sure that __new__ method that return Any are ignored when
+# determining the return type
+class A:
+    def __new__(cls):
+        pass
+
+class B:
+    def __new__(cls) -> Any:
+        pass
+
+reveal_type(A())  # N: Revealed type is '__main__.A'
+reveal_type(B())  # N: Revealed type is '__main__.B'
+
+[case testNewReturnType3]
+
+# Check for invalid __new__ typing
+
+class A:
+    def __new__(cls) -> int:  # E: Incompatible return type for "__new__" (returns "int", but must return a subtype of "A")
+        pass
+
+reveal_type(A())  # N: Revealed type is 'builtins.int'
+
+[case testNewReturnType4]
+from typing import TypeVar, Type
+
+# Check for __new__ using type vars
+
+TX = TypeVar('TX', bound='X')
+class X:
+    def __new__(lol: Type[TX], x: int) -> TX:
+        pass
+class Y(X): pass
+
+reveal_type(X(20))  # N: Revealed type is '__main__.X*'
+reveal_type(Y(20))  # N: Revealed type is '__main__.Y*'
+
+[case testNewReturnType5]
+from typing import Any, TypeVar, Generic, overload
+
+T = TypeVar('T')
+class O(Generic[T]):
+    @overload
+    def __new__(cls) -> O[int]:
+        pass
+    @overload
+    def __new__(cls, x: int) -> O[str]:
+        pass
+    def __new__(cls, x: int = 0) -> O[Any]:
+        pass
+
+reveal_type(O())  # N: Revealed type is '__main__.O[builtins.int]'
+reveal_type(O(10))  # N: Revealed type is '__main__.O[builtins.str]'
+
+[case testNewReturnType6]
+from typing import Tuple, Optional
+
+# Check for some cases that aren't allowed
+
+class X:
+    def __new__(cls) -> Optional[Y]:  # E: "__new__" must return a class instance (got "Optional[Y]")
+        pass
+class Y:
+    def __new__(cls) -> Optional[int]:  # E: "__new__" must return a class instance (got "Optional[int]")
+        pass
+
+
+[case testNewReturnType7]
+from typing import NamedTuple
+
+# ... test __new__ returning tuple type
+class A:
+    def __new__(cls) -> 'B':
+        pass
+
+N = NamedTuple('N', [('x', int)])
+class B(A, N): pass
+
+reveal_type(A())  # N: Revealed type is 'Tuple[builtins.int, fallback=__main__.B]'
+
+[case testNewReturnType8]
+from typing import TypeVar, Any
+
+# test type var from a different argument
+TX = TypeVar('TX', bound='X')
+class X:
+    def __new__(cls, x: TX) -> TX:  # E: "__new__" must return a class instance (got "TX")
+        pass