Add support for detecting overloads with overlapping arities

This commit addresses TODO 2 from python#5119
by adding support for detecting overloads with partially overlapping
arities.

It also refactors the `is_callable_compatible` method. Specifically,
this pull request...

1. Pulls out a lot of the logic for iterating over formal arguments into
   a helper method in CallableType.

2. Pulls out logic for handling varargs and kwargs outside of loops.

3. Rearranges some of the logic so we can return earlier slightly more
   frequently.

mypy/checker.py

@@ -3620,7 +3620,7 @@ def is_unsafe_overlapping_overload_signatures(signature: CallableType,
     Assumes that 'signature' appears earlier in the list of overload
     alternatives then 'other' and that their argument counts are overlapping.
     """
-    # TODO: Handle partially overlapping parameter types and argument counts
+    # TODO: Handle partially overlapping parameter types
     #
     # For example, the signatures "f(x: Union[A, B]) -> int" and "f(x: Union[B, C]) -> str"
     # is unsafe: the parameter types are partially overlapping.
@@ -3631,12 +3631,6 @@ def is_unsafe_overlapping_overload_signatures(signature: CallableType,
     #
     # (We already have a rudimentary implementation of 'is_partially_overlapping', but it only
     # attempts to handle the obvious cases -- see its docstring for more info.)
-    #
-    # Similarly, the signatures "f(x: A, y: A) -> str" and "f(*x: A) -> int" are also unsafe:
-    # the parameter *counts* or arity are partially overlapping.
-    #
-    # To fix this, we need to modify is_callable_compatible so it can optionally detect
-    # functions that are *potentially* compatible rather then *definitely* compatible.
 
     def is_more_precise_or_partially_overlapping(t: Type, s: Type) -> bool:
         return is_more_precise(t, s) or is_partially_overlapping_types(t, s)
@@ -3648,10 +3642,12 @@ def is_more_precise_or_partially_overlapping(t: Type, s: Type) -> bool:
     return (is_callable_compatible(signature, other,
                                    is_compat=is_more_precise_or_partially_overlapping,
                                    is_compat_return=lambda l, r: not is_subtype(l, r),
-                                   check_args_covariantly=True) or
+                                   check_args_covariantly=True,
+                                   allow_potential_compatibility=True) or
             is_callable_compatible(other, signature,
                                    is_compat=is_more_precise_or_partially_overlapping,
-                                   is_compat_return=lambda l, r: not is_subtype(r, l)))
+                                   is_compat_return=lambda l, r: not is_subtype(r, l),
+                                   allow_potential_compatibility=True))
 
 
 def overload_can_never_match(signature: CallableType, other: CallableType) -> bool:

mypy/subtypes.py

@@ -577,7 +577,8 @@ def is_callable_compatible(left: CallableType, right: CallableType,
                            is_compat_return: Optional[Callable[[Type, Type], bool]] = None,
                            ignore_return: bool = False,
                            ignore_pos_arg_names: bool = False,
-                           check_args_covariantly: bool = False) -> bool:
+                           check_args_covariantly: bool = False,
+                           allow_potential_compatibility: bool = False) -> bool:
     """Is the left compatible with the right, using the provided compatibility check?
 
     is_compat:
@@ -616,6 +617,30 @@ def g(x: int) -> int: ...
 
         In this case, the first call will succeed and the second will fail: f is a
         valid stand-in for g but not vice-versa.
+
+    allow_potential_compatibility:
+        By default, this function returns True if and only if left is
+        definitely compatible with right.
+
+        If this flag is set to True, we relax the checks so they return True
+        if the left is *potentially* compatible to right. For example, the
+        following two functions are normally incompatible:
+
+            f(x: int, y: str = "...", *args: bool) -> str
+            g(*args: int) -> int
+
+        However, they would be *potentially* compatible under certain conditions --
+        for example, if the user runs "f_or_g(3)". So, if this flag is
+        set to False (the default), f and g are considered incompatible; if the
+        flag is set to True they're considered compatible.
+
+        Specifically, if this flag is set to True, this function will:
+
+        -   Ignore optional arguments on the left.
+        -   No longer mandate that optional arguments on the right are
+            also optional on the left.
+        -   Ignore type mismatches between *arg and **kwarg arguments on
+            the left and the right.
     """
     if is_compat_return is None:
         is_compat_return = is_compat
@@ -657,8 +682,10 @@ def g(x: int) -> int: ...
     if right.is_ellipsis_args:
         return True
 
-    right_star_type = None   # type: Optional[Type]
-    right_star2_type = None  # type: Optional[Type]
+    left_star = left.var_arg
+    left_star2 = left.kw_arg
+    right_star = right.var_arg
+    right_star2 = right.kw_arg
 
     # Match up corresponding arguments and check them for compatibility. In
     # every pair (argL, argR) of corresponding arguments from L and R, argL must
@@ -677,94 +704,90 @@ def g(x: int) -> int: ...
 
     # Every argument in R must have a corresponding argument in L, and every
     # required argument in L must have a corresponding argument in R.
-    done_with_positional = False
-    for i in range(len(right.arg_types)):
-        right_kind = right.arg_kinds[i]
-        if right_kind in (ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT):
-            done_with_positional = True
-        right_required = right_kind in (ARG_POS, ARG_NAMED)
-        right_pos = None if done_with_positional else i
-
-        right_arg = FormalArgument(
-            right.arg_names[i],
-            right_pos,
-            right.arg_types[i],
-            right_required)
-
-        if right_kind == ARG_STAR:
-            right_star_type = right_arg.typ
-            # Right has an infinite series of optional positional arguments
-            # here.  Get all further positional arguments of left, and make sure
-            # they're more general than their corresponding member in this
-            # series.  Also make sure left has its own infinite series of
-            # optional positional arguments.
-            if not left.is_var_arg:
+
+    # Phase 1: Confirm every argument in R has a corresponding argument in L.
+
+    # Phase 1a: If right can accept an infinite number of args,
+    #           then left must also accept an infinite number of
+    #           compatible arguments.
+    #
+    #           If we relax the compatibility requirements, then the types of the
+    #           star args don't matter: left and right will be compatible if
+    #           *args or **kwargs ends up being empty.
+    if not allow_potential_compatibility:
+        if right_star is not None:
+            if left_star is None or not is_compat(right_star.typ, left_star.typ):
                 return False
-            j = i
-            while j < len(left.arg_kinds) and left.arg_kinds[j] in (ARG_POS, ARG_OPT):
-                left_by_position = left.argument_by_position(j)
-                assert left_by_position is not None
-                # This fetches the synthetic argument that's from the *args
-                right_by_position = right.argument_by_position(j)
-                assert right_by_position is not None
-                if not are_args_compatible(left_by_position, right_by_position,
-                                           ignore_pos_arg_names, is_compat):
-                    return False
-                j += 1
-            continue
-
-        if right_kind == ARG_STAR2:
-            right_star2_type = right_arg.typ
-            # Right has an infinite set of optional named arguments here.  Get
-            # all further named arguments of left and make sure they're more
-            # general than their corresponding member in this set.  Also make
-            # sure left has its own infinite set of optional named arguments.
-            if not left.is_kw_arg:
+        if right_star2 is not None:
+            if left_star2 is None or not is_compat(right_star2.typ, left_star2.typ):
                 return False
-            left_names = {name for name in left.arg_names if name is not None}
-            right_names = {name for name in right.arg_names if name is not None}
-            left_only_names = left_names - right_names
-            for name in left_only_names:
-                left_by_name = left.argument_by_name(name)
-                assert left_by_name is not None
-                # This fetches the synthetic argument that's from the **kwargs
-                right_by_name = right.argument_by_name(name)
-                assert right_by_name is not None
-                if not are_args_compatible(left_by_name, right_by_name,
-                                           ignore_pos_arg_names, is_compat):
-                    return False
-            continue
 
-        # Left must have some kind of corresponding argument.
+    # Phase 1b: Check non-star args: for every arg right can accept,
+    #           left must also accept.
+    for right_arg in right.formal_arguments():
         left_arg = left.corresponding_argument(right_arg)
         if left_arg is None:
             return False
-
-        if not are_args_compatible(left_arg, right_arg,
-                                   ignore_pos_arg_names, is_compat):
+        if not are_args_compatible(left_arg, right_arg, ignore_pos_arg_names,
+                                   allow_potential_compatibility, is_compat):
             return False
 
-    done_with_positional = False
-    for i in range(len(left.arg_types)):
-        left_kind = left.arg_kinds[i]
-        if left_kind in (ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT):
-            done_with_positional = True
-        left_arg = FormalArgument(
-            left.arg_names[i],
-            None if done_with_positional else i,
-            left.arg_types[i],
-            left_kind in (ARG_POS, ARG_NAMED))
-
-        # Check that *args and **kwargs types match in this loop
-        if left_kind == ARG_STAR:
-            if right_star_type is not None and not is_compat(right_star_type, left_arg.typ):
+    # Phase 1c: Check var args. Right has an infinite series of optional
+    #           positional arguments. Get all further positional args of left,
+    #           and make sure they're more general then the corresponding
+    #           member in right.
+    if right_star is not None:
+        # Synthesize an anonymous formal argument for the right
+        right_by_position = right.try_synthesizing_arg_from_vararg(None)
+        assert right_by_position is not None
+
+        i = right_star.pos
+        assert i is not None
+        while i < len(left.arg_kinds) and left.arg_kinds[i] in (ARG_POS, ARG_OPT):
+            if allow_potential_compatibility and left.arg_kinds[i] == ARG_OPT:
+                break
+
+            left_by_position = left.argument_by_position(i)
+            assert left_by_position is not None
+
+            if not are_args_compatible(left_by_position, right_by_position,
+                                       ignore_pos_arg_names, allow_potential_compatibility,
+                                       is_compat):
                 return False
-            continue
-        elif left_kind == ARG_STAR2:
-            if right_star2_type is not None and not is_compat(right_star2_type, left_arg.typ):
+            i += 1
+
+    # Phase 1d: Check kw args. Right has an infinite series of optional
+    #           named arguments. Get all further named args of left,
+    #           and make sure they're more general then the corresponding
+    #           member in right.
+    if right_star2 is not None:
+        right_names = {name for name in right.arg_names if name is not None}
+        left_only_names = set()
+        for name, kind in zip(left.arg_names, left.arg_kinds):
+            if name is None or kind in (ARG_STAR, ARG_STAR2) or name in right_names:
+                continue
+            left_only_names.add(name)
+
+        # Synthesize an anonymous formal argument for the right
+        right_by_name = right.try_synthesizing_arg_from_kwarg(None)
+        assert right_by_name is not None
+
+        for name in left_only_names:
+            left_by_name = left.argument_by_name(name)
+            assert left_by_name is not None
+
+            if allow_potential_compatibility and not left_by_name.required:
+                continue
+
+            if not are_args_compatible(left_by_name, right_by_name, ignore_pos_arg_names,
+                                       allow_potential_compatibility, is_compat):
                 return False
-            continue
 
+    # Phase 2: Left must not impose additional restrictions.
+    #          (Every required argument in L must have a corresponding argument in R)
+    #
+    #          Note: we already checked the *arg and **kwarg arguments in phase 1a.
+    for left_arg in left.formal_arguments():
         right_by_name = (right.argument_by_name(left_arg.name)
                          if left_arg.name is not None
                          else None)
@@ -782,7 +805,7 @@ def g(x: int) -> int: ...
             return False
 
         # All *required* left-hand arguments must have a corresponding
-        # right-hand argument.  Optional args it does not matter.
+        # right-hand argument.  Optional args do not matter.
         if left_arg.required and right_by_pos is None and right_by_name is None:
             return False
 
@@ -793,23 +816,27 @@ def are_args_compatible(
         left: FormalArgument,
         right: FormalArgument,
         ignore_pos_arg_names: bool,
+        allow_potential_compatibility: bool,
         is_compat: Callable[[Type, Type], bool]) -> bool:
     # If right has a specific name it wants this argument to be, left must
     # have the same.
     if right.name is not None and left.name != right.name:
         # But pay attention to whether we're ignoring positional arg names
         if not ignore_pos_arg_names or right.pos is None:
             return False
+
     # If right is at a specific position, left must have the same:
     if right.pos is not None and left.pos != right.pos:
         return False
-    # Left must have a more general type
-    if not is_compat(right.typ, left.typ):
-        return False
-    # If right's argument is optional, left's must also be.
-    if not right.required and left.required:
+
+    # If right's argument is optional, left's must also be
+    # (unless we're relaxing the checks to allow potential
+    # rather then definite compatibility).
+    if not allow_potential_compatibility and not right.required and left.required:
         return False
-    return True
+
+    # Left must have a more general type
+    return is_compat(right.typ, left.typ)
 
 
 def flip_compat_check(is_compat: Callable[[Type, Type], bool]) -> Callable[[Type, Type], bool]: