Use (simplified) unions instead of joins for tuple fallbacks (#17408)

Ref #12056

If `mypy_primer` will look good, I will add some logic to shorted unions
in error messages.

cc @JukkaL

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>

mypy/checker.py

@@ -49,6 +49,7 @@
     SUGGESTED_TEST_FIXTURES,
     MessageBuilder,
     append_invariance_notes,
+    append_union_note,
     format_type,
     format_type_bare,
     format_type_distinctly,
@@ -6814,6 +6815,8 @@ def check_subtype(
             )
             if isinstance(subtype, Instance) and isinstance(supertype, Instance):
                 notes = append_invariance_notes(notes, subtype, supertype)
+            if isinstance(subtype, UnionType) and isinstance(supertype, UnionType):
+                notes = append_union_note(notes, subtype, supertype, self.options)
         if extra_info:
             msg = msg.with_additional_msg(" (" + ", ".join(extra_info) + ")")
 

mypy/messages.py

@@ -90,6 +90,7 @@
     UninhabitedType,
     UnionType,
     UnpackType,
+    flatten_nested_unions,
     get_proper_type,
     get_proper_types,
 )
@@ -145,6 +146,9 @@
     "numbers.Integral",
 }
 
+MAX_TUPLE_ITEMS = 10
+MAX_UNION_ITEMS = 10
+
 
 class MessageBuilder:
     """Helper class for reporting type checker error messages with parameters.
@@ -2338,7 +2342,7 @@ def try_report_long_tuple_assignment_error(
         """
         if isinstance(subtype, TupleType):
             if (
-                len(subtype.items) > 10
+                len(subtype.items) > MAX_TUPLE_ITEMS
                 and isinstance(supertype, Instance)
                 and supertype.type.fullname == "builtins.tuple"
             ):
@@ -2347,7 +2351,7 @@ def try_report_long_tuple_assignment_error(
                 self.generate_incompatible_tuple_error(lhs_types, subtype.items, context, msg)
                 return True
             elif isinstance(supertype, TupleType) and (
-                len(subtype.items) > 10 or len(supertype.items) > 10
+                len(subtype.items) > MAX_TUPLE_ITEMS or len(supertype.items) > MAX_TUPLE_ITEMS
             ):
                 if len(subtype.items) != len(supertype.items):
                     if supertype_label is not None and subtype_label is not None:
@@ -2370,7 +2374,7 @@ def try_report_long_tuple_assignment_error(
     def format_long_tuple_type(self, typ: TupleType) -> str:
         """Format very long tuple type using an ellipsis notation"""
         item_cnt = len(typ.items)
-        if item_cnt > 10:
+        if item_cnt > MAX_TUPLE_ITEMS:
             return "{}[{}, {}, ... <{} more items>]".format(
                 "tuple" if self.options.use_lowercase_names() else "Tuple",
                 format_type_bare(typ.items[0], self.options),
@@ -2497,11 +2501,21 @@ def format(typ: Type) -> str:
     def format_list(types: Sequence[Type]) -> str:
         return ", ".join(format(typ) for typ in types)
 
-    def format_union(types: Sequence[Type]) -> str:
+    def format_union_items(types: Sequence[Type]) -> list[str]:
         formatted = [format(typ) for typ in types if format(typ) != "None"]
+        if len(formatted) > MAX_UNION_ITEMS and verbosity == 0:
+            more = len(formatted) - MAX_UNION_ITEMS // 2
+            formatted = formatted[: MAX_UNION_ITEMS // 2]
+        else:
+            more = 0
+        if more:
+            formatted.append(f"<{more} more items>")
         if any(format(typ) == "None" for typ in types):
             formatted.append("None")
-        return " | ".join(formatted)
+        return formatted
+
+    def format_union(types: Sequence[Type]) -> str:
+        return " | ".join(format_union_items(types))
 
     def format_literal_value(typ: LiteralType) -> str:
         if typ.is_enum_literal():
@@ -2605,6 +2619,9 @@ def format_literal_value(typ: LiteralType) -> str:
     elif isinstance(typ, LiteralType):
         return f"Literal[{format_literal_value(typ)}]"
     elif isinstance(typ, UnionType):
+        typ = get_proper_type(ignore_last_known_values(typ))
+        if not isinstance(typ, UnionType):
+            return format(typ)
         literal_items, union_items = separate_union_literals(typ)
 
         # Coalesce multiple Literal[] members. This also changes output order.
@@ -2624,7 +2641,7 @@ def format_literal_value(typ: LiteralType) -> str:
                 return (
                     f"{literal_str} | {format_union(union_items)}"
                     if options.use_or_syntax()
-                    else f"Union[{format_list(union_items)}, {literal_str}]"
+                    else f"Union[{', '.join(format_union_items(union_items))}, {literal_str}]"
                 )
             else:
                 return literal_str
@@ -2645,7 +2662,7 @@ def format_literal_value(typ: LiteralType) -> str:
                 s = (
                     format_union(typ.items)
                     if options.use_or_syntax()
-                    else f"Union[{format_list(typ.items)}]"
+                    else f"Union[{', '.join(format_union_items(typ.items))}]"
                 )
             return s
     elif isinstance(typ, NoneType):
@@ -3182,6 +3199,23 @@ def append_invariance_notes(
     return notes
 
 
+def append_union_note(
+    notes: list[str], arg_type: UnionType, expected_type: UnionType, options: Options
+) -> list[str]:
+    """Point to specific union item(s) that may cause failure in subtype check."""
+    non_matching = []
+    items = flatten_nested_unions(arg_type.items)
+    if len(items) < MAX_UNION_ITEMS:
+        return notes
+    for item in items:
+        if not is_subtype(item, expected_type):
+            non_matching.append(item)
+    if non_matching:
+        types = ", ".join([format_type(typ, options) for typ in non_matching])
+        notes.append(f"Item{plural_s(non_matching)} in the first union not in the second: {types}")
+    return notes
+
+
 def append_numbers_notes(
     notes: list[str], arg_type: Instance, expected_type: Instance
 ) -> list[str]:
@@ -3235,3 +3269,23 @@ def format_key_list(keys: list[str], *, short: bool = False) -> str:
         return f"{td}key {formatted_keys[0]}"
     else:
         return f"{td}keys ({', '.join(formatted_keys)})"
+
+
+def ignore_last_known_values(t: UnionType) -> Type:
+    """This will avoid types like str | str in error messages.
+
+    last_known_values are kept during union simplification, but may cause
+    weird formatting for e.g. tuples of literals.
+    """
+    union_items: list[Type] = []
+    seen_instances = set()
+    for item in t.items:
+        if isinstance(item, ProperType) and isinstance(item, Instance):
+            erased = item.copy_modified(last_known_value=None)
+            if erased in seen_instances:
+                continue
+            seen_instances.add(erased)
+            union_items.append(erased)
+        else:
+            union_items.append(item)
+    return UnionType.make_union(union_items, t.line, t.column)

mypy/semanal_shared.py

@@ -8,7 +8,6 @@
 
 from mypy_extensions import trait
 
-from mypy import join
 from mypy.errorcodes import LITERAL_REQ, ErrorCode
 from mypy.nodes import (
     CallExpr,
@@ -30,6 +29,7 @@
 from mypy.plugin import SemanticAnalyzerPluginInterface
 from mypy.tvar_scope import TypeVarLikeScope
 from mypy.type_visitor import ANY_STRATEGY, BoolTypeQuery
+from mypy.typeops import make_simplified_union
 from mypy.types import (
     TPDICT_FB_NAMES,
     AnyType,
@@ -58,7 +58,7 @@
 # Priorities for ordering of patches within the "patch" phase of semantic analysis
 # (after the main pass):
 
-# Fix fallbacks (does joins)
+# Fix fallbacks (does subtype checks).
 PRIORITY_FALLBACKS: Final = 1
 
 
@@ -304,7 +304,7 @@ def calculate_tuple_fallback(typ: TupleType) -> None:
                 raise NotImplementedError
         else:
             items.append(item)
-    fallback.args = (join.join_type_list(items),)
+    fallback.args = (make_simplified_union(items),)
 
 
 class _NamedTypeCallback(Protocol):

mypy/typeops.py

@@ -95,8 +95,6 @@ def is_recursive_pair(s: Type, t: Type) -> bool:
 
 def tuple_fallback(typ: TupleType) -> Instance:
     """Return fallback type for a tuple."""
-    from mypy.join import join_type_list
-
     info = typ.partial_fallback.type
     if info.fullname != "builtins.tuple":
         return typ.partial_fallback
@@ -115,8 +113,9 @@ def tuple_fallback(typ: TupleType) -> Instance:
                 raise NotImplementedError
         else:
             items.append(item)
-    # TODO: we should really use a union here, tuple types are special.
-    return Instance(info, [join_type_list(items)], extra_attrs=typ.partial_fallback.extra_attrs)
+    return Instance(
+        info, [make_simplified_union(items)], extra_attrs=typ.partial_fallback.extra_attrs
+    )
 
 
 def get_self_type(func: CallableType, default_self: Instance | TupleType) -> Type | None:

test-data/unit/check-enum.test

@@ -1010,7 +1010,7 @@ _empty: Final = Empty.token
 def func(x: Union[int, None, Empty] = _empty) -> int:
     boom = x + 42       # E: Unsupported left operand type for + ("None") \
                         # E: Unsupported left operand type for + ("Empty") \
-                        # N: Left operand is of type "Union[int, None, Empty]"
+                        # N: Left operand is of type "Union[int, Empty, None]"
     if x is _empty:
         reveal_type(x)  # N: Revealed type is "Literal[__main__.Empty.token]"
         return 0
@@ -1056,7 +1056,7 @@ _empty = Empty.token
 def func(x: Union[int, None, Empty] = _empty) -> int:
     boom = x + 42       # E: Unsupported left operand type for + ("None") \
                         # E: Unsupported left operand type for + ("Empty") \
-                        # N: Left operand is of type "Union[int, None, Empty]"
+                        # N: Left operand is of type "Union[int, Empty, None]"
     if x is _empty:
         reveal_type(x)  # N: Revealed type is "Literal[__main__.Empty.token]"
         return 0
@@ -1084,7 +1084,7 @@ _empty = Empty.token
 def func(x: Union[int, None, Empty] = _empty) -> int:
     boom = x + 42       # E: Unsupported left operand type for + ("None") \
                         # E: Unsupported left operand type for + ("Empty") \
-                        # N: Left operand is of type "Union[int, None, Empty]"
+                        # N: Left operand is of type "Union[int, Empty, None]"
     if x is _empty:
         reveal_type(x)  # N: Revealed type is "Literal[__main__.Empty.token]"
         return 0

test-data/unit/check-expressions.test

@@ -1640,7 +1640,7 @@ from typing import Generator
 def g() -> Generator[int, None, None]:
     x = yield from ()  # E: Function does not return a value (it only ever returns None)
     x = yield from (0, 1, 2)  # E: Function does not return a value (it only ever returns None)
-    x = yield from (0, "ERROR")  # E: Incompatible types in "yield from" (actual type "object", expected type "int") \
+    x = yield from (0, "ERROR")  # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int") \
                                  # E: Function does not return a value (it only ever returns None)
     x = yield from ("ERROR",)  # E: Incompatible types in "yield from" (actual type "str", expected type "int") \
                                # E: Function does not return a value (it only ever returns None)

test-data/unit/check-namedtuple.test

@@ -1249,7 +1249,7 @@ nti: NT[int]
 reveal_type(nti * x)  # N: Revealed type is "builtins.tuple[builtins.int, ...]"
 
 nts: NT[str]
-reveal_type(nts * x)  # N: Revealed type is "builtins.tuple[builtins.object, ...]"
+reveal_type(nts * x)  # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]"
 [builtins fixtures/tuple.pyi]
 [typing fixtures/typing-namedtuple.pyi]
 
@@ -1310,9 +1310,9 @@ reveal_type(foo(nti, nts))  # N: Revealed type is "Tuple[builtins.int, builtins.
 reveal_type(foo(nts, nti))  # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=__main__.NT[builtins.object]]"
 
 reveal_type(foo(nti, x))  # N: Revealed type is "builtins.tuple[builtins.int, ...]"
-reveal_type(foo(nts, x))  # N: Revealed type is "builtins.tuple[builtins.object, ...]"
+reveal_type(foo(nts, x))  # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]"
 reveal_type(foo(x, nti))  # N: Revealed type is "builtins.tuple[builtins.int, ...]"
-reveal_type(foo(x, nts))  # N: Revealed type is "builtins.tuple[builtins.object, ...]"
+reveal_type(foo(x, nts))  # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]"
 [builtins fixtures/tuple.pyi]
 [typing fixtures/typing-namedtuple.pyi]
 

test-data/unit/check-newsemanal.test

@@ -1947,7 +1947,7 @@ class NTStr(NamedTuple):
     y: str
 
 t1: T
-reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[__main__.A]"
+reveal_type(t1.__iter__) # N: Revealed type is "def () -> typing.Iterator[Union[__main__.B, __main__.C]]"
 
 t2: NTInt
 reveal_type(t2.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.int]"
@@ -1960,7 +1960,6 @@ t: Union[Tuple[int, int], Tuple[str, str]]
 for x in t:
     reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]"
 [builtins fixtures/for.pyi]
-[out]
 
 [case testNewAnalyzerFallbackUpperBoundCheckAndFallbacks]
 from typing import TypeVar, Generic, Tuple
@@ -1973,18 +1972,17 @@ S = TypeVar('S', bound='Tuple[G[A], ...]')
 
 class GG(Generic[S]): pass
 
-g: GG[Tuple[G[B], G[C]]] \
-  # E: Type argument "Tuple[G[B], G[C]]" of "GG" must be a subtype of "Tuple[G[A], ...]" \
-  # E: Type argument "B" of "G" must be a subtype of "A" \
-  # E: Type argument "C" of "G" must be a subtype of "A"
+g: GG[Tuple[G[B], G[C]]] # E: Type argument "Tuple[G[B], G[C]]" of "GG" must be a subtype of "Tuple[G[A], ...]" \
+                         # E: Type argument "B" of "G" must be a subtype of "A" \
+                         # E: Type argument "C" of "G" must be a subtype of "A"
 
 T = TypeVar('T', bound=A, covariant=True)
 
 class G(Generic[T]): pass
 
 t: Tuple[G[B], G[C]] # E: Type argument "B" of "G" must be a subtype of "A" \
                      # E: Type argument "C" of "G" must be a subtype of "A"
-reveal_type(t.__iter__) # N: Revealed type is "def () -> typing.Iterator[builtins.object]"
+reveal_type(t.__iter__) # N: Revealed type is "def () -> typing.Iterator[__main__.G[__main__.B]]"
 [builtins fixtures/tuple.pyi]
 
 [case testNewAnalyzerClassKeywordsForward]

test-data/unit/check-statements.test

@@ -1339,7 +1339,7 @@ from typing import Generator
 def g() -> Generator[int, None, None]:
     yield from ()
     yield from (0, 1, 2)
-    yield from (0, "ERROR")  # E: Incompatible types in "yield from" (actual type "object", expected type "int")
+    yield from (0, "ERROR")  # E: Incompatible types in "yield from" (actual type "Union[int, str]", expected type "int")
     yield from ("ERROR",)  # E: Incompatible types in "yield from" (actual type "str", expected type "int")
 [builtins fixtures/tuple.pyi]
 

test-data/unit/check-tuples.test

@@ -1408,8 +1408,8 @@ y = ""
 reveal_type(t[x])  # N: Revealed type is "Union[builtins.int, builtins.str]"
 t[y]  # E: No overload variant of "__getitem__" of "tuple" matches argument type "str" \
       # N: Possible overload variants: \
-      # N:     def __getitem__(self, int, /) -> object \
-      # N:     def __getitem__(self, slice, /) -> Tuple[object, ...]
+      # N:     def __getitem__(self, int, /) -> Union[int, str] \
+      # N:     def __getitem__(self, slice, /) -> Tuple[Union[int, str], ...]
 
 [builtins fixtures/tuple.pyi]
 

test-data/unit/check-typevar-tuple.test

@@ -24,7 +24,7 @@ def g(a: Tuple[Unpack[Ts]], b: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]:
 
 reveal_type(g(args, args))  # N: Revealed type is "Tuple[builtins.int, builtins.str]"
 reveal_type(g(args, args2))  # N: Revealed type is "Tuple[builtins.int, builtins.str]"
-reveal_type(g(args, args3))  # N: Revealed type is "builtins.tuple[builtins.object, ...]"
+reveal_type(g(args, args3))  # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.str], ...]"
 reveal_type(g(any, any))  # N: Revealed type is "builtins.tuple[Any, ...]"
 [builtins fixtures/tuple.pyi]
 
@@ -989,7 +989,7 @@ from typing_extensions import Unpack
 
 def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[float, ...]], bool]]) -> None:
     for x in xs:
-        reveal_type(x)  # N: Revealed type is "builtins.float"
+        reveal_type(x)  # N: Revealed type is "Union[builtins.int, builtins.float]"
 [builtins fixtures/tuple.pyi]
 
 [case testFixedUnpackItemInInstanceArguments]
@@ -1715,7 +1715,7 @@ vt: Tuple[int, Unpack[Tuple[float, ...]], int]
 
 reveal_type(vt + (1, 2))  # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int, Literal[1]?, Literal[2]?]"
 reveal_type((1, 2) + vt)  # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, builtins.int, Unpack[builtins.tuple[builtins.float, ...]], builtins.int]"
-reveal_type(vt + vt)  # N: Revealed type is "builtins.tuple[builtins.float, ...]"
+reveal_type(vt + vt)  # N: Revealed type is "builtins.tuple[Union[builtins.int, builtins.float], ...]"
 reveal_type(vtf + (1, 2))  # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.float, ...]], Literal[1]?, Literal[2]?]"
 reveal_type((1, 2) + vtf)  # N: Revealed type is "Tuple[Literal[1]?, Literal[2]?, Unpack[builtins.tuple[builtins.float, ...]]]"