Implement support for closed TypedDicts (PEP 728)

mypy/checker.py

@@ -6391,7 +6391,8 @@ def conditional_types_for_iterable(
     ) -> tuple[Type, Type]:
         """
         Narrows the type of `iterable_type` based on the type of `item_type`.
-        For now, we only support narrowing unions of TypedDicts based on left operand being literal string(s).
+        For now, we only support narrowing unions of TypedDicts, and TypeVars with TypedDict
+        bounds, based on left operand being literal string(s).
         """
         if_types: list[Type] = []
         else_types: list[Type] = []
@@ -6405,13 +6406,20 @@ def conditional_types_for_iterable(
         item_str_literals = try_getting_str_literals_from_type(item_type)
 
         for possible_iterable_type in possible_iterable_types:
-            if item_str_literals and isinstance(possible_iterable_type, TypedDictType):
+            bound = (
+                get_proper_type(possible_iterable_type.upper_bound)
+                if isinstance(possible_iterable_type, TypeVarType)
+                else possible_iterable_type
+            )
+
+            if item_str_literals and isinstance(bound, TypedDictType):
                 for key in item_str_literals:
-                    if key in possible_iterable_type.required_keys:
+                    if key in bound.required_keys:
                         if_types.append(possible_iterable_type)
                     elif (
-                        key in possible_iterable_type.items or not possible_iterable_type.is_final
-                    ):
+                        key in bound.items
+                        and not isinstance(get_proper_type(bound.items[key]), UninhabitedType)
+                    ) or (key not in bound.items and not bound.is_closed and not bound.is_final):
                         if_types.append(possible_iterable_type)
                         else_types.append(possible_iterable_type)
                     else:

mypy/checkexpr.py

@@ -820,8 +820,8 @@ def validate_typeddict_kwargs(
         result = defaultdict(list)
         # Keys that are guaranteed to be present no matter what (e.g. for all items of a union)
         always_present_keys = set()
-        # Indicates latest encountered ** unpack among items.
-        last_star_found = None
+        # Indicates latest encountered ** unpack of a non-closed type among items.
+        last_open_star_found = None
 
         for item_name_expr, item_arg in kwargs:
             if item_name_expr:
@@ -845,22 +845,30 @@ def validate_typeddict_kwargs(
                     result[literal_value] = [item_arg]
                     always_present_keys.add(literal_value)
             else:
-                last_star_found = item_arg
-                if not self.validate_star_typeddict_item(
+                is_valid, is_open = self.validate_star_typeddict_item(
                     item_arg, callee, result, always_present_keys
-                ):
+                )
+                if not is_valid:
                     return None
-        if self.chk.options.extra_checks and last_star_found is not None:
+                if is_open:
+                    last_open_star_found = item_arg
+        if self.chk.options.extra_checks and last_open_star_found is not None:
+            if callee.is_closed:
+                self.chk.fail(
+                    "Cannot unpack item that may contain extra keys into a closed TypedDict",
+                    last_open_star_found,
+                    code=codes.TYPEDDICT_ITEM,
+                )
             absent_keys = []
             for key in callee.items:
                 if key not in callee.required_keys and key not in result:
                     absent_keys.append(key)
             if absent_keys:
-                # Having an optional key not explicitly declared by a ** unpacked
+                # Having an optional key not explicitly declared by a ** unpacked open
                 # TypedDict is unsafe, it may be an (incompatible) subtype at runtime.
                 # TODO: catch the cases where a declared key is overridden by a subsequent
                 # ** item without it (and not again overridden with complete ** item).
-                self.msg.non_required_keys_absent_with_star(absent_keys, last_star_found)
+                self.msg.non_required_keys_absent_with_star(absent_keys, last_open_star_found)
         return result, always_present_keys
 
     def validate_star_typeddict_item(
@@ -869,14 +877,18 @@ def validate_star_typeddict_item(
         callee: TypedDictType,
         result: dict[str, list[Expression]],
         always_present_keys: set[str],
-    ) -> bool:
+    ) -> tuple[bool, bool]:
         """Update keys/expressions from a ** expression in TypedDict constructor.
 
-        Note `result` and `always_present_keys` are updated in place. Return true if the
-        expression `item_arg` may valid in `callee` TypedDict context.
+        Note `result` and `always_present_keys` are updated in place.
+
+        First tuple item returned is true if the expression `item_arg` may valid
+        in `callee` TypedDict context. Second tuple item returned is true if the
+        expression may contain other keys not explicitly declared.
         """
         inferred = get_proper_type(self.accept(item_arg, type_context=callee))
-        possible_tds = []
+        any_fallback = False
+        possible_tds: list[TypedDictType] = []
         if isinstance(inferred, TypedDictType):
             possible_tds = [inferred]
         elif isinstance(inferred, UnionType):
@@ -885,10 +897,14 @@ def validate_star_typeddict_item(
                     possible_tds.append(item)
                 elif not self.valid_unpack_fallback_item(item):
                     self.msg.unsupported_target_for_star_typeddict(item, item_arg)
-                    return False
+                    return False, True
+                else:
+                    any_fallback = True
         elif not self.valid_unpack_fallback_item(inferred):
             self.msg.unsupported_target_for_star_typeddict(inferred, item_arg)
-            return False
+            return False, True
+        else:
+            any_fallback = True
         all_keys: set[str] = set()
         for td in possible_tds:
             all_keys |= td.items.keys()
@@ -917,7 +933,8 @@ def validate_star_typeddict_item(
                 # If this key is not required at least in some item of a union
                 # it may not shadow previous item, so we need to type check both.
                 result[key].append(arg)
-        return True
+        all_closed = all(t.is_closed for t in possible_tds)
+        return True, any_fallback or not all_closed
 
     def valid_unpack_fallback_item(self, typ: ProperType) -> bool:
         if isinstance(typ, AnyType):

mypy/copytype.py

@@ -107,7 +107,7 @@ def visit_tuple_type(self, t: TupleType) -> ProperType:
 
     def visit_typeddict_type(self, t: TypedDictType) -> ProperType:
         return self.copy_common(
-            t, TypedDictType(t.items, t.required_keys, t.readonly_keys, t.fallback)
+            t, TypedDictType(t.items, t.required_keys, t.readonly_keys, t.is_closed, t.fallback)
         )
 
     def visit_literal_type(self, t: LiteralType) -> ProperType:

mypy/expandtype.py

@@ -344,7 +344,7 @@ def _possible_callable_kwargs(cls, repl: Parameters, dict_type: Instance) -> Pro
             return dict_type
         kwargs = {}
         required_names = set()
-        extra_items: Type = UninhabitedType()
+        extra_items: Type | None = None
         for kind, name, type in zip(repl.arg_kinds, repl.arg_names, repl.arg_types):
             if kind == ArgKind.ARG_NAMED and name is not None:
                 kwargs[name] = type
@@ -354,10 +354,11 @@ def _possible_callable_kwargs(cls, repl: Parameters, dict_type: Instance) -> Pro
                 extra_items = type
             elif not kind.is_star() and name is not None:
                 kwargs[name] = type
-        if not kwargs:
+        if not kwargs and extra_items is not None:
             return Instance(dict_type.type, [dict_type.args[0], extra_items])
-        # TODO: when PEP 728 is implemented, pass extra_items below.
-        return TypedDictType(kwargs, required_names, set(), fallback=dict_type)
+        # TODO: when PEP 728 `extra_items` is implemented, pass extra_items below.
+        is_closed = extra_items is None
+        return TypedDictType(kwargs, required_names, set(), is_closed, fallback=dict_type)
 
     def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type:
         # Sometimes solver may need to expand a type variable with (a copy of) itself

mypy/exprtotype.py

@@ -277,7 +277,7 @@ def expr_to_unanalyzed_type(
                 value, options, allow_new_syntax, expr, lookup_qualified=lookup_qualified
             )
         result = TypedDictType(
-            items, set(), set(), Instance(MISSING_FALLBACK, ()), expr.line, expr.column
+            items, set(), set(), False, Instance(MISSING_FALLBACK, ()), expr.line, expr.column
         )
         result.extra_items_from = extra_items_from
         return result

mypy/fastparse.py

@@ -2143,7 +2143,7 @@ def visit_Dict(self, n: ast3.Dict) -> Type:
                     continue
                 return self.invalid_type(n)
             items[item_name.value] = self.visit(value)
-        result = TypedDictType(items, set(), set(), _dummy_fallback, n.lineno, n.col_offset)
+        result = TypedDictType(items, set(), set(), False, _dummy_fallback, n.lineno, n.col_offset)
         result.extra_items_from = extra_items_from
         return result
 

mypy/join.py

@@ -610,24 +610,78 @@ def visit_tuple_type(self, t: TupleType) -> ProperType:
         else:
             return join_types(self.s, mypy.typeops.tuple_fallback(t))
 
+    def resolve_typeddict_item(
+        self,
+        item_name: str,
+        s: TypedDictType,
+        s_item_type: Type | None,
+        t: TypedDictType,
+        t_item_type: Type | None,
+    ) -> tuple[Type | None, bool, bool]:
+        """Return the type, requiredness, and readonlyness of a join item.
+
+        If the type is None, the item should be omitted from the join keys.
+        """
+        if s_item_type is None or t_item_type is None:
+            is_required = False
+            if (s_item_type is None and not s.is_closed) or (
+                t_item_type is None and not t.is_closed
+            ):
+                # Key is implicitly NotRequired[ReadOnly[object]] in one type and
+                # (object join T) == object. Omitting it in the join leaves it implicitly
+                # of object type.
+                join_type = None
+            else:
+                # Key is implicitly NotRequired[Never] in one type and (Never join T) == T
+                join_type = s_item_type if t_item_type is None else t_item_type
+            # One type is missing the setitem overload for this key, so the join supertype
+            # must also omit it
+            is_readonly = True
+        else:
+            s_required = item_name in s.required_keys
+            t_required = item_name in t.required_keys
+            is_required = s_required and t_required
+
+            if not is_equivalent(s_item_type, t_item_type):
+                join_type = join_types(s_item_type, t_item_type)
+                is_readonly = True
+            else:
+                join_type = s_item_type
+                s_readonly = item_name in s.readonly_keys
+                t_readonly = item_name in t.readonly_keys
+                if s_required != t_required:
+                    # As one of the input types marks the key as not required, it must
+                    # be not required in the join supertype. However, as the other input
+                    # type does not have a delitem overload for the key, the delitem
+                    # overload must be omitted in the join supertype too. This can only
+                    # be done by marking the key as readonly.
+                    is_readonly = True
+                else:
+                    # If either type has no setitem overload for this key,
+                    # then the join supertype must also omit it
+                    is_readonly = s_readonly or t_readonly
+
+        return join_type, is_required, is_readonly
+
     def visit_typeddict_type(self, t: TypedDictType) -> ProperType:
         if isinstance(self.s, TypedDictType):
-            items = {
-                item_name: s_item_type
-                for (item_name, s_item_type, t_item_type) in self.s.zip(t)
-                if (
-                    is_equivalent(s_item_type, t_item_type)
-                    and (item_name in t.required_keys) == (item_name in self.s.required_keys)
+            items = {}
+            required_keys = set()
+            readonly_keys = set()
+            for item_name, s_item_type, t_item_type in self.s.zipall(t):
+                item_type, is_required, is_readonly = self.resolve_typeddict_item(
+                    item_name, self.s, s_item_type, t, t_item_type
                 )
-            }
+                if item_type is not None:
+                    items[item_name] = item_type
+                    if is_required:
+                        required_keys.add(item_name)
+                    if is_readonly:
+                        readonly_keys.add(item_name)
+
             fallback = self.s.create_anonymous_fallback()
-            all_keys = set(items.keys())
-            # We need to filter by items.keys() since some required keys present in both t and
-            # self.s might be missing from the join if the types are incompatible.
-            required_keys = all_keys & t.required_keys & self.s.required_keys
-            # If one type has a key as readonly, we mark it as readonly for both:
-            readonly_keys = (t.readonly_keys | t.readonly_keys) & all_keys
-            return TypedDictType(items, required_keys, readonly_keys, fallback)
+            is_closed = self.s.is_closed and t.is_closed
+            return TypedDictType(items, required_keys, readonly_keys, is_closed, fallback)
         elif isinstance(self.s, Instance):
             return join_types(self.s, t.fallback)
         else:

mypy/meet.py

@@ -1100,26 +1100,97 @@ def visit_tuple_type(self, t: TupleType) -> ProperType:
                 return t
         return self.default(self.s)
 
+    def resolve_typeddict_item_type(
+        self,
+        name: str,
+        s: TypedDictType,
+        s_item_type: Type | None,
+        t: TypedDictType,
+        t_item_type: Type | None,
+    ) -> tuple[Type | None, bool]:
+        """Return the type and readonlyness of a meet item.
+
+        If the parent constraints are mutually incompatible, the
+        returned type will be None; the overall meet type should
+        be UninhabitedType.
+        """
+        # A missing key is implicitly ReadOnly[NotRequired[...]]
+        l_mutable = s_item_type is not None and name not in s.readonly_keys
+        r_mutable = t_item_type is not None and name not in t.readonly_keys
+        l_required = name in s.required_keys
+        r_required = name in t.required_keys
+
+        is_readonly = not l_mutable and not r_mutable
+
+        # Simplify the logic by using Never instead of None for missing keys
+        # in closed TypedDicts. Ideally we'd use builtins.object instead of
+        # None in open TypedDicts, but that is hard to get.
+        if s_item_type is None and s.is_closed:
+            s_item_type = UninhabitedType()
+        if t_item_type is None and t.is_closed:
+            t_item_type = UninhabitedType()
+
+        if t_item_type is None:
+            assert s_item_type is not None
+            meet_type = s_item_type
+        elif s_item_type is None:
+            meet_type = t_item_type
+        elif l_mutable and r_mutable:
+            if is_equivalent(s_item_type, t_item_type) and r_required == l_required:
+                meet_type = s_item_type
+            else:
+                meet_type = None
+        elif l_mutable:
+            if is_subtype(s_item_type, t_item_type) and not (r_required and not l_required):
+                meet_type = s_item_type
+            else:
+                meet_type = None
+        elif r_mutable:
+            if is_subtype(t_item_type, s_item_type) and not (l_required and not r_required):
+                meet_type = t_item_type
+            else:
+                meet_type = None
+        else:
+            m = meet_types(s_item_type, t_item_type)
+            if not isinstance(m, UninhabitedType):
+                meet_type = m
+            elif not l_required and not r_required:
+                # A non-required key can be Never
+                meet_type = m
+            else:
+                meet_type = None
+
+        return (meet_type, is_readonly)
+
     def visit_typeddict_type(self, t: TypedDictType) -> ProperType:
         if isinstance(self.s, TypedDictType):
-            for name, l, r in self.s.zip(t):
-                if not is_equivalent(l, r) or (name in t.required_keys) != (
-                    name in self.s.required_keys
-                ):
+            is_closed = self.s.is_closed or t.is_closed
+            items: dict[str, Type] = {}
+            readonly_keys: set[str] = set()
+            for name, s_item_type, t_item_type in self.s.zipall(t):
+                meet_type, is_readonly = self.resolve_typeddict_item_type(
+                    name, self.s, s_item_type, t, t_item_type
+                )
+
+                if meet_type is None:
                     return self.default(self.s)
-            item_list: list[tuple[str, Type]] = []
-            for item_name, s_item_type, t_item_type in self.s.zipall(t):
-                if s_item_type is not None:
-                    item_list.append((item_name, s_item_type))
-                else:
-                    # at least one of s_item_type and t_item_type is not None
-                    assert t_item_type is not None
-                    item_list.append((item_name, t_item_type))
-            items = dict(item_list)
+
+                if (
+                    is_closed
+                    and is_readonly
+                    and isinstance(get_proper_type(meet_type), UninhabitedType)
+                ):
+                    # Simplify emitted type by omitting redundant ReadOnly[Never] keys
+                    # from closed TypedDicts
+                    continue
+
+                items[name] = meet_type
+                if is_readonly:
+                    readonly_keys.add(name)
+
             fallback = self.s.create_anonymous_fallback()
-            required_keys = t.required_keys | self.s.required_keys
-            readonly_keys = t.readonly_keys | self.s.readonly_keys
-            return TypedDictType(items, required_keys, readonly_keys, fallback)
+            required_keys = self.s.required_keys | t.required_keys
+            return TypedDictType(items, required_keys, readonly_keys, is_closed, fallback)
         elif isinstance(self.s, Instance) and is_subtype(t, self.s):
             return t
         else:

mypy/nativeparse.py

@@ -926,7 +926,7 @@ def read_type(state: State, data: ReadBuffer) -> Type:
                 extra_items_from.append(val)
             else:
                 td_items[key] = val
-        typeddict_type = TypedDictType(td_items, set(), set(), _dummy_fallback)
+        typeddict_type = TypedDictType(td_items, set(), set(), False, _dummy_fallback)
         typeddict_type.extra_items_from = extra_items_from
         read_loc(data, typeddict_type)
         expect_end_tag(data)