From 9daef788e8b511300d1f3006adff926aa7d400c1 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Sun, 16 Jul 2023 16:37:53 +0100 Subject: [PATCH 1/5] Stubtest: Fix `__mypy-replace` false positives --- mypy/nodes.py | 7 +++++++ mypy/plugins/common.py | 2 ++ mypy/plugins/dataclasses.py | 2 ++ mypy/stubtest.py | 6 ++++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 2d763fc482d3..139c2889bfef 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3746,6 +3746,10 @@ class SymbolTableNode: implicit: Was this defined by assignment to self attribute? plugin_generated: Was this symbol generated by a plugin? (And therefore needs to be removed in aststrip.) + fictional_plugin_generated: If it was generated by a plugin, + does it actually exist at runtime? + (We generate some fictional methods for e.g. dataclasses; stubtest + needs to be able to know if they actually exist at runtime or not.) no_serialize: Do not serialize this node if True. This is used to prevent keys in the cache that refer to modules on which this file does not depend. Currently this can happen if there is a module not in build @@ -3771,6 +3775,7 @@ class SymbolTableNode: "cross_ref", "implicit", "plugin_generated", + "fictional_plugin_generated", "no_serialize", ) @@ -3783,6 +3788,7 @@ def __init__( module_hidden: bool = False, *, plugin_generated: bool = False, + fictional_plugin_generated: bool = False, no_serialize: bool = False, ) -> None: self.kind = kind @@ -3792,6 +3798,7 @@ def __init__( self.module_hidden = module_hidden self.cross_ref: str | None = None self.plugin_generated = plugin_generated + self.fictional_plugin_generated = fictional_plugin_generated self.no_serialize = no_serialize @property diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 65d967577bea..881ea6189516 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -219,6 +219,7 @@ def add_method_to_class( tvar_def: TypeVarType | None = None, is_classmethod: bool = False, is_staticmethod: bool = False, + fictional_plugin_generated: bool = False, ) -> None: """Adds a new method to a class definition.""" @@ -289,6 +290,7 @@ def add_method_to_class( else: sym = SymbolTableNode(MDEF, func) sym.plugin_generated = True + sym.fictional_plugin_generated = fictional_plugin_generated info.names[name] = sym info.defn.defs.body.append(func) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index a4babe7faf61..36da62f19eb4 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -401,6 +401,7 @@ def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> args=[attr.to_argument(self._cls.info, of="replace") for attr in attributes], return_type=NoneType(), is_staticmethod=True, + fictional_plugin_generated=True, ) def _add_internal_post_init_method(self, attributes: list[DataclassAttribute]) -> None: @@ -414,6 +415,7 @@ def _add_internal_post_init_method(self, attributes: list[DataclassAttribute]) - if attr.is_init_var ], return_type=NoneType(), + fictional_plugin_generated=True, ) def add_slots( diff --git a/mypy/stubtest.py b/mypy/stubtest.py index f06faa962b07..b58f6a74e250 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -495,8 +495,10 @@ def verify_typeinfo( stub, runtime, object_path, is_runtime_typeddict=is_runtime_typeddict ) - # Check everything already defined on the stub class itself (i.e. not inherited) - to_check = set(stub.names) + # Check everything already defined on the stub class itself (i.e. not inherited), + # as long as it wasn't a fictional method generated by a plugin + # (__mypy-replace, __mypy-post_init, etc.) + to_check = {name for name, node in stub.names.items() if not node.fictional_plugin_generated} # Check all public things on the runtime class to_check.update( m for m in vars(runtime) if not is_probably_private(m) and m not in IGNORABLE_CLASS_DUNDERS From b1c008b8b420953258df425c0c842c4919181189 Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Mon, 17 Jul 2023 00:00:59 +0100 Subject: [PATCH 2/5] Rename flag to `internal_fictional` --- mypy/nodes.py | 12 ++++++------ mypy/plugins/common.py | 4 ++-- mypy/plugins/dataclasses.py | 4 ++-- mypy/stubtest.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 139c2889bfef..b93808531e39 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3746,10 +3746,10 @@ class SymbolTableNode: implicit: Was this defined by assignment to self attribute? plugin_generated: Was this symbol generated by a plugin? (And therefore needs to be removed in aststrip.) - fictional_plugin_generated: If it was generated by a plugin, - does it actually exist at runtime? - (We generate some fictional methods for e.g. dataclasses; stubtest - needs to be able to know if they actually exist at runtime or not.) + internal_fictional: Flag to indicate that this is a node for internal + use only; it doesn't represent a method/attribute that actually + exists at runtime. (We generate some fictional methods for e.g. + dataclasses; stubtest needs to be able to spot these.) no_serialize: Do not serialize this node if True. This is used to prevent keys in the cache that refer to modules on which this file does not depend. Currently this can happen if there is a module not in build @@ -3775,7 +3775,7 @@ class SymbolTableNode: "cross_ref", "implicit", "plugin_generated", - "fictional_plugin_generated", + "internal_fictional", "no_serialize", ) @@ -3798,7 +3798,7 @@ def __init__( self.module_hidden = module_hidden self.cross_ref: str | None = None self.plugin_generated = plugin_generated - self.fictional_plugin_generated = fictional_plugin_generated + self.internal_fictional = internal_fictional self.no_serialize = no_serialize @property diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 881ea6189516..4544ccaa9cd6 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -219,7 +219,7 @@ def add_method_to_class( tvar_def: TypeVarType | None = None, is_classmethod: bool = False, is_staticmethod: bool = False, - fictional_plugin_generated: bool = False, + internal_fictional: bool = False, ) -> None: """Adds a new method to a class definition.""" @@ -290,7 +290,7 @@ def add_method_to_class( else: sym = SymbolTableNode(MDEF, func) sym.plugin_generated = True - sym.fictional_plugin_generated = fictional_plugin_generated + sym.internal_fictional = internal_fictional info.names[name] = sym info.defn.defs.body.append(func) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 36da62f19eb4..506a332d976c 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -401,7 +401,7 @@ def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> args=[attr.to_argument(self._cls.info, of="replace") for attr in attributes], return_type=NoneType(), is_staticmethod=True, - fictional_plugin_generated=True, + internal_fictional=True, ) def _add_internal_post_init_method(self, attributes: list[DataclassAttribute]) -> None: @@ -415,7 +415,7 @@ def _add_internal_post_init_method(self, attributes: list[DataclassAttribute]) - if attr.is_init_var ], return_type=NoneType(), - fictional_plugin_generated=True, + internal_fictional=True, ) def add_slots( diff --git a/mypy/stubtest.py b/mypy/stubtest.py index b58f6a74e250..19d220488232 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -498,7 +498,7 @@ def verify_typeinfo( # Check everything already defined on the stub class itself (i.e. not inherited), # as long as it wasn't a fictional method generated by a plugin # (__mypy-replace, __mypy-post_init, etc.) - to_check = {name for name, node in stub.names.items() if not node.fictional_plugin_generated} + to_check = {name for name, node in stub.names.items() if not node.internal_fictional} # Check all public things on the runtime class to_check.update( m for m in vars(runtime) if not is_probably_private(m) and m not in IGNORABLE_CLASS_DUNDERS From 54347e5258ee52eaa2c38fc3c83924eef4d53dda Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 17 Jul 2023 00:02:14 +0100 Subject: [PATCH 3/5] Update mypy/nodes.py --- mypy/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index b93808531e39..4ab1a9d79212 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3788,7 +3788,7 @@ def __init__( module_hidden: bool = False, *, plugin_generated: bool = False, - fictional_plugin_generated: bool = False, + internal_fictional: bool = False, no_serialize: bool = False, ) -> None: self.kind = kind From e83b124bf8f305c9dd4e6bc27bb4990179896fcd Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Sat, 22 Jul 2023 20:28:22 +0100 Subject: [PATCH 4/5] Revert alll changes --- mypy/nodes.py | 7 ------- mypy/plugins/common.py | 2 -- mypy/plugins/dataclasses.py | 2 -- mypy/stubtest.py | 6 ++---- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 4ab1a9d79212..2d763fc482d3 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3746,10 +3746,6 @@ class SymbolTableNode: implicit: Was this defined by assignment to self attribute? plugin_generated: Was this symbol generated by a plugin? (And therefore needs to be removed in aststrip.) - internal_fictional: Flag to indicate that this is a node for internal - use only; it doesn't represent a method/attribute that actually - exists at runtime. (We generate some fictional methods for e.g. - dataclasses; stubtest needs to be able to spot these.) no_serialize: Do not serialize this node if True. This is used to prevent keys in the cache that refer to modules on which this file does not depend. Currently this can happen if there is a module not in build @@ -3775,7 +3771,6 @@ class SymbolTableNode: "cross_ref", "implicit", "plugin_generated", - "internal_fictional", "no_serialize", ) @@ -3788,7 +3783,6 @@ def __init__( module_hidden: bool = False, *, plugin_generated: bool = False, - internal_fictional: bool = False, no_serialize: bool = False, ) -> None: self.kind = kind @@ -3798,7 +3792,6 @@ def __init__( self.module_hidden = module_hidden self.cross_ref: str | None = None self.plugin_generated = plugin_generated - self.internal_fictional = internal_fictional self.no_serialize = no_serialize @property diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 4544ccaa9cd6..65d967577bea 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -219,7 +219,6 @@ def add_method_to_class( tvar_def: TypeVarType | None = None, is_classmethod: bool = False, is_staticmethod: bool = False, - internal_fictional: bool = False, ) -> None: """Adds a new method to a class definition.""" @@ -290,7 +289,6 @@ def add_method_to_class( else: sym = SymbolTableNode(MDEF, func) sym.plugin_generated = True - sym.internal_fictional = internal_fictional info.names[name] = sym info.defn.defs.body.append(func) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 506a332d976c..a4babe7faf61 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -401,7 +401,6 @@ def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> args=[attr.to_argument(self._cls.info, of="replace") for attr in attributes], return_type=NoneType(), is_staticmethod=True, - internal_fictional=True, ) def _add_internal_post_init_method(self, attributes: list[DataclassAttribute]) -> None: @@ -415,7 +414,6 @@ def _add_internal_post_init_method(self, attributes: list[DataclassAttribute]) - if attr.is_init_var ], return_type=NoneType(), - internal_fictional=True, ) def add_slots( diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 19d220488232..f06faa962b07 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -495,10 +495,8 @@ def verify_typeinfo( stub, runtime, object_path, is_runtime_typeddict=is_runtime_typeddict ) - # Check everything already defined on the stub class itself (i.e. not inherited), - # as long as it wasn't a fictional method generated by a plugin - # (__mypy-replace, __mypy-post_init, etc.) - to_check = {name for name, node in stub.names.items() if not node.internal_fictional} + # Check everything already defined on the stub class itself (i.e. not inherited) + to_check = set(stub.names) # Check all public things on the runtime class to_check.update( m for m in vars(runtime) if not is_probably_private(m) and m not in IGNORABLE_CLASS_DUNDERS From d9ca3325f1aa492dbf500e392e760cf48829b55c Mon Sep 17 00:00:00 2001 From: AlexWaygood Date: Sat, 22 Jul 2023 20:31:43 +0100 Subject: [PATCH 5/5] Just filter out all non-identifiers from stubtest names to check --- mypy/stubtest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index f06faa962b07..906a8c923b37 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -496,7 +496,11 @@ def verify_typeinfo( ) # Check everything already defined on the stub class itself (i.e. not inherited) - to_check = set(stub.names) + # + # Filter out non-identifier names, as these are (hopefully always?) whacky/fictional things + # (like __mypy-replace or __mypy-post_init, etc.) that don't exist at runtime, + # and exist purely for internal mypy reasons + to_check = {name for name in stub.names if name.isidentifier()} # Check all public things on the runtime class to_check.update( m for m in vars(runtime) if not is_probably_private(m) and m not in IGNORABLE_CLASS_DUNDERS