From b08c8b5737b3d4aa6f5f1e45c49dc1a5316679fc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 15:58:56 -0800 Subject: [PATCH 01/32] Sync typeshed (#16721) Source commit: https://github.com/python/typeshed/commit/1d3c3265180ae4421be4ce6508810708ccd617f0 Co-authored-by: mypybot <> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: AlexWaygood --- mypy/typeshed/stdlib/VERSIONS | 445 +++++++++--------- mypy/typeshed/stdlib/_ast.pyi | 4 +- mypy/typeshed/stdlib/_thread.pyi | 5 + mypy/typeshed/stdlib/asyncore.pyi | 1 + mypy/typeshed/stdlib/base64.pyi | 4 +- .../stdlib/concurrent/futures/process.pyi | 59 ++- .../stdlib/concurrent/futures/thread.pyi | 31 +- mypy/typeshed/stdlib/http/client.pyi | 18 + mypy/typeshed/stdlib/os/__init__.pyi | 7 +- mypy/typeshed/stdlib/selectors.pyi | 38 +- mypy/typeshed/stdlib/threading.pyi | 9 +- mypy/typeshed/stdlib/unittest/_log.pyi | 5 +- mypy/typeshed/stdlib/unittest/case.pyi | 23 +- .../{zipfile.pyi => zipfile/__init__.pyi} | 5 +- mypy/typeshed/stdlib/zipfile/_path.pyi | 95 ++++ 15 files changed, 470 insertions(+), 279 deletions(-) rename mypy/typeshed/stdlib/{zipfile.pyi => zipfile/__init__.pyi} (98%) create mode 100644 mypy/typeshed/stdlib/zipfile/_path.pyi diff --git a/mypy/typeshed/stdlib/VERSIONS b/mypy/typeshed/stdlib/VERSIONS index f5e30c4602b4..5099a94c93bc 100644 --- a/mypy/typeshed/stdlib/VERSIONS +++ b/mypy/typeshed/stdlib/VERSIONS @@ -1,7 +1,7 @@ # The structure of this file is as follows: # - Blank lines and comments starting with `#` are ignored. # - Lines contain the name of a module, followed by a colon, -# a space, and a version range (for example: `symbol: 2.7-3.9`). +# a space, and a version range (for example: `symbol: 3.0-3.9`). # # Version ranges may be of the form "X.Y-A.B" or "X.Y-". The # first form means that a module was introduced in version X.Y and last @@ -12,57 +12,57 @@ # If a submodule is not listed separately, it has the same lifetime as # its parent module. # -# Python versions before 2.7 are ignored, so any module that was already -# present in 2.7 will have "2.7" as its minimum version. Version ranges +# Python versions before 3.0 are ignored, so any module that was already +# present in 3.0 will have "3.0" as its minimum version. Version ranges # for unsupported versions of Python 3 are generally accurate but we do # not guarantee their correctness. -__future__: 2.7- -__main__: 2.7- -_ast: 2.7- -_bisect: 2.7- +__future__: 3.0- +__main__: 3.0- +_ast: 3.0- +_bisect: 3.0- _bootlocale: 3.4-3.9 -_codecs: 2.7- +_codecs: 3.0- _collections_abc: 3.3- _compat_pickle: 3.1- _compression: 3.5- -_csv: 2.7- -_ctypes: 2.7- -_curses: 2.7- +_csv: 3.0- +_ctypes: 3.0- +_curses: 3.0- _decimal: 3.3- _dummy_thread: 3.0-3.8 -_dummy_threading: 2.7-3.8 -_heapq: 2.7- +_dummy_threading: 3.0-3.8 +_heapq: 3.0- _imp: 3.0- -_json: 2.7- -_locale: 2.7- -_markupbase: 2.7- -_msi: 2.7- +_json: 3.0- +_locale: 3.0- +_markupbase: 3.0- +_msi: 3.0- _operator: 3.4- -_osx_support: 2.7- +_osx_support: 3.0- _posixsubprocess: 3.2- _py_abc: 3.7- _pydecimal: 3.5- -_random: 2.7- +_random: 3.0- _sitebuiltins: 3.4- -_socket: 3.0- # present in 2.7 at runtime, but not in typeshed +_socket: 3.0- # present in 3.0 at runtime, but not in typeshed _stat: 3.4- -_thread: 2.7- -_threading_local: 2.7- -_tkinter: 2.7- +_thread: 3.0- +_threading_local: 3.0- +_tkinter: 3.0- _tracemalloc: 3.4- -_typeshed: 2.7- # not present at runtime, only for type checking -_warnings: 2.7- -_weakref: 2.7- -_weakrefset: 2.7- +_typeshed: 3.0- # not present at runtime, only for type checking +_warnings: 3.0- +_weakref: 3.0- +_weakrefset: 3.0- _winapi: 3.3- -abc: 2.7- -aifc: 2.7- -antigravity: 2.7- -argparse: 2.7- -array: 2.7- -ast: 2.7- -asynchat: 2.7-3.11 +abc: 3.0- +aifc: 3.0-3.12 +antigravity: 3.0- +argparse: 3.0- +array: 3.0- +ast: 3.0- +asynchat: 3.0-3.11 asyncio: 3.4- asyncio.mixins: 3.10- asyncio.exceptions: 3.8- @@ -73,83 +73,83 @@ asyncio.taskgroups: 3.11- asyncio.threads: 3.9- asyncio.timeouts: 3.11- asyncio.trsock: 3.8- -asyncore: 2.7-3.11 -atexit: 2.7- -audioop: 2.7- -base64: 2.7- -bdb: 2.7- -binascii: 2.7- -binhex: 2.7-3.10 -bisect: 2.7- +asyncore: 3.0-3.11 +atexit: 3.0- +audioop: 3.0-3.12 +base64: 3.0- +bdb: 3.0- +binascii: 3.0- +binhex: 3.0-3.10 +bisect: 3.0- builtins: 3.0- -bz2: 2.7- -cProfile: 2.7- -calendar: 2.7- -cgi: 2.7- -cgitb: 2.7- -chunk: 2.7- -cmath: 2.7- -cmd: 2.7- -code: 2.7- -codecs: 2.7- -codeop: 2.7- -collections: 2.7- +bz2: 3.0- +cProfile: 3.0- +calendar: 3.0- +cgi: 3.0-3.12 +cgitb: 3.0-3.12 +chunk: 3.0-3.12 +cmath: 3.0- +cmd: 3.0- +code: 3.0- +codecs: 3.0- +codeop: 3.0- +collections: 3.0- collections.abc: 3.3- -colorsys: 2.7- -compileall: 2.7- +colorsys: 3.0- +compileall: 3.0- concurrent: 3.2- configparser: 3.0- -contextlib: 2.7- +contextlib: 3.0- contextvars: 3.7- -copy: 2.7- -copyreg: 2.7- -crypt: 2.7- -csv: 2.7- -ctypes: 2.7- -curses: 2.7- +copy: 3.0- +copyreg: 3.0- +crypt: 3.0-3.12 +csv: 3.0- +ctypes: 3.0- +curses: 3.0- dataclasses: 3.7- -datetime: 2.7- -dbm: 2.7- -decimal: 2.7- -difflib: 2.7- -dis: 2.7- -distutils: 2.7-3.11 -distutils.command.bdist_msi: 2.7-3.10 -distutils.command.bdist_wininst: 2.7-3.9 -doctest: 2.7- -dummy_threading: 2.7-3.8 -email: 2.7- -encodings: 2.7- -ensurepip: 2.7- +datetime: 3.0- +dbm: 3.0- +decimal: 3.0- +difflib: 3.0- +dis: 3.0- +distutils: 3.0-3.11 +distutils.command.bdist_msi: 3.0-3.10 +distutils.command.bdist_wininst: 3.0-3.9 +doctest: 3.0- +dummy_threading: 3.0-3.8 +email: 3.0- +encodings: 3.0- +ensurepip: 3.0- enum: 3.4- -errno: 2.7- +errno: 3.0- faulthandler: 3.3- -fcntl: 2.7- -filecmp: 2.7- -fileinput: 2.7- -fnmatch: 2.7- -formatter: 2.7-3.9 -fractions: 2.7- -ftplib: 2.7- -functools: 2.7- -gc: 2.7- -genericpath: 2.7- -getopt: 2.7- -getpass: 2.7- -gettext: 2.7- -glob: 2.7- +fcntl: 3.0- +filecmp: 3.0- +fileinput: 3.0- +fnmatch: 3.0- +formatter: 3.0-3.9 +fractions: 3.0- +ftplib: 3.0- +functools: 3.0- +gc: 3.0- +genericpath: 3.0- +getopt: 3.0- +getpass: 3.0- +gettext: 3.0- +glob: 3.0- graphlib: 3.9- -grp: 2.7- -gzip: 2.7- -hashlib: 2.7- -heapq: 2.7- -hmac: 2.7- +grp: 3.0- +gzip: 3.0- +hashlib: 3.0- +heapq: 3.0- +hmac: 3.0- html: 3.0- http: 3.0- -imaplib: 2.7- -imghdr: 2.7- -imp: 2.7-3.11 -importlib: 2.7- +imaplib: 3.0- +imghdr: 3.0-3.12 +imp: 3.0-3.11 +importlib: 3.0- importlib._abc: 3.10- importlib.metadata: 3.8- importlib.metadata._meta: 3.10- @@ -159,150 +159,151 @@ importlib.resources.abc: 3.11- importlib.resources.readers: 3.11- importlib.resources.simple: 3.11- importlib.simple: 3.11- -inspect: 2.7- -io: 2.7- +inspect: 3.0- +io: 3.0- ipaddress: 3.3- -itertools: 2.7- -json: 2.7- -keyword: 2.7- -lib2to3: 2.7- -linecache: 2.7- -locale: 2.7- -logging: 2.7- +itertools: 3.0- +json: 3.0- +keyword: 3.0- +lib2to3: 3.0- +linecache: 3.0- +locale: 3.0- +logging: 3.0- lzma: 3.3- -macpath: 2.7-3.7 -mailbox: 2.7- -mailcap: 2.7- -marshal: 2.7- -math: 2.7- -mimetypes: 2.7- -mmap: 2.7- -modulefinder: 2.7- -msilib: 2.7- -msvcrt: 2.7- -multiprocessing: 2.7- +macpath: 3.0-3.7 +mailbox: 3.0- +mailcap: 3.0-3.12 +marshal: 3.0- +math: 3.0- +mimetypes: 3.0- +mmap: 3.0- +modulefinder: 3.0- +msilib: 3.0-3.12 +msvcrt: 3.0- +multiprocessing: 3.0- multiprocessing.resource_tracker: 3.8- multiprocessing.shared_memory: 3.8- -netrc: 2.7- -nis: 2.7- -nntplib: 2.7- -nt: 2.7- -ntpath: 2.7- -nturl2path: 2.7- -numbers: 2.7- -opcode: 2.7- -operator: 2.7- -optparse: 2.7- -os: 2.7- -ossaudiodev: 2.7- -parser: 2.7-3.9 +netrc: 3.0- +nis: 3.0-3.12 +nntplib: 3.0-3.12 +nt: 3.0- +ntpath: 3.0- +nturl2path: 3.0- +numbers: 3.0- +opcode: 3.0- +operator: 3.0- +optparse: 3.0- +os: 3.0- +ossaudiodev: 3.0-3.12 +parser: 3.0-3.9 pathlib: 3.4- -pdb: 2.7- -pickle: 2.7- -pickletools: 2.7- -pipes: 2.7- -pkgutil: 2.7- -platform: 2.7- -plistlib: 2.7- -poplib: 2.7- -posix: 2.7- -posixpath: 2.7- -pprint: 2.7- -profile: 2.7- -pstats: 2.7- -pty: 2.7- -pwd: 2.7- -py_compile: 2.7- -pyclbr: 2.7- -pydoc: 2.7- -pydoc_data: 2.7- -pyexpat: 2.7- +pdb: 3.0- +pickle: 3.0- +pickletools: 3.0- +pipes: 3.0-3.12 +pkgutil: 3.0- +platform: 3.0- +plistlib: 3.0- +poplib: 3.0- +posix: 3.0- +posixpath: 3.0- +pprint: 3.0- +profile: 3.0- +pstats: 3.0- +pty: 3.0- +pwd: 3.0- +py_compile: 3.0- +pyclbr: 3.0- +pydoc: 3.0- +pydoc_data: 3.0- +pyexpat: 3.0- queue: 3.0- -quopri: 2.7- -random: 2.7- -re: 2.7- -readline: 2.7- +quopri: 3.0- +random: 3.0- +re: 3.0- +readline: 3.0- reprlib: 3.0- -resource: 2.7- -rlcompleter: 2.7- -runpy: 2.7- -sched: 2.7- +resource: 3.0- +rlcompleter: 3.0- +runpy: 3.0- +sched: 3.0- secrets: 3.6- -select: 2.7- +select: 3.0- selectors: 3.4- -shelve: 2.7- -shlex: 2.7- -shutil: 2.7- -signal: 2.7- -site: 2.7- -smtpd: 2.7-3.11 -smtplib: 2.7- -sndhdr: 2.7- -socket: 2.7- +shelve: 3.0- +shlex: 3.0- +shutil: 3.0- +signal: 3.0- +site: 3.0- +smtpd: 3.0-3.11 +smtplib: 3.0- +sndhdr: 3.0-3.12 +socket: 3.0- socketserver: 3.0- -spwd: 2.7- -sqlite3: 2.7- -sre_compile: 2.7- -sre_constants: 2.7- -sre_parse: 2.7- -ssl: 2.7- -stat: 2.7- +spwd: 3.0-3.12 +sqlite3: 3.0- +sre_compile: 3.0- +sre_constants: 3.0- +sre_parse: 3.0- +ssl: 3.0- +stat: 3.0- statistics: 3.4- -string: 2.7- -stringprep: 2.7- -struct: 2.7- -subprocess: 2.7- -sunau: 2.7- -symbol: 2.7-3.9 -symtable: 2.7- -sys: 2.7- +string: 3.0- +stringprep: 3.0- +struct: 3.0- +subprocess: 3.0- +sunau: 3.0-3.12 +symbol: 3.0-3.9 +symtable: 3.0- +sys: 3.0- sys._monitoring: 3.12- # Doesn't actually exist. See comments in the stub. -sysconfig: 2.7- -syslog: 2.7- -tabnanny: 2.7- -tarfile: 2.7- -telnetlib: 2.7- -tempfile: 2.7- -termios: 2.7- -textwrap: 2.7- -this: 2.7- -threading: 2.7- -time: 2.7- -timeit: 2.7- +sysconfig: 3.0- +syslog: 3.0- +tabnanny: 3.0- +tarfile: 3.0- +telnetlib: 3.0-3.12 +tempfile: 3.0- +termios: 3.0- +textwrap: 3.0- +this: 3.0- +threading: 3.0- +time: 3.0- +timeit: 3.0- tkinter: 3.0- -token: 2.7- -tokenize: 2.7- +token: 3.0- +tokenize: 3.0- tomllib: 3.11- -trace: 2.7- -traceback: 2.7- +trace: 3.0- +traceback: 3.0- tracemalloc: 3.4- -tty: 2.7- -turtle: 2.7- -types: 2.7- +tty: 3.0- +turtle: 3.0- +types: 3.0- typing: 3.5- -typing_extensions: 2.7- -unicodedata: 2.7- -unittest: 2.7- +typing_extensions: 3.0- +unicodedata: 3.0- +unittest: 3.0- unittest._log: 3.9- unittest.async_case: 3.8- -urllib: 2.7- -uu: 2.7- -uuid: 2.7- +urllib: 3.0- +uu: 3.0-3.12 +uuid: 3.0- venv: 3.3- -warnings: 2.7- -wave: 2.7- -weakref: 2.7- -webbrowser: 2.7- +warnings: 3.0- +wave: 3.0- +weakref: 3.0- +webbrowser: 3.0- winreg: 3.0- -winsound: 2.7- -wsgiref: 2.7- +winsound: 3.0- +wsgiref: 3.0- wsgiref.types: 3.11- -xdrlib: 2.7- -xml: 2.7- +xdrlib: 3.0-3.12 +xml: 3.0- xmlrpc: 3.0- xxlimited: 3.2- zipapp: 3.5- -zipfile: 2.7- -zipimport: 2.7- -zlib: 2.7- +zipfile: 3.0- +zipfile._path: 3.12- +zipimport: 3.0- +zlib: 3.0- zoneinfo: 3.9- diff --git a/mypy/typeshed/stdlib/_ast.pyi b/mypy/typeshed/stdlib/_ast.pyi index 0302133fc6f9..0fbf66f7feda 100644 --- a/mypy/typeshed/stdlib/_ast.pyi +++ b/mypy/typeshed/stdlib/_ast.pyi @@ -586,7 +586,9 @@ if sys.version_info >= (3, 10): patterns: list[pattern] if sys.version_info >= (3, 12): - class type_param(AST): ... + class type_param(AST): + end_lineno: int + end_col_offset: int class TypeVar(type_param): __match_args__ = ("name", "bound") diff --git a/mypy/typeshed/stdlib/_thread.pyi b/mypy/typeshed/stdlib/_thread.pyi index dba8664fbf13..8bd0b179607b 100644 --- a/mypy/typeshed/stdlib/_thread.pyi +++ b/mypy/typeshed/stdlib/_thread.pyi @@ -46,3 +46,8 @@ if sys.version_info >= (3, 8): if sys.version_info >= (3, 12): def daemon_threads_allowed() -> bool: ... + +class _local: + def __getattribute__(self, __name: str) -> Any: ... + def __setattr__(self, __name: str, __value: Any) -> None: ... + def __delattr__(self, __name: str) -> None: ... diff --git a/mypy/typeshed/stdlib/asyncore.pyi b/mypy/typeshed/stdlib/asyncore.pyi index 47c8e2207022..36d1862fdda7 100644 --- a/mypy/typeshed/stdlib/asyncore.pyi +++ b/mypy/typeshed/stdlib/asyncore.pyi @@ -83,6 +83,7 @@ if sys.platform != "win32": def write(self, data: bytes, flags: int = ...) -> int: ... def close(self) -> None: ... def fileno(self) -> int: ... + def __del__(self) -> None: ... class file_dispatcher(dispatcher): def __init__(self, fd: FileDescriptorLike, map: _MapType | None = None) -> None: ... diff --git a/mypy/typeshed/stdlib/base64.pyi b/mypy/typeshed/stdlib/base64.pyi index 24830cbfba04..4629c95d0949 100644 --- a/mypy/typeshed/stdlib/base64.pyi +++ b/mypy/typeshed/stdlib/base64.pyi @@ -27,13 +27,13 @@ if sys.version_info >= (3, 10): __all__ += ["b32hexencode", "b32hexdecode"] def b64encode(s: ReadableBuffer, altchars: ReadableBuffer | None = None) -> bytes: ... -def b64decode(s: str | ReadableBuffer, altchars: ReadableBuffer | None = None, validate: bool = False) -> bytes: ... +def b64decode(s: str | ReadableBuffer, altchars: str | ReadableBuffer | None = None, validate: bool = False) -> bytes: ... def standard_b64encode(s: ReadableBuffer) -> bytes: ... def standard_b64decode(s: str | ReadableBuffer) -> bytes: ... def urlsafe_b64encode(s: ReadableBuffer) -> bytes: ... def urlsafe_b64decode(s: str | ReadableBuffer) -> bytes: ... def b32encode(s: ReadableBuffer) -> bytes: ... -def b32decode(s: str | ReadableBuffer, casefold: bool = False, map01: bytes | None = None) -> bytes: ... +def b32decode(s: str | ReadableBuffer, casefold: bool = False, map01: str | ReadableBuffer | None = None) -> bytes: ... def b16encode(s: ReadableBuffer) -> bytes: ... def b16decode(s: str | ReadableBuffer, casefold: bool = False) -> bytes: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/process.pyi b/mypy/typeshed/stdlib/concurrent/futures/process.pyi index 000e7a43503a..d3706a9c15a6 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/process.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/process.pyi @@ -5,12 +5,14 @@ from multiprocessing.context import BaseContext, Process from multiprocessing.queues import Queue, SimpleQueue from threading import Lock, Semaphore, Thread from types import TracebackType -from typing import Any, Generic, TypeVar +from typing import Any, Generic, TypeVar, overload +from typing_extensions import TypeVarTuple, Unpack from weakref import ref from ._base import BrokenExecutor, Executor, Future _T = TypeVar("_T") +_Ts = TypeVarTuple("_Ts") _threads_wakeups: MutableMapping[Any, Any] _global_shutdown: bool @@ -109,8 +111,8 @@ if sys.version_info >= (3, 11): def _process_worker( call_queue: Queue[_CallItem], result_queue: SimpleQueue[_ResultItem], - initializer: Callable[..., object] | None, - initargs: tuple[Any, ...], + initializer: Callable[[Unpack[_Ts]], object] | None, + initargs: tuple[Unpack[_Ts]], max_tasks: int | None = None, ) -> None: ... @@ -118,8 +120,8 @@ else: def _process_worker( call_queue: Queue[_CallItem], result_queue: SimpleQueue[_ResultItem], - initializer: Callable[..., object] | None, - initargs: tuple[Any, ...], + initializer: Callable[[Unpack[_Ts]], object] | None, + initargs: tuple[Unpack[_Ts]], ) -> None: ... if sys.version_info >= (3, 9): @@ -169,22 +171,61 @@ class ProcessPoolExecutor(Executor): _result_queue: SimpleQueue[Any] _work_ids: Queue[Any] if sys.version_info >= (3, 11): + @overload def __init__( self, max_workers: int | None = None, mp_context: BaseContext | None = None, - initializer: Callable[..., object] | None = None, - initargs: tuple[Any, ...] = (), + initializer: Callable[[], object] | None = None, + initargs: tuple[()] = (), + *, + max_tasks_per_child: int | None = None, + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None = None, + mp_context: BaseContext | None = None, + *, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + max_tasks_per_child: int | None = None, + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None, + mp_context: BaseContext | None, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], *, max_tasks_per_child: int | None = None, ) -> None: ... else: + @overload + def __init__( + self, + max_workers: int | None = None, + mp_context: BaseContext | None = None, + initializer: Callable[[], object] | None = None, + initargs: tuple[()] = (), + ) -> None: ... + @overload def __init__( self, max_workers: int | None = None, mp_context: BaseContext | None = None, - initializer: Callable[..., object] | None = None, - initargs: tuple[Any, ...] = (), + *, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None, + mp_context: BaseContext | None, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], ) -> None: ... if sys.version_info >= (3, 9): def _start_executor_manager_thread(self) -> None: ... diff --git a/mypy/typeshed/stdlib/concurrent/futures/thread.pyi b/mypy/typeshed/stdlib/concurrent/futures/thread.pyi index 0b00d524aa3d..f38cf2c57963 100644 --- a/mypy/typeshed/stdlib/concurrent/futures/thread.pyi +++ b/mypy/typeshed/stdlib/concurrent/futures/thread.pyi @@ -2,11 +2,14 @@ import queue import sys from collections.abc import Callable, Iterable, Mapping, Set as AbstractSet from threading import Lock, Semaphore, Thread -from typing import Any, Generic, TypeVar +from typing import Any, Generic, TypeVar, overload +from typing_extensions import TypeVarTuple, Unpack from weakref import ref from ._base import BrokenExecutor, Executor, Future +_Ts = TypeVarTuple("_Ts") + _threads_queues: Mapping[Any, Any] _shutdown: bool _global_shutdown_lock: Lock @@ -31,8 +34,8 @@ class _WorkItem(Generic[_S]): def _worker( executor_reference: ref[Any], work_queue: queue.SimpleQueue[Any], - initializer: Callable[..., object], - initargs: tuple[Any, ...], + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], ) -> None: ... class BrokenThreadPool(BrokenExecutor): ... @@ -48,12 +51,30 @@ class ThreadPoolExecutor(Executor): _initializer: Callable[..., None] | None _initargs: tuple[Any, ...] _work_queue: queue.SimpleQueue[_WorkItem[Any]] + @overload def __init__( self, max_workers: int | None = None, thread_name_prefix: str = "", - initializer: Callable[..., object] | None = None, - initargs: tuple[Any, ...] = (), + initializer: Callable[[], object] | None = None, + initargs: tuple[()] = (), + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None = None, + thread_name_prefix: str = "", + *, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], + ) -> None: ... + @overload + def __init__( + self, + max_workers: int | None, + thread_name_prefix: str, + initializer: Callable[[Unpack[_Ts]], object], + initargs: tuple[Unpack[_Ts]], ) -> None: ... def _adjust_thread_count(self) -> None: ... def _initializer_failed(self) -> None: ... diff --git a/mypy/typeshed/stdlib/http/client.pyi b/mypy/typeshed/stdlib/http/client.pyi index 20223695a1a8..fb5450730f60 100644 --- a/mypy/typeshed/stdlib/http/client.pyi +++ b/mypy/typeshed/stdlib/http/client.pyi @@ -99,6 +99,24 @@ responses: dict[int, str] class HTTPMessage(email.message.Message): def getallmatchingheaders(self, name: str) -> list[str]: ... # undocumented + # override below all of Message's methods that use `_HeaderType` / `_HeaderTypeParam` with `str` + # `HTTPMessage` breaks the Liskov substitution principle by only intending for `str` headers + # This is easier than making `Message` generic + def __getitem__(self, name: str) -> str | None: ... + def __setitem__(self, name: str, val: str) -> None: ... # type: ignore[override] + def values(self) -> list[str]: ... + def items(self) -> list[tuple[str, str]]: ... + @overload + def get(self, name: str, failobj: None = None) -> str | None: ... + @overload + def get(self, name: str, failobj: _T) -> str | _T: ... + @overload + def get_all(self, name: str, failobj: None = None) -> list[str] | None: ... + @overload + def get_all(self, name: str, failobj: _T) -> list[str] | _T: ... + def replace_header(self, _name: str, _value: str) -> None: ... # type: ignore[override] + def set_raw(self, name: str, value: str) -> None: ... # type: ignore[override] + def raw_items(self) -> Iterator[tuple[str, str]]: ... def parse_headers(fp: io.BufferedIOBase, _class: Callable[[], email.message.Message] = ...) -> HTTPMessage: ... diff --git a/mypy/typeshed/stdlib/os/__init__.pyi b/mypy/typeshed/stdlib/os/__init__.pyi index 7d4b8adcd00a..e92dd7a095d6 100644 --- a/mypy/typeshed/stdlib/os/__init__.pyi +++ b/mypy/typeshed/stdlib/os/__init__.pyi @@ -33,6 +33,9 @@ from . import path as _path if sys.version_info >= (3, 9): from types import GenericAlias +if sys.platform != "win32": + from resource import struct_rusage + # This unnecessary alias is to work around various errors path = _path @@ -962,8 +965,8 @@ else: def waitid(__idtype: int, __ident: int, __options: int) -> waitid_result | None: ... - def wait3(options: int) -> tuple[int, int, Any]: ... - def wait4(pid: int, options: int) -> tuple[int, int, Any]: ... + def wait3(options: int) -> tuple[int, int, struct_rusage]: ... + def wait4(pid: int, options: int) -> tuple[int, int, struct_rusage]: ... def WCOREDUMP(__status: int) -> bool: ... def WIFCONTINUED(status: int) -> bool: ... def WIFSTOPPED(status: int) -> bool: ... diff --git a/mypy/typeshed/stdlib/selectors.pyi b/mypy/typeshed/stdlib/selectors.pyi index 043df9253316..a857d0e242ab 100644 --- a/mypy/typeshed/stdlib/selectors.pyi +++ b/mypy/typeshed/stdlib/selectors.pyi @@ -31,49 +31,37 @@ class BaseSelector(metaclass=ABCMeta): def __enter__(self) -> Self: ... def __exit__(self, *args: Unused) -> None: ... -class SelectSelector(BaseSelector): +class _BaseSelectorImpl(BaseSelector, metaclass=ABCMeta): def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... - def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + def modify(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... +class SelectSelector(_BaseSelectorImpl): + def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + +class _PollLikeSelector(_BaseSelectorImpl): + def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... + if sys.platform != "win32": - class PollSelector(BaseSelector): - def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... - def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... - def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... - def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... + class PollSelector(_PollLikeSelector): ... if sys.platform == "linux": - class EpollSelector(BaseSelector): + class EpollSelector(_PollLikeSelector): def fileno(self) -> int: ... - def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... - def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... - def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... - def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... -class DevpollSelector(BaseSelector): +class DevpollSelector(_PollLikeSelector): def fileno(self) -> int: ... - def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = ...) -> SelectorKey: ... - def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... - def select(self, timeout: float | None = ...) -> list[tuple[SelectorKey, _EventMask]]: ... - def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... if sys.platform != "win32": - class KqueueSelector(BaseSelector): + class KqueueSelector(_BaseSelectorImpl): def fileno(self) -> int: ... - def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... - def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... - def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... # Not a real class at runtime, it is just a conditional alias to other real selectors. # The runtime logic is more fine-grained than a `sys.platform` check; # not really expressible in the stubs -class DefaultSelector(BaseSelector): - def register(self, fileobj: FileDescriptorLike, events: _EventMask, data: Any = None) -> SelectorKey: ... - def unregister(self, fileobj: FileDescriptorLike) -> SelectorKey: ... +class DefaultSelector(_BaseSelectorImpl): def select(self, timeout: float | None = None) -> list[tuple[SelectorKey, _EventMask]]: ... - def get_map(self) -> Mapping[FileDescriptorLike, SelectorKey]: ... if sys.platform != "win32": def fileno(self) -> int: ... diff --git a/mypy/typeshed/stdlib/threading.pyi b/mypy/typeshed/stdlib/threading.pyi index badd09cae051..bcd73823c63f 100644 --- a/mypy/typeshed/stdlib/threading.pyi +++ b/mypy/typeshed/stdlib/threading.pyi @@ -1,3 +1,4 @@ +import _thread import sys from _typeshed import ProfileFunction, TraceFunction from collections.abc import Callable, Iterable, Mapping @@ -68,12 +69,8 @@ def stack_size(size: int = ...) -> int: ... TIMEOUT_MAX: float -class ThreadError(Exception): ... - -class local: - def __getattribute__(self, __name: str) -> Any: ... - def __setattr__(self, __name: str, __value: Any) -> None: ... - def __delattr__(self, __name: str) -> None: ... +ThreadError = _thread.error +local = _thread._local class Thread: name: str diff --git a/mypy/typeshed/stdlib/unittest/_log.pyi b/mypy/typeshed/stdlib/unittest/_log.pyi index 4de5d502e004..011a970d8bbc 100644 --- a/mypy/typeshed/stdlib/unittest/_log.pyi +++ b/mypy/typeshed/stdlib/unittest/_log.pyi @@ -2,7 +2,7 @@ import logging import sys from types import TracebackType from typing import ClassVar, Generic, NamedTuple, TypeVar -from unittest.case import TestCase +from unittest.case import TestCase, _BaseTestCaseContext _L = TypeVar("_L", None, _LoggingWatcher) @@ -10,9 +10,8 @@ class _LoggingWatcher(NamedTuple): records: list[logging.LogRecord] output: list[str] -class _AssertLogsContext(Generic[_L]): +class _AssertLogsContext(_BaseTestCaseContext, Generic[_L]): LOGGING_FORMAT: ClassVar[str] - test_case: TestCase logger_name: str level: int msg: None diff --git a/mypy/typeshed/stdlib/unittest/case.pyi b/mypy/typeshed/stdlib/unittest/case.pyi index 7efe6cdc9662..d66f027324f1 100644 --- a/mypy/typeshed/stdlib/unittest/case.pyi +++ b/mypy/typeshed/stdlib/unittest/case.pyi @@ -25,8 +25,26 @@ _P = ParamSpec("_P") DIFF_OMITTED: str class _BaseTestCaseContext: + test_case: TestCase def __init__(self, test_case: TestCase) -> None: ... +class _AssertRaisesBaseContext(_BaseTestCaseContext): + expected: type[BaseException] | tuple[type[BaseException], ...] + expected_regex: Pattern[str] | None + obj_name: str | None + msg: str | None + + def __init__( + self, + expected: type[BaseException] | tuple[type[BaseException], ...], + test_case: TestCase, + expected_regex: str | Pattern[str] | None = None, + ) -> None: ... + + # This returns Self if args is the empty list, and None otherwise. + # but it's not possible to construct an overload which expresses that + def handle(self, name: str, args: list[Any], kwargs: dict[str, Any]) -> Any: ... + if sys.version_info >= (3, 9): from unittest._log import _AssertLogsContext, _LoggingWatcher else: @@ -41,7 +59,6 @@ else: class _AssertLogsContext(_BaseTestCaseContext, Generic[_L]): LOGGING_FORMAT: ClassVar[str] - test_case: TestCase logger_name: str level: int msg: None @@ -310,7 +327,7 @@ class FunctionTestCase(TestCase): def __hash__(self) -> int: ... def __eq__(self, other: object) -> bool: ... -class _AssertRaisesContext(Generic[_E]): +class _AssertRaisesContext(_AssertRaisesBaseContext, Generic[_E]): exception: _E def __enter__(self) -> Self: ... def __exit__( @@ -319,7 +336,7 @@ class _AssertRaisesContext(Generic[_E]): if sys.version_info >= (3, 9): def __class_getitem__(cls, item: Any) -> GenericAlias: ... -class _AssertWarnsContext: +class _AssertWarnsContext(_AssertRaisesBaseContext): warning: WarningMessage filename: str lineno: int diff --git a/mypy/typeshed/stdlib/zipfile.pyi b/mypy/typeshed/stdlib/zipfile/__init__.pyi similarity index 98% rename from mypy/typeshed/stdlib/zipfile.pyi rename to mypy/typeshed/stdlib/zipfile/__init__.pyi index 5483b84fe6f6..83982be3be08 100644 --- a/mypy/typeshed/stdlib/zipfile.pyi +++ b/mypy/typeshed/stdlib/zipfile/__init__.pyi @@ -227,7 +227,10 @@ class ZipInfo: def is_dir(self) -> bool: ... def FileHeader(self, zip64: bool | None = None) -> bytes: ... -if sys.version_info >= (3, 8): +if sys.version_info >= (3, 12): + from zipfile._path import CompleteDirs as CompleteDirs, Path as Path + +elif sys.version_info >= (3, 8): class CompleteDirs(ZipFile): def resolve_dir(self, name: str) -> str: ... @overload diff --git a/mypy/typeshed/stdlib/zipfile/_path.pyi b/mypy/typeshed/stdlib/zipfile/_path.pyi new file mode 100644 index 000000000000..3cf034aec8f4 --- /dev/null +++ b/mypy/typeshed/stdlib/zipfile/_path.pyi @@ -0,0 +1,95 @@ +import sys +from _typeshed import StrPath +from collections.abc import Iterator, Sequence +from io import TextIOWrapper +from os import PathLike +from typing import IO, overload +from typing_extensions import Literal, Self, TypeAlias +from zipfile import ZipFile + +_ReadWriteBinaryMode: TypeAlias = Literal["r", "w", "rb", "wb"] + +if sys.version_info >= (3, 12): + class InitializedState: + def __init__(self, *args: object, **kwargs: object) -> None: ... + def __getstate__(self) -> tuple[list[object], dict[object, object]]: ... + def __setstate__(self, state: Sequence[tuple[list[object], dict[object, object]]]) -> None: ... + + class CompleteDirs(InitializedState, ZipFile): + def resolve_dir(self, name: str) -> str: ... + @overload + @classmethod + def make(cls, source: ZipFile) -> CompleteDirs: ... + @overload + @classmethod + def make(cls, source: StrPath | IO[bytes]) -> Self: ... + + class Path: + root: CompleteDirs + def __init__(self, root: ZipFile | StrPath | IO[bytes], at: str = "") -> None: ... + @property + def name(self) -> str: ... + @property + def parent(self) -> PathLike[str]: ... # undocumented + if sys.version_info >= (3, 10): + @property + def filename(self) -> PathLike[str]: ... # undocumented + if sys.version_info >= (3, 11): + @property + def suffix(self) -> str: ... + @property + def suffixes(self) -> list[str]: ... + @property + def stem(self) -> str: ... + + if sys.version_info >= (3, 9): + @overload + def open( + self, + mode: Literal["r", "w"] = "r", + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + line_buffering: bool = ..., + write_through: bool = ..., + *, + pwd: bytes | None = None, + ) -> TextIOWrapper: ... + @overload + def open(self, mode: Literal["rb", "wb"], *, pwd: bytes | None = None) -> IO[bytes]: ... + else: + def open( + self, mode: _ReadWriteBinaryMode = "r", pwd: bytes | None = None, *, force_zip64: bool = False + ) -> IO[bytes]: ... + + if sys.version_info >= (3, 10): + def iterdir(self) -> Iterator[Self]: ... + else: + def iterdir(self) -> Iterator[Path]: ... + + def is_dir(self) -> bool: ... + def is_file(self) -> bool: ... + def exists(self) -> bool: ... + def read_text( + self, + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + line_buffering: bool = ..., + write_through: bool = ..., + ) -> str: ... + def read_bytes(self) -> bytes: ... + if sys.version_info >= (3, 10): + def joinpath(self, *other: StrPath) -> Path: ... + else: + def joinpath(self, add: StrPath) -> Path: ... # undocumented + if sys.version_info >= (3, 12): + def glob(self, pattern: str) -> Iterator[Self]: ... + def rglob(self, pattern: str) -> Iterator[Self]: ... + def is_symlink(self) -> Literal[False]: ... + def relative_to(self, other: Path, *extra: StrPath) -> str: ... + def match(self, path_pattern: str) -> bool: ... + def __eq__(self, other: object) -> bool: ... + def __hash__(self) -> int: ... + + def __truediv__(self, add: StrPath) -> Path: ... From 4aba5ca8450ad3f06805f2bd6dfcd06048a3f1d2 Mon Sep 17 00:00:00 2001 From: Fabian Lewis <49647154+frobian21@users.noreply.github.com> Date: Tue, 2 Jan 2024 02:26:30 +0100 Subject: [PATCH 02/32] Fix xml writing bug introduced in #16388 (#16713) #16388 introduced a bug where invalid xml could be produced by `write_junit_xml`, as special characters were no longer being escaped. The fix is to revert the removal of `from xml.sax.saxutils import escape` and `escape("\n".join(messages))` in the `write_junit_xml` function. I've added a small test case which checks that the `<`, `>`, `&` are escaped correctly in the xml. --- mypy/test/testutil.py | 22 ++++++++++++++++++++++ mypy/util.py | 6 ++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/mypy/test/testutil.py b/mypy/test/testutil.py index d0d54ffec8c6..a7c3f1c00fee 100644 --- a/mypy/test/testutil.py +++ b/mypy/test/testutil.py @@ -31,6 +31,28 @@ def test_junit_pass(self) -> None: +""" + result = _generate_junit_contents( + dt=1.23, + serious=serious, + messages_by_file=messages_by_file, + version="3.14", + platform="test-plat", + ) + assert result == expected + + def test_junit_fail_escape_xml_chars(self) -> None: + serious = False + messages_by_file: dict[str | None, list[str]] = { + "file1.py": ["Test failed", "another line < > &"] + } + expected = """ + + + Test failed +another line < > & + + """ result = _generate_junit_contents( dt=1.23, diff --git a/mypy/util.py b/mypy/util.py index be8a22d08a27..0277a96a6885 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -263,6 +263,8 @@ def _generate_junit_contents( version: str, platform: str, ) -> str: + from xml.sax.saxutils import escape + if serious: failures = 0 errors = len(messages_by_file) @@ -284,7 +286,7 @@ def _generate_junit_contents( for filename, messages in messages_by_file.items(): if filename is not None: xml += JUNIT_TESTCASE_FAIL_TEMPLATE.format( - text="\n".join(messages), + text=escape("\n".join(messages)), filename=filename, time=dt, name="mypy-py{ver}-{platform} {filename}".format( @@ -293,7 +295,7 @@ def _generate_junit_contents( ) else: xml += JUNIT_TESTCASE_FAIL_TEMPLATE.format( - text="\n".join(messages), + text=escape("\n".join(messages)), filename="mypy", time=dt, name="mypy-py{ver}-{platform}".format(ver=version, platform=platform), From d0d5876d876272c56a339c628197935906c57e3c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jan 2024 19:47:02 -0800 Subject: [PATCH 03/32] Fix various lint (#16726) --- .pre-commit-config.yaml | 2 +- mypy/binder.py | 2 +- mypy/build.py | 10 ++++------ mypy/checker.py | 7 +++---- mypy/checkexpr.py | 6 ++---- mypy/checkpattern.py | 2 +- mypy/checkstrformat.py | 4 ++-- mypy/dmypy_server.py | 2 +- mypy/dmypy_util.py | 2 +- mypy/errors.py | 2 +- mypy/main.py | 6 +++--- mypy/message_registry.py | 2 +- mypy/messages.py | 2 +- mypy/nodes.py | 6 +++--- mypy/patterns.py | 14 ++++++++------ mypy/plugins/enums.py | 4 ++-- mypy/renaming.py | 2 +- mypy/semanal.py | 10 ++++------ mypy/semanal_enum.py | 2 +- mypy/semanal_namedtuple.py | 2 +- mypy/semanal_typeddict.py | 2 +- mypy/server/astmerge.py | 4 ++-- mypy/stubdoc.py | 2 +- mypy/stubgenc.py | 8 ++++---- mypy/stubtest.py | 16 ++++++---------- mypy/stubutil.py | 2 +- mypy/test/helpers.py | 4 ++-- mypy/test/testcheck.py | 5 ++++- mypy/test/testformatter.py | 4 ++-- mypy/test/teststubtest.py | 4 ++-- mypy/typeanal.py | 2 +- mypy/types.py | 2 +- mypy/typestate.py | 4 ++-- mypy/util.py | 2 +- mypyc/codegen/emitmodule.py | 2 +- mypyc/codegen/emitwrapper.py | 4 ++-- mypyc/irbuild/prepare.py | 6 +++--- mypyc/primitives/registry.py | 20 +++++++++++++++----- pyproject.toml | 9 +++++++-- test-requirements.in | 2 +- test-requirements.txt | 2 +- 41 files changed, 102 insertions(+), 93 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bd2a09b7a8cf..4090bf0ecb4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: hooks: - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.0 # must match test-requirements.txt + rev: v0.1.4 # must match test-requirements.txt hooks: - id: ruff args: [--exit-non-zero-on-fix] diff --git a/mypy/binder.py b/mypy/binder.py index 3b67d09f16c3..9d0a33b54bc2 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -291,7 +291,7 @@ def assign_type( self.type_assignments[expr].append((type, declared_type)) return if not isinstance(expr, (IndexExpr, MemberExpr, NameExpr)): - return None + return if not literal(expr): return self.invalidate_dependencies(expr) diff --git a/mypy/build.py b/mypy/build.py index b3ca8d06916d..8049fa2d0c3f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -682,9 +682,7 @@ def __init__( # for efficient lookup self.shadow_map: dict[str, str] = {} if self.options.shadow_file is not None: - self.shadow_map = { - source_file: shadow_file for (source_file, shadow_file) in self.options.shadow_file - } + self.shadow_map = dict(self.options.shadow_file) # a mapping from each file being typechecked to its possible shadow file self.shadow_equivalence_map: dict[str, str | None] = {} self.plugin = plugin @@ -1120,7 +1118,7 @@ def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta] module_deps_metas = deps_meta["deps_meta"] assert isinstance(module_deps_metas, dict) if not manager.options.skip_cache_mtime_checks: - for id, meta in module_deps_metas.items(): + for meta in module_deps_metas.values(): try: matched = manager.getmtime(meta["path"]) == meta["mtime"] except FileNotFoundError: @@ -2093,7 +2091,7 @@ def load_tree(self, temporary: bool = False) -> None: self.meta.data_json, self.manager, "Load tree ", "Could not load tree: " ) if data is None: - return None + return t0 = time.time() # TODO: Assert data file wasn't changed. @@ -3383,7 +3381,7 @@ def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> strongly_connected_components() below for a reference. """ if len(ascc) == 1: - return [s for s in ascc] + return list(ascc) pri_spread = set() for id in ascc: state = graph[id] diff --git a/mypy/checker.py b/mypy/checker.py index 0ac8f6904973..1b57ef780104 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -632,7 +632,7 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: if not defn.items: # In this case we have already complained about none of these being # valid overloads. - return None + return if len(defn.items) == 1: self.fail(message_registry.MULTIPLE_OVERLOADS_REQUIRED, defn) @@ -676,7 +676,6 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: self.msg.no_overridable_method(defn.name, defn) self.check_explicit_override_decorator(defn, found_method_base_classes, defn.impl) self.check_inplace_operator_method(defn) - return None def extract_callable_type(self, inner_type: Type | None, ctx: Context) -> CallableType | None: """Get type as seen by an overload item caller.""" @@ -1838,7 +1837,7 @@ def check_match_args(self, var: Var, typ: Type, context: Context) -> None: return typ = get_proper_type(typ) if not isinstance(typ, TupleType) or not all( - [is_string_literal(item) for item in typ.items] + is_string_literal(item) for item in typ.items ): self.msg.note( "__match_args__ must be a tuple containing string literals for checking " @@ -5045,7 +5044,7 @@ def visit_break_stmt(self, s: BreakStmt) -> None: def visit_continue_stmt(self, s: ContinueStmt) -> None: self.binder.handle_continue() - return None + return def visit_match_stmt(self, s: MatchStmt) -> None: with self.binder.frame_context(can_skip=False, fall_through=0): diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 626584bc3a20..3af8d70e78c9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -739,7 +739,7 @@ def check_typeddict_call( context: Context, orig_callee: Type | None, ) -> Type: - if args and all([ak in (ARG_NAMED, ARG_STAR2) for ak in arg_kinds]): + if args and all(ak in (ARG_NAMED, ARG_STAR2) for ak in arg_kinds): # ex: Point(x=42, y=1337, **extras) # This is a bit ugly, but this is a price for supporting all possible syntax # variants for TypedDict constructors. @@ -4017,9 +4017,7 @@ def check_op( left_variants = [base_type] base_type = get_proper_type(base_type) if isinstance(base_type, UnionType): - left_variants = [ - item for item in flatten_nested_unions(base_type.relevant_items()) - ] + left_variants = list(flatten_nested_unions(base_type.relevant_items())) right_type = self.accept(arg) # Step 1: We first try leaving the right arguments alone and destructure diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index c0061f1c3e72..3210dcc3b7ac 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -187,7 +187,7 @@ def visit_or_pattern(self, o: OrPattern) -> PatternType: capture_types[node].append((expr, typ)) captures: dict[Expression, Type] = {} - for var, capture_list in capture_types.items(): + for capture_list in capture_types.values(): typ = UninhabitedType() for _, other in capture_list: typ = join_types(typ, other) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index 39d44e84a9c1..c63210a96c44 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -372,7 +372,7 @@ def check_specs_in_format_call( ): # TODO: add support for some custom specs like datetime? self.msg.fail( - "Unrecognized format" ' specification "{}"'.format(spec.format_spec[1:]), + f'Unrecognized format specification "{spec.format_spec[1:]}"', call, code=codes.STRING_FORMATTING, ) @@ -482,7 +482,7 @@ def find_replacements_in_call(self, call: CallExpr, keys: list[str]) -> list[Exp expr = self.get_expr_by_name(key, call) if not expr: self.msg.fail( - "Cannot find replacement for named" ' format specifier "{}"'.format(key), + f'Cannot find replacement for named format specifier "{key}"', call, code=codes.STRING_FORMATTING, ) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 42236497f275..b4c3fe8fe0dc 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -1055,7 +1055,7 @@ def fix_module_deps(graph: mypy.build.Graph) -> None: This can make some suppressed dependencies non-suppressed, and vice versa (if modules have been added to or removed from the build). """ - for module, state in graph.items(): + for state in graph.values(): new_suppressed = [] new_dependencies = [] for dep in state.dependencies + state.suppressed: diff --git a/mypy/dmypy_util.py b/mypy/dmypy_util.py index d95cba9f40b5..fe949e8fc294 100644 --- a/mypy/dmypy_util.py +++ b/mypy/dmypy_util.py @@ -43,7 +43,7 @@ def send(connection: IPCBase, data: Any) -> None: class WriteToConn: """Helper class to write to a connection instead of standard output.""" - def __init__(self, server: IPCBase, output_key: str = "stdout"): + def __init__(self, server: IPCBase, output_key: str = "stdout") -> None: self.server = server self.output_key = output_key diff --git a/mypy/errors.py b/mypy/errors.py index 6e90c28d9c03..eabe96a2dc73 100644 --- a/mypy/errors.py +++ b/mypy/errors.py @@ -170,7 +170,7 @@ def __init__( *, filter_errors: bool | Callable[[str, ErrorInfo], bool] = False, save_filtered_errors: bool = False, - ): + ) -> None: self.errors = errors self._has_new_errors = False self._filter = filter_errors diff --git a/mypy/main.py b/mypy/main.py index 2c68466ec977..b32624456ce0 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -145,7 +145,7 @@ def main( sys.exit(code) # HACK: keep res alive so that mypyc won't free it before the hard_exit - list([res]) + list([res]) # noqa: C410 def run_build( @@ -349,7 +349,7 @@ class CapturableArgumentParser(argparse.ArgumentParser): yet output must be captured to properly support mypy.api.run. """ - def __init__(self, *args: Any, **kwargs: Any): + def __init__(self, *args: Any, **kwargs: Any) -> None: self.stdout = kwargs.pop("stdout", sys.stdout) self.stderr = kwargs.pop("stderr", sys.stderr) super().__init__(*args, **kwargs) @@ -415,7 +415,7 @@ def __init__( default: str = argparse.SUPPRESS, help: str = "show program's version number and exit", stdout: IO[str] | None = None, - ): + ) -> None: super().__init__( option_strings=option_strings, dest=dest, default=default, nargs=0, help=help ) diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 8dc14e158d90..fb430b63c74b 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -52,7 +52,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage: '"return" with value in async generator is not allowed' ) INVALID_RETURN_TYPE_FOR_GENERATOR: Final = ErrorMessage( - 'The return type of a generator function should be "Generator"' " or one of its supertypes" + 'The return type of a generator function should be "Generator" or one of its supertypes' ) INVALID_RETURN_TYPE_FOR_ASYNC_GENERATOR: Final = ErrorMessage( 'The return type of an async generator function should be "AsyncGenerator" or one of its ' diff --git a/mypy/messages.py b/mypy/messages.py index 069c4d51e281..450faf4c1688 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -3099,7 +3099,7 @@ def append_invariance_notes( ): invariant_type = "Dict" covariant_suggestion = ( - 'Consider using "Mapping" instead, ' "which is covariant in the value type" + 'Consider using "Mapping" instead, which is covariant in the value type' ) if invariant_type and covariant_suggestion: notes.append( diff --git a/mypy/nodes.py b/mypy/nodes.py index 17e06613d1e3..fe41777c55f7 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3137,7 +3137,7 @@ def protocol_members(self) -> list[str]: if name in EXCLUDED_PROTOCOL_ATTRIBUTES: continue members.add(name) - return sorted(list(members)) + return sorted(members) def __getitem__(self, name: str) -> SymbolTableNode: n = self.get(name) @@ -3296,7 +3296,7 @@ def serialize(self) -> JsonDict: else self.typeddict_type.serialize(), "flags": get_flags(self, TypeInfo.FLAGS), "metadata": self.metadata, - "slots": list(sorted(self.slots)) if self.slots is not None else None, + "slots": sorted(self.slots) if self.slots is not None else None, "deletable_attributes": self.deletable_attributes, "self_type": self.self_type.serialize() if self.self_type is not None else None, "dataclass_transform_spec": ( @@ -3966,7 +3966,7 @@ def __init__( # frozen_default was added to CPythonin https://github.com/python/cpython/pull/99958 citing # positive discussion in typing-sig frozen_default: bool | None = None, - ): + ) -> None: self.eq_default = eq_default if eq_default is not None else True self.order_default = order_default if order_default is not None else False self.kw_only_default = kw_only_default if kw_only_default is not None else False diff --git a/mypy/patterns.py b/mypy/patterns.py index 839864ef5879..a01bf6acc876 100644 --- a/mypy/patterns.py +++ b/mypy/patterns.py @@ -60,7 +60,7 @@ class ValuePattern(Pattern): expr: Expression - def __init__(self, expr: Expression): + def __init__(self, expr: Expression) -> None: super().__init__() self.expr = expr @@ -72,7 +72,7 @@ class SingletonPattern(Pattern): # This can be exactly True, False or None value: bool | None - def __init__(self, value: bool | None): + def __init__(self, value: bool | None) -> None: super().__init__() self.value = value @@ -85,7 +85,7 @@ class SequencePattern(Pattern): patterns: list[Pattern] - def __init__(self, patterns: list[Pattern]): + def __init__(self, patterns: list[Pattern]) -> None: super().__init__() self.patterns = patterns @@ -98,7 +98,7 @@ class StarredPattern(Pattern): # a name. capture: NameExpr | None - def __init__(self, capture: NameExpr | None): + def __init__(self, capture: NameExpr | None) -> None: super().__init__() self.capture = capture @@ -111,7 +111,9 @@ class MappingPattern(Pattern): values: list[Pattern] rest: NameExpr | None - def __init__(self, keys: list[Expression], values: list[Pattern], rest: NameExpr | None): + def __init__( + self, keys: list[Expression], values: list[Pattern], rest: NameExpr | None + ) -> None: super().__init__() assert len(keys) == len(values) self.keys = keys @@ -136,7 +138,7 @@ def __init__( positionals: list[Pattern], keyword_keys: list[str], keyword_values: list[Pattern], - ): + ) -> None: super().__init__() assert len(keyword_keys) == len(keyword_values) self.class_ref = class_ref diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index 7869a8b5cdfa..2c568f66c62d 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -166,11 +166,11 @@ class SomeEnum: for n in stnodes if n is None or not n.implicit ) - proper_types = list( + proper_types = [ _infer_value_type_with_auto_fallback(ctx, t) for t in node_types if t is None or not isinstance(t, CallableType) - ) + ] underlying_type = _first(proper_types) if underlying_type is None: return ctx.default_attr_type diff --git a/mypy/renaming.py b/mypy/renaming.py index c960eb4b1ce8..8db336205960 100644 --- a/mypy/renaming.py +++ b/mypy/renaming.py @@ -270,7 +270,7 @@ def flush_refs(self) -> None: This will be called at the end of a scope. """ is_func = self.scope_kinds[-1] == FUNCTION - for name, refs in self.refs[-1].items(): + for refs in self.refs[-1].values(): if len(refs) == 1: # Only one definition -- no renaming needed. continue diff --git a/mypy/semanal.py b/mypy/semanal.py index 48be004daf76..e0a3db2bff1b 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4153,7 +4153,7 @@ def check_typevarlike_name(self, call: CallExpr, name: str, context: Context) -> if len(call.args) < 1: self.fail(f"Too few arguments for {typevarlike_type}()", context) return False - if not isinstance(call.args[0], StrExpr) or not call.arg_kinds[0] == ARG_POS: + if not isinstance(call.args[0], StrExpr) or call.arg_kinds[0] != ARG_POS: self.fail(f"{typevarlike_type}() expects a string literal as first argument", context) return False elif call.args[0].value != name: @@ -4961,9 +4961,7 @@ def visit_name_expr(self, expr: NameExpr) -> None: def bind_name_expr(self, expr: NameExpr, sym: SymbolTableNode) -> None: """Bind name expression to a symbol table node.""" if isinstance(sym.node, TypeVarExpr) and self.tvar_scope.get_binding(sym): - self.fail( - '"{}" is a type variable and only valid in type ' "context".format(expr.name), expr - ) + self.fail(f'"{expr.name}" is a type variable and only valid in type context', expr) elif isinstance(sym.node, PlaceholderNode): self.process_placeholder(expr.name, "name", expr) else: @@ -6809,13 +6807,13 @@ def parse_dataclass_transform_spec(self, call: CallExpr) -> DataclassTransformSp def parse_dataclass_transform_field_specifiers(self, arg: Expression) -> tuple[str, ...]: if not isinstance(arg, TupleExpr): self.fail('"field_specifiers" argument must be a tuple literal', arg) - return tuple() + return () names = [] for specifier in arg.items: if not isinstance(specifier, RefExpr): self.fail('"field_specifiers" must only contain identifiers', specifier) - return tuple() + return () names.append(specifier.fullname) return tuple(names) diff --git a/mypy/semanal_enum.py b/mypy/semanal_enum.py index 528b0519cca1..21576ab47a84 100644 --- a/mypy/semanal_enum.py +++ b/mypy/semanal_enum.py @@ -148,7 +148,7 @@ def parse_enum_call_args( Return a tuple of fields, values, was there an error. """ args = call.args - if not all([arg_kind in [ARG_POS, ARG_NAMED] for arg_kind in call.arg_kinds]): + if not all(arg_kind in [ARG_POS, ARG_NAMED] for arg_kind in call.arg_kinds): return self.fail_enum_call_arg(f"Unexpected arguments to {class_name}()", call) if len(args) < 2: return self.fail_enum_call_arg(f"Too few arguments for {class_name}()", call) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index bc3c5dd61894..9a0be9d9c14c 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -85,7 +85,7 @@ ) NAMEDTUP_CLASS_ERROR: Final = ( - "Invalid statement in NamedTuple definition; " 'expected "field_name: field_type [= default]"' + 'Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]"' ) SELF_TVAR_NAME: Final = "_NT" diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 13aab4de65e4..67c05fd74273 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -50,7 +50,7 @@ ) TPDICT_CLASS_ERROR: Final = ( - "Invalid statement in TypedDict definition; " 'expected "field_name: field_type"' + 'Invalid statement in TypedDict definition; expected "field_name: field_type"' ) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 862c3898a383..174c2922c767 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -394,7 +394,7 @@ def process_synthetic_type_info(self, info: TypeInfo) -> None: # have bodies in the AST so we need to iterate over their symbol # tables separately, unlike normal classes. self.process_type_info(info) - for name, node in info.names.items(): + for node in info.names.values(): if node.node: node.node.accept(self) @@ -549,7 +549,7 @@ def fixup(self, node: SN) -> SN: def replace_nodes_in_symbol_table( symbols: SymbolTable, replacements: dict[SymbolNode, SymbolNode] ) -> None: - for name, node in symbols.items(): + for node in symbols.values(): if node.node: if node.node in replacements: new = replacements[node.node] diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py index 86ff6e2bb540..8c0a4dab696f 100644 --- a/mypy/stubdoc.py +++ b/mypy/stubdoc.py @@ -325,7 +325,7 @@ def args_kwargs(signature: FunctionSig) -> bool: return has_arg("*args", signature) and has_arg("**kwargs", signature) # Move functions with (*args, **kwargs) in their signature to last place. - return list(sorted(self.signatures, key=lambda x: 1 if args_kwargs(x) else 0)) + return sorted(self.signatures, key=lambda x: 1 if args_kwargs(x) else 0) def infer_sig_from_docstring(docstr: str | None, name: str) -> list[FunctionSig] | None: diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 39288197f477..55e46fe0ec25 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -41,7 +41,7 @@ class ExternalSignatureGenerator(SignatureGenerator): def __init__( self, func_sigs: dict[str, str] | None = None, class_sigs: dict[str, str] | None = None - ): + ) -> None: """ Takes a mapping of function/method names to signatures and class name to class signatures (usually corresponds to __init__). @@ -187,7 +187,7 @@ class CFunctionStub: Class that mimics a C function in order to provide parseable docstrings. """ - def __init__(self, name: str, doc: str, is_abstract: bool = False): + def __init__(self, name: str, doc: str, is_abstract: bool = False) -> None: self.__name__ = name self.__doc__ = doc self.__abstractmethod__ = is_abstract @@ -404,7 +404,7 @@ def generate_module(self) -> None: if self.should_reexport(name, obj_module_name, name_is_alias=False): self.import_tracker.reexport(name) - self.set_defined_names(set([name for name, obj in all_items if not inspect.ismodule(obj)])) + self.set_defined_names({name for name, obj in all_items if not inspect.ismodule(obj)}) if self.resort_members: functions: list[str] = [] @@ -765,7 +765,7 @@ def generate_class_stub(self, class_name: str, cls: type, output: list[str]) -> items = self.get_members(cls) if self.resort_members: items = sorted(items, key=lambda x: method_name_sort_key(x[0])) - names = set(x[0] for x in items) + names = {x[0] for x in items} methods: list[str] = [] types: list[str] = [] static_properties: list[str] = [] diff --git a/mypy/stubtest.py b/mypy/stubtest.py index c02a3efd8dc0..e7cc24f33d18 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -303,11 +303,9 @@ def _verify_exported_names( # desirable in at least the `names_in_runtime_not_stub` case stub_object=MISSING, runtime_object=MISSING, - stub_desc=( - f"Names exported in the stub but not at runtime: " f"{names_in_stub_not_runtime}" - ), + stub_desc=(f"Names exported in the stub but not at runtime: {names_in_stub_not_runtime}"), runtime_desc=( - f"Names exported at runtime but not in the stub: " f"{names_in_runtime_not_stub}" + f"Names exported at runtime but not in the stub: {names_in_runtime_not_stub}" ), ) @@ -677,7 +675,7 @@ def _verify_arg_default_value( runtime_type is not None and stub_type is not None # Avoid false positives for marker objects - and type(runtime_arg.default) != object + and type(runtime_arg.default) is not object # And ellipsis and runtime_arg.default is not ... and not is_subtype_helper(runtime_type, stub_type) @@ -897,7 +895,7 @@ def _verify_signature( runtime_arg.kind == inspect.Parameter.POSITIONAL_ONLY and not stub_arg.pos_only and not stub_arg.variable.name.startswith("__") - and not stub_arg.variable.name.strip("_") == "self" + and stub_arg.variable.name.strip("_") != "self" and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods ): yield ( @@ -1812,10 +1810,8 @@ def get_importable_stdlib_modules() -> set[str]: # test.* modules do weird things like raising exceptions in __del__ methods, # leading to unraisable exceptions being logged to the terminal # as a warning at the end of the stubtest run - if ( - submodule_name.endswith(".__main__") - or submodule_name.startswith("idlelib.") - or submodule_name.startswith("test.") + if submodule_name.endswith(".__main__") or submodule_name.startswith( + ("idlelib.", "test.") ): continue diff --git a/mypy/stubutil.py b/mypy/stubutil.py index b8d601ed3c6b..e4a97964c547 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -558,7 +558,7 @@ def __init__( include_private: bool = False, export_less: bool = False, include_docstrings: bool = False, - ): + ) -> None: # Best known value of __all__. self._all_ = _all_ self._include_private = include_private diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index dc34931427ec..bae4f6e81ad1 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -145,7 +145,7 @@ def assert_module_equivalence(name: str, expected: Iterable[str], actual: Iterab assert_string_arrays_equal( expected_normalized, actual_normalized, - ("Actual modules ({}) do not match expected modules ({}) " 'for "[{} ...]"').format( + ('Actual modules ({}) do not match expected modules ({}) for "[{} ...]"').format( ", ".join(actual_normalized), ", ".join(expected_normalized), name ), ) @@ -156,7 +156,7 @@ def assert_target_equivalence(name: str, expected: list[str], actual: list[str]) assert_string_arrays_equal( expected, actual, - ("Actual targets ({}) do not match expected targets ({}) " 'for "[{} ...]"').format( + ('Actual targets ({}) do not match expected targets ({}) for "[{} ...]"').format( ", ".join(actual), ", ".join(expected), name ), ) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 3ad97ced61f2..5fba6192dcaf 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -29,6 +29,7 @@ except ImportError: lxml = None + import pytest # List of files that contain test case descriptions. @@ -99,9 +100,11 @@ def _filename(_msg: str) -> str: def run_case_once( self, testcase: DataDrivenTestCase, - operations: list[FileOperation] = [], + operations: list[FileOperation] | None = None, incremental_step: int = 0, ) -> None: + if operations is None: + operations = [] original_program_text = "\n".join(testcase.input) module_data = self.parse_module(original_program_text, incremental_step) diff --git a/mypy/test/testformatter.py b/mypy/test/testformatter.py index f64527e7804a..9f8bb5d82408 100644 --- a/mypy/test/testformatter.py +++ b/mypy/test/testformatter.py @@ -52,14 +52,14 @@ def test_trim_source(self) -> None: def test_split_words(self) -> None: assert split_words("Simple message") == ["Simple", "message"] - assert split_words('Message with "Some[Long, Types]"' " in it") == [ + assert split_words('Message with "Some[Long, Types]" in it') == [ "Message", "with", '"Some[Long, Types]"', "in", "it", ] - assert split_words('Message with "Some[Long, Types]"' " and [error-code]") == [ + assert split_words('Message with "Some[Long, Types]" and [error-code]') == [ "Message", "with", '"Some[Long, Types]"', diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 34b266115166..72b6f6620f83 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -173,7 +173,7 @@ def run_stubtest( class Case: - def __init__(self, stub: str, runtime: str, error: str | None): + def __init__(self, stub: str, runtime: str, error: str | None) -> None: self.stub = stub self.runtime = runtime self.error = error @@ -2226,7 +2226,7 @@ def also_bad(asdf): pass options=["--allowlist", allowlist.name, "--generate-allowlist"], ) assert output == ( - f"note: unused allowlist entry unused.*\n" f"{TEST_MODULE_NAME}.also_bad\n" + f"note: unused allowlist entry unused.*\n{TEST_MODULE_NAME}.also_bad\n" ) finally: os.unlink(allowlist.name) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 4d916315bddd..8a840424f76f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -2319,7 +2319,7 @@ def visit_unbound_type(self, t: UnboundType) -> None: # Special case P.args and P.kwargs for ParamSpecs only. if name.endswith("args"): - if name.endswith(".args") or name.endswith(".kwargs"): + if name.endswith((".args", ".kwargs")): base = ".".join(name.split(".")[:-1]) n = self.api.lookup_qualified(base, t) if n is not None and isinstance(n.node, ParamSpecExpr): diff --git a/mypy/types.py b/mypy/types.py index fcdb61f9719b..f02e56a677ae 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -441,7 +441,7 @@ class TypeGuardedType(Type): __slots__ = ("type_guard",) - def __init__(self, type_guard: Type): + def __init__(self, type_guard: Type) -> None: super().__init__(line=type_guard.line, column=type_guard.column) self.type_guard = type_guard diff --git a/mypy/typestate.py b/mypy/typestate.py index b32fb0ef6df1..c5a5da03eae5 100644 --- a/mypy/typestate.py +++ b/mypy/typestate.py @@ -192,7 +192,7 @@ def record_subtype_cache_entry( # These are unlikely to match, due to the large space of # possible values. Avoid uselessly increasing cache sizes. return - cache = self._subtype_caches.setdefault(right.type, dict()) + cache = self._subtype_caches.setdefault(right.type, {}) cache.setdefault(kind, set()).add((left, right)) def record_negative_subtype_cache_entry( @@ -204,7 +204,7 @@ def record_negative_subtype_cache_entry( return if len(self._negative_subtype_caches) > MAX_NEGATIVE_CACHE_TYPES: self._negative_subtype_caches.clear() - cache = self._negative_subtype_caches.setdefault(right.type, dict()) + cache = self._negative_subtype_caches.setdefault(right.type, {}) subcache = cache.setdefault(kind, set()) if len(subcache) > MAX_NEGATIVE_CACHE_ENTRIES: subcache.clear() diff --git a/mypy/util.py b/mypy/util.py index 0277a96a6885..968774ee7c98 100644 --- a/mypy/util.py +++ b/mypy/util.py @@ -451,7 +451,7 @@ def get_unique_redefinition_name(name: str, existing: Container[str]) -> str: def check_python_version(program: str) -> None: """Report issues with the Python used to run mypy, dmypy, or stubgen""" # Check for known bad Python versions. - if sys.version_info[:2] < (3, 8): + if sys.version_info[:2] < (3, 8): # noqa: UP036 sys.exit( "Running {name} with Python 3.7 or lower is not supported; " "please upgrade to 3.8 or newer".format(name=program) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index caf2058ea7c4..6c0dfd43b9af 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -993,7 +993,7 @@ def _toposort_visit(name: str) -> None: result.append(decl.declaration) decl.mark = True - for name, marked_declaration in marked_declarations.items(): + for name in marked_declarations: _toposort_visit(name) return result diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index 791e856c274a..45c6c7a05867 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -145,7 +145,7 @@ def generate_wrapper_function( real_args = list(fn.args) if fn.sig.num_bitmap_args: real_args = real_args[: -fn.sig.num_bitmap_args] - if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD: + if fn.class_name and fn.decl.kind != FUNC_STATICMETHOD: arg = real_args.pop(0) emitter.emit_line(f"PyObject *obj_{arg.name} = self;") @@ -238,7 +238,7 @@ def generate_legacy_wrapper_function( real_args = list(fn.args) if fn.sig.num_bitmap_args: real_args = real_args[: -fn.sig.num_bitmap_args] - if fn.class_name and not fn.decl.kind == FUNC_STATICMETHOD: + if fn.class_name and fn.decl.kind != FUNC_STATICMETHOD: arg = real_args.pop(0) emitter.emit_line(f"PyObject *obj_{arg.name} = self;") diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 5e6520048197..29e06439abdd 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -139,7 +139,7 @@ def is_from_module(node: SymbolNode, module: MypyFile) -> bool: def load_type_map(mapper: Mapper, modules: list[MypyFile], deser_ctx: DeserMaps) -> None: """Populate a Mapper with deserialized IR from a list of modules.""" for module in modules: - for name, node in module.names.items(): + for node in module.names.values(): if isinstance(node.node, TypeInfo) and is_from_module(node.node, module): ir = deser_ctx.classes[node.node.fullname] mapper.type_to_ir[node.node] = ir @@ -153,7 +153,7 @@ def load_type_map(mapper: Mapper, modules: list[MypyFile], deser_ctx: DeserMaps) def get_module_func_defs(module: MypyFile) -> Iterable[FuncDef]: """Collect all of the (non-method) functions declared in a module.""" - for name, node in module.names.items(): + for node in module.names.values(): # We need to filter out functions that are imported or # aliases. The best way to do this seems to be by # checking that the fullname matches. @@ -468,7 +468,7 @@ def prepare_non_ext_class_def( ir = mapper.type_to_ir[cdef.info] info = cdef.info - for name, node in info.names.items(): + for node in info.names.values(): if isinstance(node.node, (FuncDef, Decorator)): prepare_method_def(ir, module_name, cdef, mapper, node.node) elif isinstance(node.node, OverloadedFuncDef): diff --git a/mypyc/primitives/registry.py b/mypyc/primitives/registry.py index aa96b35aec56..11fca7dc2c70 100644 --- a/mypyc/primitives/registry.py +++ b/mypyc/primitives/registry.py @@ -93,7 +93,7 @@ def method_op( var_arg_type: RType | None = None, truncated_type: RType | None = None, ordering: list[int] | None = None, - extra_int_constants: list[tuple[int, RType]] = [], + extra_int_constants: list[tuple[int, RType]] | None = None, steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, @@ -122,6 +122,8 @@ def method_op( is_borrowed: if True, returned value is borrowed (no need to decrease refcount) priority: if multiple ops match, the one with the highest priority is picked """ + if extra_int_constants is None: + extra_int_constants = [] ops = method_call_ops.setdefault(name, []) desc = CFunctionDescription( name, @@ -150,7 +152,7 @@ def function_op( var_arg_type: RType | None = None, truncated_type: RType | None = None, ordering: list[int] | None = None, - extra_int_constants: list[tuple[int, RType]] = [], + extra_int_constants: list[tuple[int, RType]] | None = None, steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, @@ -165,6 +167,8 @@ def function_op( name: full name of the function arg_types: positional argument types for which this applies """ + if extra_int_constants is None: + extra_int_constants = [] ops = function_ops.setdefault(name, []) desc = CFunctionDescription( name, @@ -193,7 +197,7 @@ def binary_op( var_arg_type: RType | None = None, truncated_type: RType | None = None, ordering: list[int] | None = None, - extra_int_constants: list[tuple[int, RType]] = [], + extra_int_constants: list[tuple[int, RType]] | None = None, steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, @@ -205,6 +209,8 @@ def binary_op( Most arguments are similar to method_op(), but exactly two argument types are expected. """ + if extra_int_constants is None: + extra_int_constants = [] ops = binary_ops.setdefault(name, []) desc = CFunctionDescription( name, @@ -232,7 +238,7 @@ def custom_op( var_arg_type: RType | None = None, truncated_type: RType | None = None, ordering: list[int] | None = None, - extra_int_constants: list[tuple[int, RType]] = [], + extra_int_constants: list[tuple[int, RType]] | None = None, steals: StealsDescription = False, is_borrowed: bool = False, ) -> CFunctionDescription: @@ -240,6 +246,8 @@ def custom_op( Most arguments are similar to method_op(). """ + if extra_int_constants is None: + extra_int_constants = [] return CFunctionDescription( "", arg_types, @@ -264,7 +272,7 @@ def unary_op( error_kind: int, truncated_type: RType | None = None, ordering: list[int] | None = None, - extra_int_constants: list[tuple[int, RType]] = [], + extra_int_constants: list[tuple[int, RType]] | None = None, steals: StealsDescription = False, is_borrowed: bool = False, priority: int = 1, @@ -276,6 +284,8 @@ def unary_op( Most arguments are similar to method_op(), but exactly one argument type is expected. """ + if extra_int_constants is None: + extra_int_constants = [] ops = unary_ops.setdefault(name, []) desc = CFunctionDescription( name, diff --git a/pyproject.toml b/pyproject.toml index c43253fed982..5fc0cee3f65c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,24 +33,29 @@ fix = true select = [ "E", # pycodestyle (error) "F", # pyflakes + "W", # pycodestyle (warning) "B", # flake8-bugbear "I", # isort "RUF100", # Unused noqa comments "PGH004", # blanket noqa comments "UP", # pyupgrade + "C4", # flake8-comprehensions + "SIM201", "SIM202", # simplify comparisons involving not + "ISC001", # implicitly concatenated string + "RET501", "RET502", # better return None handling ] ignore = [ - "B006", # use of mutable defaults in function signatures "B007", # Loop control variable not used within the loop body. "B011", # Don't use assert False "B023", # Function definition does not bind loop variable - "E203", # conflicts with black + "E2", # conflicts with black "E402", # module level import not at top of file "E501", # conflicts with black "E731", # Do not assign a `lambda` expression, use a `def` "E741", # Ambiguous variable name "UP032", # 'f-string always preferable to format' is controversial + "C416", # There are a few cases where it's nice to have names for the dict items ] unfixable = [ diff --git a/test-requirements.in b/test-requirements.in index bab3ece29c02..2bf8de0aa2f5 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -14,6 +14,6 @@ psutil>=4.0 pytest>=7.4.0 pytest-xdist>=1.34.0 pytest-cov>=2.10.0 -ruff==0.1.0 # must match version in .pre-commit-config.yaml +ruff==0.1.4 # must match version in .pre-commit-config.yaml setuptools>=65.5.1 tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.7 diff --git a/test-requirements.txt b/test-requirements.txt index 3bb9cf29635f..57607c1bae57 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ ruamel-yaml==0.17.40 # via pre-commit-hooks ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.1.0 +ruff==0.1.4 # via -r test-requirements.in tomli==2.0.1 # via -r test-requirements.in From f9e8e0bda5cfbb54d6a8f9e482aa25da28a1a635 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 1 Jan 2024 20:15:15 -0800 Subject: [PATCH 04/32] Allow unary + in Literal (#16729) Implements python/typing#1550 Fixes #16727 (a trivial bug I found while implementing this feature) --- mypy/fastparse.py | 16 +++++++++++----- test-data/unit/check-literal.test | 11 ++++++++--- test-data/unit/fixtures/ops.pyi | 4 ++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index cba01eab2e4e..13b9b5c8a871 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -126,7 +126,7 @@ import ast as ast3 # TODO: Index, ExtSlice are deprecated in 3.9. -from ast import AST, Attribute, Call, FunctionType, Index, Name, Starred, UnaryOp, USub +from ast import AST, Attribute, Call, FunctionType, Index, Name, Starred, UAdd, UnaryOp, USub def ast3_parse( @@ -1940,13 +1940,19 @@ def visit_Constant(self, n: Constant) -> Type: # UnaryOp(op, operand) def visit_UnaryOp(self, n: UnaryOp) -> Type: - # We support specifically Literal[-4] and nothing else. - # For example, Literal[+4] or Literal[~6] is not supported. + # We support specifically Literal[-4], Literal[+4], and nothing else. + # For example, Literal[~6] or Literal[not False] is not supported. typ = self.visit(n.operand) - if isinstance(typ, RawExpressionType) and isinstance(n.op, USub): - if isinstance(typ.literal_value, int): + if ( + isinstance(typ, RawExpressionType) + # Use type() because we do not want to allow bools. + and type(typ.literal_value) is int # noqa: E721 + ): + if isinstance(n.op, USub): typ.literal_value *= -1 return typ + if isinstance(n.op, UAdd): + return typ return self.invalid_type(n) def numeric_type(self, value: object, n: AST) -> Type: diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index d9ad68385ad1..ea2aa5068e85 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -591,7 +591,6 @@ from typing_extensions import Literal def dummy() -> int: return 3 a: Literal[3 + 4] # E: Invalid type: Literal[...] cannot contain arbitrary expressions b: Literal[" foo ".trim()] # E: Invalid type: Literal[...] cannot contain arbitrary expressions -c: Literal[+42] # E: Invalid type: Literal[...] cannot contain arbitrary expressions d: Literal[~12] # E: Invalid type: Literal[...] cannot contain arbitrary expressions e: Literal[dummy()] # E: Invalid type: Literal[...] cannot contain arbitrary expressions [builtins fixtures/tuple.pyi] @@ -2739,7 +2738,7 @@ def test() -> None: ... [builtins fixtures/bool.pyi] -[case testNegativeIntLiteral] +[case testUnaryOpLiteral] from typing_extensions import Literal a: Literal[-2] = -2 @@ -2747,8 +2746,14 @@ b: Literal[-1] = -1 c: Literal[0] = 0 d: Literal[1] = 1 e: Literal[2] = 2 +f: Literal[+1] = 1 +g: Literal[+2] = 2 + +x: Literal[+True] = True # E: Invalid type: Literal[...] cannot contain arbitrary expressions +y: Literal[-True] = -1 # E: Invalid type: Literal[...] cannot contain arbitrary expressions +z: Literal[~0] = 0 # E: Invalid type: Literal[...] cannot contain arbitrary expressions [out] -[builtins fixtures/float.pyi] +[builtins fixtures/ops.pyi] [case testNegativeIntLiteralWithFinal] from typing_extensions import Literal, Final diff --git a/test-data/unit/fixtures/ops.pyi b/test-data/unit/fixtures/ops.pyi index 9cc4d22eb0a7..df3b163166ad 100644 --- a/test-data/unit/fixtures/ops.pyi +++ b/test-data/unit/fixtures/ops.pyi @@ -24,8 +24,6 @@ class tuple(Sequence[Tco]): class function: pass -class bool: pass - class str: def __init__(self, x: 'int') -> None: pass def __add__(self, x: 'str') -> 'str': pass @@ -54,6 +52,8 @@ class int: def __gt__(self, x: 'int') -> bool: pass def __ge__(self, x: 'int') -> bool: pass +class bool(int): pass + class float: def __add__(self, x: 'float') -> 'float': pass def __radd__(self, x: 'float') -> 'float': pass From a1b4e32785b69291748579d97cb9456e1198ec21 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:02:15 -0800 Subject: [PATCH 05/32] Document how evil `--no-strict-optional` is (#16731) On multiple occasions, I've encountered folks using this, running into issues and then being perplexed when they figure out what `--no-strict-optional` actually does. Most recently in https://github.com/python/mypy/issues/16718#issuecomment-1873374058 This is a non-standard, dangerous option that should not be used in modern typed Python. It's been five and a half years since it was the default behaviour in mypy, so we should deemphasise and warn about its existence. --- docs/source/command_line.rst | 16 ++++---- docs/source/common_issues.rst | 2 +- docs/source/config_file.rst | 9 ++++- docs/source/kinds_of_types.rst | 68 ---------------------------------- 4 files changed, 17 insertions(+), 78 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index c8e0ef3df11f..4a7ead3e8724 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -415,7 +415,6 @@ None and Optional handling ************************** The following flags adjust how mypy handles values of type ``None``. -For more details, see :ref:`no_strict_optional`. .. _implicit-optional: @@ -435,16 +434,19 @@ For more details, see :ref:`no_strict_optional`. **Note:** This was disabled by default starting in mypy 0.980. +.. _no_strict_optional: + .. option:: --no-strict-optional - This flag disables strict checking of :py:data:`~typing.Optional` + This flag effectively disables checking of :py:data:`~typing.Optional` types and ``None`` values. With this option, mypy doesn't - generally check the use of ``None`` values -- they are valid - everywhere. See :ref:`no_strict_optional` for more about this feature. + generally check the use of ``None`` values -- it is treated + as compatible with every type. + + .. warning:: - **Note:** Strict optional checking was enabled by default starting in - mypy 0.600, and in previous versions it had to be explicitly enabled - using ``--strict-optional`` (which is still accepted). + ``--no-strict-optional`` is evil. Avoid using it and definitely do + not use it without understanding what it does. .. _configuring-warnings: diff --git a/docs/source/common_issues.rst b/docs/source/common_issues.rst index e6570b7eef5b..8cc18c863e45 100644 --- a/docs/source/common_issues.rst +++ b/docs/source/common_issues.rst @@ -119,7 +119,7 @@ and mypy doesn't complain**. return None # No error! You may have disabled strict optional checking (see -:ref:`no_strict_optional` for more). +:ref:`--no-strict-optional ` for more). .. _silencing_checker: diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index de769200bf2b..910b015df658 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -580,10 +580,15 @@ section of the command line docs. :type: boolean :default: True - Enables or disables strict Optional checks. If False, mypy treats ``None`` + Effectively disables checking of :py:data:`~typing.Optional` + types and ``None`` values. With this option, mypy doesn't + generally check the use of ``None`` values -- it is treated as compatible with every type. - **Note:** This was False by default in mypy versions earlier than 0.600. + .. warning:: + + ``strict_optional = false`` is evil. Avoid using it and definitely do + not use it without understanding what it does. Configuring warnings diff --git a/docs/source/kinds_of_types.rst b/docs/source/kinds_of_types.rst index c3180850f119..d07eb40753f3 100644 --- a/docs/source/kinds_of_types.rst +++ b/docs/source/kinds_of_types.rst @@ -429,74 +429,6 @@ the runtime with some limitations (see :ref:`runtime_troubles`). t2: int | None # equivalent to Optional[int] -.. _no_strict_optional: - -Disabling strict optional checking -********************************** - -Mypy also has an option to treat ``None`` as a valid value for every -type (in case you know Java, it's useful to think of it as similar to -the Java ``null``). In this mode ``None`` is also valid for primitive -types such as ``int`` and ``float``, and :py:data:`~typing.Optional` types are -not required. - -The mode is enabled through the :option:`--no-strict-optional ` command-line -option. In mypy versions before 0.600 this was the default mode. You -can enable this option explicitly for backward compatibility with -earlier mypy versions, in case you don't want to introduce optional -types to your codebase yet. - -It will cause mypy to silently accept some buggy code, such as -this example -- it's not recommended if you can avoid it: - -.. code-block:: python - - def inc(x: int) -> int: - return x + 1 - - x = inc(None) # No error reported by mypy if strict optional mode disabled! - -However, making code "optional clean" can take some work! You can also use -:ref:`the mypy configuration file ` to migrate your code -to strict optional checking one file at a time, since there exists -the per-module flag -:confval:`strict_optional` to control strict optional mode. - -Often it's still useful to document whether a variable can be -``None``. For example, this function accepts a ``None`` argument, -but it's not obvious from its signature: - -.. code-block:: python - - def greeting(name: str) -> str: - if name: - return f'Hello, {name}' - else: - return 'Hello, stranger' - - print(greeting('Python')) # Okay! - print(greeting(None)) # Also okay! - -You can still use :py:data:`Optional[t] ` to document that ``None`` is a -valid argument type, even if strict ``None`` checking is not -enabled: - -.. code-block:: python - - from typing import Optional - - def greeting(name: Optional[str]) -> str: - if name: - return f'Hello, {name}' - else: - return 'Hello, stranger' - -Mypy treats this as semantically equivalent to the previous example -if strict optional checking is disabled, since ``None`` is implicitly -valid for any type, but it's much more -useful for a programmer who is reading the code. This also makes -it easier to migrate to strict ``None`` checking in the future. - .. _type-aliases: Type aliases From fbb738a4626976f83a7412df73533d87483a31b7 Mon Sep 17 00:00:00 2001 From: Ihor <31508183+nautics889@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:40:24 +0200 Subject: [PATCH 06/32] [minor] Fix arg name in classmethod (#16741) --- mypy/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index fe41777c55f7..6c23c51dfcd3 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1155,7 +1155,7 @@ def serialize(self) -> JsonDict: } @classmethod - def deserialize(self, data: JsonDict) -> ClassDef: + def deserialize(cls, data: JsonDict) -> ClassDef: assert data[".class"] == "ClassDef" res = ClassDef( data["name"], From 35f402c74205d373b81076ea82f507eb85161a25 Mon Sep 17 00:00:00 2001 From: WeilerMarcel Date: Mon, 8 Jan 2024 10:08:32 +0100 Subject: [PATCH 07/32] Support type stub generation for `staticmethod` (#14934) Fixes #13574 This PR fixes the generation of type hints for static methods of pybind11 classes. The code changes are based on the suggestions in #13574. The fix introduces an additional check if the property under inspection is of type `staticmethod`. If it is, the type information is read from the staticmethod's `__func__` attribute, instead of the staticmethod instance itself. I added a test for C++ classes with static methods bound using pybind11. Both, an overloaded and a non-overloaded static method are tested. --- mypy/stubgenc.py | 16 ++++++---- test-data/pybind11_mypy_demo/src/main.cpp | 15 ++++++++++ .../pybind11_mypy_demo/basics.pyi | 30 +++++++++++++++++++ .../stubgen/pybind11_mypy_demo/basics.pyi | 11 +++++++ 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py index 55e46fe0ec25..3bec0c246d9a 100755 --- a/mypy/stubgenc.py +++ b/mypy/stubgenc.py @@ -530,12 +530,14 @@ def is_classmethod(self, class_info: ClassInfo, name: str, obj: object) -> bool: return inspect.ismethod(obj) def is_staticmethod(self, class_info: ClassInfo | None, name: str, obj: object) -> bool: - if self.is_c_module: + if class_info is None: return False + elif self.is_c_module: + raw_lookup: Mapping[str, Any] = getattr(class_info.cls, "__dict__") # noqa: B009 + raw_value = raw_lookup.get(name, obj) + return isinstance(raw_value, staticmethod) else: - return class_info is not None and isinstance( - inspect.getattr_static(class_info.cls, name), staticmethod - ) + return isinstance(inspect.getattr_static(class_info.cls, name), staticmethod) @staticmethod def is_abstract_method(obj: object) -> bool: @@ -761,7 +763,7 @@ def generate_class_stub(self, class_name: str, cls: type, output: list[str]) -> The result lines will be appended to 'output'. If necessary, any required names will be added to 'imports'. """ - raw_lookup = getattr(cls, "__dict__") # noqa: B009 + raw_lookup: Mapping[str, Any] = getattr(cls, "__dict__") # noqa: B009 items = self.get_members(cls) if self.resort_members: items = sorted(items, key=lambda x: method_name_sort_key(x[0])) @@ -793,7 +795,9 @@ def generate_class_stub(self, class_name: str, cls: type, output: list[str]) -> continue attr = "__init__" # FIXME: make this nicer - if self.is_classmethod(class_info, attr, value): + if self.is_staticmethod(class_info, attr, value): + class_info.self_var = "" + elif self.is_classmethod(class_info, attr, value): class_info.self_var = "cls" else: class_info.self_var = "self" diff --git a/test-data/pybind11_mypy_demo/src/main.cpp b/test-data/pybind11_mypy_demo/src/main.cpp index 192a90cf8e30..8be759d51671 100644 --- a/test-data/pybind11_mypy_demo/src/main.cpp +++ b/test-data/pybind11_mypy_demo/src/main.cpp @@ -118,6 +118,13 @@ const Point Point::y_axis = Point(0, 1); Point::LengthUnit Point::length_unit = Point::LengthUnit::mm; Point::AngleUnit Point::angle_unit = Point::AngleUnit::radian; +struct Foo +{ + static int some_static_method(int a, int b) { return a * 42 + b; } + static int overloaded_static_method(int value) { return value * 42; } + static double overloaded_static_method(double value) { return value * 42; } +}; + } // namespace: basics void bind_basics(py::module& basics) { @@ -166,6 +173,14 @@ void bind_basics(py::module& basics) { .value("radian", Point::AngleUnit::radian) .value("degree", Point::AngleUnit::degree); + // Static methods + py::class_ pyFoo(basics, "Foo"); + + pyFoo + .def_static("some_static_method", &Foo::some_static_method, R"#(None)#", py::arg("a"), py::arg("b")) + .def_static("overloaded_static_method", py::overload_cast(&Foo::overloaded_static_method), py::arg("value")) + .def_static("overloaded_static_method", py::overload_cast(&Foo::overloaded_static_method), py::arg("value")); + // Module-level attributes basics.attr("PI") = std::acos(-1); basics.attr("__version__") = "0.0.1"; diff --git a/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi b/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi index b761291e11f3..3047da622a3e 100644 --- a/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi +++ b/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi @@ -3,6 +3,36 @@ from typing import ClassVar, List, overload PI: float __version__: str +class Foo: + def __init__(self, *args, **kwargs) -> None: + """Initialize self. See help(type(self)) for accurate signature.""" + @overload + @staticmethod + def overloaded_static_method(value: int) -> int: + """overloaded_static_method(*args, **kwargs) + Overloaded function. + + 1. overloaded_static_method(value: int) -> int + + 2. overloaded_static_method(value: float) -> float + """ + @overload + @staticmethod + def overloaded_static_method(value: float) -> float: + """overloaded_static_method(*args, **kwargs) + Overloaded function. + + 1. overloaded_static_method(value: int) -> int + + 2. overloaded_static_method(value: float) -> float + """ + @staticmethod + def some_static_method(a: int, b: int) -> int: + """some_static_method(a: int, b: int) -> int + + None + """ + class Point: class AngleUnit: __members__: ClassVar[dict] = ... # read-only diff --git a/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi b/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi index 6f164a03edcc..f3acb1677e68 100644 --- a/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi +++ b/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi @@ -3,6 +3,17 @@ from typing import ClassVar, List, overload PI: float __version__: str +class Foo: + def __init__(self, *args, **kwargs) -> None: ... + @overload + @staticmethod + def overloaded_static_method(value: int) -> int: ... + @overload + @staticmethod + def overloaded_static_method(value: float) -> float: ... + @staticmethod + def some_static_method(a: int, b: int) -> int: ... + class Point: class AngleUnit: __members__: ClassVar[dict] = ... # read-only From edebe024cbe8406cdf191cefe111683d0c2849ea Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 11 Jan 2024 19:08:49 -0800 Subject: [PATCH 08/32] Accept multiline quoted annotations (#16765) See discussion in https://discuss.python.org/t/newlines-within-triple-quoted-type-expressions/42833 --- mypy/fastparse.py | 2 +- test-data/unit/check-basic.test | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 13b9b5c8a871..b3e0fc70a9d6 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -318,7 +318,7 @@ def parse_type_string( string expression "blah" using this function. """ try: - _, node = parse_type_comment(expr_string.strip(), line=line, column=column, errors=None) + _, node = parse_type_comment(f"({expr_string})", line=line, column=column, errors=None) if isinstance(node, UnboundType) and node.original_str_expr is None: node.original_str_expr = expr_string node.original_str_fallback = expr_fallback_name diff --git a/test-data/unit/check-basic.test b/test-data/unit/check-basic.test index 61a7160ce4f4..7a426c3eca9f 100644 --- a/test-data/unit/check-basic.test +++ b/test-data/unit/check-basic.test @@ -502,3 +502,19 @@ s2: str = 42 # E: Incompatible types in assignment (expression has type "int", s3: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") [file c.py] s3: str = 'foo' + +[case testMultilineQuotedAnnotation] +x: """ + + int | + str + +""" +reveal_type(x) # N: Revealed type is "Union[builtins.int, builtins.str]" + +y: """( + int | + str +) +""" +reveal_type(y) # N: Revealed type is "Union[builtins.int, builtins.str]" From 1fd29ac54a2135ec1d1d30289c625d12e6198eac Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 11 Jan 2024 19:27:46 -0800 Subject: [PATCH 09/32] stubtest: fix pos-only handling in overload resolution (#16750) --- mypy/stubtest.py | 7 ++++- mypy/test/teststubtest.py | 58 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index e7cc24f33d18..6061e98bd7cd 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -826,7 +826,10 @@ def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> Signature[nodes.Arg # argument. To accomplish this, we just make up a fake index-based name. name = ( f"__{index}" - if arg.variable.name.startswith("__") or assume_positional_only + if arg.variable.name.startswith("__") + or arg.pos_only + or assume_positional_only + or arg.variable.name.strip("_") == "self" else arg.variable.name ) all_args.setdefault(name, []).append((arg, index)) @@ -870,6 +873,7 @@ def get_kind(arg_name: str) -> nodes.ArgKind: type_annotation=None, initializer=None, kind=get_kind(arg_name), + pos_only=all(arg.pos_only for arg, _ in all_args[arg_name]), ) if arg.kind.is_positional(): sig.pos.append(arg) @@ -905,6 +909,7 @@ def _verify_signature( if ( runtime_arg.kind != inspect.Parameter.POSITIONAL_ONLY and (stub_arg.pos_only or stub_arg.variable.name.startswith("__")) + and stub_arg.variable.name.strip("_") != "self" and not is_dunder(function_name, exclude_special=True) # noisy for dunder methods ): yield ( diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 72b6f6620f83..c0bb2eb6f9da 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -210,7 +210,13 @@ def test(*args: Any, **kwargs: Any) -> None: ) actual_errors = set(output.splitlines()) - assert actual_errors == expected_errors, output + if actual_errors != expected_errors: + output = run_stubtest( + stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases), + runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases), + options=[], + ) + assert actual_errors == expected_errors, output return test @@ -660,6 +666,56 @@ def f6(self, x, /): pass """, error=None, ) + yield Case( + stub=""" + @overload + def f7(a: int, /) -> int: ... + @overload + def f7(b: str, /) -> str: ... + """, + runtime="def f7(x, /): pass", + error=None, + ) + yield Case( + stub=""" + @overload + def f8(a: int, c: int = 0, /) -> int: ... + @overload + def f8(b: str, d: int, /) -> str: ... + """, + runtime="def f8(x, y, /): pass", + error="f8", + ) + yield Case( + stub=""" + @overload + def f9(a: int, c: int = 0, /) -> int: ... + @overload + def f9(b: str, d: int, /) -> str: ... + """, + runtime="def f9(x, y=0, /): pass", + error=None, + ) + yield Case( + stub=""" + class Bar: + @overload + def f1(self) -> int: ... + @overload + def f1(self, a: int, /) -> int: ... + + @overload + def f2(self, a: int, /) -> int: ... + @overload + def f2(self, a: str, /) -> int: ... + """, + runtime=""" + class Bar: + def f1(self, *a) -> int: ... + def f2(self, *a) -> int: ... + """, + error=None, + ) @collect_cases def test_property(self) -> Iterator[Case]: From a8741d8fda2695e189add60b1cfc4adebd2c3e1e Mon Sep 17 00:00:00 2001 From: Fabian Keller Date: Sat, 13 Jan 2024 16:14:16 +0100 Subject: [PATCH 10/32] Improve stubgen tests (#16760) Improve test cases around #16486 This PR does not change any actual mypy behavior, only hardens the stubgen tests. The specific changes are: - **dedicated test cases**: The existing pybind11 test cases originate from a pybind11 demo. They cover a specific topic involving geometry types and semi-implemented logic related to it. This somehow distracts from the aspects we are trying to test here from the mypy stubgen perspective, because it hides the actual intent of the bindings. I've simply started adding new test cases that clearly express via their name what the test case is addressing. I've kept the original demo stuff for now, so that the new cases are just an addition (overhead is negligible). - **running mypy on the generated stubs**: In general it is crucial that the output produced by the stubgen can actually be type checked by mypy (this was the regression in #18486). This wasn't covered by the CI check so far. I've added check now, which would have avoided the regression. My goal for follow up PRs would be that we can use `mypy --disallow-untyped-defs` or even `mypy --strict` on the output. - **minor directory restructuring**: So far the expected stub outputs were stored in folder names `stubgen` and `stubgen-include-docs`. This was a bit confusing, because the folder `stubgen` suggested it contains _the_ stubgen (implementation). I went for `expected_stubs_no_docs` and `expected_stubs_with_docs` to make the role of the two folders more explicit. - **minor script bugfix**: Fix a bug in `test-stubgen.sh`: The pre-delete functionality was broken, because the `*` was quoted and therefore did not match. --- .github/workflows/test_stubgenc.yml | 2 +- misc/test-stubgenc.sh | 20 ++- .../pybind11_fixtures/__init__.pyi | 27 ++++ .../pybind11_fixtures/demo.pyi} | 11 -- .../pybind11_fixtures/__init__.pyi | 52 +++++++ .../pybind11_fixtures/demo.pyi} | 60 ++------ .../pyproject.toml | 0 .../setup.py | 4 +- .../src/main.cpp | 143 ++++++++++++++---- .../pybind11_mypy_demo/__init__.pyi | 1 - .../stubgen/pybind11_mypy_demo/__init__.pyi | 1 - 11 files changed, 227 insertions(+), 94 deletions(-) create mode 100644 test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi rename test-data/{pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi => pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi} (86%) create mode 100644 test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi rename test-data/{pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi => pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi} (57%) rename test-data/{pybind11_mypy_demo => pybind11_fixtures}/pyproject.toml (100%) rename test-data/{pybind11_mypy_demo => pybind11_fixtures}/setup.py (85%) rename test-data/{pybind11_mypy_demo => pybind11_fixtures}/src/main.cpp (61%) delete mode 100644 test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/__init__.pyi delete mode 100644 test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/__init__.pyi diff --git a/.github/workflows/test_stubgenc.yml b/.github/workflows/test_stubgenc.yml index a2fb3e9dce6b..3daf01372d2d 100644 --- a/.github/workflows/test_stubgenc.yml +++ b/.github/workflows/test_stubgenc.yml @@ -1,4 +1,4 @@ -name: Test stubgenc on pybind11-mypy-demo +name: Test stubgenc on pybind11_fixtures on: workflow_dispatch: diff --git a/misc/test-stubgenc.sh b/misc/test-stubgenc.sh index 5cb5140eba76..ad66722628d8 100755 --- a/misc/test-stubgenc.sh +++ b/misc/test-stubgenc.sh @@ -7,7 +7,7 @@ cd "$(dirname "$0")/.." # Install dependencies, demo project and mypy python -m pip install -r test-requirements.txt -python -m pip install ./test-data/pybind11_mypy_demo +python -m pip install ./test-data/pybind11_fixtures python -m pip install . EXIT=0 @@ -17,19 +17,29 @@ EXIT=0 # everything else is passed to stubgen as its arguments function stubgenc_test() { # Remove expected stubs and generate new inplace - STUBGEN_OUTPUT_FOLDER=./test-data/pybind11_mypy_demo/$1 - rm -rf "${STUBGEN_OUTPUT_FOLDER:?}/*" + STUBGEN_OUTPUT_FOLDER=./test-data/pybind11_fixtures/$1 + rm -rf "${STUBGEN_OUTPUT_FOLDER:?}" + stubgen -o "$STUBGEN_OUTPUT_FOLDER" "${@:2}" + # Check if generated stubs can actually be type checked by mypy + if ! mypy "$STUBGEN_OUTPUT_FOLDER"; + then + echo "Stubgen test failed, because generated stubs failed to type check." + EXIT=1 + fi + # Compare generated stubs to expected ones if ! git diff --exit-code "$STUBGEN_OUTPUT_FOLDER"; then + echo "Stubgen test failed, because generated stubs differ from expected outputs." EXIT=1 fi } # create stubs without docstrings -stubgenc_test stubgen -p pybind11_mypy_demo +stubgenc_test expected_stubs_no_docs -p pybind11_fixtures # create stubs with docstrings -stubgenc_test stubgen-include-docs -p pybind11_mypy_demo --include-docstrings +stubgenc_test expected_stubs_with_docs -p pybind11_fixtures --include-docstrings + exit $EXIT diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi new file mode 100644 index 000000000000..e113d8a69a5d --- /dev/null +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi @@ -0,0 +1,27 @@ +import os +from . import demo as demo +from typing import List, Optional, Tuple, overload + +class StaticMethods: + def __init__(self, *args, **kwargs) -> None: ... + @overload + @staticmethod + def overloaded_static_method(value: int) -> int: ... + @overload + @staticmethod + def overloaded_static_method(value: float) -> float: ... + @staticmethod + def some_static_method(a: int, b: int) -> int: ... + +class TestStruct: + field_readwrite: int + field_readwrite_docstring: int + def __init__(self, *args, **kwargs) -> None: ... + @property + def field_readonly(self) -> int: ... + +def func_incomplete_signature(*args, **kwargs): ... +def func_returning_optional() -> Optional[int]: ... +def func_returning_pair() -> Tuple[int, float]: ... +def func_returning_path() -> os.PathLike: ... +def func_returning_vector() -> List[float]: ... diff --git a/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi similarity index 86% rename from test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi rename to test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi index f3acb1677e68..6f164a03edcc 100644 --- a/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/basics.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/demo.pyi @@ -3,17 +3,6 @@ from typing import ClassVar, List, overload PI: float __version__: str -class Foo: - def __init__(self, *args, **kwargs) -> None: ... - @overload - @staticmethod - def overloaded_static_method(value: int) -> int: ... - @overload - @staticmethod - def overloaded_static_method(value: float) -> float: ... - @staticmethod - def some_static_method(a: int, b: int) -> int: ... - class Point: class AngleUnit: __members__: ClassVar[dict] = ... # read-only diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi new file mode 100644 index 000000000000..1dabb0d9a330 --- /dev/null +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi @@ -0,0 +1,52 @@ +import os +from . import demo as demo +from typing import List, Optional, Tuple, overload + +class StaticMethods: + def __init__(self, *args, **kwargs) -> None: + """Initialize self. See help(type(self)) for accurate signature.""" + @overload + @staticmethod + def overloaded_static_method(value: int) -> int: + """overloaded_static_method(*args, **kwargs) + Overloaded function. + + 1. overloaded_static_method(value: int) -> int + + 2. overloaded_static_method(value: float) -> float + """ + @overload + @staticmethod + def overloaded_static_method(value: float) -> float: + """overloaded_static_method(*args, **kwargs) + Overloaded function. + + 1. overloaded_static_method(value: int) -> int + + 2. overloaded_static_method(value: float) -> float + """ + @staticmethod + def some_static_method(a: int, b: int) -> int: + """some_static_method(a: int, b: int) -> int + + None + """ + +class TestStruct: + field_readwrite: int + field_readwrite_docstring: int + def __init__(self, *args, **kwargs) -> None: + """Initialize self. See help(type(self)) for accurate signature.""" + @property + def field_readonly(self) -> int: ... + +def func_incomplete_signature(*args, **kwargs): + """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding""" +def func_returning_optional() -> Optional[int]: + """func_returning_optional() -> Optional[int]""" +def func_returning_pair() -> Tuple[int, float]: + """func_returning_pair() -> Tuple[int, float]""" +def func_returning_path() -> os.PathLike: + """func_returning_path() -> os.PathLike""" +def func_returning_vector() -> List[float]: + """func_returning_vector() -> List[float]""" diff --git a/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi similarity index 57% rename from test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi rename to test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi index 3047da622a3e..1527225ed009 100644 --- a/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/basics.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi @@ -3,36 +3,6 @@ from typing import ClassVar, List, overload PI: float __version__: str -class Foo: - def __init__(self, *args, **kwargs) -> None: - """Initialize self. See help(type(self)) for accurate signature.""" - @overload - @staticmethod - def overloaded_static_method(value: int) -> int: - """overloaded_static_method(*args, **kwargs) - Overloaded function. - - 1. overloaded_static_method(value: int) -> int - - 2. overloaded_static_method(value: float) -> float - """ - @overload - @staticmethod - def overloaded_static_method(value: float) -> float: - """overloaded_static_method(*args, **kwargs) - Overloaded function. - - 1. overloaded_static_method(value: int) -> int - - 2. overloaded_static_method(value: float) -> float - """ - @staticmethod - def some_static_method(a: int, b: int) -> int: - """some_static_method(a: int, b: int) -> int - - None - """ - class Point: class AngleUnit: __members__: ClassVar[dict] = ... # read-only @@ -40,15 +10,15 @@ class Point: degree: ClassVar[Point.AngleUnit] = ... radian: ClassVar[Point.AngleUnit] = ... def __init__(self, value: int) -> None: - """__init__(self: pybind11_mypy_demo.basics.Point.AngleUnit, value: int) -> None""" + """__init__(self: pybind11_fixtures.demo.Point.AngleUnit, value: int) -> None""" def __eq__(self, other: object) -> bool: """__eq__(self: object, other: object) -> bool""" def __hash__(self) -> int: """__hash__(self: object) -> int""" def __index__(self) -> int: - """__index__(self: pybind11_mypy_demo.basics.Point.AngleUnit) -> int""" + """__index__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" def __int__(self) -> int: - """__int__(self: pybind11_mypy_demo.basics.Point.AngleUnit) -> int""" + """__int__(self: pybind11_fixtures.demo.Point.AngleUnit) -> int""" def __ne__(self, other: object) -> bool: """__ne__(self: object, other: object) -> bool""" @property @@ -63,15 +33,15 @@ class Point: mm: ClassVar[Point.LengthUnit] = ... pixel: ClassVar[Point.LengthUnit] = ... def __init__(self, value: int) -> None: - """__init__(self: pybind11_mypy_demo.basics.Point.LengthUnit, value: int) -> None""" + """__init__(self: pybind11_fixtures.demo.Point.LengthUnit, value: int) -> None""" def __eq__(self, other: object) -> bool: """__eq__(self: object, other: object) -> bool""" def __hash__(self) -> int: """__hash__(self: object) -> int""" def __index__(self) -> int: - """__index__(self: pybind11_mypy_demo.basics.Point.LengthUnit) -> int""" + """__index__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" def __int__(self) -> int: - """__int__(self: pybind11_mypy_demo.basics.Point.LengthUnit) -> int""" + """__int__(self: pybind11_fixtures.demo.Point.LengthUnit) -> int""" def __ne__(self, other: object) -> bool: """__ne__(self: object, other: object) -> bool""" @property @@ -90,38 +60,38 @@ class Point: """__init__(*args, **kwargs) Overloaded function. - 1. __init__(self: pybind11_mypy_demo.basics.Point) -> None + 1. __init__(self: pybind11_fixtures.demo.Point) -> None - 2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None + 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None """ @overload def __init__(self, x: float, y: float) -> None: """__init__(*args, **kwargs) Overloaded function. - 1. __init__(self: pybind11_mypy_demo.basics.Point) -> None + 1. __init__(self: pybind11_fixtures.demo.Point) -> None - 2. __init__(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> None + 2. __init__(self: pybind11_fixtures.demo.Point, x: float, y: float) -> None """ def as_list(self) -> List[float]: - """as_list(self: pybind11_mypy_demo.basics.Point) -> List[float]""" + """as_list(self: pybind11_fixtures.demo.Point) -> List[float]""" @overload def distance_to(self, x: float, y: float) -> float: """distance_to(*args, **kwargs) Overloaded function. - 1. distance_to(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> float + 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float - 2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float + 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float """ @overload def distance_to(self, other: Point) -> float: """distance_to(*args, **kwargs) Overloaded function. - 1. distance_to(self: pybind11_mypy_demo.basics.Point, x: float, y: float) -> float + 1. distance_to(self: pybind11_fixtures.demo.Point, x: float, y: float) -> float - 2. distance_to(self: pybind11_mypy_demo.basics.Point, other: pybind11_mypy_demo.basics.Point) -> float + 2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float """ @property def length(self) -> float: ... diff --git a/test-data/pybind11_mypy_demo/pyproject.toml b/test-data/pybind11_fixtures/pyproject.toml similarity index 100% rename from test-data/pybind11_mypy_demo/pyproject.toml rename to test-data/pybind11_fixtures/pyproject.toml diff --git a/test-data/pybind11_mypy_demo/setup.py b/test-data/pybind11_fixtures/setup.py similarity index 85% rename from test-data/pybind11_mypy_demo/setup.py rename to test-data/pybind11_fixtures/setup.py index 0da1cfbcef19..e227b49935ea 100644 --- a/test-data/pybind11_mypy_demo/setup.py +++ b/test-data/pybind11_fixtures/setup.py @@ -5,14 +5,14 @@ # Documentation: https://pybind11.readthedocs.io/en/stable/compiling.html ext_modules = [ Pybind11Extension( - "pybind11_mypy_demo", + "pybind11_fixtures", ["src/main.cpp"], cxx_std=17, ), ] setup( - name="pybind11-mypy-demo", + name="pybind11_fixtures", version="0.0.1", ext_modules=ext_modules, ) diff --git a/test-data/pybind11_mypy_demo/src/main.cpp b/test-data/pybind11_fixtures/src/main.cpp similarity index 61% rename from test-data/pybind11_mypy_demo/src/main.cpp rename to test-data/pybind11_fixtures/src/main.cpp index 8be759d51671..4d275ab1fd70 100644 --- a/test-data/pybind11_mypy_demo/src/main.cpp +++ b/test-data/pybind11_fixtures/src/main.cpp @@ -43,12 +43,106 @@ */ #include +#include +#include +#include +#include + #include #include +#include namespace py = pybind11; -namespace basics { +// ---------------------------------------------------------------------------- +// Dedicated test cases +// ---------------------------------------------------------------------------- + +std::vector funcReturningVector() +{ + return std::vector{1.0, 2.0, 3.0}; +} + +std::pair funcReturningPair() +{ + return std::pair{42, 1.0}; +} + +std::optional funcReturningOptional() +{ + return std::nullopt; +} + +std::filesystem::path funcReturningPath() +{ + return std::filesystem::path{"foobar"}; +} + +namespace dummy_sub_namespace { + struct HasNoBinding{}; +} + +// We can enforce the case of an incomplete signature by referring to a type in +// some namespace that doesn't have a pybind11 binding. +dummy_sub_namespace::HasNoBinding funcIncompleteSignature() +{ + return dummy_sub_namespace::HasNoBinding{}; +} + +struct TestStruct +{ + int field_readwrite; + int field_readwrite_docstring; + int field_readonly; +}; + +struct StaticMethods +{ + static int some_static_method(int a, int b) { return 42; } + static int overloaded_static_method(int value) { return 42; } + static double overloaded_static_method(double value) { return 1.0; } +}; + +// Bindings + +void bind_test_cases(py::module& m) { + m.def("func_returning_vector", &funcReturningVector); + m.def("func_returning_pair", &funcReturningPair); + m.def("func_returning_optional", &funcReturningOptional); + m.def("func_returning_path", &funcReturningPath); + + m.def("func_incomplete_signature", &funcIncompleteSignature); + + py::class_(m, "TestStruct") + .def_readwrite("field_readwrite", &TestStruct::field_readwrite) + .def_readwrite("field_readwrite_docstring", &TestStruct::field_readwrite_docstring, "some docstring") + .def_property_readonly( + "field_readonly", + [](const TestStruct& x) { + return x.field_readonly; + }, + "some docstring"); + + // Static methods + py::class_ pyStaticMethods(m, "StaticMethods"); + + pyStaticMethods + .def_static( + "some_static_method", + &StaticMethods::some_static_method, R"#(None)#", py::arg("a"), py::arg("b")) + .def_static( + "overloaded_static_method", + py::overload_cast(&StaticMethods::overloaded_static_method), py::arg("value")) + .def_static( + "overloaded_static_method", + py::overload_cast(&StaticMethods::overloaded_static_method), py::arg("value")); +} + +// ---------------------------------------------------------------------------- +// Original demo +// ---------------------------------------------------------------------------- + +namespace demo { int answer() { return 42; @@ -118,27 +212,22 @@ const Point Point::y_axis = Point(0, 1); Point::LengthUnit Point::length_unit = Point::LengthUnit::mm; Point::AngleUnit Point::angle_unit = Point::AngleUnit::radian; -struct Foo -{ - static int some_static_method(int a, int b) { return a * 42 + b; } - static int overloaded_static_method(int value) { return value * 42; } - static double overloaded_static_method(double value) { return value * 42; } -}; +} // namespace: demo -} // namespace: basics +// Bindings -void bind_basics(py::module& basics) { +void bind_demo(py::module& m) { - using namespace basics; + using namespace demo; // Functions - basics.def("answer", &answer, "answer docstring, with end quote\""); // tests explicit docstrings - basics.def("sum", &sum, "multiline docstring test, edge case quotes \"\"\"'''"); - basics.def("midpoint", &midpoint, py::arg("left"), py::arg("right")); - basics.def("weighted_midpoint", weighted_midpoint, py::arg("left"), py::arg("right"), py::arg("alpha")=0.5); + m.def("answer", &answer, "answer docstring, with end quote\""); // tests explicit docstrings + m.def("sum", &sum, "multiline docstring test, edge case quotes \"\"\"'''"); + m.def("midpoint", &midpoint, py::arg("left"), py::arg("right")); + m.def("weighted_midpoint", weighted_midpoint, py::arg("left"), py::arg("right"), py::arg("alpha")=0.5); // Classes - py::class_ pyPoint(basics, "Point"); + py::class_ pyPoint(m, "Point"); py::enum_ pyLengthUnit(pyPoint, "LengthUnit"); py::enum_ pyAngleUnit(pyPoint, "AngleUnit"); @@ -173,20 +262,18 @@ void bind_basics(py::module& basics) { .value("radian", Point::AngleUnit::radian) .value("degree", Point::AngleUnit::degree); - // Static methods - py::class_ pyFoo(basics, "Foo"); - - pyFoo - .def_static("some_static_method", &Foo::some_static_method, R"#(None)#", py::arg("a"), py::arg("b")) - .def_static("overloaded_static_method", py::overload_cast(&Foo::overloaded_static_method), py::arg("value")) - .def_static("overloaded_static_method", py::overload_cast(&Foo::overloaded_static_method), py::arg("value")); - // Module-level attributes - basics.attr("PI") = std::acos(-1); - basics.attr("__version__") = "0.0.1"; + m.attr("PI") = std::acos(-1); + m.attr("__version__") = "0.0.1"; } -PYBIND11_MODULE(pybind11_mypy_demo, m) { - auto basics = m.def_submodule("basics"); - bind_basics(basics); +// ---------------------------------------------------------------------------- +// Module entry point +// ---------------------------------------------------------------------------- + +PYBIND11_MODULE(pybind11_fixtures, m) { + bind_test_cases(m); + + auto demo = m.def_submodule("demo"); + bind_demo(demo); } diff --git a/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/__init__.pyi b/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/__init__.pyi deleted file mode 100644 index 0cb252f00259..000000000000 --- a/test-data/pybind11_mypy_demo/stubgen-include-docs/pybind11_mypy_demo/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import basics as basics diff --git a/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/__init__.pyi b/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/__init__.pyi deleted file mode 100644 index 0cb252f00259..000000000000 --- a/test-data/pybind11_mypy_demo/stubgen/pybind11_mypy_demo/__init__.pyi +++ /dev/null @@ -1 +0,0 @@ -from . import basics as basics From e28925ddd34fd128d509b5b4741789578462cbe0 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 13 Jan 2024 16:58:37 +0100 Subject: [PATCH 11/32] stubgen: use PEP 604 unions everywhere (#16519) Fixes #12920 --- mypy/stubgen.py | 22 ++++++++++------------ mypy/stubutil.py | 25 +++++++++++++++++-------- test-data/unit/stubgen.test | 30 ++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 7ec3d7069fb2..e753fd1e73d5 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -22,7 +22,7 @@ => Generate out/urllib/parse.pyi. $ stubgen -p urllib - => Generate stubs for whole urlib package (recursively). + => Generate stubs for whole urllib package (recursively). For C modules, you can get more precise function signatures by parsing .rst (Sphinx) documentation for extra information. For this, use the --doc-dir option: @@ -306,6 +306,13 @@ def visit_str_expr(self, node: StrExpr) -> str: return repr(node.value) def visit_index_expr(self, node: IndexExpr) -> str: + base_fullname = self.stubgen.get_fullname(node.base) + if base_fullname == "typing.Union": + if isinstance(node.index, TupleExpr): + return " | ".join([item.accept(self) for item in node.index.items]) + return node.index.accept(self) + if base_fullname == "typing.Optional": + return f"{node.index.accept(self)} | None" base = node.base.accept(self) index = node.index.accept(self) if len(index) > 2 and index.startswith("(") and index.endswith(")"): @@ -682,7 +689,7 @@ def process_decorator(self, o: Decorator) -> None: self.add_decorator(qualname, require_name=False) def get_fullname(self, expr: Expression) -> str: - """Return the full name resolving imports and import aliases.""" + """Return the expression's full name.""" if ( self.analyzed and isinstance(expr, (NameExpr, MemberExpr)) @@ -691,16 +698,7 @@ def get_fullname(self, expr: Expression) -> str: ): return expr.fullname name = get_qualified_name(expr) - if "." not in name: - real_module = self.import_tracker.module_for.get(name) - real_short = self.import_tracker.reverse_alias.get(name, name) - if real_module is None and real_short not in self.defined_names: - real_module = "builtins" # not imported and not defined, must be a builtin - else: - name_module, real_short = name.split(".", 1) - real_module = self.import_tracker.reverse_alias.get(name_module, name_module) - resolved_name = real_short if real_module is None else f"{real_module}.{real_short}" - return resolved_name + return self.resolve_name(name) def visit_class_def(self, o: ClassDef) -> None: self._current_class = o diff --git a/mypy/stubutil.py b/mypy/stubutil.py index e4a97964c547..b7f6131c003d 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -226,6 +226,11 @@ def visit_any(self, t: AnyType) -> str: def visit_unbound_type(self, t: UnboundType) -> str: s = t.name + fullname = self.stubgen.resolve_name(s) + if fullname == "typing.Union": + return " | ".join([item.accept(self) for item in t.args]) + if fullname == "typing.Optional": + return f"{t.args[0].accept(self)} | None" if self.known_modules is not None and "." in s: # see if this object is from any of the modules that we're currently processing. # reverse sort so that subpackages come before parents: e.g. "foo.bar" before "foo". @@ -588,14 +593,18 @@ def __init__( def get_sig_generators(self) -> list[SignatureGenerator]: return [] - def refers_to_fullname(self, name: str, fullname: str | tuple[str, ...]) -> bool: - """Return True if the variable name identifies the same object as the given fullname(s).""" - if isinstance(fullname, tuple): - return any(self.refers_to_fullname(name, fname) for fname in fullname) - module, short = fullname.rsplit(".", 1) - return self.import_tracker.module_for.get(name) == module and ( - name == short or self.import_tracker.reverse_alias.get(name) == short - ) + def resolve_name(self, name: str) -> str: + """Return the full name resolving imports and import aliases.""" + if "." not in name: + real_module = self.import_tracker.module_for.get(name) + real_short = self.import_tracker.reverse_alias.get(name, name) + if real_module is None and real_short not in self.defined_names: + real_module = "builtins" # not imported and not defined, must be a builtin + else: + name_module, real_short = name.split(".", 1) + real_module = self.import_tracker.reverse_alias.get(name_module, name_module) + resolved_name = real_short if real_module is None else f"{real_module}.{real_short}" + return resolved_name def add_name(self, fullname: str, require: bool = True) -> str: """Add a name to be imported and return the name reference. diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index cd38242ce031..50437c903530 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -4139,3 +4139,33 @@ from dataclasses import dataclass class X(missing.Base): a: int def __init__(self, *selfa_, a, **selfa__) -> None: ... + +[case testAlwaysUsePEP604Union] +import typing +import typing as t +from typing import Optional, Union, Optional as O, Union as U +import x + +union = Union[int, str] +bad_union = Union[int] +nested_union = Optional[Union[int, str]] +not_union = x.Union[int, str] +u = U[int, str] +o = O[int] + +def f1(a: Union["int", Optional[tuple[int, t.Optional[int]]]]) -> int: ... +def f2(a: typing.Union[int | x.Union[int, int], O[float]]) -> int: ... + +[out] +import x +from _typeshed import Incomplete + +union = int | str +bad_union = int +nested_union = int | str | None +not_union: Incomplete +u = int | str +o = int | None + +def f1(a: int | tuple[int, int | None] | None) -> int: ... +def f2(a: int | x.Union[int, int] | float | None) -> int: ... From e64fb9f7fe75d5e5aa722f80576bd2b67b442a4e Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 13 Jan 2024 21:19:44 +0100 Subject: [PATCH 12/32] Fix failing stubgen tests (#16779) --- .github/workflows/test_stubgenc.yml | 1 + .pre-commit-config.yaml | 1 + .../expected_stubs_no_docs/pybind11_fixtures/__init__.pyi | 4 ++-- .../expected_stubs_with_docs/pybind11_fixtures/__init__.pyi | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_stubgenc.yml b/.github/workflows/test_stubgenc.yml index 3daf01372d2d..7bdcfdb305bb 100644 --- a/.github/workflows/test_stubgenc.yml +++ b/.github/workflows/test_stubgenc.yml @@ -10,6 +10,7 @@ on: - 'misc/test-stubgenc.sh' - 'mypy/stubgenc.py' - 'mypy/stubdoc.py' + - 'mypy/stubutil.py' - 'test-data/stubgen/**' permissions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4090bf0ecb4c..0bbd7b3ce382 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,7 @@ repos: rev: 23.9.1 # must match test-requirements.txt hooks: - id: black + exclude: '^(test-data/)' - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.4 # must match test-requirements.txt hooks: diff --git a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi index e113d8a69a5d..bb939aa5a5e7 100644 --- a/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_no_docs/pybind11_fixtures/__init__.pyi @@ -1,6 +1,6 @@ import os from . import demo as demo -from typing import List, Optional, Tuple, overload +from typing import List, Tuple, overload class StaticMethods: def __init__(self, *args, **kwargs) -> None: ... @@ -21,7 +21,7 @@ class TestStruct: def field_readonly(self) -> int: ... def func_incomplete_signature(*args, **kwargs): ... -def func_returning_optional() -> Optional[int]: ... +def func_returning_optional() -> int | None: ... def func_returning_pair() -> Tuple[int, float]: ... def func_returning_path() -> os.PathLike: ... def func_returning_vector() -> List[float]: ... diff --git a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi index 1dabb0d9a330..622e5881a147 100644 --- a/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi +++ b/test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi @@ -1,6 +1,6 @@ import os from . import demo as demo -from typing import List, Optional, Tuple, overload +from typing import List, Tuple, overload class StaticMethods: def __init__(self, *args, **kwargs) -> None: @@ -42,7 +42,7 @@ class TestStruct: def func_incomplete_signature(*args, **kwargs): """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding""" -def func_returning_optional() -> Optional[int]: +def func_returning_optional() -> int | None: """func_returning_optional() -> Optional[int]""" def func_returning_pair() -> Tuple[int, float]: """func_returning_pair() -> Tuple[int, float]""" From 186ace34432014e668de2ed1449f9262c1e7f056 Mon Sep 17 00:00:00 2001 From: Makonnen Makonnen Date: Sat, 13 Jan 2024 22:05:38 -0500 Subject: [PATCH 13/32] Improve mypy daemon documentation note about local partial types (#16782) The existing documentation almost makes it seem like the user has the option to toggle this. --- docs/source/mypy_daemon.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/mypy_daemon.rst b/docs/source/mypy_daemon.rst index 7586026b6c81..6c511e14eb95 100644 --- a/docs/source/mypy_daemon.rst +++ b/docs/source/mypy_daemon.rst @@ -61,7 +61,7 @@ you have a large codebase.) .. note:: - The mypy daemon automatically enables ``--local-partial-types`` by default. + The mypy daemon requires ``--local-partial-types`` and automatically enables it. Daemon client commands From b1fe23f5a3a4457cae3430acca6e99a36e3efd00 Mon Sep 17 00:00:00 2001 From: anniel-stripe <97691964+anniel-stripe@users.noreply.github.com> Date: Sat, 13 Jan 2024 19:09:24 -0800 Subject: [PATCH 14/32] Support TypedDict functional syntax as class base type (#16703) Fixes https://github.com/python/mypy/issues/16701 This PR allows `TypedDict(...)` calls to be used as a base class. This fixes the error emitted by mypy described in https://github.com/python/mypy/issues/16701 . --- mypy/semanal.py | 8 ++++++++ mypy/semanal_typeddict.py | 19 +++++++++++++++---- test-data/unit/check-typeddict.test | 10 ++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e0a3db2bff1b..4bf9f0c3eabb 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -2169,8 +2169,16 @@ def analyze_base_classes( if ( isinstance(base_expr, RefExpr) and base_expr.fullname in TYPED_NAMEDTUPLE_NAMES + TPDICT_NAMES + ) or ( + isinstance(base_expr, CallExpr) + and isinstance(base_expr.callee, RefExpr) + and base_expr.callee.fullname in TPDICT_NAMES ): # Ignore magic bases for now. + # For example: + # class Foo(TypedDict): ... # RefExpr + # class Foo(NamedTuple): ... # RefExpr + # class Foo(TypedDict("Foo", {"a": int})): ... # CallExpr continue try: diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 67c05fd74273..dbec981bdc96 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -79,6 +79,8 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N """ possible = False for base_expr in defn.base_type_exprs: + if isinstance(base_expr, CallExpr): + base_expr = base_expr.callee if isinstance(base_expr, IndexExpr): base_expr = base_expr.base if isinstance(base_expr, RefExpr): @@ -117,7 +119,13 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N typeddict_bases: list[Expression] = [] typeddict_bases_set = set() for expr in defn.base_type_exprs: - if isinstance(expr, RefExpr) and expr.fullname in TPDICT_NAMES: + ok, maybe_type_info, _ = self.check_typeddict(expr, None, False) + if ok and maybe_type_info is not None: + # expr is a CallExpr + info = maybe_type_info + typeddict_bases_set.add(info.fullname) + typeddict_bases.append(expr) + elif isinstance(expr, RefExpr) and expr.fullname in TPDICT_NAMES: if "TypedDict" not in typeddict_bases_set: typeddict_bases_set.add("TypedDict") else: @@ -176,12 +184,11 @@ def add_keys_and_types_from_base( required_keys: set[str], ctx: Context, ) -> None: + base_args: list[Type] = [] if isinstance(base, RefExpr): assert isinstance(base.node, TypeInfo) info = base.node - base_args: list[Type] = [] - else: - assert isinstance(base, IndexExpr) + elif isinstance(base, IndexExpr): assert isinstance(base.base, RefExpr) assert isinstance(base.base.node, TypeInfo) info = base.base.node @@ -189,6 +196,10 @@ def add_keys_and_types_from_base( if args is None: return base_args = args + else: + assert isinstance(base, CallExpr) + assert isinstance(base.analyzed, TypedDictExpr) + info = base.analyzed.info assert info.typeddict_type is not None base_typed_dict = info.typeddict_type diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index d8022f85574c..625b82936e8c 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -3438,3 +3438,13 @@ class TotalInTheMiddle(TypedDict, a=1, total=True, b=2, c=3): # E: Unexpected k ... [builtins fixtures/dict.pyi] [typing fixtures/typing-typeddict.pyi] + +[case testCanCreateClassWithFunctionBasedTypedDictBase] +from mypy_extensions import TypedDict + +class Params(TypedDict("Params", {'x': int})): + pass + +p: Params = {'x': 2} +reveal_type(p) # N: Revealed type is "TypedDict('__main__.Params', {'x': builtins.int})" +[builtins fixtures/dict.pyi] From 261e569e2c256451692dd57c05295ef0bcfb34fd Mon Sep 17 00:00:00 2001 From: Christoph Tyralla Date: Sun, 14 Jan 2024 04:15:33 +0100 Subject: [PATCH 15/32] Support narrowing unions that include type[None] (#16315) Fixes #16279 See my comment in the referenced issue. --- mypy/checker.py | 18 +++++--- mypy/checkexpr.py | 19 +++++--- test-data/unit/check-narrowing.test | 70 +++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 12 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 1b57ef780104..cd23e74a8dac 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -7121,9 +7121,10 @@ def conditional_types_with_intersection( possible_target_types = [] for tr in type_ranges: item = get_proper_type(tr.item) - if not isinstance(item, Instance) or tr.is_upper_bound: - return yes_type, no_type - possible_target_types.append(item) + if isinstance(item, (Instance, NoneType)): + possible_target_types.append(item) + if not possible_target_types: + return yes_type, no_type out = [] errors: list[tuple[str, str]] = [] @@ -7131,6 +7132,9 @@ def conditional_types_with_intersection( if not isinstance(v, Instance): return yes_type, no_type for t in possible_target_types: + if isinstance(t, NoneType): + errors.append((f'"{v.type.name}" and "NoneType"', '"NoneType" is final')) + continue intersection = self.intersect_instances((v, t), errors) if intersection is None: continue @@ -7174,7 +7178,11 @@ def get_isinstance_type(self, expr: Expression) -> list[TypeRange] | None: elif isinstance(typ, TypeType): # Type[A] means "any type that is a subtype of A" rather than "precisely type A" # we indicate this by setting is_upper_bound flag - types.append(TypeRange(typ.item, is_upper_bound=True)) + is_upper_bound = True + if isinstance(typ.item, NoneType): + # except for Type[None], because "'NoneType' is not an acceptable base type" + is_upper_bound = False + types.append(TypeRange(typ.item, is_upper_bound=is_upper_bound)) elif isinstance(typ, Instance) and typ.type.fullname == "builtins.type": object_type = Instance(typ.type.mro[-1], []) types.append(TypeRange(object_type, is_upper_bound=True)) @@ -7627,7 +7635,7 @@ def convert_to_typetype(type_map: TypeMap) -> TypeMap: if isinstance(t, TypeVarType): t = t.upper_bound # TODO: should we only allow unions of instances as per PEP 484? - if not isinstance(get_proper_type(t), (UnionType, Instance)): + if not isinstance(get_proper_type(t), (UnionType, Instance, NoneType)): # unknown type; error was likely reported earlier return {} converted_type_map[expr] = TypeType.make_normalized(typ) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 3af8d70e78c9..fbedd95e8fd2 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -385,6 +385,9 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type: if node.typeddict_type: # We special-case TypedDict, because they don't define any constructor. result = self.typeddict_callable(node) + elif node.fullname == "types.NoneType": + # We special case NoneType, because its stub definition is not related to None. + result = TypeType(NoneType()) else: result = type_object_type(node, self.named_type) if isinstance(result, CallableType) and isinstance( # type: ignore[misc] @@ -511,13 +514,13 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> if is_expr_literal_type(typ): self.msg.cannot_use_function_with_type(e.callee.name, "Literal", e) continue - if ( - node - and isinstance(node.node, TypeAlias) - and isinstance(get_proper_type(node.node.target), AnyType) - ): - self.msg.cannot_use_function_with_type(e.callee.name, "Any", e) - continue + if node and isinstance(node.node, TypeAlias): + target = get_proper_type(node.node.target) + if isinstance(target, AnyType): + self.msg.cannot_use_function_with_type(e.callee.name, "Any", e) + continue + if isinstance(target, NoneType): + continue if ( isinstance(typ, IndexExpr) and isinstance(typ.analyzed, (TypeApplication, TypeAliasExpr)) @@ -4731,6 +4734,8 @@ class LongName(Generic[T]): ... return type_object_type(tuple_fallback(item).type, self.named_type) elif isinstance(item, TypedDictType): return self.typeddict_callable_from_context(item) + elif isinstance(item, NoneType): + return TypeType(item, line=item.line, column=item.column) elif isinstance(item, AnyType): return AnyType(TypeOfAny.from_another_any, source_any=item) else: diff --git a/test-data/unit/check-narrowing.test b/test-data/unit/check-narrowing.test index a2859dfffa3a..d50d1f508b85 100644 --- a/test-data/unit/check-narrowing.test +++ b/test-data/unit/check-narrowing.test @@ -2022,3 +2022,73 @@ def f(x: Union[int, Sequence[int]]) -> None: ): reveal_type(x) # N: Revealed type is "Tuple[builtins.int, builtins.int]" [builtins fixtures/len.pyi] + +[case testNarrowingIsSubclassNoneType1] +from typing import Type, Union + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, int): + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" + else: + reveal_type(cls) # N: Revealed type is "Type[None]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsSubclassNoneType2] +from typing import Type, Union + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, type(None)): + reveal_type(cls) # N: Revealed type is "Type[None]" + else: + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsSubclassNoneType3] +from typing import Type, Union + +NoneType_ = type(None) + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, NoneType_): + reveal_type(cls) # N: Revealed type is "Type[None]" + else: + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsSubclassNoneType4] +# flags: --python-version 3.10 + +from types import NoneType +from typing import Type, Union + +def f(cls: Type[Union[None, int]]) -> None: + if issubclass(cls, NoneType): + reveal_type(cls) # N: Revealed type is "Type[None]" + else: + reveal_type(cls) # N: Revealed type is "Type[builtins.int]" +[builtins fixtures/isinstance.pyi] + +[case testNarrowingIsInstanceNoIntersectionWithFinalTypeAndNoneType] +# flags: --warn-unreachable --python-version 3.10 + +from types import NoneType +from typing import final + +class X: ... +class Y: ... +@final +class Z: ... + +x: X + +if isinstance(x, (Y, Z)): + reveal_type(x) # N: Revealed type is "__main__." +if isinstance(x, (Y, NoneType)): + reveal_type(x) # N: Revealed type is "__main__.1" +if isinstance(x, (Y, Z, NoneType)): + reveal_type(x) # N: Revealed type is "__main__.2" +if isinstance(x, (Z, NoneType)): # E: Subclass of "X" and "Z" cannot exist: "Z" is final \ + # E: Subclass of "X" and "NoneType" cannot exist: "NoneType" is final + reveal_type(x) # E: Statement is unreachable + +[builtins fixtures/isinstance.pyi] From 75b68fa2c0d32bd08b98b00aece20c22d82b1b15 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 14 Jan 2024 18:19:12 +0100 Subject: [PATCH 16/32] stubgen: Do not ignore property deleter (#16781) Fixes #16690 --- mypy/stubgen.py | 2 +- test-data/unit/stubgen.test | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index e753fd1e73d5..1f7a01e6ae3c 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -685,7 +685,7 @@ def process_decorator(self, o: Decorator) -> None: elif fullname in OVERLOAD_NAMES: self.add_decorator(qualname, require_name=True) o.func.is_overload = True - elif qualname.endswith(".setter"): + elif qualname.endswith((".setter", ".deleter")): self.add_decorator(qualname, require_name=False) def get_fullname(self, expr: Expression) -> str: diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test index 50437c903530..751a1a7fbbcf 100644 --- a/test-data/unit/stubgen.test +++ b/test-data/unit/stubgen.test @@ -368,6 +368,8 @@ class A: return 1 @f.setter def f(self, x): ... + @f.deleter + def f(self): ... def h(self): self.f = 1 @@ -377,6 +379,8 @@ class A: def f(self): ... @f.setter def f(self, x) -> None: ... + @f.deleter + def f(self) -> None: ... def h(self) -> None: ... [case testProperty_semanal] @@ -386,6 +390,8 @@ class A: return 1 @f.setter def f(self, x): ... + @f.deleter + def f(self): ... def h(self): self.f = 1 @@ -395,6 +401,8 @@ class A: def f(self): ... @f.setter def f(self, x) -> None: ... + @f.deleter + def f(self) -> None: ... def h(self) -> None: ... -- a read/write property is treated the same as an attribute @@ -2338,10 +2346,12 @@ class B: @property def x(self): return 'x' - @x.setter def x(self, value): self.y = 'y' + @x.deleter + def x(self): + del self.y [out] class A: @@ -2355,6 +2365,8 @@ class B: y: str @x.setter def x(self, value) -> None: ... + @x.deleter + def x(self) -> None: ... [case testMisplacedTypeComment] def f(): From 7eab8a429187ffd2b64bbb9ca29f7d24b262d269 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 21 Jan 2024 21:31:53 -0800 Subject: [PATCH 17/32] stubtest: Add support for setting enum members to "..." (#16807) Unblock python/typeshed#11299 --- mypy/stubtest.py | 7 +++++++ mypy/test/teststubtest.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 6061e98bd7cd..9a038105dc82 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1093,6 +1093,13 @@ def verify_var( runtime_type = get_mypy_type_of_runtime_value(runtime.value) if runtime_type is not None and is_subtype_helper(runtime_type, stub.type): should_error = False + # We always allow setting the stub value to ... + proper_type = mypy.types.get_proper_type(stub.type) + if ( + isinstance(proper_type, mypy.types.Instance) + and proper_type.type.fullname == "builtins.ellipsis" + ): + should_error = False if should_error: yield Error( diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index c0bb2eb6f9da..3f231d8afbcc 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1139,6 +1139,25 @@ def baz(x=Flags3(0)): pass """, error=None, ) + yield Case( + runtime=""" + import enum + class SomeObject: ... + + class WeirdEnum(enum.Enum): + a = SomeObject() + b = SomeObject() + """, + stub=""" + import enum + class SomeObject: ... + class WeirdEnum(enum.Enum): + _value_: SomeObject + a = ... + b = ... + """, + error=None, + ) yield Case( stub=""" class Flags4(enum.Flag): From ec06e004426d92637cd0ddd266a464cea723f952 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 24 Jan 2024 19:30:30 +0100 Subject: [PATCH 18/32] Use TypeVar defaults instead of Any when fixing instance types (PEP 696) (#16812) Start using TypeVar defaults when fixing instance types, instead of filling those with `Any`. This PR preserves the way an invalid amount of args is handled. I.e. filling all with `Any` / defaults, instead of cutting off additional args. Thus preserving full backwards compatibility. This can be easily changed later if necessary. `TypeVarTuple` defaults aren't handled correctly yet. Those will require additional logic which would have complicated the change here and made it more difficult to review. Ref: https://github.com/python/mypy/issues/14851 --- mypy/messages.py | 15 ++- mypy/typeanal.py | 87 ++++++++++----- test-data/unit/check-typevar-defaults.test | 123 +++++++++++++++++++++ 3 files changed, 189 insertions(+), 36 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 450faf4c1688..75b8eeb3174c 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -3017,12 +3017,15 @@ def for_function(callee: CallableType) -> str: return "" -def wrong_type_arg_count(n: int, act: str, name: str) -> str: - s = f"{n} type arguments" - if n == 0: - s = "no type arguments" - elif n == 1: - s = "1 type argument" +def wrong_type_arg_count(low: int, high: int, act: str, name: str) -> str: + if low == high: + s = f"{low} type arguments" + if low == 0: + s = "no type arguments" + elif low == 1: + s = "1 type argument" + else: + s = f"between {low} and {high} type arguments" if act == "0": act = "none" return f'"{name}" expects {s}, but {act} given' diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 8a840424f76f..d10f26f5199a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -9,6 +9,7 @@ from mypy import errorcodes as codes, message_registry, nodes from mypy.errorcodes import ErrorCode +from mypy.expandtype import expand_type from mypy.messages import MessageBuilder, format_type_bare, quote_type_string, wrong_type_arg_count from mypy.nodes import ( ARG_NAMED, @@ -75,6 +76,7 @@ TypeOfAny, TypeQuery, TypeType, + TypeVarId, TypeVarLikeType, TypeVarTupleType, TypeVarType, @@ -1834,14 +1836,14 @@ def get_omitted_any( return any_type -def fix_type_var_tuple_argument(any_type: Type, t: Instance) -> None: +def fix_type_var_tuple_argument(t: Instance) -> None: if t.type.has_type_var_tuple_type: args = list(t.args) assert t.type.type_var_tuple_prefix is not None tvt = t.type.defn.type_vars[t.type.type_var_tuple_prefix] assert isinstance(tvt, TypeVarTupleType) args[t.type.type_var_tuple_prefix] = UnpackType( - Instance(tvt.tuple_fallback.type, [any_type]) + Instance(tvt.tuple_fallback.type, [args[t.type.type_var_tuple_prefix]]) ) t.args = tuple(args) @@ -1855,26 +1857,42 @@ def fix_instance( use_generic_error: bool = False, unexpanded_type: Type | None = None, ) -> None: - """Fix a malformed instance by replacing all type arguments with Any. + """Fix a malformed instance by replacing all type arguments with TypeVar default or Any. Also emit a suitable error if this is not due to implicit Any's. """ - if len(t.args) == 0: - if use_generic_error: - fullname: str | None = None - else: - fullname = t.type.fullname - any_type = get_omitted_any(disallow_any, fail, note, t, options, fullname, unexpanded_type) - t.args = (any_type,) * len(t.type.type_vars) - fix_type_var_tuple_argument(any_type, t) - return - # Construct the correct number of type arguments, as - # otherwise the type checker may crash as it expects - # things to be right. - any_type = AnyType(TypeOfAny.from_error) - t.args = tuple(any_type for _ in t.type.type_vars) - fix_type_var_tuple_argument(any_type, t) - t.invalid = True + arg_count = len(t.args) + min_tv_count = sum(not tv.has_default() for tv in t.type.defn.type_vars) + max_tv_count = len(t.type.type_vars) + if arg_count < min_tv_count or arg_count > max_tv_count: + # Don't use existing args if arg_count doesn't match + t.args = () + + args: list[Type] = [*(t.args[:max_tv_count])] + any_type: AnyType | None = None + env: dict[TypeVarId, Type] = {} + + for tv, arg in itertools.zip_longest(t.type.defn.type_vars, t.args, fillvalue=None): + if tv is None: + continue + if arg is None: + if tv.has_default(): + arg = tv.default + else: + if any_type is None: + fullname = None if use_generic_error else t.type.fullname + any_type = get_omitted_any( + disallow_any, fail, note, t, options, fullname, unexpanded_type + ) + arg = any_type + args.append(arg) + env[tv.id] = arg + t.args = tuple(args) + fix_type_var_tuple_argument(t) + if not t.type.has_type_var_tuple_type: + fixed = expand_type(t, env) + assert isinstance(fixed, Instance) + t.args = fixed.args def instantiate_type_alias( @@ -1963,7 +1981,7 @@ def instantiate_type_alias( if use_standard_error: # This is used if type alias is an internal representation of another type, # for example a generic TypedDict or NamedTuple. - msg = wrong_type_arg_count(exp_len, str(act_len), node.name) + msg = wrong_type_arg_count(exp_len, exp_len, str(act_len), node.name) else: if node.tvar_tuple_index is not None: exp_len_str = f"at least {exp_len - 1}" @@ -2217,24 +2235,27 @@ def validate_instance(t: Instance, fail: MsgCallback, empty_tuple_index: bool) - # TODO: is it OK to fill with TypeOfAny.from_error instead of special form? return False if t.type.has_type_var_tuple_type: - correct = len(t.args) >= len(t.type.type_vars) - 1 + min_tv_count = sum( + not tv.has_default() and not isinstance(tv, TypeVarTupleType) + for tv in t.type.defn.type_vars + ) + correct = len(t.args) >= min_tv_count if any( isinstance(a, UnpackType) and isinstance(get_proper_type(a.type), Instance) for a in t.args ): correct = True - if not correct: - exp_len = f"at least {len(t.type.type_vars) - 1}" + if not t.args: + if not (empty_tuple_index and len(t.type.type_vars) == 1): + # The Any arguments should be set by the caller. + return False + elif not correct: fail( - f"Bad number of arguments, expected: {exp_len}, given: {len(t.args)}", + f"Bad number of arguments, expected: at least {min_tv_count}, given: {len(t.args)}", t, code=codes.TYPE_ARG, ) return False - elif not t.args: - if not (empty_tuple_index and len(t.type.type_vars) == 1): - # The Any arguments should be set by the caller. - return False else: # We also need to check if we are not performing a type variable tuple split. unpack = find_unpack_in_list(t.args) @@ -2254,15 +2275,21 @@ def validate_instance(t: Instance, fail: MsgCallback, empty_tuple_index: bool) - elif any(isinstance(a, UnpackType) for a in t.args): # A variadic unpack in fixed size instance (fixed unpacks must be flattened by the caller) fail(message_registry.INVALID_UNPACK_POSITION, t, code=codes.VALID_TYPE) + t.args = () return False elif len(t.args) != len(t.type.type_vars): # Invalid number of type parameters. - if t.args: + arg_count = len(t.args) + min_tv_count = sum(not tv.has_default() for tv in t.type.defn.type_vars) + max_tv_count = len(t.type.type_vars) + if arg_count and (arg_count < min_tv_count or arg_count > max_tv_count): fail( - wrong_type_arg_count(len(t.type.type_vars), str(len(t.args)), t.type.name), + wrong_type_arg_count(min_tv_count, max_tv_count, str(arg_count), t.type.name), t, code=codes.TYPE_ARG, ) + t.args = () + t.invalid = True return False return True diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 9015d353fa08..c4d258d50ee5 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -116,3 +116,126 @@ def func_c1(x: Union[int, Callable[[Unpack[Ts1]], None]]) -> Tuple[Unpack[Ts1]]: # reveal_type(func_c1(callback1)) # Revealed type is "builtins.tuple[str]" # TODO # reveal_type(func_c1(2)) # Revealed type is "builtins.tuple[builtins.int, builtins.str]" # TODO [builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsClass1] +from typing import Generic, TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=int) +T3 = TypeVar("T3", default=str) + +class ClassA1(Generic[T2, T3]): ... + +def func_a1( + a: ClassA1, + b: ClassA1[float], + c: ClassA1[float, float], + d: ClassA1[float, float, float], # E: "ClassA1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + +class ClassA2(Generic[T1, T2, T3]): ... + +def func_a2( + a: ClassA2, + b: ClassA2[float], + c: ClassA2[float, float], + d: ClassA2[float, float, float], + e: ClassA2[float, float, float, float], # E: "ClassA2" expects between 1 and 3 type arguments, but 4 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA2[Any, builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.str]" + reveal_type(d) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.float]" + reveal_type(e) # N: Revealed type is "__main__.ClassA2[Any, builtins.int, builtins.str]" + +[case testTypeVarDefaultsClass2] +from typing import Generic, ParamSpec + +P1 = ParamSpec("P1") +P2 = ParamSpec("P2", default=[int, str]) +P3 = ParamSpec("P3", default=...) + +class ClassB1(Generic[P2, P3]): ... + +def func_b1( + a: ClassB1, + b: ClassB1[[float]], + c: ClassB1[[float], [float]], + d: ClassB1[[float], [float], [float]], # E: "ClassB1" expects between 0 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], ...]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + +class ClassB2(Generic[P1, P2]): ... + +def func_b2( + a: ClassB2, + b: ClassB2[[float]], + c: ClassB2[[float], [float]], + d: ClassB2[[float], [float], [float]], # E: "ClassB2" expects between 1 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.int, builtins.str]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + +[case testTypeVarDefaultsClass3] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1") +T3 = TypeVar("T3", default=str) + +Ts1 = TypeVarTuple("Ts1") +Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, str]]) +Ts3 = TypeVarTuple("Ts3", default=Unpack[Tuple[float, ...]]) +Ts4 = TypeVarTuple("Ts4", default=Unpack[Tuple[()]]) + +class ClassC1(Generic[Unpack[Ts2]]): ... + +def func_c1( + a: ClassC1, + b: ClassC1[float], +) -> None: + # reveal_type(a) # Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "__main__.ClassC1[builtins.float]" + +class ClassC2(Generic[T3, Unpack[Ts3]]): ... + +def func_c2( + a: ClassC2, + b: ClassC2[int], + c: ClassC2[int, Unpack[Tuple[()]]], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassC2[builtins.str, Unpack[builtins.tuple[builtins.float, ...]]]" + # reveal_type(b) # Revealed type is "__main__.ClassC2[builtins.int, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + reveal_type(c) # N: Revealed type is "__main__.ClassC2[builtins.int]" + +class ClassC3(Generic[T3, Unpack[Ts4]]): ... + +def func_c3( + a: ClassC3, + b: ClassC3[int], + c: ClassC3[int, Unpack[Tuple[float]]] +) -> None: + # reveal_type(a) # Revealed type is "__main__.ClassC3[builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "__main__.ClassC3[builtins.int]" + reveal_type(c) # N: Revealed type is "__main__.ClassC3[builtins.int, builtins.float]" + +class ClassC4(Generic[T1, Unpack[Ts1], T3]): ... + +def func_c4( + a: ClassC4, + b: ClassC4[int], + c: ClassC4[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassC4[Any, Unpack[builtins.tuple[Any, ...]], builtins.str]" + # reveal_type(b) # Revealed type is "__main__.ClassC4[builtins.int, builtins.str]" # TODO + reveal_type(c) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" +[builtins fixtures/tuple.pyi] From 3838bff555de4237cb77ef2a191a6791a4d0ae7a Mon Sep 17 00:00:00 2001 From: Aleksi Tarvainen Date: Thu, 25 Jan 2024 15:24:09 +0200 Subject: [PATCH 19/32] Docs: Add missing class instantiation to cheat sheet (#16817) --- docs/source/cheat_sheet_py3.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/cheat_sheet_py3.rst b/docs/source/cheat_sheet_py3.rst index fe5761ca6187..7ae8eeb59d66 100644 --- a/docs/source/cheat_sheet_py3.rst +++ b/docs/source/cheat_sheet_py3.rst @@ -209,6 +209,7 @@ Classes # This will allow access to any A.x, if x is compatible with the return type def __getattr__(self, name: str) -> int: ... + a = A() a.foo = 42 # Works a.bar = 'Ex-parrot' # Fails type checking From 717a263fcb594689ff48b1d1a88a8bc4f10c2652 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 27 Jan 2024 01:20:24 -0800 Subject: [PATCH 20/32] stubtest: adjust symtable logic (#16823) Fixes https://github.com/python/typeshed/issues/11318 --- mypy/stubtest.py | 59 ++++++++++++++++++++------------------- mypy/test/teststubtest.py | 18 ++++++++++++ 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 9a038105dc82..7ab3a2b1e5d0 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -10,6 +10,7 @@ import collections.abc import copy import enum +import functools import importlib import importlib.machinery import inspect @@ -310,35 +311,23 @@ def _verify_exported_names( ) -def _get_imported_symbol_names(runtime: types.ModuleType) -> frozenset[str] | None: - """Retrieve the names in the global namespace which are known to be imported. +@functools.lru_cache +def _module_symbol_table(runtime: types.ModuleType) -> symtable.SymbolTable | None: + """Retrieve the symbol table for the module (or None on failure). - 1). Use inspect to retrieve the source code of the module - 2). Use symtable to parse the source and retrieve names that are known to be imported - from other modules. - - If either of the above steps fails, return `None`. - - Note that if a set of names is returned, - it won't include names imported via `from foo import *` imports. + 1) Use inspect to retrieve the source code of the module + 2) Use symtable to parse the source (and use what symtable knows for its purposes) """ try: source = inspect.getsource(runtime) except (OSError, TypeError, SyntaxError): return None - if not source.strip(): - # The source code for the module was an empty file, - # no point in parsing it with symtable - return frozenset() - try: - module_symtable = symtable.symtable(source, runtime.__name__, "exec") + return symtable.symtable(source, runtime.__name__, "exec") except SyntaxError: return None - return frozenset(sym.get_name() for sym in module_symtable.get_symbols() if sym.is_imported()) - @verify.register(nodes.MypyFile) def verify_mypyfile( @@ -369,25 +358,37 @@ def verify_mypyfile( if not o.module_hidden and (not is_probably_private(m) or hasattr(runtime, m)) } - imported_symbols = _get_imported_symbol_names(runtime) - def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool: """Heuristics to determine whether a name originates from another module.""" obj = getattr(r, attr) if isinstance(obj, types.ModuleType): return False - if callable(obj): - # It's highly likely to be a class or a function if it's callable, - # so the __module__ attribute will give a good indication of which module it comes from + + symbol_table = _module_symbol_table(r) + if symbol_table is not None: try: - obj_mod = obj.__module__ - except Exception: + symbol = symbol_table.lookup(attr) + except KeyError: pass else: - if isinstance(obj_mod, str): - return bool(obj_mod == r.__name__) - if imported_symbols is not None: - return attr not in imported_symbols + if symbol.is_imported(): + # symtable says we got this from another module + return False + # But we can't just return True here, because symtable doesn't know about symbols + # that come from `from module import *` + if symbol.is_assigned(): + # symtable knows we assigned this symbol in the module + return True + + # The __module__ attribute is unreliable for anything except functions and classes, + # but it's our best guess at this point + try: + obj_mod = obj.__module__ + except Exception: + pass + else: + if isinstance(obj_mod, str): + return bool(obj_mod == r.__name__) return True runtime_public_contents = ( diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py index 3f231d8afbcc..55f35200a7f5 100644 --- a/mypy/test/teststubtest.py +++ b/mypy/test/teststubtest.py @@ -1285,6 +1285,24 @@ def test_missing_no_runtime_all(self) -> Iterator[Case]: yield Case(stub="", runtime="from json.scanner import NUMBER_RE", error=None) yield Case(stub="", runtime="from string import ascii_letters", error=None) + @collect_cases + def test_missing_no_runtime_all_terrible(self) -> Iterator[Case]: + yield Case( + stub="", + runtime=""" +import sys +import types +import __future__ +_m = types.SimpleNamespace() +_m.annotations = __future__.annotations +sys.modules["_terrible_stubtest_test_module"] = _m + +from _terrible_stubtest_test_module import * +assert annotations +""", + error=None, + ) + @collect_cases def test_non_public_1(self) -> Iterator[Case]: yield Case( From 09490c890590d1ead414b06a332c7570fd589342 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sun, 28 Jan 2024 02:37:14 +0100 Subject: [PATCH 21/32] Use TypeVar defaults instead of Any when fixing TypeAlias types (PEP 696) (#16825) This PR applies the TypeVar defaults to `TypeAlias` types instead of using `Any` exclusively, similar to https://github.com/python/mypy/pull/16812. Again `TypeVarTuple` defaults aren't handled correctly yet. Ref: https://github.com/python/mypy/issues/14851 --- mypy/checkexpr.py | 1 + mypy/typeanal.py | 104 ++++++++----- test-data/unit/check-typevar-defaults.test | 161 +++++++++++++++++++++ 3 files changed, 232 insertions(+), 34 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index fbedd95e8fd2..a4b66bb9932c 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4711,6 +4711,7 @@ class LongName(Generic[T]): ... item = get_proper_type( set_any_tvars( alias, + [], ctx.line, ctx.column, self.chk.options, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d10f26f5199a..678407acc3ac 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1927,18 +1927,19 @@ def instantiate_type_alias( if any(unknown_unpack(a) for a in args): # This type is not ready to be validated, because of unknown total count. # Note that we keep the kind of Any for consistency. - return set_any_tvars(node, ctx.line, ctx.column, options, special_form=True) + return set_any_tvars(node, [], ctx.line, ctx.column, options, special_form=True) - exp_len = len(node.alias_tvars) + max_tv_count = len(node.alias_tvars) act_len = len(args) if ( - exp_len > 0 + max_tv_count > 0 and act_len == 0 and not (empty_tuple_index and node.tvar_tuple_index is not None) ): # Interpret bare Alias same as normal generic, i.e., Alias[Any, Any, ...] return set_any_tvars( node, + args, ctx.line, ctx.column, options, @@ -1946,7 +1947,7 @@ def instantiate_type_alias( fail=fail, unexpanded_type=unexpanded_type, ) - if exp_len == 0 and act_len == 0: + if max_tv_count == 0 and act_len == 0: if no_args: assert isinstance(node.target, Instance) # type: ignore[misc] # Note: this is the only case where we use an eager expansion. See more info about @@ -1954,7 +1955,7 @@ def instantiate_type_alias( return Instance(node.target.type, [], line=ctx.line, column=ctx.column) return TypeAliasType(node, [], line=ctx.line, column=ctx.column) if ( - exp_len == 0 + max_tv_count == 0 and act_len > 0 and isinstance(node.target, Instance) # type: ignore[misc] and no_args @@ -1967,32 +1968,48 @@ def instantiate_type_alias( if any(isinstance(a, UnpackType) for a in args): # A variadic unpack in fixed size alias (fixed unpacks must be flattened by the caller) fail(message_registry.INVALID_UNPACK_POSITION, ctx, code=codes.VALID_TYPE) - return set_any_tvars(node, ctx.line, ctx.column, options, from_error=True) - correct = act_len == exp_len + return set_any_tvars(node, [], ctx.line, ctx.column, options, from_error=True) + min_tv_count = sum(not tv.has_default() for tv in node.alias_tvars) + fill_typevars = act_len != max_tv_count + correct = min_tv_count <= act_len <= max_tv_count else: - correct = act_len >= exp_len - 1 + min_tv_count = sum( + not tv.has_default() and not isinstance(tv, TypeVarTupleType) + for tv in node.alias_tvars + ) + correct = act_len >= min_tv_count for a in args: if isinstance(a, UnpackType): unpacked = get_proper_type(a.type) if isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple": # Variadic tuple is always correct. correct = True - if not correct: - if use_standard_error: - # This is used if type alias is an internal representation of another type, - # for example a generic TypedDict or NamedTuple. - msg = wrong_type_arg_count(exp_len, exp_len, str(act_len), node.name) - else: - if node.tvar_tuple_index is not None: - exp_len_str = f"at least {exp_len - 1}" + fill_typevars = not correct + if fill_typevars: + if not correct: + if use_standard_error: + # This is used if type alias is an internal representation of another type, + # for example a generic TypedDict or NamedTuple. + msg = wrong_type_arg_count(max_tv_count, max_tv_count, str(act_len), node.name) else: - exp_len_str = str(exp_len) - msg = ( - "Bad number of arguments for type alias," - f" expected: {exp_len_str}, given: {act_len}" - ) - fail(msg, ctx, code=codes.TYPE_ARG) - return set_any_tvars(node, ctx.line, ctx.column, options, from_error=True) + if node.tvar_tuple_index is not None: + msg = ( + "Bad number of arguments for type alias," + f" expected: at least {min_tv_count}, given: {act_len}" + ) + elif min_tv_count != max_tv_count: + msg = ( + "Bad number of arguments for type alias," + f" expected between {min_tv_count} and {max_tv_count}, given: {act_len}" + ) + else: + msg = ( + "Bad number of arguments for type alias," + f" expected: {min_tv_count}, given: {act_len}" + ) + fail(msg, ctx, code=codes.TYPE_ARG) + args = [] + return set_any_tvars(node, args, ctx.line, ctx.column, options, from_error=True) elif node.tvar_tuple_index is not None: # We also need to check if we are not performing a type variable tuple split. unpack = find_unpack_in_list(args) @@ -2006,7 +2023,7 @@ def instantiate_type_alias( act_suffix = len(args) - unpack - 1 if act_prefix < exp_prefix or act_suffix < exp_suffix: fail("TypeVarTuple cannot be split", ctx, code=codes.TYPE_ARG) - return set_any_tvars(node, ctx.line, ctx.column, options, from_error=True) + return set_any_tvars(node, [], ctx.line, ctx.column, options, from_error=True) # TODO: we need to check args validity w.r.t alias.alias_tvars. # Otherwise invalid instantiations will be allowed in runtime context. # Note: in type context, these will be still caught by semanal_typeargs. @@ -2025,6 +2042,7 @@ def instantiate_type_alias( def set_any_tvars( node: TypeAlias, + args: list[Type], newline: int, newcolumn: int, options: Options, @@ -2041,7 +2059,33 @@ def set_any_tvars( type_of_any = TypeOfAny.special_form else: type_of_any = TypeOfAny.from_omitted_generics - if disallow_any and node.alias_tvars: + any_type = AnyType(type_of_any, line=newline, column=newcolumn) + + env: dict[TypeVarId, Type] = {} + used_any_type = False + has_type_var_tuple_type = False + for tv, arg in itertools.zip_longest(node.alias_tvars, args, fillvalue=None): + if tv is None: + continue + if arg is None: + if tv.has_default(): + arg = tv.default + else: + arg = any_type + used_any_type = True + if isinstance(tv, TypeVarTupleType): + # TODO Handle TypeVarTuple defaults + has_type_var_tuple_type = True + arg = UnpackType(Instance(tv.tuple_fallback.type, [any_type])) + args.append(arg) + env[tv.id] = arg + t = TypeAliasType(node, args, newline, newcolumn) + if not has_type_var_tuple_type: + fixed = expand_type(t, env) + assert isinstance(fixed, TypeAliasType) + t.args = fixed.args + + if used_any_type and disallow_any and node.alias_tvars: assert fail is not None if unexpanded_type: type_str = ( @@ -2057,15 +2101,7 @@ def set_any_tvars( Context(newline, newcolumn), code=codes.TYPE_ARG, ) - any_type = AnyType(type_of_any, line=newline, column=newcolumn) - - args: list[Type] = [] - for tv in node.alias_tvars: - if isinstance(tv, TypeVarTupleType): - args.append(UnpackType(Instance(tv.tuple_fallback.type, [any_type]))) - else: - args.append(any_type) - return TypeAliasType(node, args, newline, newcolumn) + return t def flatten_tvars(lists: list[list[T]]) -> list[T]: diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index c4d258d50ee5..0c531dd3f18b 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -239,3 +239,164 @@ def func_c4( # reveal_type(b) # Revealed type is "__main__.ClassC4[builtins.int, builtins.str]" # TODO reveal_type(c) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsTypeAlias1] +# flags: --disallow-any-generics +from typing import Any, Dict, List, Tuple, TypeVar, Union + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=int) +T3 = TypeVar("T3", default=str) +T4 = TypeVar("T4") + +TA1 = Dict[T2, T3] + +def func_a1( + a: TA1, + b: TA1[float], + c: TA1[float, float], + d: TA1[float, float, float], # E: Bad number of arguments for type alias, expected between 0 and 2, given: 3 +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.float, builtins.str]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + +TA2 = Tuple[T1, T2, T3] + +def func_a2( + a: TA2, # E: Missing type parameters for generic type "TA2" + b: TA2[float], + c: TA2[float, float], + d: TA2[float, float, float], + e: TA2[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given: 4 +) -> None: + reveal_type(a) # N: Revealed type is "Tuple[Any, builtins.int, builtins.str]" + reveal_type(b) # N: Revealed type is "Tuple[builtins.float, builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.str]" + reveal_type(d) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.float]" + reveal_type(e) # N: Revealed type is "Tuple[Any, builtins.int, builtins.str]" + +TA3 = Union[Dict[T1, T2], List[T3]] + +def func_a3( + a: TA3, # E: Missing type parameters for generic type "TA3" + b: TA3[float], + c: TA3[float, float], + d: TA3[float, float, float], + e: TA3[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given: 4 +) -> None: + reveal_type(a) # N: Revealed type is "Union[builtins.dict[Any, builtins.int], builtins.list[builtins.str]]" + reveal_type(b) # N: Revealed type is "Union[builtins.dict[builtins.float, builtins.int], builtins.list[builtins.str]]" + reveal_type(c) # N: Revealed type is "Union[builtins.dict[builtins.float, builtins.float], builtins.list[builtins.str]]" + reveal_type(d) # N: Revealed type is "Union[builtins.dict[builtins.float, builtins.float], builtins.list[builtins.float]]" + reveal_type(e) # N: Revealed type is "Union[builtins.dict[Any, builtins.int], builtins.list[builtins.str]]" + +TA4 = Tuple[T1, T4, T2] + +def func_a4( + a: TA4, # E: Missing type parameters for generic type "TA4" + b: TA4[float], # E: Bad number of arguments for type alias, expected between 2 and 3, given: 1 + c: TA4[float, float], + d: TA4[float, float, float], + e: TA4[float, float, float, float], # E: Bad number of arguments for type alias, expected between 2 and 3, given: 4 +) -> None: + reveal_type(a) # N: Revealed type is "Tuple[Any, Any, builtins.int]" + reveal_type(b) # N: Revealed type is "Tuple[Any, Any, builtins.int]" + reveal_type(c) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.int]" + reveal_type(d) # N: Revealed type is "Tuple[builtins.float, builtins.float, builtins.float]" + reveal_type(e) # N: Revealed type is "Tuple[Any, Any, builtins.int]" +[builtins fixtures/dict.pyi] + +[case testTypeVarDefaultsTypeAlias2] +# flags: --disallow-any-generics +from typing import Any, Generic, ParamSpec + +P1 = ParamSpec("P1") +P2 = ParamSpec("P2", default=[int, str]) +P3 = ParamSpec("P3", default=...) + +class ClassB1(Generic[P2, P3]): ... +TB1 = ClassB1[P2, P3] + +def func_b1( + a: TB1, + b: TB1[[float]], + c: TB1[[float], [float]], + d: TB1[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 0 and 2, given: 3 +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + +class ClassB2(Generic[P1, P2]): ... +TB2 = ClassB2[P1, P2] + +def func_b2( + a: TB2, # E: Missing type parameters for generic type "TB2" + b: TB2[[float]], + c: TB2[[float], [float]], + d: TB2[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 1 and 2, given: 3 +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + reveal_type(b) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.int, builtins.str]]" + reveal_type(c) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.float]]" + reveal_type(d) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsTypeAlias3] +# flags: --disallow-any-generics +from typing import Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1") +T3 = TypeVar("T3", default=str) + +Ts1 = TypeVarTuple("Ts1") +Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, str]]) +Ts3 = TypeVarTuple("Ts3", default=Unpack[Tuple[float, ...]]) +Ts4 = TypeVarTuple("Ts4", default=Unpack[Tuple[()]]) + +TC1 = Tuple[Unpack[Ts2]] + +def func_c1( + a: TC1, + b: TC1[float], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "Tuple[builtins.float]" + +TC2 = Tuple[T3, Unpack[Ts3]] + +def func_c2( + a: TC2, + b: TC2[int], + c: TC2[int, Unpack[Tuple[()]]], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.str, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + # reveal_type(b) # Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + reveal_type(c) # N: Revealed type is "Tuple[builtins.int]" + +TC3 = Tuple[T3, Unpack[Ts4]] + +def func_c3( + a: TC3, + b: TC3[int], + c: TC3[int, Unpack[Tuple[float]]], +) -> None: + # reveal_type(a) # Revealed type is "Tuple[builtins.str]" # TODO + reveal_type(b) # N: Revealed type is "Tuple[builtins.int]" + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, builtins.float]" + +TC4 = Tuple[T1, Unpack[Ts1], T3] + +def func_c4( + a: TC4, # E: Missing type parameters for generic type "TC4" + b: TC4[int], + c: TC4[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "Tuple[Any, Unpack[builtins.tuple[Any, ...]], builtins.str]" + # reveal_type(b) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO + reveal_type(c) # N: Revealed type is "Tuple[builtins.int, builtins.float]" +[builtins fixtures/tuple.pyi] From 1da0ebe5ab96cb4982184e405260822a18b3502c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 29 Jan 2024 21:27:19 -0800 Subject: [PATCH 22/32] Various docs improvements (#16836) Recommend `--disable-error-code=import-untyped`. It's probably strictly better than `--ignore-missing-imports` for most users. Remove the displaying error codes section, since it's on by default. Some more advice and discoverability for "using mypy with an existing codebase". --- docs/source/config_file.rst | 4 ++++ docs/source/error_codes.rst | 23 +++++++---------------- docs/source/existing_code.rst | 21 ++++++++++++++------- docs/source/running_mypy.rst | 18 +++++++++++------- 4 files changed, 36 insertions(+), 30 deletions(-) diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 910b015df658..ac110cbed9f1 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -3,6 +3,10 @@ The mypy configuration file =========================== +Mypy is very configurable. This is most useful when introducing typing to +an existing codebase. See :ref:`existing-code` for concrete advice for +that situation. + Mypy supports reading configuration settings from a file with the following precedence order: 1. ``./mypy.ini`` diff --git a/docs/source/error_codes.rst b/docs/source/error_codes.rst index a71168cadf30..35fad161f8a2 100644 --- a/docs/source/error_codes.rst +++ b/docs/source/error_codes.rst @@ -19,22 +19,6 @@ Most error codes are shared between multiple related error messages. Error codes may change in future mypy releases. - -Displaying error codes ----------------------- - -Error codes are displayed by default. Use :option:`--hide-error-codes ` -or config ``hide_error_codes = True`` to hide error codes. Error codes are shown inside square brackets: - -.. code-block:: text - - $ mypy prog.py - prog.py:1: error: "str" has no attribute "trim" [attr-defined] - -It's also possible to require error codes for ``type: ignore`` comments. -See :ref:`ignore-without-code` for more information. - - .. _silence-error-codes: Silencing errors based on error codes @@ -121,3 +105,10 @@ Similar logic works for disabling error codes globally. If a given error code is a subcode of another one, it will be mentioned in the documentation for the narrower code. This hierarchy is not nested: there cannot be subcodes of other subcodes. + + +Requiring error codes +--------------------- + +It's possible to require error codes be specified in ``type: ignore`` comments. +See :ref:`ignore-without-code` for more information. diff --git a/docs/source/existing_code.rst b/docs/source/existing_code.rst index c66008f4b782..0a5ac2bfa8f6 100644 --- a/docs/source/existing_code.rst +++ b/docs/source/existing_code.rst @@ -31,8 +31,8 @@ invocation to your codebase, or adding your mypy invocation to existing tools you use to run tests, like ``tox``. * Make sure everyone runs mypy with the same options. Checking a mypy - :ref:`configuration file ` into your codebase can help - with this. + :ref:`configuration file ` into your codebase is the + easiest way to do this. * Make sure everyone type checks the same set of files. See :ref:`specifying-code-to-be-checked` for details. @@ -48,7 +48,7 @@ A simple CI script could look something like this: .. code-block:: text - python3 -m pip install mypy==0.971 + python3 -m pip install mypy==1.8 # Run your standardised mypy invocation, e.g. mypy my_project # This could also look like `scripts/run_mypy.sh`, `tox run -e mypy`, `make mypy`, etc @@ -74,6 +74,11 @@ You could even invert this, by setting ``ignore_errors = True`` in your global config section and only enabling error reporting with ``ignore_errors = False`` for the set of modules you are ready to type check. +The per-module configuration that mypy's configuration file allows can be +extremely useful. Many configuration options can be enabled or disabled +only for specific modules. In particular, you can also enable or disable +various error codes on a per-module basis, see :ref:`error-codes`. + Fixing errors related to imports -------------------------------- @@ -89,7 +94,7 @@ that it can't find, that don't have types, or don't have stub files: Sometimes these can be fixed by installing the relevant packages or stub libraries in the environment you're running ``mypy`` in. -See :ref:`ignore-missing-imports` for a complete reference on these errors +See :ref:`fix-missing-imports` for a complete reference on these errors and the ways in which you can fix them. You'll likely find that you want to suppress all errors from importing @@ -118,13 +123,15 @@ codebase, use a config like this: ignore_missing_imports = True If you get a large number of errors, you may want to ignore all errors -about missing imports, for instance by setting :confval:`ignore_missing_imports` -to true globally. This can hide errors later on, so we recommend avoiding this +about missing imports, for instance by setting +:option:`--disable-error-code=import-untyped `. +or setting :confval:`ignore_missing_imports` to true globally. +This can hide errors later on, so we recommend avoiding this if possible. Finally, mypy allows fine-grained control over specific import following behaviour. It's very easy to silently shoot yourself in the foot when playing -around with these, so it's mostly recommended as a last resort. For more +around with these, so this should be a last resort. For more details, look :ref:`here `. Prioritise annotating widely imported modules diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index b0cefec9dafa..f959e9af2391 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -305,16 +305,20 @@ not catch errors in its use. The ``.*`` after ``foobar`` will ignore imports of ``foobar`` modules and subpackages in addition to the ``foobar`` top-level package namespace. -3. To suppress *all* missing import errors for *all* libraries in your codebase, - invoke mypy with the :option:`--ignore-missing-imports ` command line flag or set - the :confval:`ignore_missing_imports` - config file option to True - in the *global* section of your mypy config file:: +3. To suppress *all* missing import errors for *all* untyped libraries + in your codebase, use :option:`--disable-error-code=import-untyped `. + See :ref:`code-import-untyped` for more details on this error code. + + You can also set :confval:`disable_error_code`, like so:: [mypy] - ignore_missing_imports = True + disable_error_code = import-untyped + - We recommend using this approach only as a last resort: it's equivalent + You can also set the :option:`--ignore-missing-imports ` + command line flag or set the :confval:`ignore_missing_imports` config file + option to True in the *global* section of your mypy config file. We + recommend avoiding ``--ignore-missing-imports`` if possible: it's equivalent to adding a ``# type: ignore`` to all unresolved imports in your codebase. From 418377892f3220958f0d71c3429a9e7e638b8ca0 Mon Sep 17 00:00:00 2001 From: Stefanie Molin <24376333+stefmolin@users.noreply.github.com> Date: Tue, 30 Jan 2024 00:27:56 -0500 Subject: [PATCH 23/32] Fix numbering error in docs (#16838) --- docs/source/running_mypy.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst index f959e9af2391..25b34b247b4b 100644 --- a/docs/source/running_mypy.rst +++ b/docs/source/running_mypy.rst @@ -391,11 +391,11 @@ this error, try: installing into the environment you expect by running pip like ``python -m pip ...``. -2. Reading the :ref:`finding-imports` section below to make sure you +3. Reading the :ref:`finding-imports` section below to make sure you understand how exactly mypy searches for and finds modules and modify how you're invoking mypy accordingly. -3. Directly specifying the directory containing the module you want to +4. Directly specifying the directory containing the module you want to type check from the command line, by using the :confval:`mypy_path` or :confval:`files` config file options, or by using the ``MYPYPATH`` environment variable. From 7a746c4dbd8a0d35e9a3889c9fc861b63aec66cc Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 30 Jan 2024 06:42:46 +0100 Subject: [PATCH 24/32] Consider TypeVarTuple to be invariant (#16759) The TypeVarTuple equality checks mentioned in PEP 646 assume TypeVarTuple to be `invariant`. https://peps.python.org/pep-0646/#type-variable-tuple-equality Fixes: #16739 --- mypy/constraints.py | 5 +++-- mypy/test/testconstraints.py | 24 +++++++++++++++++----- test-data/unit/check-typevar-tuple.test | 27 +++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index d6a4b28799e5..c4eba2ca1ede 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -894,8 +894,9 @@ def visit_instance(self, template: Instance) -> list[Constraint]: res.append(Constraint(template_arg, SUBTYPE_OF, suffix)) res.append(Constraint(template_arg, SUPERTYPE_OF, suffix)) elif isinstance(tvar, TypeVarTupleType): - # Handle variadic type variables covariantly for consistency. - res.extend(infer_constraints(template_arg, mapped_arg, self.direction)) + # Consider variadic type variables to be invariant. + res.extend(infer_constraints(template_arg, mapped_arg, SUBTYPE_OF)) + res.extend(infer_constraints(template_arg, mapped_arg, SUPERTYPE_OF)) return res if ( template.type.is_protocol diff --git a/mypy/test/testconstraints.py b/mypy/test/testconstraints.py index 5ec292f07056..a701a173cbaa 100644 --- a/mypy/test/testconstraints.py +++ b/mypy/test/testconstraints.py @@ -30,13 +30,18 @@ def test_basic_type_var_tuple_subtype(self) -> None: def test_basic_type_var_tuple(self) -> None: fx = self.fx - assert infer_constraints( - Instance(fx.gvi, [UnpackType(fx.ts)]), Instance(fx.gvi, [fx.a, fx.b]), SUPERTYPE_OF - ) == [ + assert set( + infer_constraints( + Instance(fx.gvi, [UnpackType(fx.ts)]), Instance(fx.gvi, [fx.a, fx.b]), SUPERTYPE_OF + ) + ) == { Constraint( type_var=fx.ts, op=SUPERTYPE_OF, target=TupleType([fx.a, fx.b], fx.std_tuple) - ) - ] + ), + Constraint( + type_var=fx.ts, op=SUBTYPE_OF, target=TupleType([fx.a, fx.b], fx.std_tuple) + ), + } def test_type_var_tuple_with_prefix_and_suffix(self) -> None: fx = self.fx @@ -51,6 +56,9 @@ def test_type_var_tuple_with_prefix_and_suffix(self) -> None: Constraint( type_var=fx.ts, op=SUPERTYPE_OF, target=TupleType([fx.b, fx.c], fx.std_tuple) ), + Constraint( + type_var=fx.ts, op=SUBTYPE_OF, target=TupleType([fx.b, fx.c], fx.std_tuple) + ), Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.d), } @@ -64,7 +72,9 @@ def test_unpack_homogenous_tuple(self) -> None: ) ) == { Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), + Constraint(type_var=fx.t, op=SUBTYPE_OF, target=fx.a), Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.t, op=SUBTYPE_OF, target=fx.b), } def test_unpack_homogenous_tuple_with_prefix_and_suffix(self) -> None: @@ -78,7 +88,9 @@ def test_unpack_homogenous_tuple_with_prefix_and_suffix(self) -> None: ) == { Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.s, op=SUBTYPE_OF, target=fx.b), Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.c), + Constraint(type_var=fx.s, op=SUBTYPE_OF, target=fx.c), Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), } @@ -93,7 +105,9 @@ def test_unpack_with_prefix_and_suffix(self) -> None: ) == { Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.a), Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.b), + Constraint(type_var=fx.t, op=SUBTYPE_OF, target=fx.b), Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.c), + Constraint(type_var=fx.s, op=SUBTYPE_OF, target=fx.c), Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), } diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 9c8d21114d4c..70229f5d9a62 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -2277,3 +2277,30 @@ higher_order(bad2) # E: Argument 1 to "higher_order" has incompatible type "Cal higher_order(bad3) # E: Argument 1 to "higher_order" has incompatible type "Callable[[NamedArg(str, 'd')], int]"; expected "Callable[[int, str, VarArg(Unpack[Tuple[Unpack[Tuple[Any, ...]], int]])], Any]" higher_order(bad4) # E: Argument 1 to "higher_order" has incompatible type "Callable[[KwArg(None)], None]"; expected "Callable[[int, str, VarArg(Unpack[Tuple[Unpack[Tuple[Any, ...]], int]])], Any]" [builtins fixtures/tuple.pyi] + +[case testTypeVarTupleInvariant] +from typing import Generic, Tuple +from typing_extensions import Unpack, TypeVarTuple +Ts = TypeVarTuple("Ts") + +class Array(Generic[Unpack[Ts]]): ... + +def pointwise_multiply(x: Array[Unpack[Ts]], y: Array[Unpack[Ts]]) -> Array[Unpack[Ts]]: ... + +def a1(x: Array[int], y: Array[str], z: Array[int, str]) -> None: + reveal_type(pointwise_multiply(x, x)) # N: Revealed type is "__main__.Array[builtins.int]" + reveal_type(pointwise_multiply(x, y)) # E: Cannot infer type argument 1 of "pointwise_multiply" \ + # N: Revealed type is "__main__.Array[Unpack[builtins.tuple[Any, ...]]]" + reveal_type(pointwise_multiply(x, z)) # E: Cannot infer type argument 1 of "pointwise_multiply" \ + # N: Revealed type is "__main__.Array[Unpack[builtins.tuple[Any, ...]]]" + +def func(x: Array[Unpack[Ts]], *args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: + ... + +def a2(x: Array[int, str]) -> None: + reveal_type(func(x, 2, "Hello")) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + reveal_type(func(x, 2)) # E: Cannot infer type argument 1 of "func" \ + # N: Revealed type is "builtins.tuple[Any, ...]" + reveal_type(func(x, 2, "Hello", True)) # E: Cannot infer type argument 1 of "func" \ + # N: Revealed type is "builtins.tuple[Any, ...]" +[builtins fixtures/tuple.pyi] From ed50208055e8c5f99caf7da0cea8c99fcfdc5670 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 30 Jan 2024 06:44:09 +0100 Subject: [PATCH 25/32] Update TypeAlias error messages to remove colon (#16831) Small update to adjust the error messages following the suggestion in https://github.com/python/mypy/pull/16825#discussion_r1468507923. --- mypy/typeanal.py | 6 +++--- test-data/unit/check-generics.test | 9 ++++----- test-data/unit/check-parameter-specification.test | 5 +++-- test-data/unit/check-type-aliases.test | 5 ++--- test-data/unit/check-typevar-defaults.test | 14 +++++++------- test-data/unit/check-typevar-tuple.test | 6 +++--- test-data/unit/fine-grained.test | 8 ++++---- 7 files changed, 26 insertions(+), 27 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 678407acc3ac..1bcba5f0ca88 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1995,17 +1995,17 @@ def instantiate_type_alias( if node.tvar_tuple_index is not None: msg = ( "Bad number of arguments for type alias," - f" expected: at least {min_tv_count}, given: {act_len}" + f" expected at least {min_tv_count}, given {act_len}" ) elif min_tv_count != max_tv_count: msg = ( "Bad number of arguments for type alias," - f" expected between {min_tv_count} and {max_tv_count}, given: {act_len}" + f" expected between {min_tv_count} and {max_tv_count}, given {act_len}" ) else: msg = ( "Bad number of arguments for type alias," - f" expected: {min_tv_count}, given: {act_len}" + f" expected {min_tv_count}, given {act_len}" ) fail(msg, ctx, code=codes.TYPE_ARG) args = [] diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index e2f65ed39c1e..337e92daf365 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -630,7 +630,7 @@ main:11:5: error: "Node" expects 2 type arguments, but 3 given main:15:10: error: "list" expects 1 type argument, but 2 given main:16:19: error: "list" expects 1 type argument, but 2 given main:17:25: error: "Node" expects 2 type arguments, but 1 given -main:19:5: error: Bad number of arguments for type alias, expected: 1, given: 2 +main:19:5: error: Bad number of arguments for type alias, expected 1, given 2 main:22:13: note: Revealed type is "__main__.Node[builtins.int, builtins.str]" main:24:13: note: Revealed type is "__main__.Node[__main__.Node[builtins.int, builtins.int], builtins.list[builtins.int]]" main:26:5: error: Type variable "__main__.T" is invalid as target for type alias @@ -944,7 +944,7 @@ Transform = Callable[[T, int], Tuple[T, R]] [case testGenericTypeAliasesImportingWithoutTypeVarError] from a import Alias -x: Alias[int, str] # E: Bad number of arguments for type alias, expected: 1, given: 2 +x: Alias[int, str] # E: Bad number of arguments for type alias, expected 1, given 2 reveal_type(x) # N: Revealed type is "builtins.list[builtins.list[Any]]" [file a.py] @@ -953,7 +953,6 @@ T = TypeVar('T') Alias = List[List[T]] [builtins fixtures/list.pyi] -[out] [case testGenericAliasWithTypeVarsFromDifferentModules] from mod import Alias, TypeVar @@ -1000,8 +999,8 @@ reveal_type(x) # N: Revealed type is "Union[builtins.int, None]" reveal_type(y) # N: Revealed type is "builtins.int" U[int] # E: Type application targets a non-generic function or class -O[int] # E: Bad number of arguments for type alias, expected: 0, given: 1 # E: Type application is only supported for generic classes -[out] +O[int] # E: Bad number of arguments for type alias, expected 0, given 1 \ + # E: Type application is only supported for generic classes [case testAliasesInClassBodyNormalVsSubscripted] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index af2be84f5412..7a5c5934a94e 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1506,9 +1506,10 @@ def g(x: A[P]) -> None: ... # E: Invalid location for ParamSpec "P" \ # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' C = Callable[P, T] -x: C[int] # E: Bad number of arguments for type alias, expected: 2, given: 1 +x: C[int] # E: Bad number of arguments for type alias, expected 2, given 1 y: C[int, str] # E: Can only replace ParamSpec with a parameter types list or another ParamSpec, got "int" -z: C[int, str, bytes] # E: Bad number of arguments for type alias, expected: 2, given: 3 +z: C[int, str, bytes] # E: Bad number of arguments for type alias, expected 2, given 3 + [builtins fixtures/paramspec.pyi] [case testTrivialParametersHandledCorrectly] diff --git a/test-data/unit/check-type-aliases.test b/test-data/unit/check-type-aliases.test index 4364a9bfa9dc..a43233eed973 100644 --- a/test-data/unit/check-type-aliases.test +++ b/test-data/unit/check-type-aliases.test @@ -330,10 +330,9 @@ c: C t: T reveal_type(c) # N: Revealed type is "def (*Any, **Any) -> Any" reveal_type(t) # N: Revealed type is "builtins.tuple[Any, ...]" -bad: C[int] # E: Bad number of arguments for type alias, expected: 0, given: 1 -also_bad: T[int] # E: Bad number of arguments for type alias, expected: 0, given: 1 +bad: C[int] # E: Bad number of arguments for type alias, expected 0, given 1 +also_bad: T[int] # E: Bad number of arguments for type alias, expected 0, given 1 [builtins fixtures/tuple.pyi] -[out] [case testAliasRefOnClass] from typing import Generic, TypeVar, Type diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 0c531dd3f18b..4b509cd4fc40 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -255,7 +255,7 @@ def func_a1( a: TA1, b: TA1[float], c: TA1[float, float], - d: TA1[float, float, float], # E: Bad number of arguments for type alias, expected between 0 and 2, given: 3 + d: TA1[float, float, float], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 ) -> None: reveal_type(a) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" reveal_type(b) # N: Revealed type is "builtins.dict[builtins.float, builtins.str]" @@ -269,7 +269,7 @@ def func_a2( b: TA2[float], c: TA2[float, float], d: TA2[float, float, float], - e: TA2[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given: 4 + e: TA2[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given 4 ) -> None: reveal_type(a) # N: Revealed type is "Tuple[Any, builtins.int, builtins.str]" reveal_type(b) # N: Revealed type is "Tuple[builtins.float, builtins.int, builtins.str]" @@ -284,7 +284,7 @@ def func_a3( b: TA3[float], c: TA3[float, float], d: TA3[float, float, float], - e: TA3[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given: 4 + e: TA3[float, float, float, float], # E: Bad number of arguments for type alias, expected between 1 and 3, given 4 ) -> None: reveal_type(a) # N: Revealed type is "Union[builtins.dict[Any, builtins.int], builtins.list[builtins.str]]" reveal_type(b) # N: Revealed type is "Union[builtins.dict[builtins.float, builtins.int], builtins.list[builtins.str]]" @@ -296,10 +296,10 @@ TA4 = Tuple[T1, T4, T2] def func_a4( a: TA4, # E: Missing type parameters for generic type "TA4" - b: TA4[float], # E: Bad number of arguments for type alias, expected between 2 and 3, given: 1 + b: TA4[float], # E: Bad number of arguments for type alias, expected between 2 and 3, given 1 c: TA4[float, float], d: TA4[float, float, float], - e: TA4[float, float, float, float], # E: Bad number of arguments for type alias, expected between 2 and 3, given: 4 + e: TA4[float, float, float, float], # E: Bad number of arguments for type alias, expected between 2 and 3, given 4 ) -> None: reveal_type(a) # N: Revealed type is "Tuple[Any, Any, builtins.int]" reveal_type(b) # N: Revealed type is "Tuple[Any, Any, builtins.int]" @@ -323,7 +323,7 @@ def func_b1( a: TB1, b: TB1[[float]], c: TB1[[float], [float]], - d: TB1[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 0 and 2, given: 3 + d: TB1[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 0 and 2, given 3 ) -> None: reveal_type(a) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" reveal_type(b) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" @@ -337,7 +337,7 @@ def func_b2( a: TB2, # E: Missing type parameters for generic type "TB2" b: TB2[[float]], c: TB2[[float], [float]], - d: TB2[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 1 and 2, given: 3 + d: TB2[[float], [float], [float]], # E: Bad number of arguments for type alias, expected between 1 and 2, given 3 ) -> None: reveal_type(a) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" reveal_type(b) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.int, builtins.str]]" diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 70229f5d9a62..cc3dc4ed9f39 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -769,15 +769,15 @@ Ts = TypeVarTuple("Ts") class G(Generic[Unpack[Ts]]): ... A = List[Tuple[T, Unpack[Ts], S]] -x: A[int] # E: Bad number of arguments for type alias, expected: at least 2, given: 1 +x: A[int] # E: Bad number of arguments for type alias, expected at least 2, given 1 reveal_type(x) # N: Revealed type is "builtins.list[Tuple[Any, Unpack[builtins.tuple[Any, ...]], Any]]" B = Callable[[T, S, Unpack[Ts]], int] -y: B[int] # E: Bad number of arguments for type alias, expected: at least 2, given: 1 +y: B[int] # E: Bad number of arguments for type alias, expected at least 2, given 1 reveal_type(y) # N: Revealed type is "def (Any, Any, *Any) -> builtins.int" C = G[T, Unpack[Ts], S] -z: C[int] # E: Bad number of arguments for type alias, expected: at least 2, given: 1 +z: C[int] # E: Bad number of arguments for type alias, expected at least 2, given 1 reveal_type(z) # N: Revealed type is "__main__.G[Any, Unpack[builtins.tuple[Any, ...]], Any]" [builtins fixtures/tuple.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 165a2089b466..266d9a9efd01 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -4032,7 +4032,7 @@ def f(x: a.A[str]): [builtins fixtures/dict.pyi] [out] == -b.py:2: error: Bad number of arguments for type alias, expected: 2, given: 1 +b.py:2: error: Bad number of arguments for type alias, expected 2, given 1 [case testAliasFineAdded] import b @@ -5542,7 +5542,7 @@ main:9: error: Function "a.T" is not valid as a type main:9: note: Perhaps you need "Callable[...]" or a callback protocol? main:12: error: Function "a.T" is not valid as a type main:12: note: Perhaps you need "Callable[...]" or a callback protocol? -main:12: error: Bad number of arguments for type alias, expected: 0, given: 1 +main:12: error: Bad number of arguments for type alias, expected 0, given 1 [case testChangeTypeVarToModule] @@ -5576,7 +5576,7 @@ main:9: error: Module "T" is not valid as a type main:9: note: Perhaps you meant to use a protocol matching the module structure? main:12: error: Module "T" is not valid as a type main:12: note: Perhaps you meant to use a protocol matching the module structure? -main:12: error: Bad number of arguments for type alias, expected: 0, given: 1 +main:12: error: Bad number of arguments for type alias, expected 0, given 1 [case testChangeClassToModule] @@ -5628,7 +5628,7 @@ T = int == main:5: error: Free type variable expected in Generic[...] main:9: error: "C" expects no type arguments, but 1 given -main:12: error: Bad number of arguments for type alias, expected: 0, given: 1 +main:12: error: Bad number of arguments for type alias, expected 0, given 1 [case testChangeTypeAliasToModule] From 06b01c80a1a365cb8af80b9eecfbe4b14f04fc3e Mon Sep 17 00:00:00 2001 From: Charlie Denton Date: Tue, 30 Jan 2024 06:39:02 +0000 Subject: [PATCH 26/32] Fix `'WriteToConn' object has no attribute 'flush'` (#16801) `WriteToConn` replaces stdout and stderr to capture output, but causes issues because it doesn't implement the `TextIO` API (as expected of `sys.stdout` and `sys.stderr`). By stubbing the rest of the `TextIO` API we prevent issues with other code which uses more of the API than we had previously accounted for. Fixes https://github.com/python/mypy/issues/16678 --- mypy/dmypy_server.py | 4 +-- mypy/dmypy_util.py | 65 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index b4c3fe8fe0dc..3d337eedbf1c 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -221,8 +221,8 @@ def serve(self) -> None: while True: with server: data = receive(server) - sys.stdout = WriteToConn(server, "stdout") # type: ignore[assignment] - sys.stderr = WriteToConn(server, "stderr") # type: ignore[assignment] + sys.stdout = WriteToConn(server, "stdout", sys.stdout.isatty()) + sys.stderr = WriteToConn(server, "stderr", sys.stderr.isatty()) resp: dict[str, Any] = {} if "command" not in data: resp = {"error": "No command found in request"} diff --git a/mypy/dmypy_util.py b/mypy/dmypy_util.py index fe949e8fc294..0baff863b3c3 100644 --- a/mypy/dmypy_util.py +++ b/mypy/dmypy_util.py @@ -5,8 +5,10 @@ from __future__ import annotations +import io import json -from typing import Any, Final, Iterable +from types import TracebackType +from typing import Any, Final, Iterable, Iterator, TextIO from mypy.ipc import IPCBase @@ -40,12 +42,66 @@ def send(connection: IPCBase, data: Any) -> None: connection.write(json.dumps(data)) -class WriteToConn: +class WriteToConn(TextIO): """Helper class to write to a connection instead of standard output.""" - def __init__(self, server: IPCBase, output_key: str = "stdout") -> None: + def __init__(self, server: IPCBase, output_key: str, isatty: bool) -> None: self.server = server self.output_key = output_key + self._isatty = isatty + + def __enter__(self) -> TextIO: + return self + + def __exit__( + self, + t: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + pass + + def __iter__(self) -> Iterator[str]: + raise io.UnsupportedOperation + + def __next__(self) -> str: + raise io.UnsupportedOperation + + def close(self) -> None: + pass + + def fileno(self) -> int: + raise OSError + + def flush(self) -> None: + pass + + def isatty(self) -> bool: + return self._isatty + + def read(self, n: int = 0) -> str: + raise io.UnsupportedOperation + + def readable(self) -> bool: + return False + + def readline(self, limit: int = 0) -> str: + raise io.UnsupportedOperation + + def readlines(self, hint: int = 0) -> list[str]: + raise io.UnsupportedOperation + + def seek(self, offset: int, whence: int = 0) -> int: + raise io.UnsupportedOperation + + def seekable(self) -> bool: + return False + + def tell(self) -> int: + raise io.UnsupportedOperation + + def truncate(self, size: int | None = 0) -> int: + raise io.UnsupportedOperation def write(self, output: str) -> int: resp: dict[str, Any] = {} @@ -53,6 +109,9 @@ def write(self, output: str) -> int: send(self.server, resp) return len(output) + def writable(self) -> bool: + return True + def writelines(self, lines: Iterable[str]) -> None: for s in lines: self.write(s) From e40935e5e0b55bee9379e35aa27216c3e0287b13 Mon Sep 17 00:00:00 2001 From: thomaswhaley <39359869+thomas-whaley@users.noreply.github.com> Date: Wed, 31 Jan 2024 19:26:22 +1300 Subject: [PATCH 27/32] Update new type system discussion links (#16841) As of the 10th of September 2023 the typing-sig mail list is now replaced with [discuss.python.org](https://discuss.python.org/c/typing/32) as explained [here](https://mail.python.org/archives/list/typing-sig@python.org/thread/BENDBBUDRCRMTTK2B5HJ7RCKENFD6DW2/) and [here](https://discuss.python.org/t/about-the-typing-category/34155) The README.md file was referencing the old link. --------- Co-authored-by: Jelle Zijlstra --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8b1ebbc0f2cb..07c170d46cb3 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ To report a bug or request an enhancement: To discuss a new type system feature: -- discuss at [typing-sig mailing list](https://mail.python.org/archives/list/typing-sig@python.org/) -- there is also some historical discussion [here](https://github.com/python/typing/issues) +- discuss at [discuss.python.org](https://discuss.python.org/c/typing/32) +- there is also some historical discussion at the [typing-sig mailing list](https://mail.python.org/archives/list/typing-sig@python.org/) and the [python/typing repo](https://github.com/python/typing/issues) What is mypy? ------------- From 55247c469077963a389941a2386ff2747abe517a Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 31 Jan 2024 17:09:44 +0100 Subject: [PATCH 28/32] Apply TypeVar defaults to callables (PEP 696) (#16842) Implement type application for callables with TypeVar defaults. Similar to previous PRs, support for TypeVarTuples is still TODO. Ref: https://github.com/python/mypy/issues/14851 --- mypy/applytype.py | 3 +- mypy/checkexpr.py | 24 ++++-- mypy/messages.py | 21 ++--- test-data/unit/check-typevar-defaults.test | 93 +++++++++++++++++++++- 4 files changed, 122 insertions(+), 19 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index c7da67d6140b..b00372855d9c 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -93,7 +93,8 @@ def apply_generic_arguments( bound or constraints, instead of giving an error. """ tvars = callable.variables - assert len(tvars) == len(orig_types) + min_arg_count = sum(not tv.has_default() for tv in tvars) + assert min_arg_count <= len(orig_types) <= len(tvars) # Check that inferred type variable values are compatible with allowed # values and bounds. Also, promote subtype values to allowed values. # Create a map from type variable id to target type. diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index a4b66bb9932c..e04d413eab8d 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4809,21 +4809,29 @@ def apply_type_arguments_to_callable( tp = get_proper_type(tp) if isinstance(tp, CallableType): - if len(tp.variables) != len(args) and not any( - isinstance(v, TypeVarTupleType) for v in tp.variables - ): + min_arg_count = sum(not v.has_default() for v in tp.variables) + has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in tp.variables) + if ( + len(args) < min_arg_count or len(args) > len(tp.variables) + ) and not has_type_var_tuple: if tp.is_type_obj() and tp.type_object().fullname == "builtins.tuple": # TODO: Specialize the callable for the type arguments return tp - self.msg.incompatible_type_application(len(tp.variables), len(args), ctx) + self.msg.incompatible_type_application( + min_arg_count, len(tp.variables), len(args), ctx + ) return AnyType(TypeOfAny.from_error) return self.apply_generic_arguments(tp, self.split_for_callable(tp, args, ctx), ctx) if isinstance(tp, Overloaded): for it in tp.items: - if len(it.variables) != len(args) and not any( - isinstance(v, TypeVarTupleType) for v in it.variables - ): - self.msg.incompatible_type_application(len(it.variables), len(args), ctx) + min_arg_count = sum(not v.has_default() for v in it.variables) + has_type_var_tuple = any(isinstance(v, TypeVarTupleType) for v in it.variables) + if ( + len(args) < min_arg_count or len(args) > len(it.variables) + ) and not has_type_var_tuple: + self.msg.incompatible_type_application( + min_arg_count, len(it.variables), len(args), ctx + ) return AnyType(TypeOfAny.from_error) return Overloaded( [ diff --git a/mypy/messages.py b/mypy/messages.py index 75b8eeb3174c..2f2d346c2c51 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1347,18 +1347,21 @@ def override_target(self, name: str, name_in_super: str, supertype: str) -> str: return target def incompatible_type_application( - self, expected_arg_count: int, actual_arg_count: int, context: Context + self, min_arg_count: int, max_arg_count: int, actual_arg_count: int, context: Context ) -> None: - if expected_arg_count == 0: + if max_arg_count == 0: self.fail("Type application targets a non-generic function or class", context) - elif actual_arg_count > expected_arg_count: - self.fail( - f"Type application has too many types ({expected_arg_count} expected)", context - ) + return + + if min_arg_count == max_arg_count: + s = f"{max_arg_count} expected" else: - self.fail( - f"Type application has too few types ({expected_arg_count} expected)", context - ) + s = f"expected between {min_arg_count} and {max_arg_count}" + + if actual_arg_count > max_arg_count: + self.fail(f"Type application has too many types ({s})", context) + else: + self.fail(f"Type application has too few types ({s})", context) def could_not_infer_type_arguments( self, callee_type: CallableType, n: int, context: Context diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 4b509cd4fc40..7c748821401a 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -118,7 +118,7 @@ def func_c1(x: Union[int, Callable[[Unpack[Ts1]], None]]) -> Tuple[Unpack[Ts1]]: [builtins fixtures/tuple.pyi] [case testTypeVarDefaultsClass1] -from typing import Generic, TypeVar +from typing import Generic, TypeVar, Union, overload T1 = TypeVar("T1") T2 = TypeVar("T2", default=int) @@ -137,6 +137,15 @@ def func_a1( reveal_type(c) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.float]" reveal_type(d) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + k = ClassA1() + reveal_type(k) # N: Revealed type is "__main__.ClassA1[builtins.int, builtins.str]" + l = ClassA1[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.str]" + m = ClassA1[float, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassA1[builtins.float, builtins.float]" + n = ClassA1[float, float, float]() # E: Type application has too many types (expected between 0 and 2) + reveal_type(n) # N: Revealed type is "Any" + class ClassA2(Generic[T1, T2, T3]): ... def func_a2( @@ -152,6 +161,44 @@ def func_a2( reveal_type(d) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.float]" reveal_type(e) # N: Revealed type is "__main__.ClassA2[Any, builtins.int, builtins.str]" + k = ClassA2() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassA2[Any, builtins.int, builtins.str]" + l = ClassA2[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.int, builtins.str]" + m = ClassA2[float, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.str]" + n = ClassA2[float, float, float]() + reveal_type(n) # N: Revealed type is "__main__.ClassA2[builtins.float, builtins.float, builtins.float]" + o = ClassA2[float, float, float, float]() # E: Type application has too many types (expected between 1 and 3) + reveal_type(o) # N: Revealed type is "Any" + +class ClassA3(Generic[T1, T2]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, var: int) -> None: ... + def __init__(self, var: Union[int, None] = None) -> None: ... + +def func_a3( + a: ClassA3, + b: ClassA3[float], + c: ClassA3[float, float], + d: ClassA3[float, float, float], # E: "ClassA3" expects between 1 and 2 type arguments, but 3 given +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassA3[Any, builtins.int]" + reveal_type(b) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.int]" + reveal_type(c) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.float]" + reveal_type(d) # N: Revealed type is "__main__.ClassA3[Any, builtins.int]" + + k = ClassA3() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassA3[Any, builtins.int]" + l = ClassA3[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.int]" + m = ClassA3[float, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassA3[builtins.float, builtins.float]" + n = ClassA3[float, float, float]() # E: Type application has too many types (expected between 1 and 2) + reveal_type(n) # N: Revealed type is "Any" + [case testTypeVarDefaultsClass2] from typing import Generic, ParamSpec @@ -172,6 +219,15 @@ def func_b1( reveal_type(c) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" reveal_type(d) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], ...]" + k = ClassB1() + reveal_type(k) # N: Revealed type is "__main__.ClassB1[[builtins.int, builtins.str], [*Any, **Any]]" + l = ClassB1[[float]]() + reveal_type(l) # N: Revealed type is "__main__.ClassB1[[builtins.float], [*Any, **Any]]" + m = ClassB1[[float], [float]]() + reveal_type(m) # N: Revealed type is "__main__.ClassB1[[builtins.float], [builtins.float]]" + n = ClassB1[[float], [float], [float]]() # E: Type application has too many types (expected between 0 and 2) + reveal_type(n) # N: Revealed type is "Any" + class ClassB2(Generic[P1, P2]): ... def func_b2( @@ -185,6 +241,15 @@ def func_b2( reveal_type(c) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.float]]" reveal_type(d) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + k = ClassB2() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassB2[Any, [builtins.int, builtins.str]]" + l = ClassB2[[float]]() + reveal_type(l) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.int, builtins.str]]" + m = ClassB2[[float], [float]]() + reveal_type(m) # N: Revealed type is "__main__.ClassB2[[builtins.float], [builtins.float]]" + n = ClassB2[[float], [float], [float]]() # E: Type application has too many types (expected between 1 and 2) + reveal_type(n) # N: Revealed type is "Any" + [case testTypeVarDefaultsClass3] from typing import Generic, Tuple, TypeVar from typing_extensions import TypeVarTuple, Unpack @@ -206,6 +271,11 @@ def func_c1( # reveal_type(a) # Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" # TODO reveal_type(b) # N: Revealed type is "__main__.ClassC1[builtins.float]" + # k = ClassC1() # TODO + # reveal_type(k) # Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" # TODO + l = ClassC1[float]() + reveal_type(l) # N: Revealed type is "__main__.ClassC1[builtins.float]" + class ClassC2(Generic[T3, Unpack[Ts3]]): ... def func_c2( @@ -217,6 +287,13 @@ def func_c2( # reveal_type(b) # Revealed type is "__main__.ClassC2[builtins.int, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO reveal_type(c) # N: Revealed type is "__main__.ClassC2[builtins.int]" + # k = ClassC2() # TODO + # reveal_type(k) # Revealed type is "__main__.ClassC2[builtins.str, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + l = ClassC2[int]() + # reveal_type(l) # Revealed type is "__main__.ClassC2[builtins.int, Unpack[builtins.tuple[builtins.float, ...]]]" # TODO + m = ClassC2[int, Unpack[Tuple[()]]]() + reveal_type(m) # N: Revealed type is "__main__.ClassC2[builtins.int]" + class ClassC3(Generic[T3, Unpack[Ts4]]): ... def func_c3( @@ -228,6 +305,13 @@ def func_c3( reveal_type(b) # N: Revealed type is "__main__.ClassC3[builtins.int]" reveal_type(c) # N: Revealed type is "__main__.ClassC3[builtins.int, builtins.float]" + # k = ClassC3() # TODO + # reveal_type(k) # Revealed type is "__main__.ClassC3[builtins.str]" # TODO + l = ClassC3[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassC3[builtins.int]" + m = ClassC3[int, Unpack[Tuple[float]]]() + reveal_type(m) # N: Revealed type is "__main__.ClassC3[builtins.int, builtins.float]" + class ClassC4(Generic[T1, Unpack[Ts1], T3]): ... def func_c4( @@ -238,6 +322,13 @@ def func_c4( reveal_type(a) # N: Revealed type is "__main__.ClassC4[Any, Unpack[builtins.tuple[Any, ...]], builtins.str]" # reveal_type(b) # Revealed type is "__main__.ClassC4[builtins.int, builtins.str]" # TODO reveal_type(c) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" + + k = ClassC4() # E: Need type annotation for "k" + reveal_type(k) # N: Revealed type is "__main__.ClassC4[Any, Unpack[builtins.tuple[Any, ...]], builtins.str]" + l = ClassC4[int]() + # reveal_type(l) # Revealed type is "__main__.ClassC4[builtins.int, builtins.str]" # TODO + m = ClassC4[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassC4[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] [case testTypeVarDefaultsTypeAlias1] From 5bf774234da052efffd14e6c3dd26d93a14e3cc8 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 31 Jan 2024 22:41:49 +0100 Subject: [PATCH 29/32] Fix missing type store for overloads (#16803) Add missing call to store inferred types if an overload match is found early. All other code paths already do that. ### Some background on the issue this fixes I recently saw an interesting pattern in `aiohttp` to type values in an `dict[str, Any]` by subclassing dict. ```py T = TypeVar("T") U = TypeVar("U") class Key(Generic[T]): ... class CustomDict(dict[Key[Any] | str, Any]): @overload # type: ignore[override] def get(self, __key: Key[T]) -> T | None: ... @overload def get(self, __key: Key[T], __default: U) -> T | U: ... @overload def get(self, __key: str) -> Any | None: ... @overload def get(self, __key: str, __default: Any) -> Any: ... def get(self, __key: Key[Any] | str, __default: Any = None) -> Any: """Forward to super implementation.""" return super().get(__key, __default) # overloads for __getitem__, setdefault, pop # ... @overload # type: ignore[override] def __setitem__(self, key: Key[T], value: T) -> None: ... @overload def __setitem__(self, key: str, value: Any) -> None: ... def __setitem__(self, key: Key[Any] | str, value: Any) -> None: """Forward to super implementation.""" return super().__setitem__(key, value) ``` With the exception that these overloads aren't technically compatible with the supertype, they do the job. ```py d = CustomDict() key = Key[int]() other_key = "other" assert_type(d.get(key), int | None) assert_type(d.get("other"), Any | None) ``` The issue exists for the `__setitem__` case. Without this PR the following would create an issue. Here `var` would be inferred as `dict[Never, Never]`, even though it should be `dict[Any, Any]` which is the case for non-subclassed dicts. ```py def a2(d: CustomDict) -> None: if (var := d.get("arg")) is None: var = d["arg"] = {} reveal_type(var) ``` --- mypy/checkexpr.py | 1 + test-data/unit/check-generics.test | 24 ++++++++++++++++++++++++ test-data/unit/typexport-basic.test | 21 +++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index e04d413eab8d..c628a3398b59 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2825,6 +2825,7 @@ def infer_overload_return_type( # Return early if possible; otherwise record info, so we can # check for ambiguity due to 'Any' below. if not args_contain_any: + self.chk.store_types(m) return ret_type, infer_type p_infer_type = get_proper_type(infer_type) if isinstance(p_infer_type, CallableType): diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 337e92daf365..469cedb8f6f7 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -1480,6 +1480,30 @@ if int(): b = f(b) [builtins fixtures/list.pyi] +[case testGenericDictWithOverload] +from typing import Dict, Generic, TypeVar, Any, overload +T = TypeVar("T") + +class Key(Generic[T]): ... +class CustomDict(dict): + @overload # type: ignore[override] + def __setitem__(self, key: Key[T], value: T) -> None: ... + @overload + def __setitem__(self, key: str, value: Any) -> None: ... + def __setitem__(self, key, value): + return super().__setitem__(key, value) + +def a1(d: Dict[str, Any]) -> None: + if (var := d.get("arg")) is None: + var = d["arg"] = {} + reveal_type(var) # N: Revealed type is "builtins.dict[Any, Any]" + +def a2(d: CustomDict) -> None: + if (var := d.get("arg")) is None: + var = d["arg"] = {} + reveal_type(var) # N: Revealed type is "builtins.dict[Any, Any]" +[builtins fixtures/dict.pyi] + -- Type variable scoping -- --------------------- diff --git a/test-data/unit/typexport-basic.test b/test-data/unit/typexport-basic.test index c4c3a1d36f83..d78cf0f179f2 100644 --- a/test-data/unit/typexport-basic.test +++ b/test-data/unit/typexport-basic.test @@ -1236,6 +1236,27 @@ LambdaExpr(10) : def (x: builtins.int) -> builtins.int LambdaExpr(12) : def (y: builtins.str) -> builtins.str LambdaExpr(13) : def (x: builtins.str) -> builtins.str +[case testExportOverloadArgTypeDict] +## DictExpr +from typing import TypeVar, Generic, Any, overload, Dict +T = TypeVar("T") +class Key(Generic[T]): ... +@overload +def f(x: Key[T], y: T) -> T: ... +@overload +def f(x: int, y: Any) -> Any: ... +def f(x, y): ... +d: Dict = {} +d.get( + "", {}) +f( + 2, {}) +[builtins fixtures/dict.pyi] +[out] +DictExpr(10) : builtins.dict[Any, Any] +DictExpr(12) : builtins.dict[Any, Any] +DictExpr(14) : builtins.dict[Any, Any] + -- TODO -- -- test expressions From c3cd83a23dcdbd72e6e2a4c2d6c605eadc7bccf4 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 31 Jan 2024 23:21:57 +0100 Subject: [PATCH 30/32] Fix disallow-any errors for Instance types (PEP 696) (#16832) Similar to TypeAlias types `Missing type parameters for generic type` should not be emitted if too many arguments are given. There is a separate error message for that. Ref: https://github.com/python/mypy/issues/14851 --- mypy/messages.py | 8 ++++---- mypy/typeanal.py | 5 ++++- test-data/unit/check-typevar-defaults.test | 11 +++++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 2f2d346c2c51..c107e874f4fc 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2515,10 +2515,10 @@ def format_literal_value(typ: LiteralType) -> str: else: base_str = itype.type.name if not itype.args: - if not itype.type.has_type_var_tuple_type: - # No type arguments, just return the type name - return base_str - return base_str + "[()]" + if itype.type.has_type_var_tuple_type and len(itype.type.type_vars) == 1: + return base_str + "[()]" + # No type arguments, just return the type name + return base_str elif itype.type.fullname == "builtins.tuple": item_type_str = format(itype.args[0]) return f"{'tuple' if options.use_lowercase_names() else 'Tuple'}[{item_type_str}, ...]" diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1bcba5f0ca88..601f98e958e2 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1866,7 +1866,11 @@ def fix_instance( max_tv_count = len(t.type.type_vars) if arg_count < min_tv_count or arg_count > max_tv_count: # Don't use existing args if arg_count doesn't match + if arg_count > max_tv_count: + # Already wrong arg count error, don't emit missing type parameters error as well. + disallow_any = False t.args = () + arg_count = 0 args: list[Type] = [*(t.args[:max_tv_count])] any_type: AnyType | None = None @@ -2324,7 +2328,6 @@ def validate_instance(t: Instance, fail: MsgCallback, empty_tuple_index: bool) - t, code=codes.TYPE_ARG, ) - t.args = () t.invalid = True return False return True diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 7c748821401a..6136746cbd0a 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -118,6 +118,7 @@ def func_c1(x: Union[int, Callable[[Unpack[Ts1]], None]]) -> Tuple[Unpack[Ts1]]: [builtins fixtures/tuple.pyi] [case testTypeVarDefaultsClass1] +# flags: --disallow-any-generics from typing import Generic, TypeVar, Union, overload T1 = TypeVar("T1") @@ -149,7 +150,7 @@ def func_a1( class ClassA2(Generic[T1, T2, T3]): ... def func_a2( - a: ClassA2, + a: ClassA2, # E: Missing type parameters for generic type "ClassA2" b: ClassA2[float], c: ClassA2[float, float], d: ClassA2[float, float, float], @@ -180,7 +181,7 @@ class ClassA3(Generic[T1, T2]): def __init__(self, var: Union[int, None] = None) -> None: ... def func_a3( - a: ClassA3, + a: ClassA3, # E: Missing type parameters for generic type "ClassA3" b: ClassA3[float], c: ClassA3[float, float], d: ClassA3[float, float, float], # E: "ClassA3" expects between 1 and 2 type arguments, but 3 given @@ -200,6 +201,7 @@ def func_a3( reveal_type(n) # N: Revealed type is "Any" [case testTypeVarDefaultsClass2] +# flags: --disallow-any-generics from typing import Generic, ParamSpec P1 = ParamSpec("P1") @@ -231,7 +233,7 @@ def func_b1( class ClassB2(Generic[P1, P2]): ... def func_b2( - a: ClassB2, + a: ClassB2, # E: Missing type parameters for generic type "ClassB2" b: ClassB2[[float]], c: ClassB2[[float], [float]], d: ClassB2[[float], [float], [float]], # E: "ClassB2" expects between 1 and 2 type arguments, but 3 given @@ -251,6 +253,7 @@ def func_b2( reveal_type(n) # N: Revealed type is "Any" [case testTypeVarDefaultsClass3] +# flags: --disallow-any-generics from typing import Generic, Tuple, TypeVar from typing_extensions import TypeVarTuple, Unpack @@ -315,7 +318,7 @@ def func_c3( class ClassC4(Generic[T1, Unpack[Ts1], T3]): ... def func_c4( - a: ClassC4, + a: ClassC4, # E: Missing type parameters for generic type "ClassC4" b: ClassC4[int], c: ClassC4[int, float], ) -> None: From 8107e53158d83d30bb04d290ac10d8d3ccd344f8 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:46:07 +0100 Subject: [PATCH 31/32] Update black to 24.1.1 (#16847) --- .pre-commit-config.yaml | 2 +- mypy/build.py | 1 + mypy/checker.py | 42 +++++++++++++++--------------- mypy/checkexpr.py | 16 ++++++------ mypy/checkpattern.py | 9 +++---- mypy/evalexpr.py | 1 + mypy/expandtype.py | 24 ++++++----------- mypy/fastparse.py | 21 +++++++-------- mypy/join.py | 6 ++--- mypy/main.py | 2 -- mypy/metastore.py | 3 +-- mypy/nodes.py | 12 ++++----- mypy/plugins/attrs.py | 16 +++++++----- mypy/plugins/enums.py | 1 + mypy/plugins/functools.py | 1 + mypy/semanal_shared.py | 9 +++---- mypy/semanal_typeddict.py | 9 +++---- mypy/stubtest.py | 8 +++--- mypy/stubutil.py | 6 ++--- mypy/subtypes.py | 8 +++--- mypy/test/meta/test_parse_data.py | 1 + mypy/test/meta/test_update_data.py | 1 + mypy/test/testargs.py | 1 + mypy/test/testerrorstream.py | 1 + mypy/test/testreports.py | 1 + mypy/test/teststubgen.py | 3 +-- mypy/typeanal.py | 3 +-- mypy/typeops.py | 8 +++--- mypy/types.py | 23 +++++++--------- mypyc/analysis/ircheck.py | 1 + mypyc/codegen/emitfunc.py | 4 +-- mypyc/ir/class_ir.py | 6 ++--- mypyc/irbuild/builder.py | 7 +++-- mypyc/irbuild/nonlocalcontrol.py | 3 +-- test-requirements.in | 2 +- test-requirements.txt | 2 +- 36 files changed, 126 insertions(+), 138 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bbd7b3ce382..6566bb0297d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 # must match test-requirements.txt + rev: 24.1.1 # must match test-requirements.txt hooks: - id: black exclude: '^(test-data/)' diff --git a/mypy/build.py b/mypy/build.py index 8049fa2d0c3f..65a06211c87e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -8,6 +8,7 @@ The function build() is the main interface to this module. """ + # TODO: More consistent terminology, e.g. path/fnam, module/id, state/file from __future__ import annotations diff --git a/mypy/checker.py b/mypy/checker.py index cd23e74a8dac..391f28e93b1d 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -526,12 +526,16 @@ def check_second_pass( # print("XXX in pass %d, class %s, function %s" % # (self.pass_num, type_name, node.fullname or node.name)) done.add(node) - with self.tscope.class_scope( - active_typeinfo - ) if active_typeinfo else nullcontext(): - with self.scope.push_class( - active_typeinfo - ) if active_typeinfo else nullcontext(): + with ( + self.tscope.class_scope(active_typeinfo) + if active_typeinfo + else nullcontext() + ): + with ( + self.scope.push_class(active_typeinfo) + if active_typeinfo + else nullcontext() + ): self.check_partial(node) return True @@ -3802,9 +3806,11 @@ def check_multi_assignment_from_tuple( if star_lv: list_expr = ListExpr( [ - self.temp_node(rv_type, context) - if not isinstance(rv_type, UnpackType) - else StarExpr(self.temp_node(rv_type.type, context)) + ( + self.temp_node(rv_type, context) + if not isinstance(rv_type, UnpackType) + else StarExpr(self.temp_node(rv_type.type, context)) + ) for rv_type in star_rv_types ] ) @@ -6593,8 +6599,7 @@ def check_subtype( notes: list[str] | None = None, code: ErrorCode | None = None, outer_context: Context | None = None, - ) -> bool: - ... + ) -> bool: ... @overload def check_subtype( @@ -6608,8 +6613,7 @@ def check_subtype( *, notes: list[str] | None = None, outer_context: Context | None = None, - ) -> bool: - ... + ) -> bool: ... def check_subtype( self, @@ -7083,14 +7087,12 @@ def conditional_types_with_intersection( type_ranges: list[TypeRange] | None, ctx: Context, default: None = None, - ) -> tuple[Type | None, Type | None]: - ... + ) -> tuple[Type | None, Type | None]: ... @overload def conditional_types_with_intersection( self, expr_type: Type, type_ranges: list[TypeRange] | None, ctx: Context, default: Type - ) -> tuple[Type, Type]: - ... + ) -> tuple[Type, Type]: ... def conditional_types_with_intersection( self, @@ -7348,15 +7350,13 @@ def visit_type_var(self, t: TypeVarType) -> None: @overload def conditional_types( current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: None = None -) -> tuple[Type | None, Type | None]: - ... +) -> tuple[Type | None, Type | None]: ... @overload def conditional_types( current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type -) -> tuple[Type, Type]: - ... +) -> tuple[Type, Type]: ... def conditional_types( diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c628a3398b59..ff7b7fa2ff58 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2132,11 +2132,13 @@ def infer_function_type_arguments( unknown = UninhabitedType() unknown.ambiguous = True inferred_args = [ - expand_type( - a, {v.id: unknown for v in list(callee_type.variables) + free_vars} + ( + expand_type( + a, {v.id: unknown for v in list(callee_type.variables) + free_vars} + ) + if a is not None + else None ) - if a is not None - else None for a in poly_inferred_args ] else: @@ -6042,14 +6044,12 @@ def bool_type(self) -> Instance: return self.named_type("builtins.bool") @overload - def narrow_type_from_binder(self, expr: Expression, known_type: Type) -> Type: - ... + def narrow_type_from_binder(self, expr: Expression, known_type: Type) -> Type: ... @overload def narrow_type_from_binder( self, expr: Expression, known_type: Type, skip_non_overlapping: bool - ) -> Type | None: - ... + ) -> Type | None: ... def narrow_type_from_binder( self, expr: Expression, known_type: Type, skip_non_overlapping: bool = False diff --git a/mypy/checkpattern.py b/mypy/checkpattern.py index 3210dcc3b7ac..7b6a55324741 100644 --- a/mypy/checkpattern.py +++ b/mypy/checkpattern.py @@ -305,11 +305,10 @@ def visit_sequence_pattern(self, o: SequencePattern) -> PatternType: narrowed_inner_types = [] inner_rest_types = [] for inner_type, new_inner_type in zip(inner_types, new_inner_types): - ( - narrowed_inner_type, - inner_rest_type, - ) = self.chk.conditional_types_with_intersection( - new_inner_type, [get_type_range(inner_type)], o, default=new_inner_type + (narrowed_inner_type, inner_rest_type) = ( + self.chk.conditional_types_with_intersection( + new_inner_type, [get_type_range(inner_type)], o, default=new_inner_type + ) ) narrowed_inner_types.append(narrowed_inner_type) inner_rest_types.append(inner_rest_type) diff --git a/mypy/evalexpr.py b/mypy/evalexpr.py index 4b3abb1be3e2..e39c5840d47a 100644 --- a/mypy/evalexpr.py +++ b/mypy/evalexpr.py @@ -6,6 +6,7 @@ put it in a mypyc-compiled file. """ + import ast from typing import Final diff --git a/mypy/expandtype.py b/mypy/expandtype.py index f6aa74add9d8..b4bc1aa9b9a5 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -50,18 +50,15 @@ @overload -def expand_type(typ: CallableType, env: Mapping[TypeVarId, Type]) -> CallableType: - ... +def expand_type(typ: CallableType, env: Mapping[TypeVarId, Type]) -> CallableType: ... @overload -def expand_type(typ: ProperType, env: Mapping[TypeVarId, Type]) -> ProperType: - ... +def expand_type(typ: ProperType, env: Mapping[TypeVarId, Type]) -> ProperType: ... @overload -def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: - ... +def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: ... def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: @@ -72,18 +69,15 @@ def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: @overload -def expand_type_by_instance(typ: CallableType, instance: Instance) -> CallableType: - ... +def expand_type_by_instance(typ: CallableType, instance: Instance) -> CallableType: ... @overload -def expand_type_by_instance(typ: ProperType, instance: Instance) -> ProperType: - ... +def expand_type_by_instance(typ: ProperType, instance: Instance) -> ProperType: ... @overload -def expand_type_by_instance(typ: Type, instance: Instance) -> Type: - ... +def expand_type_by_instance(typ: Type, instance: Instance) -> Type: ... def expand_type_by_instance(typ: Type, instance: Instance) -> Type: @@ -470,13 +464,11 @@ def expand_types(self, types: Iterable[Type]) -> list[Type]: @overload -def expand_self_type(var: Var, typ: ProperType, replacement: ProperType) -> ProperType: - ... +def expand_self_type(var: Var, typ: ProperType, replacement: ProperType) -> ProperType: ... @overload -def expand_self_type(var: Var, typ: Type, replacement: Type) -> Type: - ... +def expand_self_type(var: Var, typ: Type, replacement: Type) -> Type: ... def expand_self_type(var: Var, typ: Type, replacement: Type) -> Type: diff --git a/mypy/fastparse.py b/mypy/fastparse.py index b3e0fc70a9d6..a155187992ec 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -608,10 +608,9 @@ def fix_function_overloads(self, stmts: list[Statement]) -> list[Statement]: # Check IfStmt block to determine if function overloads can be merged if_overload_name = self._check_ifstmt_for_overloads(stmt, current_overload_name) if if_overload_name is not None: - ( - if_block_with_overload, - if_unknown_truth_value, - ) = self._get_executable_if_block_with_overloads(stmt) + (if_block_with_overload, if_unknown_truth_value) = ( + self._get_executable_if_block_with_overloads(stmt) + ) if ( current_overload_name is not None @@ -911,9 +910,11 @@ def do_func_def( # PEP 484 disallows both type annotations and type comments self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) arg_types = [ - a.type_annotation - if a.type_annotation is not None - else AnyType(TypeOfAny.unannotated) + ( + a.type_annotation + if a.type_annotation is not None + else AnyType(TypeOfAny.unannotated) + ) for a in args ] else: @@ -1790,12 +1791,10 @@ def invalid_type(self, node: AST, note: str | None = None) -> RawExpressionType: ) @overload - def visit(self, node: ast3.expr) -> ProperType: - ... + def visit(self, node: ast3.expr) -> ProperType: ... @overload - def visit(self, node: AST | None) -> ProperType | None: - ... + def visit(self, node: AST | None) -> ProperType | None: ... def visit(self, node: AST | None) -> ProperType | None: """Modified visit -- keep track of the stack of nodes""" diff --git a/mypy/join.py b/mypy/join.py index d33cbd98726d..bf88f43d88fe 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -237,13 +237,11 @@ def trivial_join(s: Type, t: Type) -> Type: @overload def join_types( s: ProperType, t: ProperType, instance_joiner: InstanceJoiner | None = None -) -> ProperType: - ... +) -> ProperType: ... @overload -def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> Type: - ... +def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> Type: ... def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None) -> Type: diff --git a/mypy/main.py b/mypy/main.py index b32624456ce0..c2df79d51e83 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -342,7 +342,6 @@ def infer_python_executable(options: Options, special_opts: argparse.Namespace) class CapturableArgumentParser(argparse.ArgumentParser): - """Override ArgumentParser methods that use sys.stdout/sys.stderr directly. This is needed because hijacking sys.std* is not thread-safe, @@ -396,7 +395,6 @@ def error(self, message: str) -> NoReturn: class CapturableVersionAction(argparse.Action): - """Supplement CapturableArgumentParser to handle --version. This is nearly identical to argparse._VersionAction except, diff --git a/mypy/metastore.py b/mypy/metastore.py index 0547f94cd671..4caa7d7f0534 100644 --- a/mypy/metastore.py +++ b/mypy/metastore.py @@ -63,8 +63,7 @@ def commit(self) -> None: """ @abstractmethod - def list_all(self) -> Iterable[str]: - ... + def list_all(self) -> Iterable[str]: ... def random_string() -> str: diff --git a/mypy/nodes.py b/mypy/nodes.py index 6c23c51dfcd3..1c781320580a 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3287,13 +3287,13 @@ def serialize(self) -> JsonDict: "declared_metaclass": ( None if self.declared_metaclass is None else self.declared_metaclass.serialize() ), - "metaclass_type": None - if self.metaclass_type is None - else self.metaclass_type.serialize(), + "metaclass_type": ( + None if self.metaclass_type is None else self.metaclass_type.serialize() + ), "tuple_type": None if self.tuple_type is None else self.tuple_type.serialize(), - "typeddict_type": None - if self.typeddict_type is None - else self.typeddict_type.serialize(), + "typeddict_type": ( + None if self.typeddict_type is None else self.typeddict_type.serialize() + ), "flags": get_flags(self, TypeInfo.FLAGS), "metadata": self.metadata, "slots": sorted(self.slots) if self.slots is not None else None, diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index 19a402492aef..345ea822ed94 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -185,9 +185,11 @@ def serialize(self) -> JsonDict: "init": self.init, "kw_only": self.kw_only, "has_converter": self.converter is not None, - "converter_init_type": self.converter.init_type.serialize() - if self.converter and self.converter.init_type - else None, + "converter_init_type": ( + self.converter.init_type.serialize() + if self.converter and self.converter.init_type + else None + ), "context_line": self.context.line, "context_column": self.context.column, "init_type": self.init_type.serialize() if self.init_type else None, @@ -1073,9 +1075,11 @@ def _meet_fields(types: list[Mapping[str, Type]]) -> Mapping[str, Type]: field_to_types[name].append(typ) return { - name: get_proper_type(reduce(meet_types, f_types)) - if len(f_types) == len(types) - else UninhabitedType() + name: ( + get_proper_type(reduce(meet_types, f_types)) + if len(f_types) == len(types) + else UninhabitedType() + ) for name, f_types in field_to_types.items() } diff --git a/mypy/plugins/enums.py b/mypy/plugins/enums.py index 2c568f66c62d..83350fe2fe11 100644 --- a/mypy/plugins/enums.py +++ b/mypy/plugins/enums.py @@ -10,6 +10,7 @@ we actually bake some of it directly in to the semantic analysis layer (see semanal_enum.py). """ + from __future__ import annotations from typing import Final, Iterable, Sequence, TypeVar, cast diff --git a/mypy/plugins/functools.py b/mypy/plugins/functools.py index 0aa2824c9b51..792ed6669503 100644 --- a/mypy/plugins/functools.py +++ b/mypy/plugins/functools.py @@ -1,4 +1,5 @@ """Plugin for supporting the functools standard library module.""" + from __future__ import annotations from typing import Final, NamedTuple diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index e8edfe65c8d4..b5ec2bb52a0d 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -308,8 +308,7 @@ def calculate_tuple_fallback(typ: TupleType) -> None: class _NamedTypeCallback(Protocol): - def __call__(self, fully_qualified_name: str, args: list[Type] | None = None) -> Instance: - ... + def __call__(self, fully_qualified_name: str, args: list[Type] | None = None) -> Instance: ... def paramspec_args( @@ -453,8 +452,7 @@ def require_bool_literal_argument( expression: Expression, name: str, default: Literal[True] | Literal[False], -) -> bool: - ... +) -> bool: ... @overload @@ -463,8 +461,7 @@ def require_bool_literal_argument( expression: Expression, name: str, default: None = None, -) -> bool | None: - ... +) -> bool | None: ... def require_bool_literal_argument( diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index dbec981bdc96..eee98d4d20fa 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -156,12 +156,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N # Iterate over bases in reverse order so that leftmost base class' keys take precedence for base in reversed(typeddict_bases): self.add_keys_and_types_from_base(base, keys, types, required_keys, defn) - ( - new_keys, - new_types, - new_statements, - new_required_keys, - ) = self.analyze_typeddict_classdef_fields(defn, keys) + (new_keys, new_types, new_statements, new_required_keys) = ( + self.analyze_typeddict_classdef_fields(defn, keys) + ) if new_keys is None: return True, None # Defer keys.extend(new_keys) diff --git a/mypy/stubtest.py b/mypy/stubtest.py index 7ab3a2b1e5d0..225c1c35c1be 100644 --- a/mypy/stubtest.py +++ b/mypy/stubtest.py @@ -1519,9 +1519,11 @@ def safe_inspect_signature(runtime: Any) -> inspect.Signature | None: sig = inspect._signature_fromstr(inspect.Signature, runtime, sig) # type: ignore[attr-defined] assert isinstance(sig, inspect.Signature) new_params = [ - parameter.replace(default=UNREPRESENTABLE) - if parameter.default is ... - else parameter + ( + parameter.replace(default=UNREPRESENTABLE) + if parameter.default is ... + else parameter + ) for parameter in sig.parameters.values() ] return sig.replace(parameters=new_params) diff --git a/mypy/stubutil.py b/mypy/stubutil.py index b7f6131c003d..1a9c2357c58e 100644 --- a/mypy/stubutil.py +++ b/mypy/stubutil.py @@ -140,13 +140,11 @@ def fail_missing(mod: str, reason: ModuleNotFoundReason) -> None: @overload -def remove_misplaced_type_comments(source: bytes) -> bytes: - ... +def remove_misplaced_type_comments(source: bytes) -> bytes: ... @overload -def remove_misplaced_type_comments(source: str) -> str: - ... +def remove_misplaced_type_comments(source: str) -> str: ... def remove_misplaced_type_comments(source: str | bytes) -> str | bytes: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 4fd3f8ff98ca..2d536f892a2a 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -693,9 +693,11 @@ def visit_callable_type(self, left: CallableType) -> bool: is_compat=self._is_subtype, is_proper_subtype=self.proper_subtype, ignore_pos_arg_names=self.subtype_context.ignore_pos_arg_names, - strict_concatenate=(self.options.extra_checks or self.options.strict_concatenate) - if self.options - else False, + strict_concatenate=( + (self.options.extra_checks or self.options.strict_concatenate) + if self.options + else False + ), ) elif isinstance(right, Overloaded): return all(self._is_subtype(left, item) for item in right.items) diff --git a/mypy/test/meta/test_parse_data.py b/mypy/test/meta/test_parse_data.py index 797fdd7b2c8c..bff2d6977612 100644 --- a/mypy/test/meta/test_parse_data.py +++ b/mypy/test/meta/test_parse_data.py @@ -2,6 +2,7 @@ A "meta test" which tests the parsing of .test files. This is not meant to become exhaustive but to ensure we maintain a basic level of ergonomics for mypy contributors. """ + from mypy.test.helpers import Suite from mypy.test.meta._pytest import PytestResult, run_pytest_data_suite diff --git a/mypy/test/meta/test_update_data.py b/mypy/test/meta/test_update_data.py index 40b70157a0e3..820fd359893e 100644 --- a/mypy/test/meta/test_update_data.py +++ b/mypy/test/meta/test_update_data.py @@ -3,6 +3,7 @@ Updating the expected output, especially when it's in the form of inline (comment) assertions, can be brittle, which is why we're "meta-testing" here. """ + from mypy.test.helpers import Suite from mypy.test.meta._pytest import PytestResult, dedent_docstring, run_pytest_data_suite diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index b0cc6b19aa80..7c139902fe90 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -4,6 +4,7 @@ defaults, and that argparse doesn't assign any new members to the Options object it creates. """ + from __future__ import annotations import argparse diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index 5ed112fd31e7..a54a3495ddb2 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -1,4 +1,5 @@ """Tests for mypy incremental error output.""" + from __future__ import annotations from mypy import build diff --git a/mypy/test/testreports.py b/mypy/test/testreports.py index 5ff315f83ba8..f638756ad819 100644 --- a/mypy/test/testreports.py +++ b/mypy/test/testreports.py @@ -1,4 +1,5 @@ """Test cases for reports generated by mypy.""" + from __future__ import annotations import textwrap diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index ace0b4d95573..c138f9953918 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -1099,8 +1099,7 @@ def test(arg0: str) -> None: assert_equal(gen.get_imports().splitlines(), ["import foo", "import other"]) def test_generate_c_function_no_crash_for_non_str_docstring(self) -> None: - def test(arg0: str) -> None: - ... + def test(arg0: str) -> None: ... test.__doc__ = property(lambda self: "test(arg0: str) -> None") # type: ignore[assignment] diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 601f98e958e2..2b37d10e2aff 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1775,8 +1775,7 @@ def tuple_type(self, items: list[Type], line: int, column: int) -> TupleType: class MsgCallback(Protocol): - def __call__(self, __msg: str, __ctx: Context, *, code: ErrorCode | None = None) -> None: - ... + def __call__(self, __msg: str, __ctx: Context, *, code: ErrorCode | None = None) -> None: ... def get_omitted_any( diff --git a/mypy/typeops.py b/mypy/typeops.py index 2bf8ffbf47ab..5b396308d955 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -915,9 +915,11 @@ def try_contracting_literals_in_union(types: Sequence[Type]) -> list[ProperType] if typ.fallback.type.is_enum or isinstance(typ.value, bool): if fullname not in sum_types: sum_types[fullname] = ( - set(typ.fallback.get_enum_values()) - if typ.fallback.type.is_enum - else {True, False}, + ( + set(typ.fallback.get_enum_values()) + if typ.fallback.type.is_enum + else {True, False} + ), [], ) literals, indexes = sum_types[fullname] diff --git a/mypy/types.py b/mypy/types.py index f02e56a677ae..0e87ec701386 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1490,9 +1490,9 @@ def copy_modified( args if args is not _dummy else self.args, self.line, self.column, - last_known_value=last_known_value - if last_known_value is not _dummy - else self.last_known_value, + last_known_value=( + last_known_value if last_known_value is not _dummy else self.last_known_value + ), ) # We intentionally don't copy the extra_attrs here, so they will be erased. new.can_be_true = self.can_be_true @@ -2834,13 +2834,13 @@ def __eq__(self, other: object) -> bool: @overload @staticmethod - def make_union(items: Sequence[ProperType], line: int = -1, column: int = -1) -> ProperType: - ... + def make_union( + items: Sequence[ProperType], line: int = -1, column: int = -1 + ) -> ProperType: ... @overload @staticmethod - def make_union(items: Sequence[Type], line: int = -1, column: int = -1) -> Type: - ... + def make_union(items: Sequence[Type], line: int = -1, column: int = -1) -> Type: ... @staticmethod def make_union(items: Sequence[Type], line: int = -1, column: int = -1) -> Type: @@ -3052,13 +3052,11 @@ def serialize(self) -> str: @overload -def get_proper_type(typ: None) -> None: - ... +def get_proper_type(typ: None) -> None: ... @overload -def get_proper_type(typ: Type) -> ProperType: - ... +def get_proper_type(typ: Type) -> ProperType: ... def get_proper_type(typ: Type | None) -> ProperType | None: @@ -3088,8 +3086,7 @@ def get_proper_types(types: list[Type] | tuple[Type, ...]) -> list[ProperType]: @overload def get_proper_types( types: list[Type | None] | tuple[Type | None, ...] -) -> list[ProperType | None]: - ... +) -> list[ProperType | None]: ... def get_proper_types( diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index a31b1517b036..127047e02ff5 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -1,4 +1,5 @@ """Utilities for checking that internal ir is valid and consistent.""" + from __future__ import annotations from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncIR diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 3bce84d3ea59..c08f1f840fa4 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -535,9 +535,7 @@ def visit_method_call(self, op: MethodCall) -> None: obj_args = ( [] if method.decl.kind == FUNC_STATICMETHOD - else [f"(PyObject *)Py_TYPE({obj})"] - if method.decl.kind == FUNC_CLASSMETHOD - else [obj] + else [f"(PyObject *)Py_TYPE({obj})"] if method.decl.kind == FUNC_CLASSMETHOD else [obj] ) args = ", ".join(obj_args + [self.reg(arg) for arg in op.args]) mtype = native_function_type(method, self.emitter) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 61f0fc36e1b3..18f3cbcff987 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -383,9 +383,9 @@ def serialize(self) -> JsonDict: "traits": [cir.fullname for cir in self.traits], "mro": [cir.fullname for cir in self.mro], "base_mro": [cir.fullname for cir in self.base_mro], - "children": [cir.fullname for cir in self.children] - if self.children is not None - else None, + "children": ( + [cir.fullname for cir in self.children] if self.children is not None else None + ), "deletable": self.deletable, "attrs_with_defaults": sorted(self.attrs_with_defaults), "_always_initialized_attrs": sorted(self._always_initialized_attrs), diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 5ed617aa925f..9d160b08505d 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -10,6 +10,7 @@ example, expressions are transformed in mypyc.irbuild.expression and functions are transformed in mypyc.irbuild.function. """ + from __future__ import annotations from contextlib import contextmanager @@ -231,12 +232,10 @@ def set_module(self, module_name: str, module_path: str) -> None: self.builder.set_module(module_name, module_path) @overload - def accept(self, node: Expression, *, can_borrow: bool = False) -> Value: - ... + def accept(self, node: Expression, *, can_borrow: bool = False) -> Value: ... @overload - def accept(self, node: Statement) -> None: - ... + def accept(self, node: Statement) -> None: ... def accept(self, node: Statement | Expression, *, can_borrow: bool = False) -> Value | None: """Transform an expression or a statement. diff --git a/mypyc/irbuild/nonlocalcontrol.py b/mypyc/irbuild/nonlocalcontrol.py index 02dd51283e95..0ac9bd3cee31 100644 --- a/mypyc/irbuild/nonlocalcontrol.py +++ b/mypyc/irbuild/nonlocalcontrol.py @@ -120,8 +120,7 @@ def __init__(self, outer: NonlocalControl) -> None: self.outer = outer @abstractmethod - def gen_cleanup(self, builder: IRBuilder, line: int) -> None: - ... + def gen_cleanup(self, builder: IRBuilder, line: int) -> None: ... def gen_break(self, builder: IRBuilder, line: int) -> None: self.gen_cleanup(builder, line) diff --git a/test-requirements.in b/test-requirements.in index 2bf8de0aa2f5..5971074a00be 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -4,7 +4,7 @@ -r mypy-requirements.txt -r build-requirements.txt attrs>=18.0 -black==23.9.1 # must match version in .pre-commit-config.yaml +black==24.1.1 # must match version in .pre-commit-config.yaml filelock>=3.3.0 # lxml 4.9.3 switched to manylinux_2_28, the wheel builder still uses manylinux2014 lxml>=4.9.1,<4.9.3; (python_version<'3.11' or sys_platform!='win32') and python_version<'3.12' diff --git a/test-requirements.txt b/test-requirements.txt index 57607c1bae57..683efa2a7334 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ # attrs==23.1.0 # via -r test-requirements.in -black==23.9.1 +black==24.1.1 # via -r test-requirements.in cfgv==3.4.0 # via pre-commit From d94b972ad3a2f61076ce1a70cf7e1dca1c9fe289 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:46:29 +0100 Subject: [PATCH 32/32] Update ruff to 0.1.15 (#16846) --- .pre-commit-config.yaml | 2 +- mypy/types.py | 2 +- test-requirements.in | 2 +- test-requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6566bb0297d8..eec410457641 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: black exclude: '^(test-data/)' - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.4 # must match test-requirements.txt + rev: v0.1.15 # must match test-requirements.txt hooks: - id: ruff args: [--exit-non-zero-on-fix] diff --git a/mypy/types.py b/mypy/types.py index 0e87ec701386..b1119c9447e2 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3108,7 +3108,7 @@ def get_proper_types( # to make it easier to gradually get modules working with mypyc. # Import them here, after the types are defined. # This is intended as a re-export also. -from mypy.type_visitor import ( # noqa: F811 +from mypy.type_visitor import ( ALL_STRATEGY as ALL_STRATEGY, ANY_STRATEGY as ANY_STRATEGY, BoolTypeQuery as BoolTypeQuery, diff --git a/test-requirements.in b/test-requirements.in index 5971074a00be..8f9d2cae0cce 100644 --- a/test-requirements.in +++ b/test-requirements.in @@ -14,6 +14,6 @@ psutil>=4.0 pytest>=7.4.0 pytest-xdist>=1.34.0 pytest-cov>=2.10.0 -ruff==0.1.4 # must match version in .pre-commit-config.yaml +ruff==0.1.15 # must match version in .pre-commit-config.yaml setuptools>=65.5.1 tomli>=1.1.0 # needed even on py311+ so the self check passes with --python-version 3.7 diff --git a/test-requirements.txt b/test-requirements.txt index 683efa2a7334..dcd250970a15 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -67,7 +67,7 @@ ruamel-yaml==0.17.40 # via pre-commit-hooks ruamel-yaml-clib==0.2.8 # via ruamel-yaml -ruff==0.1.4 +ruff==0.1.15 # via -r test-requirements.in tomli==2.0.1 # via -r test-requirements.in