From 23638375a14c558f5991890e0dcef6c19207ff66 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 28 Mar 2023 17:47:06 -0600 Subject: [PATCH 1/4] Update stubs for upstream updates we haven't taken yet: lowlevel.open_process(), nursery.start(run_process, ...), run(strict_exception_groups=...), DTLS --- trio-stubs/__init__.pyi | 99 ++++++++++++++++++++++------------------- trio-stubs/lowlevel.pyi | 62 ++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 47 deletions(-) diff --git a/trio-stubs/__init__.pyi b/trio-stubs/__init__.pyi index cb27bfe..889cbff 100644 --- a/trio-stubs/__init__.pyi +++ b/trio-stubs/__init__.pyi @@ -183,6 +183,7 @@ def run( clock: Optional[trio.abc.Clock] = ..., instruments: Sequence[trio.abc.Instrument] = ..., restrict_keyboard_interrupt_to_checkpoints: bool = ..., + strict_exception_groups: bool = ..., ) -> T: ... # _timeouts @@ -336,6 +337,53 @@ class SocketListener(trio.abc.Listener[SocketStream]): async def accept(self) -> SocketStream: ... async def aclose(self) -> None: ... +# _dtls +class DTLSEndpoint: + socket: trio.socket.SocketType + incoming_packets_buffer: int + def __init__(self, socket: trio.socket.SocketType, *, incoming_packets_buffer: int = ...) -> None: ... + # ssl_context is an OpenSSL.SSL.Context from PyOpenSSL, but we can't + # import that name because we don't depend on PyOpenSSL + def connect(self, address: address: Union[Tuple[Any, ...], str, bytes], ssl_context: Any) -> DTLSChannel: ... + @takes_callable_and_args + async def serve( + self, + ssl_context: Any, + async_fn: Union[ + Callable[..., Awaitable[Any]], + Callable[[DTLSChannel, VarArg()], Awaitable[Any]], + ], + *args: Any, + task_status: TaskStatus[None] = ..., + ) -> NoReturn: ... + def close(self) -> None: ... + def __enter__(self) -> DTLSEndpoint: ... + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: ... + +class DTLSChannel(_NotConstructible, trio.abc.Channel[bytes]): + endpoint: DTLSEndpoint + peer_address: Union[Tuple[Any, ...], str, bytes] + async def do_handshake(self, *, initial_retransmit_timeout: float = ...) -> None: ... + async def send(self, data: bytes) -> None: ... + async def receive(self) -> bytes: ... + def set_ciphertext_mtu(self, new_mtu: int) -> None: ... + def get_cleartext_mtu(self) -> int: ... + def statistics(self) -> Any: ... + async def aclose(self) -> None: ... + def close(self) -> None: ... + def __enter__(self) -> DTLSChannel: ... + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: ... + # _file_io # The actual Trio I/O classes use metaprogramming to forward their methods @@ -697,7 +745,7 @@ class _HasFileno(Protocol): _Redirect = Union[int, _HasFileno, None] -class Process(trio.abc.AsyncResource, _NotConstructible, metaclass=ABCMeta): +class Process(_NotConstructible, metaclass=ABCMeta): stdin: Optional[trio.abc.SendStream] stdout: Optional[trio.abc.ReceiveStream] stderr: Optional[trio.abc.ReceiveStream] @@ -706,7 +754,6 @@ class Process(trio.abc.AsyncResource, _NotConstructible, metaclass=ABCMeta): pid: int @property def returncode(self) -> Optional[int]: ... - async def aclose(self) -> None: ... async def wait(self) -> int: ... def poll(self) -> Optional[int]: ... def send_signal(self, sig: signal.Signals) -> None: ... @@ -725,22 +772,10 @@ class Process(trio.abc.AsyncResource, _NotConstructible, metaclass=ABCMeta): # bytes as stdin if sys.platform == "win32": - async def open_process( - command: Union[StrOrBytesPath, Sequence[StrOrBytesPath]], - *, - stdin: _Redirect = ..., - stdout: _Redirect = ..., - stderr: _Redirect = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[StrOrBytesPath] = ..., - env: Optional[Mapping[str, str]] = ..., - startupinfo: subprocess.STARTUPINFO = ..., - creationflags: int = ..., - ) -> Process: ... async def run_process( command: Union[StrOrBytesPath, Sequence[StrOrBytesPath]], *, + task_status: TaskStatus[Process] = ..., stdin: Union[bytes, _Redirect] = ..., capture_stdout: bool = ..., capture_stderr: bool = ..., @@ -757,42 +792,11 @@ if sys.platform == "win32": ) -> subprocess.CompletedProcess[bytes]: ... else: - @overload - async def open_process( - command: StrOrBytesPath, - *, - stdin: _Redirect = ..., - stdout: _Redirect = ..., - stderr: _Redirect = ..., - close_fds: bool = ..., - shell: Literal[True], - cwd: Optional[StrOrBytesPath] = ..., - env: Optional[Mapping[str, str]] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Sequence[int] = ..., - ) -> Process: ... - @overload - async def open_process( - command: Sequence[StrOrBytesPath], - *, - stdin: _Redirect = ..., - stdout: _Redirect = ..., - stderr: _Redirect = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[StrOrBytesPath] = ..., - env: Optional[Mapping[str, str]] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Sequence[int] = ..., - ) -> Process: ... @overload async def run_process( command: StrOrBytesPath, *, + task_status: TaskStatus[Process] = ..., stdin: Union[bytes, _Redirect] = ..., capture_stdout: bool = ..., capture_stderr: bool = ..., @@ -813,6 +817,7 @@ else: async def run_process( command: Sequence[StrOrBytesPath], *, + task_status: TaskStatus[Process] = ..., stdin: Union[bytes, _Redirect] = ..., capture_stdout: bool = ..., capture_stderr: bool = ..., diff --git a/trio-stubs/lowlevel.pyi b/trio-stubs/lowlevel.pyi index 8f4d5ef..fe31521 100644 --- a/trio-stubs/lowlevel.pyi +++ b/trio-stubs/lowlevel.pyi @@ -165,6 +165,68 @@ def start_thread_soon( fn: Callable[[], T], deliver: Callable[[outcome.Outcome[T]], None] ) -> None: ... +# _subprocess + +# There's a lot of duplication here because mypy doesn't +# have a good way to represent overloads that differ only +# slightly. A cheat sheet: +# - on Windows, command is Union[str, Sequence[str]]; +# on Unix, command is str if shell=True and Sequence[str] otherwise +# - on Windows, there are startupinfo and creationflags options; +# on Unix, there are preexec_fn, restore_signals, start_new_session, and pass_fds +# - run_process() has the signature of open_process() plus arguments +# capture_stdout, capture_stderr, check, deliver_cancel, and the ability to pass +# bytes as stdin + +if sys.platform == "win32": + async def open_process( + command: Union[StrOrBytesPath, Sequence[StrOrBytesPath]], + *, + stdin: _Redirect = ..., + stdout: _Redirect = ..., + stderr: _Redirect = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[StrOrBytesPath] = ..., + env: Optional[Mapping[str, str]] = ..., + startupinfo: subprocess.STARTUPINFO = ..., + creationflags: int = ..., + ) -> Process: ... + +else: + @overload + async def open_process( + command: StrOrBytesPath, + *, + stdin: _Redirect = ..., + stdout: _Redirect = ..., + stderr: _Redirect = ..., + close_fds: bool = ..., + shell: Literal[True], + cwd: Optional[StrOrBytesPath] = ..., + env: Optional[Mapping[str, str]] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Sequence[int] = ..., + ) -> Process: ... + @overload + async def open_process( + command: Sequence[StrOrBytesPath], + *, + stdin: _Redirect = ..., + stdout: _Redirect = ..., + stderr: _Redirect = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[StrOrBytesPath] = ..., + env: Optional[Mapping[str, str]] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Sequence[int] = ..., + ) -> Process: ... + # _unix_pipes class FdStream(trio.abc.Stream): def __init__(self, fd: int): ... From b95d2d58292e64ee6cfc752d93b1282db7bebf9f Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 28 Mar 2023 17:48:31 -0600 Subject: [PATCH 2/4] Run black and fix typo --- trio-stubs/__init__.pyi | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/trio-stubs/__init__.pyi b/trio-stubs/__init__.pyi index 889cbff..0616041 100644 --- a/trio-stubs/__init__.pyi +++ b/trio-stubs/__init__.pyi @@ -341,10 +341,14 @@ class SocketListener(trio.abc.Listener[SocketStream]): class DTLSEndpoint: socket: trio.socket.SocketType incoming_packets_buffer: int - def __init__(self, socket: trio.socket.SocketType, *, incoming_packets_buffer: int = ...) -> None: ... + def __init__( + self, socket: trio.socket.SocketType, *, incoming_packets_buffer: int = ... + ) -> None: ... # ssl_context is an OpenSSL.SSL.Context from PyOpenSSL, but we can't # import that name because we don't depend on PyOpenSSL - def connect(self, address: address: Union[Tuple[Any, ...], str, bytes], ssl_context: Any) -> DTLSChannel: ... + def connect( + self, address: Union[Tuple[Any, ...], str, bytes], ssl_context: Any + ) -> DTLSChannel: ... @takes_callable_and_args async def serve( self, @@ -368,7 +372,9 @@ class DTLSEndpoint: class DTLSChannel(_NotConstructible, trio.abc.Channel[bytes]): endpoint: DTLSEndpoint peer_address: Union[Tuple[Any, ...], str, bytes] - async def do_handshake(self, *, initial_retransmit_timeout: float = ...) -> None: ... + async def do_handshake( + self, *, initial_retransmit_timeout: float = ... + ) -> None: ... async def send(self, data: bytes) -> None: ... async def receive(self) -> bytes: ... def set_ciphertext_mtu(self, new_mtu: int) -> None: ... From 5f1a9759d4d3c8d18b9d48a604aabaf4b113c266 Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 28 Mar 2023 18:11:56 -0600 Subject: [PATCH 3/4] Fix mypy errors --- trio-stubs/__init__.pyi | 2 +- trio-stubs/lowlevel.pyi | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/trio-stubs/__init__.pyi b/trio-stubs/__init__.pyi index 0616041..bdf34dc 100644 --- a/trio-stubs/__init__.pyi +++ b/trio-stubs/__init__.pyi @@ -369,7 +369,7 @@ class DTLSEndpoint: exc_tb: TracebackType | None, ) -> None: ... -class DTLSChannel(_NotConstructible, trio.abc.Channel[bytes]): +class DTLSChannel(_NotConstructible, trio.abc.Channel[bytes], metaclass=ABCMeta): endpoint: DTLSEndpoint peer_address: Union[Tuple[Any, ...], str, bytes] async def do_handshake( diff --git a/trio-stubs/lowlevel.pyi b/trio-stubs/lowlevel.pyi index fe31521..d84dda8 100644 --- a/trio-stubs/lowlevel.pyi +++ b/trio-stubs/lowlevel.pyi @@ -7,6 +7,7 @@ from typing import ( ContextManager, Coroutine, Generic, + Mapping, NoReturn, Optional, Sequence, @@ -14,8 +15,11 @@ from typing import ( Sequence, TypeVar, Tuple, + overload, ) +from _typeshed import StrOrBytesPath from trio_typing import Nursery, takes_callable_and_args +from typing_extensions import Literal, Protocol from mypy_extensions import VarArg import trio import outcome @@ -178,6 +182,11 @@ def start_thread_soon( # capture_stdout, capture_stderr, check, deliver_cancel, and the ability to pass # bytes as stdin +class _HasFileno(Protocol): + def fileno(self) -> int: ... + +_Redirect = Union[int, _HasFileno, None] + if sys.platform == "win32": async def open_process( command: Union[StrOrBytesPath, Sequence[StrOrBytesPath]], @@ -191,7 +200,7 @@ if sys.platform == "win32": env: Optional[Mapping[str, str]] = ..., startupinfo: subprocess.STARTUPINFO = ..., creationflags: int = ..., - ) -> Process: ... + ) -> trio.Process: ... else: @overload @@ -209,7 +218,7 @@ else: restore_signals: bool = ..., start_new_session: bool = ..., pass_fds: Sequence[int] = ..., - ) -> Process: ... + ) -> trio.Process: ... @overload async def open_process( command: Sequence[StrOrBytesPath], @@ -225,7 +234,7 @@ else: restore_signals: bool = ..., start_new_session: bool = ..., pass_fds: Sequence[int] = ..., - ) -> Process: ... + ) -> trio.Process: ... # _unix_pipes class FdStream(trio.abc.Stream): From 0373f91e4232205abd6ad12487b1fc240796f86e Mon Sep 17 00:00:00 2001 From: Joshua Oreman Date: Tue, 28 Mar 2023 18:17:07 -0600 Subject: [PATCH 4/4] Tweak stubtest allowlist --- allowlist.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/allowlist.txt b/allowlist.txt index e01df85..b622582 100644 --- a/allowlist.txt +++ b/allowlist.txt @@ -111,22 +111,21 @@ trio.socket.fromshare trio.socket.ntohl # Not present in stub (but maybe should be?) -trio.DTLSChannel -trio.DTLSEndpoint trio.MultiError.derive trio.NeedHandshakeError trio.Path.hardlink_to trio.Path.is_mount trio.Path.link_to trio.Path.readlink +trio.Process.aclose trio.Process.encoding trio.Process.errors trio.Process.universal_newlines trio.TrioDeprecationWarning trio.lowlevel.FdStream.close -trio.lowlevel.open_process # NoPublicConstructor +trio.DTLSChannel.__init__ trio.MemoryReceiveChannel.__init__ trio.MemorySendChannel.__init__ trio.Nursery.__init__