From d62b607fdbd884bbed83db36672922e246ea5f93 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Fri, 3 Oct 2025 13:38:43 +0200 Subject: [PATCH 1/4] upath.types: adjust PathLike types --- upath/types/__init__.py | 54 ++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/upath/types/__init__.py b/upath/types/__init__.py index b7e0bed8..2262f44e 100644 --- a/upath/types/__init__.py +++ b/upath/types/__init__.py @@ -1,7 +1,6 @@ from __future__ import annotations import enum -import os import pathlib import sys from collections.abc import Iterator @@ -13,7 +12,6 @@ from typing import Literal from typing import Protocol from typing import TextIO -from typing import Union from typing import overload from typing import runtime_checkable @@ -25,6 +23,8 @@ from upath.types._abc import vfsopen if TYPE_CHECKING: + from os import PathLike # noqa: F401 + if sys.version_info > (3, 11): from typing import Self else: @@ -43,6 +43,7 @@ "JoinablePathLike", "ReadablePathLike", "WritablePathLike", + "SupportsPathLike", "CompatJoinablePath", "CompatReadablePath", "CompatWritablePath", @@ -54,9 +55,18 @@ "UNSET_DEFAULT", ] -JoinablePathLike: TypeAlias = Union[str, JoinablePath] -ReadablePathLike: TypeAlias = Union[str, ReadablePath] -WritablePathLike: TypeAlias = Union[str, WritablePath] + +class VFSPathLike(Protocol): + def __vfspath__(self) -> str: ... + + +SupportsPathLike: TypeAlias = "VFSPathLike | PathLike[str]" +JoinablePathLike: TypeAlias = "JoinablePath | SupportsPathLike | str" +ReadablePathLike: TypeAlias = "ReadablePath | SupportsPathLike | str" +WritablePathLike: TypeAlias = "WritablePath | SupportsPathLike | str" +CompatJoinablePathLike: TypeAlias = "CompatJoinablePath | SupportsPathLike | str" +CompatReadablePathLike: TypeAlias = "CompatReadablePath | SupportsPathLike | str" +CompatWritablePathLike: TypeAlias = "CompatWritablePath | SupportsPathLike | str" class _DefaultValue(enum.Enum): @@ -139,16 +149,16 @@ def suffixes(self) -> Sequence[str]: ... @property def stem(self) -> str: ... - def with_name(self, name) -> Self: ... - def with_stem(self, stem) -> Self: ... - def with_suffix(self, suffix) -> Self: ... + def with_name(self, name: str) -> Self: ... + def with_stem(self, stem: str) -> Self: ... + def with_suffix(self, suffix: str) -> Self: ... @property def parts(self) -> Sequence[str]: ... - def joinpath(self, *pathsegments: str | Self) -> Self: ... - def __truediv__(self, key: str | Self) -> Self: ... - def __rtruediv__(self, key: str | Self) -> Self: ... + def joinpath(self, *pathsegments: str) -> Self: ... + def __truediv__(self, key: str) -> Self: ... + def __rtruediv__(self, key: str) -> Self: ... @property def parent(self) -> Self: ... @@ -190,7 +200,7 @@ class CompatWritablePath(CompatJoinablePath, Protocol): __slots__ = () def symlink_to( - self, target: WritablePath, target_is_directory: bool = ... + self, target: str | Self, target_is_directory: bool = ... ) -> None: ... def mkdir(self) -> None: ... @@ -280,20 +290,20 @@ def st_birthtime_ns(self) -> int: ... class UPathParser(PathParser, Protocol): """duck-type for upath.core.UPathParser""" - def strip_protocol(self, path: JoinablePath | str) -> str: ... + def split(self, path: JoinablePathLike) -> tuple[str, str]: ... + def splitext(self, path: JoinablePathLike) -> tuple[str, str]: ... + def normcase(self, path: JoinablePathLike) -> str: ... + + def strip_protocol(self, path: JoinablePathLike) -> str: ... def join( self, - path: JoinablePath | os.PathLike[str] | str, - *paths: JoinablePath | os.PathLike[str] | str, + path: JoinablePathLike, + *paths: JoinablePathLike, ) -> str: ... - def isabs(self, path: JoinablePath | os.PathLike[str] | str) -> bool: ... + def isabs(self, path: JoinablePathLike) -> bool: ... - def splitdrive( - self, path: JoinablePath | os.PathLike[str] | str - ) -> tuple[str, str]: ... + def splitdrive(self, path: JoinablePathLike) -> tuple[str, str]: ... - def splitroot( - self, path: JoinablePath | os.PathLike[str] | str - ) -> tuple[str, str, str]: ... + def splitroot(self, path: JoinablePathLike) -> tuple[str, str, str]: ... From 689324b54b3f6147368e270b3ebbdbb6c2c8e4fc Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Fri, 3 Oct 2025 13:39:23 +0200 Subject: [PATCH 2/4] upath.types._abc: stricter types for pathlib_abc base types --- upath/types/_abc.pyi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/upath/types/_abc.pyi b/upath/types/_abc.pyi index 6964b2cc..ea0810e7 100644 --- a/upath/types/_abc.pyi +++ b/upath/types/_abc.pyi @@ -26,7 +26,7 @@ class JoinablePath(ABC): @abstractmethod def parser(self) -> PathParser: ... @abstractmethod - def with_segments(self, *pathsegments: str | Self) -> Self: ... + def with_segments(self, *pathsegments: str) -> Self: ... @abstractmethod def __vfspath__(self) -> str: ... @property @@ -45,8 +45,8 @@ class JoinablePath(ABC): @property def parts(self) -> Sequence[str]: ... def joinpath(self, *pathsegments: str) -> Self: ... - def __truediv__(self, key: str | Self) -> Self: ... - def __rtruediv__(self, key: str | Self) -> Self: ... + def __truediv__(self, key: str) -> Self: ... + def __rtruediv__(self, key: str) -> Self: ... @property def parent(self) -> Self: ... @property From 13baf76afade80572247a5685f242abd22d041eb Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Fri, 3 Oct 2025 15:07:15 +0200 Subject: [PATCH 3/4] upath.types: remove Compat* Protocols and Openable* convenience classes --- upath/core.py | 5 +- upath/tests/test_core.py | 16 ---- upath/types/__init__.py | 190 --------------------------------------- 3 files changed, 3 insertions(+), 208 deletions(-) diff --git a/upath/core.py b/upath/core.py index 1c86665c..9e010b01 100644 --- a/upath/core.py +++ b/upath/core.py @@ -38,10 +38,11 @@ from upath.registry import get_upath_class from upath.types import UNSET_DEFAULT from upath.types import JoinablePathLike -from upath.types import OpenablePath from upath.types import PathInfo +from upath.types import ReadablePath from upath.types import ReadablePathLike from upath.types import UPathParser +from upath.types import WritablePath from upath.types import WritablePathLike if TYPE_CHECKING: @@ -427,7 +428,7 @@ def _url(self) -> SplitResult: return urlsplit(self.__str__()) -class UPath(_UPathMixin, OpenablePath): +class UPath(_UPathMixin, WritablePath, ReadablePath): __slots__ = ( "_chain", "_chain_parser", diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index 4033e616..9aabd2f6 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -11,10 +11,6 @@ from upath import UPath from upath.implementations.cloud import GCSPath from upath.implementations.cloud import S3Path -from upath.types import CompatOpenablePath -from upath.types import CompatReadablePath -from upath.types import CompatWritablePath -from upath.types import OpenablePath from upath.types import ReadablePath from upath.types import WritablePath @@ -123,10 +119,6 @@ def test_subclass_with_gcs(): assert isinstance(path, UPath) assert isinstance(path, ReadablePath) assert isinstance(path, WritablePath) - assert isinstance(path, OpenablePath) - assert isinstance(path, CompatReadablePath) - assert isinstance(path, CompatWritablePath) - assert isinstance(path, CompatOpenablePath) assert not isinstance(path, os.PathLike) assert not isinstance(path, pathlib.Path) @@ -137,10 +129,6 @@ def test_instance_check(local_testdir): assert isinstance(path, UPath) assert isinstance(path, ReadablePath) assert isinstance(path, WritablePath) - assert isinstance(path, OpenablePath) - assert isinstance(path, CompatReadablePath) - assert isinstance(path, CompatWritablePath) - assert isinstance(path, CompatOpenablePath) assert isinstance(path, os.PathLike) assert isinstance(path, pathlib.Path) @@ -150,10 +138,6 @@ def test_instance_check_local_uri(local_testdir): assert isinstance(path, UPath) assert isinstance(path, ReadablePath) assert isinstance(path, WritablePath) - assert isinstance(path, OpenablePath) - assert isinstance(path, CompatReadablePath) - assert isinstance(path, CompatWritablePath) - assert isinstance(path, CompatOpenablePath) assert isinstance(path, os.PathLike) assert not isinstance(path, pathlib.Path) diff --git a/upath/types/__init__.py b/upath/types/__init__.py index 2262f44e..01e8fe0f 100644 --- a/upath/types/__init__.py +++ b/upath/types/__init__.py @@ -3,16 +3,9 @@ import enum import pathlib import sys -from collections.abc import Iterator -from collections.abc import Sequence -from typing import IO from typing import TYPE_CHECKING from typing import Any -from typing import BinaryIO -from typing import Literal from typing import Protocol -from typing import TextIO -from typing import overload from typing import runtime_checkable from upath.types._abc import JoinablePath @@ -20,16 +13,10 @@ from upath.types._abc import PathParser from upath.types._abc import ReadablePath from upath.types._abc import WritablePath -from upath.types._abc import vfsopen if TYPE_CHECKING: from os import PathLike # noqa: F401 - if sys.version_info > (3, 11): - from typing import Self - else: - from typing_extensions import Self - if sys.version_info >= (3, 12): from typing import TypeAlias else: @@ -39,15 +26,10 @@ "JoinablePath", "ReadablePath", "WritablePath", - "OpenablePath", "JoinablePathLike", "ReadablePathLike", "WritablePathLike", "SupportsPathLike", - "CompatJoinablePath", - "CompatReadablePath", - "CompatWritablePath", - "CompatOpenablePath", "PathInfo", "StatResultType", "PathParser", @@ -64,9 +46,6 @@ def __vfspath__(self) -> str: ... JoinablePathLike: TypeAlias = "JoinablePath | SupportsPathLike | str" ReadablePathLike: TypeAlias = "ReadablePath | SupportsPathLike | str" WritablePathLike: TypeAlias = "WritablePath | SupportsPathLike | str" -CompatJoinablePathLike: TypeAlias = "CompatJoinablePath | SupportsPathLike | str" -CompatReadablePathLike: TypeAlias = "CompatReadablePath | SupportsPathLike | str" -CompatWritablePathLike: TypeAlias = "CompatWritablePath | SupportsPathLike | str" class _DefaultValue(enum.Enum): @@ -76,179 +55,10 @@ class _DefaultValue(enum.Enum): UNSET_DEFAULT: Any = _DefaultValue.UNSET -class OpenablePath(ReadablePath, WritablePath): - """Helper class to annotate read/writable paths which have an .open() method.""" - - __slots__ = () - - @overload - def open( - self, - mode: Literal["r", "w", "a"] = ..., - buffering: int = ..., - encoding: str = ..., - errors: str = ..., - newline: str = ..., - ) -> TextIO: ... - - @overload - def open( - self, - mode: Literal["rb", "wb", "ab"] = ..., - buffering: int = ..., - encoding: str = ..., - errors: str = ..., - newline: str = ..., - ) -> BinaryIO: ... - - @overload - def open( - self, - mode: str = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - ) -> IO[Any]: ... - - def open( - self, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> IO[Any]: - return vfsopen(self, mode, buffering, encoding, errors, newline) - - if sys.version_info >= (3, 14): JoinablePath.register(pathlib.PurePath) ReadablePath.register(pathlib.Path) WritablePath.register(pathlib.Path) - OpenablePath.register(pathlib.Path) - - -@runtime_checkable -class CompatJoinablePath(Protocol): - # not available in Python 3.9.* pathlib: - # - `parser` - # - `with_segments` - # - `__vfspath__` - # - `full_match` - __slots__ = () - - @property - def anchor(self) -> str: ... - @property - def name(self) -> str: ... - @property - def suffix(self) -> str: ... - @property - def suffixes(self) -> Sequence[str]: ... - @property - def stem(self) -> str: ... - - def with_name(self, name: str) -> Self: ... - def with_stem(self, stem: str) -> Self: ... - def with_suffix(self, suffix: str) -> Self: ... - - @property - def parts(self) -> Sequence[str]: ... - - def joinpath(self, *pathsegments: str) -> Self: ... - def __truediv__(self, key: str) -> Self: ... - def __rtruediv__(self, key: str) -> Self: ... - - @property - def parent(self) -> Self: ... - @property - def parents(self) -> Sequence[Self]: ... - - -@runtime_checkable -class CompatReadablePath(CompatJoinablePath, Protocol): - # not available in Python 3.9.* pathlib: - # - `info` - # - `__open_reader__` - # - `copy` - # - `copy_into` - # - `walk` - __slots__ = () - - def read_bytes(self) -> bytes: ... - - def read_text( - self, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> str: ... - - def iterdir(self) -> Iterator[Self]: ... - - def glob(self, pattern: str, *, recurse_symlinks: bool = ...) -> Iterator[Self]: ... - - def readlink(self) -> Self: ... - - -@runtime_checkable -class CompatWritablePath(CompatJoinablePath, Protocol): - # not available in Python 3.9.* pathlib: - # - `__open_writer__` - # - `_copy_from` - __slots__ = () - - def symlink_to( - self, target: str | Self, target_is_directory: bool = ... - ) -> None: ... - def mkdir(self) -> None: ... - - def write_bytes(self, data: bytes) -> int: ... - - def write_text( - self, - data: str, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> int: ... - - -@runtime_checkable -class CompatOpenablePath(CompatReadablePath, CompatWritablePath, Protocol): - """A path that can be read from and written to.""" - - __slots__ = () - - @overload - def open( - self, - mode: Literal["r", "w", "a"] = "r", - buffering: int = ..., - encoding: str = ..., - errors: str = ..., - newline: str = ..., - ) -> TextIO: ... - - @overload - def open( - self, - mode: Literal["rb", "wb", "ab"], - buffering: int = ..., - encoding: str = ..., - errors: str = ..., - newline: str = ..., - ) -> BinaryIO: ... - - def open( - self, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> IO[Any]: ... class StatResultType(Protocol): From c13b45f3ee1623f4bc5c39b31d1d1e07e53767dd Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Fri, 3 Oct 2025 15:14:08 +0200 Subject: [PATCH 4/4] upath._protocol and upath._flavour type adjustments --- upath/_flavour.py | 30 ++++++++++++++---------------- upath/_protocol.py | 9 +++++---- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/upath/_flavour.py b/upath/_flavour.py index ed107c60..cbd921f2 100644 --- a/upath/_flavour.py +++ b/upath/_flavour.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING from typing import Any from typing import TypedDict -from typing import Union from urllib.parse import SplitResult from urllib.parse import urlsplit @@ -22,7 +21,7 @@ from upath._flavour_sources import flavour_registry from upath._protocol import get_upath_protocol from upath._protocol import normalize_empty_netloc -from upath.types import JoinablePath +from upath.types import JoinablePathLike from upath.types import UPathParser if TYPE_CHECKING: @@ -42,7 +41,6 @@ ] class_registry: Mapping[str, type[AbstractFileSystem]] = _class_registry -PathOrStr: TypeAlias = Union[str, os.PathLike[str], JoinablePath] class AnyProtocolFileSystemFlavour(FileSystemFlavourBase): @@ -237,7 +235,7 @@ def local_file(self) -> bool: return bool(getattr(self._spec, "local_file", False)) @staticmethod - def stringify_path(pth: PathOrStr) -> str: + def stringify_path(pth: JoinablePathLike) -> str: if isinstance(pth, str): out = pth elif isinstance(pth, upath.UPath) and not pth.is_absolute(): @@ -253,18 +251,18 @@ def stringify_path(pth: PathOrStr) -> str: out = str(pth) return normalize_empty_netloc(out) - def strip_protocol(self, pth: PathOrStr) -> str: + def strip_protocol(self, pth: JoinablePathLike) -> str: pth = self.stringify_path(pth) return self._spec._strip_protocol(pth) - def get_kwargs_from_url(self, url: PathOrStr) -> dict[str, Any]: + def get_kwargs_from_url(self, url: JoinablePathLike) -> dict[str, Any]: # NOTE: the public variant is _from_url not _from_urls if hasattr(url, "storage_options"): return dict(url.storage_options) url = self.stringify_path(url) return self._spec._get_kwargs_from_urls(url) - def parent(self, path: PathOrStr) -> str: + def parent(self, path: JoinablePathLike) -> str: path = self.stringify_path(path) return self._spec._parent(path) @@ -278,14 +276,14 @@ def sep(self) -> str: # type: ignore[override] def altsep(self) -> str | None: # type: ignore[override] return None - def isabs(self, path: PathOrStr) -> bool: + def isabs(self, path: JoinablePathLike) -> bool: path = self.strip_protocol(path) if self.local_file: return os.path.isabs(path) else: return path.startswith(self.root_marker) - def join(self, path: PathOrStr, *paths: PathOrStr) -> str: + def join(self, path: JoinablePathLike, *paths: JoinablePathLike) -> str: if not paths: return self.strip_protocol(path) or self.root_marker if self.local_file: @@ -309,7 +307,7 @@ def join(self, path: PathOrStr, *paths: PathOrStr) -> str: else: return drv + posixpath.join(p0, *pN) - def split(self, path: PathOrStr): + def split(self, path: JoinablePathLike) -> tuple[str, str]: stripped_path = self.strip_protocol(path) if self.local_file: return os.path.split(stripped_path) @@ -328,7 +326,7 @@ def split(self, path: PathOrStr): return self.split(head) return head, tail - def splitdrive(self, path: PathOrStr) -> tuple[str, str]: + def splitdrive(self, path: JoinablePathLike) -> tuple[str, str]: path = self.strip_protocol(path) if self.netloc_is_anchor: u = urlsplit(path) @@ -353,13 +351,13 @@ def splitdrive(self, path: PathOrStr) -> tuple[str, str]: # all other cases don't have a drive return "", path - def normcase(self, path: PathOrStr) -> str: + def normcase(self, path: JoinablePathLike) -> str: if self.local_file: return os.path.normcase(self.stringify_path(path)) else: return self.stringify_path(path) - def splitext(self, path: PathOrStr) -> tuple[str, str]: + def splitext(self, path: JoinablePathLike) -> tuple[str, str]: path = self.stringify_path(path) if self.local_file: return os.path.splitext(path) @@ -375,7 +373,7 @@ def splitext(self, path: PathOrStr) -> tuple[str, str]: # === Python3.12 pathlib flavour ================================== - def splitroot(self, path: PathOrStr) -> tuple[str, str, str]: + def splitroot(self, path: JoinablePathLike) -> tuple[str, str, str]: drive, tail = self.splitdrive(path) if self.netloc_is_anchor: root_marker = self.root_marker or self.sep @@ -422,13 +420,13 @@ def __repr__(self): return f"<{cls_name} of {self._owner.__name__}>" -def upath_strip_protocol(pth: PathOrStr) -> str: +def upath_strip_protocol(pth: JoinablePathLike) -> str: if protocol := get_upath_protocol(pth): return WrappedFileSystemFlavour.from_protocol(protocol).strip_protocol(pth) return WrappedFileSystemFlavour.stringify_path(pth) -def upath_get_kwargs_from_url(url: PathOrStr) -> dict[str, Any]: +def upath_get_kwargs_from_url(url: JoinablePathLike) -> dict[str, Any]: if protocol := get_upath_protocol(url): return WrappedFileSystemFlavour.from_protocol(protocol).get_kwargs_from_url(url) return {} diff --git a/upath/_protocol.py b/upath/_protocol.py index 51588c1d..fd92024b 100644 --- a/upath/_protocol.py +++ b/upath/_protocol.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re from collections import ChainMap from pathlib import PurePath @@ -11,7 +10,7 @@ from fsspec.registry import registry as _registry if TYPE_CHECKING: - from upath.types import JoinablePath + from upath.types import JoinablePathLike __all__ = [ "get_upath_protocol", @@ -60,7 +59,7 @@ def _fsspec_protocol_equals(p0: str, p1: str) -> bool: def get_upath_protocol( - pth: str | os.PathLike[str] | PurePath | JoinablePath, + pth: JoinablePathLike, *, protocol: str | None = None, storage_options: dict[str, Any] | None = None, @@ -74,6 +73,8 @@ def get_upath_protocol( pth_protocol = pth.protocol elif isinstance(pth, PurePath): pth_protocol = getattr(pth, "protocol", "") + elif hasattr(pth, "__vfspath__"): + pth_protocol = _match_protocol(pth.__vfspath__()) elif hasattr(pth, "__fspath__"): pth_protocol = _match_protocol(pth.__fspath__()) else: @@ -102,7 +103,7 @@ def normalize_empty_netloc(pth: str) -> str: def compatible_protocol( protocol: str, - *args: str | os.PathLike[str] | PurePath | JoinablePath, + *args: JoinablePathLike, ) -> bool: """check if UPath protocols are compatible""" from upath.core import UPath