From 6fe5f8028a529a98c7f04e6a12c6c82321b587f8 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Thu, 11 May 2023 02:31:40 -0700 Subject: [PATCH 1/6] Fix CloudPathMeta.__call__ return type Since pyright==1.1.307, instances of the subclasses of CloudPath stopped being recognized as instances of their own class; a possible fix is to type hint CloudPathMeta.__call__, but it creates a host of mypy errors --- cloudpathlib/cloudpath.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index c37979ad..412a264a 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -98,10 +98,11 @@ def path_class(self) -> Type["CloudPath"]: implementation_registry: Dict[str, CloudImplementation] = defaultdict(CloudImplementation) -def register_path_class(key: str) -> Callable: - T = TypeVar("T", bound=Type[CloudPath]) +CloudPathT = TypeVar("CloudPathT", bound="CloudPath") - def decorator(cls: Type[T]) -> Type[T]: + +def register_path_class(key: str) -> Callable[[Type[CloudPathT]], Type[CloudPathT]]: + def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]: if not issubclass(cls, CloudPath): raise TypeError("Only subclasses of CloudPath can be registered.") implementation_registry[key]._path_class = cls @@ -112,7 +113,7 @@ def decorator(cls: Type[T]) -> Type[T]: class CloudPathMeta(abc.ABCMeta): - def __call__(cls, cloud_path, *args, **kwargs): + def __call__(cls, cloud_path: Union[str, CloudPathT], *args, **kwargs) -> CloudPathT: # cls is a class that is the instance of this metaclass, e.g., CloudPath # Dispatch to subclass if base CloudPath From 755cef4d42c09a5b85802d8d929b5296acdb3c50 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 27 May 2023 19:22:09 -0700 Subject: [PATCH 2/6] Refactor CloudPathMeta.__call__ Fix mypy errors by making use of object.__new__ and type assertions --- cloudpathlib/cloudpath.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 412a264a..1777021d 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -113,20 +113,19 @@ def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]: class CloudPathMeta(abc.ABCMeta): - def __call__(cls, cloud_path: Union[str, CloudPathT], *args, **kwargs) -> CloudPathT: + def __call__(cls, cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any) -> CloudPathT: # cls is a class that is the instance of this metaclass, e.g., CloudPath # Dispatch to subclass if base CloudPath - if cls == CloudPath: + if cls is CloudPath: for implementation in implementation_registry.values(): path_class = implementation._path_class if path_class is not None and path_class.is_valid_cloudpath( cloud_path, raise_on_error=False ): # Instantiate path_class instance - new_obj = path_class.__new__(path_class, cloud_path, *args, **kwargs) - if isinstance(new_obj, path_class): - path_class.__init__(new_obj, cloud_path, *args, **kwargs) + new_obj = object.__new__(path_class) + path_class.__init__(new_obj, cloud_path, *args, **kwargs) return new_obj valid = [ impl._path_class.cloud_prefix @@ -136,11 +135,9 @@ def __call__(cls, cloud_path: Union[str, CloudPathT], *args, **kwargs) -> CloudP raise InvalidPrefixError( f"Path {cloud_path} does not begin with a known prefix {valid}." ) - - # Otherwise instantiate as normal - new_obj = cls.__new__(cls, cloud_path, *args, **kwargs) - if isinstance(new_obj, cls): - cls.__init__(new_obj, cloud_path, *args, **kwargs) + assert issubclass(cls, CloudPath) + new_obj = object.__new__(cls) + new_obj.__init__(cloud_path, *args, **kwargs) return new_obj def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> None: @@ -180,7 +177,7 @@ class CloudPath(metaclass=CloudPathMeta): def __init__( self, - cloud_path: Union[str, Self], + cloud_path: Union[str, CloudPathT], client: Optional["Client"] = None, ) -> None: # handle if local file gets opened. must be set at the top of the method in case any code @@ -450,7 +447,7 @@ def open( newline: Optional[str] = None, force_overwrite_from_cloud: bool = False, # extra kwarg not in pathlib force_overwrite_to_cloud: bool = False, # extra kwarg not in pathlib - ) -> IO: + ) -> IO[Any]: # if trying to call open on a directory that exists if self.exists() and not self.is_file(): raise CloudPathIsADirectoryError( From bd7f5578f826ed8661b8cddce01734b02e7f3406 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Sat, 27 May 2023 19:35:05 -0700 Subject: [PATCH 3/6] Add type ignore on CloudPathMeta.__call__ return Incompatible return value type (got "CloudPath", expected "CloudPathT"); not a clear fix here without a more general refactor --- cloudpathlib/cloudpath.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 1777021d..b94eae4b 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -126,7 +126,7 @@ def __call__(cls, cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any) # Instantiate path_class instance new_obj = object.__new__(path_class) path_class.__init__(new_obj, cloud_path, *args, **kwargs) - return new_obj + return new_obj # type: ignore[return-value] valid = [ impl._path_class.cloud_prefix for impl in implementation_registry.values() @@ -137,8 +137,8 @@ def __call__(cls, cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any) ) assert issubclass(cls, CloudPath) new_obj = object.__new__(cls) - new_obj.__init__(cloud_path, *args, **kwargs) - return new_obj + cls.__init__(new_obj, cloud_path, *args, **kwargs) + return new_obj # type: ignore[return-value] def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> None: # Copy docstring from pathlib.Path From c12d8f2a64b11183334fe665e7e2199aae4969c4 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 31 May 2023 19:29:19 -0700 Subject: [PATCH 4/6] Refactor CloudPathMeta.__call__ signature to use overload Also make issubclass(cls, CloudPath) explicit by raising TypeError --- cloudpathlib/cloudpath.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index b94eae4b..f1d4f883 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -98,6 +98,7 @@ def path_class(self) -> Type["CloudPath"]: implementation_registry: Dict[str, CloudImplementation] = defaultdict(CloudImplementation) +T = TypeVar("T") CloudPathT = TypeVar("CloudPathT", bound="CloudPath") @@ -113,8 +114,26 @@ def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]: class CloudPathMeta(abc.ABCMeta): - def __call__(cls, cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any) -> CloudPathT: + @overload + def __call__( + cls: Type[T], cloud_path: CloudPathT, *args: Any, **kwargs: Any + ) -> CloudPathT: + ... + + @overload + def __call__( + cls: Type[T], cloud_path: Union[str, "CloudPath"], *args: Any, **kwargs: Any + ) -> T: + ... + + def __call__( + cls: Type[T], cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any + ) -> Union[T, "CloudPath", CloudPathT]: # cls is a class that is the instance of this metaclass, e.g., CloudPath + if not issubclass(cls, CloudPath): + raise TypeError( + f"Only subclasses of {CloudPath.__name__} can be instantiated from its meta class." + ) # Dispatch to subclass if base CloudPath if cls is CloudPath: @@ -126,19 +145,19 @@ def __call__(cls, cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any) # Instantiate path_class instance new_obj = object.__new__(path_class) path_class.__init__(new_obj, cloud_path, *args, **kwargs) - return new_obj # type: ignore[return-value] - valid = [ + return new_obj + valid_prefixes = [ impl._path_class.cloud_prefix for impl in implementation_registry.values() if impl._path_class is not None ] raise InvalidPrefixError( - f"Path {cloud_path} does not begin with a known prefix {valid}." + f"Path {cloud_path} does not begin with a known prefix {valid_prefixes}." ) - assert issubclass(cls, CloudPath) + new_obj = object.__new__(cls) cls.__init__(new_obj, cloud_path, *args, **kwargs) - return new_obj # type: ignore[return-value] + return new_obj def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> None: # Copy docstring from pathlib.Path From d2010fdd677510f131a6aba2f3ac20151bb28e8b Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 31 May 2023 19:31:10 -0700 Subject: [PATCH 5/6] Run black --- cloudpathlib/cloudpath.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index f1d4f883..7f6cc4cc 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -115,9 +115,7 @@ def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]: class CloudPathMeta(abc.ABCMeta): @overload - def __call__( - cls: Type[T], cloud_path: CloudPathT, *args: Any, **kwargs: Any - ) -> CloudPathT: + def __call__(cls: Type[T], cloud_path: CloudPathT, *args: Any, **kwargs: Any) -> CloudPathT: ... @overload From af7c3d7462711aa4a1407aae0efeeb3ee95d64f3 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Wed, 31 May 2023 21:25:57 -0700 Subject: [PATCH 6/6] Change back __init__ to use Self type For some reason this leads to mypy type-var errors in the meta __call__ (probably erroneous) --- cloudpathlib/cloudpath.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudpathlib/cloudpath.py b/cloudpathlib/cloudpath.py index 7f6cc4cc..a744ae50 100644 --- a/cloudpathlib/cloudpath.py +++ b/cloudpathlib/cloudpath.py @@ -142,7 +142,7 @@ def __call__( ): # Instantiate path_class instance new_obj = object.__new__(path_class) - path_class.__init__(new_obj, cloud_path, *args, **kwargs) + path_class.__init__(new_obj, cloud_path, *args, **kwargs) # type: ignore[type-var] return new_obj valid_prefixes = [ impl._path_class.cloud_prefix @@ -154,7 +154,7 @@ def __call__( ) new_obj = object.__new__(cls) - cls.__init__(new_obj, cloud_path, *args, **kwargs) + cls.__init__(new_obj, cloud_path, *args, **kwargs) # type: ignore[type-var] return new_obj def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> None: @@ -194,7 +194,7 @@ class CloudPath(metaclass=CloudPathMeta): def __init__( self, - cloud_path: Union[str, CloudPathT], + cloud_path: Union[str, Self], client: Optional["Client"] = None, ) -> None: # handle if local file gets opened. must be set at the top of the method in case any code