From 3ba1dfd6eb9d1bd46d23eca0ef1b45a32a4614fd Mon Sep 17 00:00:00 2001 From: Khue Ngoc Dang Date: Sun, 17 Jul 2022 15:14:15 +0700 Subject: [PATCH 1/4] Added stub file for s3's _list.py --- awswrangler/s3/_list.pyi | 198 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 awswrangler/s3/_list.pyi diff --git a/awswrangler/s3/_list.pyi b/awswrangler/s3/_list.pyi new file mode 100644 index 000000000..a318b91f0 --- /dev/null +++ b/awswrangler/s3/_list.pyi @@ -0,0 +1,198 @@ +import datetime +from typing import Any, Dict, Iterator, List, Literal, Optional, Sequence, Union, overload + +import boto3 + +def _path2list( + path: Union[str, Sequence[str]], + boto3_session: boto3.Session, + s3_additional_kwargs: Optional[Dict[str, Any]] = ..., + last_modified_begin: Optional[datetime.datetime] = ..., + last_modified_end: Optional[datetime.datetime] = ..., + suffix: Union[str, List[str], None] = ..., + ignore_suffix: Union[str, List[str], None] = ..., + ignore_empty: bool = ..., +) -> List[str]: ... +def _prefix_cleanup(prefix: str) -> str: ... +def does_object_exist( + path: str, + s3_additional_kwargs: Optional[Dict[str, Any]] = ..., + boto3_session: Optional[boto3.Session] = ..., + version_id: Optional[str] = ..., +) -> bool: ... +def list_buckets(boto3_session: Optional[boto3.Session] = ...) -> List[str]: ... +@overload +def list_directories( + path: str, + chunked: Literal[False], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session], +) -> List[str]: ... +@overload +def list_directories( + path: str, + chunked: Literal[False], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session] = ..., +) -> List[str]: ... +@overload +def list_directories( + path: str, + chunked: Literal[False], + *, + boto3_session: Optional[boto3.Session], +) -> List[str]: ... +@overload +def list_directories( + path: str, + chunked: Literal[False], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> List[str]: ... +@overload +def list_directories( + path: str, + chunked: Literal[True], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session], +) -> Iterator[List[str]]: ... +@overload +def list_directories( + path: str, + chunked: Literal[True], + *, + boto3_session: Optional[boto3.Session], +) -> Iterator[List[str]]: ... +@overload +def list_directories( + path: str, + chunked: Literal[True], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session] = ..., +) -> Iterator[List[str]]: ... +@overload +def list_directories( + path: str, + chunked: Literal[True], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> Iterator[List[str]]: ... +@overload +def list_directories( + path: str, + chunked: bool, + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session], +) -> Union[List[str], Iterator[List[str]]]: ... +@overload +def list_directories( + path: str, + chunked: bool, + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session] = ..., +) -> Union[List[str], Iterator[List[str]]]: ... +@overload +def list_directories( + path: str, + chunked: bool, + *, + boto3_session: Optional[boto3.Session], +) -> Union[List[str], Iterator[List[str]]]: ... +@overload +def list_directories( + path: str, + chunked: bool, + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> Union[List[str], Iterator[List[str]]]: ... +@overload +def list_objects( + path: str, + *, + boto3_session: Optional[boto3.Session], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], +) -> List[str]: ... +@overload +def list_objects( + path: str, + chunked: Literal[False], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session], +) -> List[str]: ... +@overload +def list_objects( + path: str, + chunked: Literal[False], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session] = ..., +) -> List[str]: ... +@overload +def list_objects( + path: str, + chunked: Literal[False], + *, + boto3_session: Optional[boto3.Session], +) -> List[str]: ... +@overload +def list_objects( + path: str, + chunked: Literal[False], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> List[str]: ... +@overload +def list_objects( + path: str, + chunked: Literal[True], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session], +) -> Iterator[List[str]]: ... +@overload +def list_objects( + path: str, + chunked: Literal[True], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session] = ..., +) -> Iterator[List[str]]: ... +@overload +def list_objects( + path: str, + chunked: Literal[True], + *, + boto3_session: Optional[boto3.Session], +) -> Iterator[List[str]]: ... +@overload +def list_objects( + path: str, + chunked: Literal[True], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> Iterator[List[str]]: ... +@overload +def list_objects( + path: str, + chunked: bool, + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session], +) -> Union[List[str], Iterator[List[str]]]: ... +@overload +def list_objects( + path: str, + chunked: bool, + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + boto3_session: Optional[boto3.Session] = ..., +) -> Union[List[str], Iterator[List[str]]]: ... +@overload +def list_objects( + path: str, + chunked: bool, + *, + boto3_session: Optional[boto3.Session], +) -> Union[List[str], Iterator[List[str]]]: ... +@overload +def list_objects( + path: str, + chunked: bool, + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> Union[List[str], Iterator[List[str]]]: ... From c16c1e6aa41aa49080a33091b3bbf76dfb38f6a5 Mon Sep 17 00:00:00 2001 From: Khue Ngoc Dang Date: Sun, 17 Jul 2022 15:43:42 +0700 Subject: [PATCH 2/4] Added additional overload for list_directories --- awswrangler/s3/_list.pyi | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/awswrangler/s3/_list.pyi b/awswrangler/s3/_list.pyi index a318b91f0..355cf0eb4 100644 --- a/awswrangler/s3/_list.pyi +++ b/awswrangler/s3/_list.pyi @@ -50,6 +50,20 @@ def list_directories( boto3_session: Optional[boto3.Session] = ..., ) -> List[str]: ... @overload +def list_directories( + path: str, + chunked: Literal[False] = ..., + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> List[str]: ... +@overload +def list_directories( + path: str, + chunked: Literal[False] = ..., + boto3_session: Optional[boto3.Session] = ..., + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., +) -> List[str]: ... +@overload def list_directories( path: str, chunked: Literal[True], From 7918334f97f45d4a8e92dbe2e7c1891b072e6c56 Mon Sep 17 00:00:00 2001 From: Khue Ngoc Dang Date: Sun, 17 Jul 2022 15:50:54 +0700 Subject: [PATCH 3/4] Added additionals overload --- awswrangler/s3/_list.pyi | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/awswrangler/s3/_list.pyi b/awswrangler/s3/_list.pyi index 355cf0eb4..d9b36c6f8 100644 --- a/awswrangler/s3/_list.pyi +++ b/awswrangler/s3/_list.pyi @@ -52,14 +52,14 @@ def list_directories( @overload def list_directories( path: str, - chunked: Literal[False] = ..., + *, s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., boto3_session: Optional[boto3.Session] = ..., ) -> List[str]: ... @overload def list_directories( path: str, - chunked: Literal[False] = ..., + *, boto3_session: Optional[boto3.Session] = ..., s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., ) -> List[str]: ... @@ -123,8 +123,15 @@ def list_directories( def list_objects( path: str, *, - boto3_session: Optional[boto3.Session], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., + boto3_session: Optional[boto3.Session] = ..., +) -> List[str]: ... +@overload +def list_objects( + path: str, + *, + boto3_session: Optional[boto3.Session] = ..., + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., ) -> List[str]: ... @overload def list_objects( From 7133a968d21cc7ccbe274d0eedcd9c2401952f50 Mon Sep 17 00:00:00 2001 From: Khue Ngoc Dang Date: Sun, 17 Jul 2022 17:35:49 +0700 Subject: [PATCH 4/4] Cleaned up smaller overloads --- awswrangler/s3/_list.pyi | 142 +-------------------------------------- 1 file changed, 1 insertion(+), 141 deletions(-) diff --git a/awswrangler/s3/_list.pyi b/awswrangler/s3/_list.pyi index d9b36c6f8..76ff194b5 100644 --- a/awswrangler/s3/_list.pyi +++ b/awswrangler/s3/_list.pyi @@ -22,27 +22,6 @@ def does_object_exist( ) -> bool: ... def list_buckets(boto3_session: Optional[boto3.Session] = ...) -> List[str]: ... @overload -def list_directories( - path: str, - chunked: Literal[False], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session], -) -> List[str]: ... -@overload -def list_directories( - path: str, - chunked: Literal[False], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session] = ..., -) -> List[str]: ... -@overload -def list_directories( - path: str, - chunked: Literal[False], - *, - boto3_session: Optional[boto3.Session], -) -> List[str]: ... -@overload def list_directories( path: str, chunked: Literal[False], @@ -50,13 +29,6 @@ def list_directories( boto3_session: Optional[boto3.Session] = ..., ) -> List[str]: ... @overload -def list_directories( - path: str, - *, - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., - boto3_session: Optional[boto3.Session] = ..., -) -> List[str]: ... -@overload def list_directories( path: str, *, @@ -64,27 +36,6 @@ def list_directories( s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., ) -> List[str]: ... @overload -def list_directories( - path: str, - chunked: Literal[True], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session], -) -> Iterator[List[str]]: ... -@overload -def list_directories( - path: str, - chunked: Literal[True], - *, - boto3_session: Optional[boto3.Session], -) -> Iterator[List[str]]: ... -@overload -def list_directories( - path: str, - chunked: Literal[True], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session] = ..., -) -> Iterator[List[str]]: ... -@overload def list_directories( path: str, chunked: Literal[True], @@ -92,27 +43,6 @@ def list_directories( boto3_session: Optional[boto3.Session] = ..., ) -> Iterator[List[str]]: ... @overload -def list_directories( - path: str, - chunked: bool, - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session], -) -> Union[List[str], Iterator[List[str]]]: ... -@overload -def list_directories( - path: str, - chunked: bool, - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session] = ..., -) -> Union[List[str], Iterator[List[str]]]: ... -@overload -def list_directories( - path: str, - chunked: bool, - *, - boto3_session: Optional[boto3.Session], -) -> Union[List[str], Iterator[List[str]]]: ... -@overload def list_directories( path: str, chunked: bool, @@ -120,69 +50,20 @@ def list_directories( boto3_session: Optional[boto3.Session] = ..., ) -> Union[List[str], Iterator[List[str]]]: ... @overload -def list_objects( - path: str, - *, - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., - boto3_session: Optional[boto3.Session] = ..., -) -> List[str]: ... -@overload -def list_objects( - path: str, - *, - boto3_session: Optional[boto3.Session] = ..., - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., -) -> List[str]: ... -@overload -def list_objects( - path: str, - chunked: Literal[False], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session], -) -> List[str]: ... -@overload def list_objects( path: str, chunked: Literal[False], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], + s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., boto3_session: Optional[boto3.Session] = ..., ) -> List[str]: ... @overload def list_objects( path: str, - chunked: Literal[False], *, - boto3_session: Optional[boto3.Session], -) -> List[str]: ... -@overload -def list_objects( - path: str, - chunked: Literal[False], s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None] = ..., boto3_session: Optional[boto3.Session] = ..., ) -> List[str]: ... @overload -def list_objects( - path: str, - chunked: Literal[True], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session], -) -> Iterator[List[str]]: ... -@overload -def list_objects( - path: str, - chunked: Literal[True], - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session] = ..., -) -> Iterator[List[str]]: ... -@overload -def list_objects( - path: str, - chunked: Literal[True], - *, - boto3_session: Optional[boto3.Session], -) -> Iterator[List[str]]: ... -@overload def list_objects( path: str, chunked: Literal[True], @@ -190,27 +71,6 @@ def list_objects( boto3_session: Optional[boto3.Session] = ..., ) -> Iterator[List[str]]: ... @overload -def list_objects( - path: str, - chunked: bool, - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session], -) -> Union[List[str], Iterator[List[str]]]: ... -@overload -def list_objects( - path: str, - chunked: bool, - s3_additional_kwargs: Union[Dict[str, Any], Dict[str, str], None], - boto3_session: Optional[boto3.Session] = ..., -) -> Union[List[str], Iterator[List[str]]]: ... -@overload -def list_objects( - path: str, - chunked: bool, - *, - boto3_session: Optional[boto3.Session], -) -> Union[List[str], Iterator[List[str]]]: ... -@overload def list_objects( path: str, chunked: bool,