Skip to content

Commit

Permalink
use distinct filesystem instances in pipeline threads
Browse files Browse the repository at this point in the history
- clearer naming in FileItemDict
  • Loading branch information
deanja committed Feb 16, 2024
1 parent 62790c1 commit b473fb5
Showing 1 changed file with 10 additions and 8 deletions.
18 changes: 10 additions & 8 deletions dlt/common/storages/fsspec_filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,29 +199,31 @@ class FileItemDict(DictStrAny):
def __init__(
self,
mapping: FileItem,
credentials: Optional[Union[FileSystemCredentials, AbstractFileSystem]] = None,
fs_details: Optional[Union[AbstractFileSystem, FilesystemConfiguration, FileSystemCredentials]] = None,
):
"""Create a dictionary with the filesystem client.
Args:
mapping (FileItem): The file item TypedDict.
credentials (Optional[FileSystemCredentials], optional): The credentials to the
fs_details (Optional[AbstractFileSystem, FilesystemConfiguration, FileSystemCredentials], optional): Details to help get a
filesystem. Defaults to None.
"""
self.credentials = credentials
self.fs_details = fs_details
super().__init__(**mapping)

@property
def fsspec(self) -> AbstractFileSystem:
"""The filesystem client is based on the given credentials.
"""The filesystem client is based on the given details.
Returns:
AbstractFileSystem: The fsspec client.
AbstractFileSystem: An fsspec client.
"""
if isinstance(self.credentials, AbstractFileSystem):
return self.credentials
if isinstance(self.fs_details, AbstractFileSystem):
return self.fs_details
elif isinstance(self.fs_details, FilesystemConfiguration):
return fsspec_from_config(self.fs_details)[0]
else:
return fsspec_filesystem(self["file_url"], self.credentials)[0]
return fsspec_filesystem(self["file_url"], self.fs_details)[0]

def open( # noqa: A003
self,
Expand Down

0 comments on commit b473fb5

Please sign in to comment.