From db3f0a17c5cd520aed1743063697b1a82f7eb887 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 19 Feb 2025 09:23:16 -0500 Subject: [PATCH 1/4] Make asynchronous optional in async_wrapper --- fsspec/implementations/asyn_wrapper.py | 5 ++--- fsspec/implementations/tests/test_asyn_wrapper.py | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fsspec/implementations/asyn_wrapper.py b/fsspec/implementations/asyn_wrapper.py index f58b0b612..0da0914c1 100644 --- a/fsspec/implementations/asyn_wrapper.py +++ b/fsspec/implementations/asyn_wrapper.py @@ -42,9 +42,8 @@ class AsyncFileSystemWrapper(AsyncFileSystem): The synchronous filesystem instance to wrap. """ - def __init__(self, sync_fs, *args, **kwargs): - super().__init__(*args, **kwargs) - self.asynchronous = True + def __init__(self, sync_fs, *args, asynchronous=True, **kwargs): + super().__init__(*args, asynchronous=asynchronous, **kwargs) self.sync_fs = sync_fs self.protocol = self.sync_fs.protocol self._wrap_all_sync_methods() diff --git a/fsspec/implementations/tests/test_asyn_wrapper.py b/fsspec/implementations/tests/test_asyn_wrapper.py index 82e9f3d4c..d69cc815d 100644 --- a/fsspec/implementations/tests/test_asyn_wrapper.py +++ b/fsspec/implementations/tests/test_asyn_wrapper.py @@ -10,10 +10,13 @@ from .test_local import csv_files, filetexts -def test_is_async(): +def test_is_async_default(): fs = fsspec.filesystem("file") async_fs = AsyncFileSystemWrapper(fs) assert async_fs.async_impl + assert async_fs.asynchronous + async_fs = AsyncFileSystemWrapper(fs, asynchronous=False) + assert not async_fs.asynchronous def test_class_wrapper(): @@ -53,6 +56,7 @@ async def test_cats(): assert result == b"a,b\n1,2\n"[1:-2] # test synchronous API is available as expected + async_fs = AsyncFileSystemWrapper(fs, asynchronous=False) result = async_fs.cat(".test.fakedata.1.csv", start=1, end=-2) assert result == b"a,b\n1,2\n"[1:-2] From d275921ea10670108daa8a0aa772696d5e1397a6 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 19 Feb 2025 11:51:48 -0500 Subject: [PATCH 2/4] Take asynchronous from coroutine if not given --- fsspec/implementations/asyn_wrapper.py | 10 +++++++--- fsspec/implementations/dirfs.py | 2 -- .../implementations/tests/test_asyn_wrapper.py | 17 ++++++++++++++++- fsspec/registry.py | 3 +++ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/fsspec/implementations/asyn_wrapper.py b/fsspec/implementations/asyn_wrapper.py index 0da0914c1..5b618061d 100644 --- a/fsspec/implementations/asyn_wrapper.py +++ b/fsspec/implementations/asyn_wrapper.py @@ -2,7 +2,7 @@ import functools import inspect -from fsspec.asyn import AsyncFileSystem +from fsspec.asyn import AsyncFileSystem, running_async def async_wrapper(func, obj=None): @@ -42,9 +42,13 @@ class AsyncFileSystemWrapper(AsyncFileSystem): The synchronous filesystem instance to wrap. """ - def __init__(self, sync_fs, *args, asynchronous=True, **kwargs): + protocol = "async_wrapper" + + def __init__(self, fs, *args, asynchronous=None, **kwargs): + if asynchronous is None: + asynchronous = running_async() super().__init__(*args, asynchronous=asynchronous, **kwargs) - self.sync_fs = sync_fs + self.sync_fs = fs self.protocol = self.sync_fs.protocol self._wrap_all_sync_methods() diff --git a/fsspec/implementations/dirfs.py b/fsspec/implementations/dirfs.py index 0bee86469..a94445280 100644 --- a/fsspec/implementations/dirfs.py +++ b/fsspec/implementations/dirfs.py @@ -36,8 +36,6 @@ def __init__( super().__init__(**storage_options) if fs is None: fs = filesystem(protocol=target_protocol, **(target_options or {})) - if (path is not None) ^ (fo is not None) is False: - raise ValueError("Provide path or fo, not both") path = path or fo if self.asynchronous and not fs.async_impl: diff --git a/fsspec/implementations/tests/test_asyn_wrapper.py b/fsspec/implementations/tests/test_asyn_wrapper.py index d69cc815d..13b766fb1 100644 --- a/fsspec/implementations/tests/test_asyn_wrapper.py +++ b/fsspec/implementations/tests/test_asyn_wrapper.py @@ -10,7 +10,8 @@ from .test_local import csv_files, filetexts -def test_is_async_default(): +@pytest.mark.asyncio +async def test_is_async_default(): fs = fsspec.filesystem("file") async_fs = AsyncFileSystemWrapper(fs) assert async_fs.async_impl @@ -146,3 +147,17 @@ async def test_batch_operations(): await async_fs._rm([".test.fakedata.1.csv", ".test.fakedata.2.csv"]) assert not await async_fs._exists(".test.fakedata.1.csv") assert not await async_fs._exists(".test.fakedata.2.csv") + + +def test_open(tmpdir): + fn = f"{tmpdir}/afile" + with open(fn, "wb") as f: + f.write(b"hello") + of = fsspec.open( + "dir://afile::async_wrapper::file", + mode="rb", + async_wrapper={"asynchronous": False}, + dir={"path": str(tmpdir)}, + ) + with of as f: + assert f.read() == b"hello" diff --git a/fsspec/registry.py b/fsspec/registry.py index 5d104f266..59be4db7b 100644 --- a/fsspec/registry.py +++ b/fsspec/registry.py @@ -72,6 +72,9 @@ def register_implementation(name, cls, clobber=False, errtxt=None): "class": "fsspec.implementations.arrow.HadoopFileSystem", "err": "pyarrow and local java libraries required for HDFS", }, + "async_wrapper": { + "class": "morefs.asyn_wrapper.AsyncWrapperFileSystem", + }, "asynclocal": { "class": "morefs.asyn_local.AsyncLocalFileSystem", "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem", From 81198f5dd9ca45f51e77bcfcf3cdf2fc17553880 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 19 Feb 2025 12:49:09 -0500 Subject: [PATCH 3/4] pure wrapper should not be cachable --- fsspec/implementations/asyn_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fsspec/implementations/asyn_wrapper.py b/fsspec/implementations/asyn_wrapper.py index 5b618061d..ee009f3bb 100644 --- a/fsspec/implementations/asyn_wrapper.py +++ b/fsspec/implementations/asyn_wrapper.py @@ -43,6 +43,7 @@ class AsyncFileSystemWrapper(AsyncFileSystem): """ protocol = "async_wrapper" + cachable = False def __init__(self, fs, *args, asynchronous=None, **kwargs): if asynchronous is None: From 41de77feb1f7b0f187750ee6dafea3fdcd44d26e Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 19 Feb 2025 13:09:40 -0500 Subject: [PATCH 4/4] last fix --- fsspec/implementations/tests/test_reference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fsspec/implementations/tests/test_reference.py b/fsspec/implementations/tests/test_reference.py index 3d3a6140b..52d39fb3a 100644 --- a/fsspec/implementations/tests/test_reference.py +++ b/fsspec/implementations/tests/test_reference.py @@ -515,10 +515,10 @@ def test_fss_has_defaults(m): assert fs.fss[None] is m fs = fsspec.filesystem("reference", fo={"key": ["memory://a"]}) - assert fs.fss[None] is fs.fss["memory"] + assert fs.fss[None] == fs.fss["memory"] fs = fsspec.filesystem("reference", fo={"key": ["memory://a"], "blah": ["path"]}) - assert fs.fss[None] is fs.fss["memory"] + assert fs.fss[None] == fs.fss["memory"] def test_merging(m):