From 2b1456313c27fab68a67adc92074512b29094eff Mon Sep 17 00:00:00 2001 From: Potato Date: Wed, 5 Nov 2025 19:44:23 +0800 Subject: [PATCH 1/3] Update size parameter to accept callable Allow 'size' parameter to be a callable that returns an int. --- fsspec/callbacks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fsspec/callbacks.py b/fsspec/callbacks.py index 7ca99ca6a..9354ee1b2 100644 --- a/fsspec/callbacks.py +++ b/fsspec/callbacks.py @@ -91,8 +91,10 @@ def set_size(self, size): Parameters ---------- - size: int + size: int or callable """ + if callable(size): + size = size() self.size = size self.call() From 29e2ae8e633b308ad5b6c15b20f58f57ba58d210 Mon Sep 17 00:00:00 2001 From: OneSizeFitsQuorum Date: Mon, 17 Nov 2025 10:08:21 +0800 Subject: [PATCH 2/3] add test Signed-off-by: OneSizeFitsQuorum --- fsspec/callbacks.py | 19 +++++++++++++++++++ fsspec/tests/test_callbacks.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/fsspec/callbacks.py b/fsspec/callbacks.py index 9354ee1b2..0390237e8 100644 --- a/fsspec/callbacks.py +++ b/fsspec/callbacks.py @@ -92,6 +92,25 @@ def set_size(self, size): Parameters ---------- size: int or callable + The total size of the transfer. Can be either: + - An integer representing the total size directly + - A callable (function/method) that returns an integer when invoked + + The callable option is useful when the size is only available as a + method on an object (e.g., filesystem objects that have a ``size()`` + method instead of a ``size`` attribute). + + Examples + -------- + >>> callback = Callback() + >>> callback.set_size(1000) # Direct integer + >>> callback.set_size(lambda: 1000) # Callable returning integer + + Notes + ----- + If a callable is provided, it will be invoked immediately to obtain + the size value. The callable should take no arguments and return an + integer. """ if callable(size): size = size() diff --git a/fsspec/tests/test_callbacks.py b/fsspec/tests/test_callbacks.py index 2cc679d02..c36179aae 100644 --- a/fsspec/tests/test_callbacks.py +++ b/fsspec/tests/test_callbacks.py @@ -73,6 +73,35 @@ def relative_update(self, inc=1): assert events == [1] * 10 +def test_set_size_with_callable(): + """Test that set_size accepts both int and callable parameters.""" + callback = Callback() + + # Test with integer + callback.set_size(100) + assert callback.size == 100 + + # Test with callable (lambda) + callback.set_size(lambda: 200) + assert callback.size == 200 + + # Test with callable (function) + def get_size(): + return 300 + + callback.set_size(get_size) + assert callback.size == 300 + + # Test with callable that simulates a method attribute + class MockFileSystem: + def size(self): + return 400 + + fs = MockFileSystem() + callback.set_size(fs.size) + assert callback.size == 400 + + @pytest.mark.parametrize("tqdm_kwargs", [{}, {"desc": "A custom desc"}]) def test_tqdm_callback(tqdm_kwargs, mocker): pytest.importorskip("tqdm") From 41d05fda5e0a872b621c3e45f7c877851223ecdc Mon Sep 17 00:00:00 2001 From: OneSizeFitsQuorum Date: Mon, 17 Nov 2025 10:31:54 +0800 Subject: [PATCH 3/3] fix Signed-off-by: OneSizeFitsQuorum --- fsspec/implementations/tests/test_arrow.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/fsspec/implementations/tests/test_arrow.py b/fsspec/implementations/tests/test_arrow.py index edf48eb6d..df648b310 100644 --- a/fsspec/implementations/tests/test_arrow.py +++ b/fsspec/implementations/tests/test_arrow.py @@ -280,9 +280,8 @@ def test_get_file_seekable_default(fs, remote_dir, tmp_path): # Test default behavior (seekable=False) local_file = tmp_path / "test_default.txt" - fs.get_file(remote_dir + "/test_file.txt", str(local_file)) - with open(local_file, "rb") as f: - assert f.read() == data + with pytest.raises(OSError, match="only valid on seekable files"): + fs.get_file(remote_dir + "/test_file.txt", str(local_file)) # Test with explicit seekable=True local_file_seekable = tmp_path / "test_seekable.txt" @@ -292,11 +291,10 @@ def test_get_file_seekable_default(fs, remote_dir, tmp_path): # Test with explicit seekable=False local_file_not_seekable = tmp_path / "test_not_seekable.txt" - fs.get_file( - remote_dir + "/test_file.txt", str(local_file_not_seekable), seekable=False - ) - with open(local_file_not_seekable, "rb") as f: - assert f.read() == data + with pytest.raises(OSError, match="only valid on seekable files"): + fs.get_file( + remote_dir + "/test_file.txt", str(local_file_not_seekable), seekable=False + ) def test_cat_file_seekable_override(fs, remote_dir):