From 41c7498a0275541fe497602df156d1f5bb369821 Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Tue, 30 May 2023 09:58:45 +0100 Subject: [PATCH 1/2] Create parent directories in get_file --- .../tests/memory/memory_test.py | 18 +----------------- fsspec/spec.py | 2 ++ 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/fsspec/implementations/tests/memory/memory_test.py b/fsspec/implementations/tests/memory/memory_test.py index 98bf063ef..fd0ebaac8 100644 --- a/fsspec/implementations/tests/memory/memory_test.py +++ b/fsspec/implementations/tests/memory/memory_test.py @@ -1,5 +1,3 @@ -import pytest - import fsspec.tests.abstract as abstract from fsspec.implementations.tests.memory.memory_fixtures import MemoryFixtures @@ -9,21 +7,7 @@ class TestMemoryCopy(abstract.AbstractCopyTests, MemoryFixtures): class TestMemoryGet(abstract.AbstractGetTests, MemoryFixtures): - @pytest.mark.skip(reason="Bug: does not auto-create new directory") - def test_get_file_to_new_directory(self): - pass - - @pytest.mark.skip(reason="Bug: does not auto-create new directory") - def test_get_file_to_file_in_new_directory(self): - pass - - @pytest.mark.skip(reason="Bug: does not auto-create new directory") - def test_get_glob_to_new_directory(self): - pass - - @pytest.mark.skip(reason="Bug: does not auto-create new directory") - def test_get_list_of_files_to_new_directory(self): - pass + pass class TestMemoryPut(abstract.AbstractPutTests, MemoryFixtures): diff --git a/fsspec/spec.py b/fsspec/spec.py index 8b051b94b..0e9b92501 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -867,6 +867,8 @@ def get_file( os.makedirs(lpath, exist_ok=True) return None + os.makedirs(self._parent(lpath), exist_ok=True) + with self.open(rpath, "rb", **kwargs) as f1: if outfile is None: outfile = open(lpath, "wb") From 96e04ca88a5716793cc2a22ae3085c65777e8444 Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Wed, 31 May 2023 10:34:35 +0100 Subject: [PATCH 2/2] Use LocalFileSystem.makedirs --- fsspec/spec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index 0e9b92501..2bdfa3854 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -861,13 +861,15 @@ def get_file( self, rpath, lpath, callback=_DEFAULT_CALLBACK, outfile=None, **kwargs ): """Copy single remote file to local""" + from .implementations.local import LocalFileSystem + if isfilelike(lpath): outfile = lpath elif self.isdir(rpath): os.makedirs(lpath, exist_ok=True) return None - os.makedirs(self._parent(lpath), exist_ok=True) + LocalFileSystem(auto_mkdir=True).makedirs(self._parent(lpath), exist_ok=True) with self.open(rpath, "rb", **kwargs) as f1: if outfile is None: