From 6dbda8b2f60a1d8dcfdd7f4443a110ac519dbcad Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Thu, 28 Dec 2017 13:44:23 +0000 Subject: [PATCH 1/8] added extract method --- fs/base.py | 28 +++++++++++++++++++++++++++- fs/test.py | 10 ++++++++++ tests/test_ftpfs.py | 1 - 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/fs/base.py b/fs/base.py index 4672161d..a051428c 100644 --- a/fs/base.py +++ b/fs/base.py @@ -43,7 +43,7 @@ class FS(object): # This is the "standard" meta namespace. _meta = {} - + # most FS will use default walking algorithms walker_class=Walker @@ -404,6 +404,32 @@ def exists(self, path): else: return True + def extract(self, path, file, **options): + """Copies a file from the filesystem to an binary file-like + object. + + This may be more efficient that opening and copying files + manually if the filesystem supplies an optimized method. + + Arguments: + path (str): Path to a resource + file (file-link): A file-like object open for writing in + binary mode. + options: Implementation specific options required to open + the source file. + + Note that the file object ``file`` will *not* be closed by this + method. Take care to close it after this method completes + (ideally with a context manager). + + Example: + >>> with open('starwars.mov', 'wb') as write_file: + ... my_fs.extract('starwars.mov', write_file) + + """ + with self.openbin(path, **options) as read_file: + tools.copy_file_data(read_file, file) + def filterdir(self, path, files=None, diff --git a/fs/test.py b/fs/test.py index 4e3be148..f2fda2b8 100644 --- a/fs/test.py +++ b/fs/test.py @@ -520,6 +520,16 @@ def test_exists(self): self.assertTrue(self.fs.exists('/')) self.assertTrue(self.fs.exists('')) + def test_extract(self): + test_bytes = b'Hello, World' + self.fs.setbytes('hello.bin', test_bytes) + write_file = io.BytesIO() + self.fs.extract('hello.bin', write_file) + self.assertEqual(write_file.getvalue(), test_bytes) + + with self.assertRaises(errors.ResourceNotFound): + self.fs.extract('foo.bin', write_file) + def test_listdir(self): # Check listing directory that doesn't exist with self.assertRaises(errors.ResourceNotFound): diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 92e43dbd..6a137ae9 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -79,7 +79,6 @@ def test_manager(self): with ftp_errors(mem_fs): raise error_perm('999 foo') - class TestFTPFS(FSTestCases, unittest.TestCase): user = 'user' From b77348f78de30eff62d1a5829cc5575da9f9aa4e Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Thu, 28 Dec 2017 13:54:55 +0000 Subject: [PATCH 2/8] added chunk_size and docs --- fs/base.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/fs/base.py b/fs/base.py index a051428c..0a46405e 100644 --- a/fs/base.py +++ b/fs/base.py @@ -404,17 +404,20 @@ def exists(self, path): else: return True - def extract(self, path, file, **options): - """Copies a file from the filesystem to an binary file-like - object. + def extract(self, path, file, chunk_size=None, **options): + """Copies a file from the filesystem to a file-like object, + open for writing in binary mode. This may be more efficient that opening and copying files manually if the filesystem supplies an optimized method. Arguments: - path (str): Path to a resource + path (str): Path to a resource. file (file-link): A file-like object open for writing in binary mode. + chunk_size (int, optional): Number of bytes to read at a + time, if a simple copy is used, or `None` to use + sensible default. options: Implementation specific options required to open the source file. @@ -422,13 +425,17 @@ def extract(self, path, file, **options): method. Take care to close it after this method completes (ideally with a context manager). - Example: + Example: >>> with open('starwars.mov', 'wb') as write_file: ... my_fs.extract('starwars.mov', write_file) """ with self.openbin(path, **options) as read_file: - tools.copy_file_data(read_file, file) + tools.copy_file_data( + read_file, + file, + chunk_size=chunk_size + ) def filterdir(self, path, From 801c78de53beb54329a685be551fd2733757681b Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Thu, 28 Dec 2017 13:56:37 +0000 Subject: [PATCH 3/8] typo --- fs/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index 0a46405e..faf3920d 100644 --- a/fs/base.py +++ b/fs/base.py @@ -413,7 +413,7 @@ def extract(self, path, file, chunk_size=None, **options): Arguments: path (str): Path to a resource. - file (file-link): A file-like object open for writing in + file (file-like): A file-like object open for writing in binary mode. chunk_size (int, optional): Number of bytes to read at a time, if a simple copy is used, or `None` to use From 9ea6066af42260cab4ec6879e72684eb0a5c4143 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 31 Dec 2017 20:11:01 +0000 Subject: [PATCH 4/8] renamed extract to getfile --- fs/base.py | 65 +++++++++++++++++++++++++++--------------------------- fs/test.py | 20 ++++++++--------- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/fs/base.py b/fs/base.py index faf3920d..3f2eadbd 100644 --- a/fs/base.py +++ b/fs/base.py @@ -404,39 +404,6 @@ def exists(self, path): else: return True - def extract(self, path, file, chunk_size=None, **options): - """Copies a file from the filesystem to a file-like object, - open for writing in binary mode. - - This may be more efficient that opening and copying files - manually if the filesystem supplies an optimized method. - - Arguments: - path (str): Path to a resource. - file (file-like): A file-like object open for writing in - binary mode. - chunk_size (int, optional): Number of bytes to read at a - time, if a simple copy is used, or `None` to use - sensible default. - options: Implementation specific options required to open - the source file. - - Note that the file object ``file`` will *not* be closed by this - method. Take care to close it after this method completes - (ideally with a context manager). - - Example: - >>> with open('starwars.mov', 'wb') as write_file: - ... my_fs.extract('starwars.mov', write_file) - - """ - with self.openbin(path, **options) as read_file: - tools.copy_file_data( - read_file, - file, - chunk_size=chunk_size - ) - def filterdir(self, path, files=None, @@ -534,6 +501,38 @@ def getbytes(self, path): contents = read_file.read() return contents + def getfile(self, path, file, chunk_size=None, **options): + """Copies a file from the filesystem to a file-like object. + + This may be more efficient that opening and copying files + manually if the filesystem supplies an optimized method. + + Arguments: + path (str): Path to a resource. + file (file-like): A file-like object open for writing in + binary mode. + chunk_size (int, optional): Number of bytes to read at a + time, if a simple copy is used, or `None` to use + sensible default. + **options: Implementation specific options required to open + the source file. + + Note that the file object ``file`` will *not* be closed by this + method. Take care to close it after this method completes + (ideally with a context manager). + + Example: + >>> with open('starwars.mov', 'wb') as write_file: + ... my_fs.getfile('/movies/starwars.mov', write_file) + + """ + with self.openbin(path, **options) as read_file: + tools.copy_file_data( + read_file, + file, + chunk_size=chunk_size + ) + def gettext(self, path, encoding=None, errors=None, newline=''): """Get the contents of a file as a string. diff --git a/fs/test.py b/fs/test.py index f2fda2b8..db399b20 100644 --- a/fs/test.py +++ b/fs/test.py @@ -520,16 +520,6 @@ def test_exists(self): self.assertTrue(self.fs.exists('/')) self.assertTrue(self.fs.exists('')) - def test_extract(self): - test_bytes = b'Hello, World' - self.fs.setbytes('hello.bin', test_bytes) - write_file = io.BytesIO() - self.fs.extract('hello.bin', write_file) - self.assertEqual(write_file.getvalue(), test_bytes) - - with self.assertRaises(errors.ResourceNotFound): - self.fs.extract('foo.bin', write_file) - def test_listdir(self): # Check listing directory that doesn't exist with self.assertRaises(errors.ResourceNotFound): @@ -1341,6 +1331,16 @@ def test_getbytes(self): with self.assertRaises(errors.FileExpected): self.fs.getbytes('baz') + def test_getfile(self): + test_bytes = b'Hello, World' + self.fs.setbytes('hello.bin', test_bytes) + write_file = io.BytesIO() + self.fs.getfile('hello.bin', write_file) + self.assertEqual(write_file.getvalue(), test_bytes) + + with self.assertRaises(errors.ResourceNotFound): + self.fs.getfile('foo.bin', write_file) + def test_isempty(self): self.assertTrue(self.fs.isempty('/')) self.fs.makedir('foo') From b02b4244596599ad4b8c9081e2645e21f9f9203b Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 31 Dec 2017 20:23:06 +0000 Subject: [PATCH 5/8] used openbin --- fs/copy.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 5130fa18..601a8a95 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -103,7 +103,7 @@ def copy_file(src_fs, src_path, dst_fs, dst_path): else: # Standard copy with src_fs.lock(), dst_fs.lock(): - with src_fs.open(src_path, 'rb') as read_file: + with src_fs.openbin(src_path) as read_file: # There may be an optimized copy available on # dst_fs dst_fs.setbinfile(dst_path, read_file) @@ -143,7 +143,7 @@ def copy_file_if_newer(src_fs, src_path, dst_fs, dst_path): with src_fs.lock(), dst_fs.lock(): if _source_is_newer(src_fs, src_path, dst_fs, dst_path): - with src_fs.open(src_path, 'rb') as read_file: + with src_fs.openbin(src_path) as read_file: # There may be an optimized copy available # on dst_fs dst_fs.setbinfile(dst_path, read_file) @@ -152,7 +152,6 @@ def copy_file_if_newer(src_fs, src_path, dst_fs, dst_path): return False - def copy_structure(src_fs, dst_fs, walker=None): """Copy directories (but not files) from ``src_fs`` to ``dst_fs``. From dd3a536594e16845bd627ca5b95bad1e594cc409 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 31 Dec 2017 20:30:16 +0000 Subject: [PATCH 6/8] locked getfile, added to docs --- docs/source/implementers.rst | 9 +++++---- fs/base.py | 13 +++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/source/implementers.rst b/docs/source/implementers.rst index b0971fc5..281a5366 100644 --- a/docs/source/implementers.rst +++ b/docs/source/implementers.rst @@ -93,10 +93,11 @@ In the general case, it is a good idea to look at how these methods are implemen * :meth:`~fs.base.FS.exists` * :meth:`~fs.base.FS.filterdir` * :meth:`~fs.base.FS.getbytes` -* :meth:`~fs.base.FS.gettext` +* :meth:`~fs.base.FS.getfile` * :meth:`~fs.base.FS.getmeta` * :meth:`~fs.base.FS.getsize` * :meth:`~fs.base.FS.getsyspath` +* :meth:`~fs.base.FS.gettext` * :meth:`~fs.base.FS.gettype` * :meth:`~fs.base.FS.geturl` * :meth:`~fs.base.FS.hassyspath` @@ -105,18 +106,18 @@ In the general case, it is a good idea to look at how these methods are implemen * :meth:`~fs.base.FS.isempty` * :meth:`~fs.base.FS.isfile` * :meth:`~fs.base.FS.lock` -* :meth:`~fs.base.FS.movedir` * :meth:`~fs.base.FS.makedirs` * :meth:`~fs.base.FS.move` +* :meth:`~fs.base.FS.movedir` * :meth:`~fs.base.FS.open` * :meth:`~fs.base.FS.opendir` * :meth:`~fs.base.FS.removetree` * :meth:`~fs.base.FS.scandir` -* :meth:`~fs.base.FS.setbytes` * :meth:`~fs.base.FS.setbin` +* :meth:`~fs.base.FS.setbytes` * :meth:`~fs.base.FS.setfile` -* :meth:`~fs.base.FS.settimes` * :meth:`~fs.base.FS.settext` +* :meth:`~fs.base.FS.settimes` * :meth:`~fs.base.FS.touch` * :meth:`~fs.base.FS.validatepath` diff --git a/fs/base.py b/fs/base.py index 3f2eadbd..230ffeb4 100644 --- a/fs/base.py +++ b/fs/base.py @@ -526,12 +526,13 @@ def getfile(self, path, file, chunk_size=None, **options): ... my_fs.getfile('/movies/starwars.mov', write_file) """ - with self.openbin(path, **options) as read_file: - tools.copy_file_data( - read_file, - file, - chunk_size=chunk_size - ) + with self._lock: + with self.openbin(path, **options) as read_file: + tools.copy_file_data( + read_file, + file, + chunk_size=chunk_size + ) def gettext(self, path, encoding=None, errors=None, newline=''): """Get the contents of a file as a string. From b153a9d1eb392e19acbdf67be260186136d758b3 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 3 Jan 2018 17:12:55 +0000 Subject: [PATCH 7/8] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b101d52a..8c485cc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [Unrealeased] + +### Added + +- fs.getfile function + ## [2.0.17] - 2017-11-20 ### Fixed From db28164c9f586faa83efad6a17c284fef65d0a4d Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 3 Jan 2018 17:13:11 +0000 Subject: [PATCH 8/8] version bump --- fs/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/_version.py b/fs/_version.py index acde5fbe..d5f052d0 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.0.17" +__version__ = "2.0.18a0"