From 810ee9b4cc80b1b79f952168b18248c31a58cc25 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 20 Dec 2017 12:37:50 +0100 Subject: [PATCH 01/12] Use @pjdelport backports.os on Py2 #120 Signed-off-by: Philippe Ombredanne --- fs/_fscompat.py | 34 +--------------------------------- requirements.txt | 1 + setup.py | 3 ++- 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/fs/_fscompat.py b/fs/_fscompat.py index b450fff1..c81bb17b 100644 --- a/fs/_fscompat.py +++ b/fs/_fscompat.py @@ -5,40 +5,8 @@ try: from os import fsencode, fsdecode except ImportError: - def _fscodec(): - encoding = sys.getfilesystemencoding() - errors = 'strict' if encoding == 'mbcs' else 'surrogateescape' + from backports.os import fsencode, fsdecode - def fsencode(filename): - """ - Encode filename to the filesystem encoding with 'surrogateescape' error - handler, return bytes unchanged. On Windows, use 'strict' error handler if - the file system encoding is 'mbcs' (which is the default encoding). - """ - if isinstance(filename, bytes): - return filename - elif isinstance(filename, six.text_type): - return filename.encode(encoding, errors) - else: - raise TypeError("expect string type, not %s" % type(filename).__name__) - - def fsdecode(filename): - """ - Decode filename from the filesystem encoding with 'surrogateescape' error - handler, return str unchanged. On Windows, use 'strict' error handler if - the file system encoding is 'mbcs' (which is the default encoding). - """ - if isinstance(filename, six.text_type): - return filename - elif isinstance(filename, bytes): - return filename.decode(encoding, errors) - else: - raise TypeError("expect string type, not %s" % type(filename).__name__) - - return fsencode, fsdecode - - fsencode, fsdecode = _fscodec() - del _fscodec try: from os import fspath diff --git a/requirements.txt b/requirements.txt index 5325081b..32e3697f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ enum34==1.1.6 ; python_version < '3.4' pytz setuptools six==1.10.0 +backports.os diff --git a/setup.py b/setup.py index 485ad179..0be05f5d 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,8 @@ install_requires=REQUIREMENTS, extras_require={ "scandir :python_version < '3.5'": ['scandir~=1.5'], - ":python_version < '3.4'": ['enum34~=1.1.6'] + ":python_version < '3.4'": ['enum34~=1.1.6'], + ":python_version < '3.0'": ['backports.os'], }, entry_points={'fs.opener': [ 'ftp = fs.opener.ftpfs:FTPOpener', From b3c22c7adf1d5f83e891d2c70b3bc81451f88211 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 20 Dec 2017 16:59:42 +0100 Subject: [PATCH 02/12] Add new tests for non-unicode bytes paths #120 Signed-off-by: Philippe Ombredanne --- tests/test_osfs.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 99630121..206c6f2e 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -13,7 +13,7 @@ from fs.test import FSTestCases - +from six import binary_type from six import text_type @@ -95,6 +95,34 @@ def test_unicode_paths(self): finally: shutil.rmtree(dir_path) + def test_non_decodable_unicode_paths(self): + dir_path = tempfile.mkdtemp() + try: + fs_dir_bytes = os.path.join(bytes(dir_path), b'some') + fs_dir_unicode = unicode(fs_dir_bytes) + os.mkdir(fs_dir_bytes) + with open(os.path.join(fs_dir_bytes, b'foo\xb1bar'), 'wb') as uf: + uf.write(b'') + with osfs.OSFS(fs_dir_unicode) as tfs: + f = tfs.listdir(u'.')[0] + self.assertFalse(isinstance(f , binary_type)) + finally: + shutil.rmtree(dir_path) + + def test_can_open_non_decodable_unicode_paths(self): + dir_path = tempfile.mkdtemp() + try: + fs_dir_bytes = os.path.join(bytes(dir_path), b'some') + fs_dir_unicode = unicode(fs_dir_bytes) + os.mkdir(fs_dir_bytes) + with open(os.path.join(fs_dir_bytes, b'foo\xb1bar'), 'wb') as uf: + uf.write(b'') + with osfs.OSFS(fs_dir_unicode) as tfs: + with tfs.open(b'foo\xb1bar') as tf: + tf.read() + finally: + shutil.rmtree(dir_path) + def test_symlinks(self): with open(self._get_real_path('foo'), 'wb') as f: f.write(b'foobar') From ce6ae0823eb404e5c540a649d9831bbba00a5241 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 20 Dec 2017 17:00:18 +0100 Subject: [PATCH 03/12] Add optional "as_bytes" arg #120 Signed-off-by: Philippe Ombredanne --- fs/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/base.py b/fs/base.py index b0ae0b8c..69bf936d 100644 --- a/fs/base.py +++ b/fs/base.py @@ -599,14 +599,16 @@ def getsize(self, path): size = self.getdetails(path).size return size - def getsyspath(self, path): + def getsyspath(self, path, as_bytes=False): """Get the *system path* of a resource. Parameters: path (str): A path on the filesystem. + as_bytes (bool, optional): If `True`, return the path as bytes using + the native filesystem encoding. (defaults to `False`). Returns: - str: the *system path* of the resource, if any. + str: the *system path* of the resource, if any, always as Unicode. Raises: fs.errors.NoSysPath: If there is no corresponding system path. From 8f13427a2239cee9ac0ed85caf056a349c8ecd79 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 20 Dec 2017 17:37:59 +0100 Subject: [PATCH 04/12] Use fsencoded paths on *nix #120 The approach is that unicode is used everywhere unless when on *nix and that real access to files is needed. In this case the patch is encoded to bytes using the filesystem encoding. Signed-off-by: Philippe Ombredanne --- fs/osfs.py | 132 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 92 insertions(+), 40 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index 09cf2bf5..42c002f6 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -15,7 +15,6 @@ import os import platform import stat -import sys import six @@ -41,8 +40,11 @@ log = logging.getLogger('fs.osfs') - -_WINDOWS_PLATFORM = platform.system() == 'Windows' +ps = platform.system() +_WINDOWS_PLATFORM = ps == 'Windows' +_MAC_PLATFORM = ps == 'Darwin' +_NIX_PLATFORM = ps != _WINDOWS_PLATFORM and ps != _MAC_PLATFORM +del ps @six.python_2_unicode_compatible @@ -77,23 +79,27 @@ def __init__(self, """Create an OSFS instance. """ super(OSFS, self).__init__() - root_path = fsdecode(fspath(root_path)) - _root_path = os.path.expanduser(os.path.expandvars(root_path)) - _root_path = os.path.normpath(os.path.abspath(_root_path)) - self.root_path = _root_path + _root_path_native = fsencode(fspath(root_path)) + _root_path_native = os.path.expandvars(_root_path_native) + _root_path_native = os.path.expanduser(_root_path_native) + _root_path_native = os.path.abspath(_root_path_native) + _root_path_native = os.path.normpath(_root_path_native) + + self.root_path_native = _root_path_native + self.root_path = fsdecode(_root_path_native) if create: try: - if not os.path.isdir(_root_path): - os.makedirs(_root_path, mode=create_mode) + if not os.path.isdir(_root_path_native): + os.makedirs(_root_path_native, mode=create_mode) except OSError as error: raise errors.CreateFailed( 'unable to create {} ({})'.format(root_path, error) ) else: - if not os.path.isdir(_root_path): + if not os.path.isdir(_root_path_native): raise errors.CreateFailed( - 'root path does not exist' + 'root path does not exist or is file' ) _meta = self._meta = { @@ -107,7 +113,7 @@ def __init__(self, } if _WINDOWS_PLATFORM: # pragma: nocover - _meta["invalid_path_chars"] =\ + _meta["invalid_path_chars"] = \ ''.join(six.unichr(n) for n in range(31)) + '\\:*?"<>|' else: _meta["invalid_path_chars"] = '\0' @@ -115,7 +121,7 @@ def __init__(self, if 'PC_PATH_MAX' in os.pathconf_names: _meta['max_sys_path_length'] = ( os.pathconf( - fsencode(_root_path), + _root_path_native, os.pathconf_names['PC_PATH_MAX'] ) ) @@ -130,13 +136,32 @@ def __str__(self): return fmt.format(self.__class__.__name__.lower(), self.root_path) - def _to_sys_path(self, path): + def _to_sys_path(self, path, as_bytes=False): """Convert a FS path to a path on the OS. + If `as_bytes` is True, return bytes on *nix and unicode elsewhere. """ + root_path = self.root_path + sep = '/' + os_sep = os.sep + + if _NIX_PLATFORM: + root_path = self.root_path_native + path = path and fsencode(path) or path + sep = six.binary_type(sep) + os_sep = six.binary_type(os_sep) + sys_path = os.path.join( - self.root_path, - path.lstrip('/').replace('/', os.sep) + root_path, + path.lstrip(sep).replace(sep, os_sep) ) + if as_bytes: + if _NIX_PLATFORM: + return sys_path + else: + return fsencode(sys_path) + + if _NIX_PLATFORM: + return fsdecode(sys_path) return sys_path @classmethod @@ -209,25 +234,30 @@ def _get_type_from_stat(cls, _stat): # -------------------------------------------------------- def _gettarget(self, sys_path): + if _NIX_PLATFORM: + sys_path = fsencode(sys_path) try: target = os.readlink(sys_path) except OSError: return None else: + if _NIX_PLATFORM and target: + target = fsdecode(target) return target def _make_link_info(self, sys_path): _target = self._gettarget(sys_path) link = { - 'target': _target, + 'target': _target and fsdecode(_target) or _target, } return link def getinfo(self, path, namespaces=None): self.check() namespaces = namespaces or () + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self.getsyspath(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) _lstat = None with convert_os_errors('getinfo', path): _stat = os.stat(sys_path) @@ -261,17 +291,19 @@ def getinfo(self, path, namespaces=None): def listdir(self, path): self.check() + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) with convert_os_errors('listdir', path, directory=True): - names = os.listdir(sys_path) + names = [fsdecode(f) for f in os.listdir(sys_path)] return names def makedir(self, path, permissions=None, recreate=False): self.check() mode = Permissions.get_mode(permissions) + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) with convert_os_errors('makedir', path, directory=True): try: os.mkdir(sys_path, mode) @@ -288,8 +320,9 @@ def openbin(self, path, mode="r", buffering=-1, **options): _mode = Mode(mode) _mode.validate_bin() self.check() + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) with convert_os_errors('openbin', path): if six.PY2 and _mode.exclusive and self.exists(path): raise errors.FileExists(path) @@ -303,17 +336,18 @@ def openbin(self, path, mode="r", buffering=-1, **options): def remove(self, path): self.check() + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) with convert_os_errors('remove', path): try: os.remove(sys_path) except OSError as error: - if error.errno == errno.EACCES and sys.platform == "win32": + if error.errno == errno.EACCES and _WINDOWS_PLATFORM: # sometimes windows says this for attempts to remove a dir if os.path.isdir(sys_path): # pragma: nocover raise errors.FileExpected(path) - if error.errno == errno.EPERM and sys.platform == "darwin": + if error.errno == errno.EPERM and _MAC_PLATFORM: # sometimes OSX says this for attempts to remove a dir if os.path.isdir(sys_path): # pragma: nocover raise errors.FileExpected(path) @@ -321,10 +355,11 @@ def remove(self, path): def removedir(self, path): self.check() + path = path and fsdecode(path) or path _path = self.validatepath(path) if _path == '/': raise errors.RemoveRootError() - sys_path = self._to_sys_path(path) + sys_path = self._to_sys_path(path, as_bytes=_NIX_PLATFORM) with convert_os_errors('removedir', path, directory=True): os.rmdir(sys_path) @@ -332,18 +367,25 @@ def removedir(self, path): # Optional Methods # -------------------------------------------------------- - def getsyspath(self, path): - sys_path = self._to_sys_path(path) + def getsyspath(self, path, as_bytes=False): + path = path and fsdecode(path) or path + sys_path = self._to_sys_path(path, as_bytes=as_bytes) return sys_path def geturl(self, path, purpose='download'): if purpose != 'download': raise NoURL(path, purpose) + path = path and fsdecode(path) or path + syspath = self.getsyspath(path, as_bytes=_NIX_PLATFORM) + if _NIX_PLATFORM: + syspath = fsdecode(syspath) + # FIXME: segments might need to be URL/percent-encoded instead return "file://" + self.getsyspath(path) def gettype(self, path): self.check() - sys_path = self._to_sys_path(path) + path = path and fsdecode(path) or path + sys_path = self._to_sys_path(path, as_bytes=_NIX_PLATFORM) with convert_os_errors('gettype', path): stat = os.stat(sys_path) resource_type = self._get_type_from_stat(stat) @@ -351,8 +393,9 @@ def gettype(self, path): def islink(self, path): self.check() + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) if not self.exists(path): raise errors.ResourceNotFound(path) with convert_os_errors('islink', path): @@ -370,8 +413,9 @@ def open(self, _mode = Mode(mode) validate_open_mode(mode) self.check() + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) with convert_os_errors('open', path): if six.PY2 and _mode.exclusive and self.exists(path): raise FileExists(path) @@ -388,8 +432,9 @@ def open(self, def setinfo(self, path, info): self.check() + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) if not os.path.exists(sys_path): raise errors.ResourceNotFound(path) if 'details' in info: @@ -407,13 +452,14 @@ def setinfo(self, path, info): def _scandir(self, path, namespaces=None): self.check() namespaces = namespaces or () + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) with convert_os_errors('scandir', path, directory=True): for dir_entry in scandir(sys_path): info = { "basic": { - "name": dir_entry.name, + "name": fsdecode(dir_entry.name), "is_dir": dir_entry.is_dir() } } @@ -434,12 +480,15 @@ def _scandir(self, path, namespaces=None): for k in dir(lstat_result) if k.startswith('st_') } if 'link' in namespaces: + dir_entry_name = dir_entry.name + if _NIX_PLATFORM: + dir_entry_name = fsencode(dir_entry_name) info['link'] = self._make_link_info( - os.path.join(sys_path, dir_entry.name) + fsdecode(os.path.join(sys_path, dir_entry_name)) ) if 'access' in namespaces: stat_result = dir_entry.stat() - info['access'] =\ + info['access'] = \ self._make_access_from_stat(stat_result) yield Info(info) @@ -449,15 +498,17 @@ def _scandir(self, path, namespaces=None): def _scandir(self, path, namespaces=None): self.check() namespaces = namespaces or () + path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) with convert_os_errors('scandir', path, directory=True): for entry_name in os.listdir(sys_path): + entry_name_u = fsdecode(entry_name) entry_path = os.path.join(sys_path, entry_name) stat_result = os.stat(entry_path) info = { "basic": { - "name": entry_name, + "name": entry_name_u, "is_dir": stat.S_ISDIR(stat_result.st_mode), } } @@ -477,16 +528,17 @@ def _scandir(self, path, namespaces=None): } if 'link' in namespaces: info['link'] = self._make_link_info( - os.path.join(sys_path, entry_name) + fsdecode(os.path.join(sys_path, entry_name)) ) if 'access' in namespaces: - info['access'] =\ + info['access'] = \ self._make_access_from_stat(stat_result) yield Info(info) def scandir(self, path, namespaces=None, page=None): + path = path and fsdecode(path) or path iter_info = self._scandir(path, namespaces=namespaces) if page is not None: start, end = page From 7c7886f4be08905a67d9d4355acee13cf2f76388 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 20 Dec 2017 17:42:45 +0100 Subject: [PATCH 05/12] Ignore tmp dir Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3df8aa55..f68834f1 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,4 @@ ENV/ #PyCharm .idea/ +/tmp/ From 116c5d68f5b9c79adc9aa510a58d3ac6b626063b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 20 Dec 2017 18:24:10 +0100 Subject: [PATCH 06/12] Ensure tests pass on Python3 #120 Signed-off-by: Philippe Ombredanne --- fs/osfs.py | 6 +++--- tests/test_osfs.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index 42c002f6..9bc5608f 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -146,9 +146,9 @@ def _to_sys_path(self, path, as_bytes=False): if _NIX_PLATFORM: root_path = self.root_path_native - path = path and fsencode(path) or path - sep = six.binary_type(sep) - os_sep = six.binary_type(os_sep) + path = fsencode(path) + sep = b'/' + os_sep = fsencode(os_sep) sys_path = os.path.join( root_path, diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 206c6f2e..6a5ce4f1 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -96,10 +96,12 @@ def test_unicode_paths(self): shutil.rmtree(dir_path) def test_non_decodable_unicode_paths(self): + from fs._fscompat import fsencode, fsdecode + dir_path = tempfile.mkdtemp() try: - fs_dir_bytes = os.path.join(bytes(dir_path), b'some') - fs_dir_unicode = unicode(fs_dir_bytes) + fs_dir_bytes = os.path.join(fsencode(dir_path), b'some') + fs_dir_unicode = fsdecode(fs_dir_bytes) os.mkdir(fs_dir_bytes) with open(os.path.join(fs_dir_bytes, b'foo\xb1bar'), 'wb') as uf: uf.write(b'') @@ -110,10 +112,12 @@ def test_non_decodable_unicode_paths(self): shutil.rmtree(dir_path) def test_can_open_non_decodable_unicode_paths(self): + from fs._fscompat import fsencode, fsdecode + dir_path = tempfile.mkdtemp() try: - fs_dir_bytes = os.path.join(bytes(dir_path), b'some') - fs_dir_unicode = unicode(fs_dir_bytes) + fs_dir_bytes = os.path.join(fsencode(dir_path), b'some') + fs_dir_unicode = fsdecode(fs_dir_bytes) os.mkdir(fs_dir_bytes) with open(os.path.join(fs_dir_bytes, b'foo\xb1bar'), 'wb') as uf: uf.write(b'') From 9f3626def08f66f7e8533f31c18832a32a33dedf Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 21 Dec 2017 09:55:03 +0100 Subject: [PATCH 07/12] Remove trailing line return Signed-off-by: Philippe Ombredanne --- tests/test_osfs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 6a5ce4f1..66c08c0c 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -144,4 +144,3 @@ def test_symlinks(self): bar_info = self.fs.getinfo('bar', namespaces=['link', 'lstat']) self.assertIn('link', bar_info.raw) self.assertIn('lstat', bar_info.raw) - From 257cc1fc6f81e190e4487ee887852484e7bc9747 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 21 Dec 2017 09:56:02 +0100 Subject: [PATCH 08/12] fsencode only on *nix and Python2 #120 Signed-off-by: Philippe Ombredanne --- fs/osfs.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index 9bc5608f..5798b923 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -45,6 +45,7 @@ _MAC_PLATFORM = ps == 'Darwin' _NIX_PLATFORM = ps != _WINDOWS_PLATFORM and ps != _MAC_PLATFORM del ps +_NIX_PY2 = _NIX_PLATFORM and six.PY2 @six.python_2_unicode_compatible @@ -138,13 +139,13 @@ def __str__(self): def _to_sys_path(self, path, as_bytes=False): """Convert a FS path to a path on the OS. - If `as_bytes` is True, return bytes on *nix and unicode elsewhere. + If `as_bytes` is True, return fsencoded-bytes instead of Unicode. """ root_path = self.root_path sep = '/' os_sep = os.sep - if _NIX_PLATFORM: + if _NIX_PY2: root_path = self.root_path_native path = fsencode(path) sep = b'/' @@ -154,14 +155,13 @@ def _to_sys_path(self, path, as_bytes=False): root_path, path.lstrip(sep).replace(sep, os_sep) ) + if as_bytes: - if _NIX_PLATFORM: - return sys_path - else: - return fsencode(sys_path) + return fsencode(sys_path) - if _NIX_PLATFORM: + if _NIX_PY2: return fsdecode(sys_path) + return sys_path @classmethod @@ -234,14 +234,14 @@ def _get_type_from_stat(cls, _stat): # -------------------------------------------------------- def _gettarget(self, sys_path): - if _NIX_PLATFORM: + if _NIX_PY2: sys_path = fsencode(sys_path) try: target = os.readlink(sys_path) except OSError: return None else: - if _NIX_PLATFORM and target: + if _NIX_PY2 and target: target = fsdecode(target) return target @@ -257,7 +257,7 @@ def getinfo(self, path, namespaces=None): namespaces = namespaces or () path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) _lstat = None with convert_os_errors('getinfo', path): _stat = os.stat(sys_path) @@ -293,7 +293,7 @@ def listdir(self, path): self.check() path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) with convert_os_errors('listdir', path, directory=True): names = [fsdecode(f) for f in os.listdir(sys_path)] return names @@ -303,7 +303,7 @@ def makedir(self, path, permissions=None, recreate=False): mode = Permissions.get_mode(permissions) path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) with convert_os_errors('makedir', path, directory=True): try: os.mkdir(sys_path, mode) @@ -322,7 +322,7 @@ def openbin(self, path, mode="r", buffering=-1, **options): self.check() path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) with convert_os_errors('openbin', path): if six.PY2 and _mode.exclusive and self.exists(path): raise errors.FileExists(path) @@ -338,7 +338,7 @@ def remove(self, path): self.check() path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) with convert_os_errors('remove', path): try: os.remove(sys_path) @@ -359,7 +359,7 @@ def removedir(self, path): _path = self.validatepath(path) if _path == '/': raise errors.RemoveRootError() - sys_path = self._to_sys_path(path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(path, as_bytes=_NIX_PY2) with convert_os_errors('removedir', path, directory=True): os.rmdir(sys_path) @@ -376,8 +376,8 @@ def geturl(self, path, purpose='download'): if purpose != 'download': raise NoURL(path, purpose) path = path and fsdecode(path) or path - syspath = self.getsyspath(path, as_bytes=_NIX_PLATFORM) - if _NIX_PLATFORM: + syspath = self.getsyspath(path, as_bytes=_NIX_PY2) + if _NIX_PY2: syspath = fsdecode(syspath) # FIXME: segments might need to be URL/percent-encoded instead return "file://" + self.getsyspath(path) @@ -385,7 +385,7 @@ def geturl(self, path, purpose='download'): def gettype(self, path): self.check() path = path and fsdecode(path) or path - sys_path = self._to_sys_path(path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(path, as_bytes=_NIX_PY2) with convert_os_errors('gettype', path): stat = os.stat(sys_path) resource_type = self._get_type_from_stat(stat) @@ -395,7 +395,7 @@ def islink(self, path): self.check() path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) if not self.exists(path): raise errors.ResourceNotFound(path) with convert_os_errors('islink', path): @@ -415,7 +415,7 @@ def open(self, self.check() path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) with convert_os_errors('open', path): if six.PY2 and _mode.exclusive and self.exists(path): raise FileExists(path) @@ -434,7 +434,7 @@ def setinfo(self, path, info): self.check() path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) if not os.path.exists(sys_path): raise errors.ResourceNotFound(path) if 'details' in info: @@ -454,7 +454,7 @@ def _scandir(self, path, namespaces=None): namespaces = namespaces or () path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) with convert_os_errors('scandir', path, directory=True): for dir_entry in scandir(sys_path): info = { @@ -481,7 +481,7 @@ def _scandir(self, path, namespaces=None): } if 'link' in namespaces: dir_entry_name = dir_entry.name - if _NIX_PLATFORM: + if _NIX_PY2: dir_entry_name = fsencode(dir_entry_name) info['link'] = self._make_link_info( fsdecode(os.path.join(sys_path, dir_entry_name)) @@ -500,7 +500,7 @@ def _scandir(self, path, namespaces=None): namespaces = namespaces or () path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PLATFORM) + sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) with convert_os_errors('scandir', path, directory=True): for entry_name in os.listdir(sys_path): entry_name_u = fsdecode(entry_name) From 6d5181b93ff6129d6ac0f88179d80240e5377f01 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 21 Dec 2017 10:18:52 +0100 Subject: [PATCH 09/12] Remove as_bytes arg from getsyspath #120 Instead I added doc to explain that fsencode how can be used if needed. Signed-off-by: Philippe Ombredanne --- fs/base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fs/base.py b/fs/base.py index 69bf936d..7efc76e4 100644 --- a/fs/base.py +++ b/fs/base.py @@ -604,8 +604,6 @@ def getsyspath(self, path, as_bytes=False): Parameters: path (str): A path on the filesystem. - as_bytes (bool, optional): If `True`, return the path as bytes using - the native filesystem encoding. (defaults to `False`). Returns: str: the *system path* of the resource, if any, always as Unicode. @@ -631,6 +629,14 @@ def getsyspath(self, path, as_bytes=False): resource is referenced by that path -- as long as it can be certain what that system path would be. + Note: + The returned value is always Unicode. Some filesystems + such as on Linux only deal with bytes. If you need to use + a path outside of Python, or if you are running Python 2 + on Linux, you can get a proper byte value from a Unicode + path using the os.fsencode function on Python 3 or its + Pythyon 2 backport available here. + """ raise errors.NoSysPath(path=path) From 0ebc6318e283fcabb2dd5004e3465058537fcd84 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 21 Dec 2017 10:20:21 +0100 Subject: [PATCH 10/12] Refactor common checks in _get_validated_syspath() #120 * Avoid code duplication with a new _get_validated_syspath() method * Remove as_bytes arg from getsyspath #120 Signed-off-by: Philippe Ombredanne --- fs/osfs.py | 59 +++++++++++++++++++++--------------------------------- 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index 5798b923..fb21de9f 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -252,12 +252,18 @@ def _make_link_info(self, sys_path): } return link + def _get_validated_syspath(self, path): + """Return a validated, normalized and eventually encoded string or byte + path. + """ + path = path and fsdecode(path) or path + _path = self.validatepath(path) + return self._to_sys_path(_path, as_bytes=_NIX_PY2) + def getinfo(self, path, namespaces=None): self.check() namespaces = namespaces or () - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) _lstat = None with convert_os_errors('getinfo', path): _stat = os.stat(sys_path) @@ -266,7 +272,7 @@ def getinfo(self, path, namespaces=None): info = { 'basic': { - 'name': basename(_path), + 'name': fsdecode(basename(sys_path)), 'is_dir': stat.S_ISDIR(_stat.st_mode) } } @@ -291,9 +297,7 @@ def getinfo(self, path, namespaces=None): def listdir(self, path): self.check() - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) with convert_os_errors('listdir', path, directory=True): names = [fsdecode(f) for f in os.listdir(sys_path)] return names @@ -303,7 +307,7 @@ def makedir(self, path, permissions=None, recreate=False): mode = Permissions.get_mode(permissions) path = path and fsdecode(path) or path _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(_path) with convert_os_errors('makedir', path, directory=True): try: os.mkdir(sys_path, mode) @@ -321,8 +325,7 @@ def openbin(self, path, mode="r", buffering=-1, **options): _mode.validate_bin() self.check() path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) with convert_os_errors('openbin', path): if six.PY2 and _mode.exclusive and self.exists(path): raise errors.FileExists(path) @@ -336,9 +339,7 @@ def openbin(self, path, mode="r", buffering=-1, **options): def remove(self, path): self.check() - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) with convert_os_errors('remove', path): try: os.remove(sys_path) @@ -367,25 +368,20 @@ def removedir(self, path): # Optional Methods # -------------------------------------------------------- - def getsyspath(self, path, as_bytes=False): + def getsyspath(self, path): path = path and fsdecode(path) or path - sys_path = self._to_sys_path(path, as_bytes=as_bytes) + sys_path = self._to_sys_path(path, as_bytes=False) return sys_path def geturl(self, path, purpose='download'): if purpose != 'download': raise NoURL(path, purpose) - path = path and fsdecode(path) or path - syspath = self.getsyspath(path, as_bytes=_NIX_PY2) - if _NIX_PY2: - syspath = fsdecode(syspath) # FIXME: segments might need to be URL/percent-encoded instead return "file://" + self.getsyspath(path) def gettype(self, path): self.check() - path = path and fsdecode(path) or path - sys_path = self._to_sys_path(path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) with convert_os_errors('gettype', path): stat = os.stat(sys_path) resource_type = self._get_type_from_stat(stat) @@ -393,9 +389,7 @@ def gettype(self, path): def islink(self, path): self.check() - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) if not self.exists(path): raise errors.ResourceNotFound(path) with convert_os_errors('islink', path): @@ -411,11 +405,10 @@ def open(self, line_buffering=False, **options): _mode = Mode(mode) + validate_open_mode(mode) self.check() - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) with convert_os_errors('open', path): if six.PY2 and _mode.exclusive and self.exists(path): raise FileExists(path) @@ -432,9 +425,7 @@ def open(self, def setinfo(self, path, info): self.check() - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) if not os.path.exists(sys_path): raise errors.ResourceNotFound(path) if 'details' in info: @@ -452,9 +443,7 @@ def setinfo(self, path, info): def _scandir(self, path, namespaces=None): self.check() namespaces = namespaces or () - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) with convert_os_errors('scandir', path, directory=True): for dir_entry in scandir(sys_path): info = { @@ -498,9 +487,7 @@ def _scandir(self, path, namespaces=None): def _scandir(self, path, namespaces=None): self.check() namespaces = namespaces or () - path = path and fsdecode(path) or path - _path = self.validatepath(path) - sys_path = self._to_sys_path(_path, as_bytes=_NIX_PY2) + sys_path = self._get_validated_syspath(path) with convert_os_errors('scandir', path, directory=True): for entry_name in os.listdir(sys_path): entry_name_u = fsdecode(entry_name) From a48d13430fd8c3e4634835b7e3a20ec2df23e1b8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 21 Dec 2017 11:29:15 +0100 Subject: [PATCH 11/12] Remove as_bytes arg #120 * This was mistakenly left over * Remove as_bytes arg from getsyspath #120 Signed-off-by: Philippe Ombredanne --- fs/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index 7efc76e4..37bb99aa 100644 --- a/fs/base.py +++ b/fs/base.py @@ -599,7 +599,7 @@ def getsize(self, path): size = self.getdetails(path).size return size - def getsyspath(self, path, as_bytes=False): + def getsyspath(self, path): """Get the *system path* of a resource. Parameters: From 3504187ca7af3ca6e32defb0d399989e049fe008 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 21 Dec 2017 11:33:35 +0100 Subject: [PATCH 12/12] Pin backports.os with correct Python version #120 Signed-off-by: Philippe Ombredanne --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 32e3697f..5f1c61c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ enum34==1.1.6 ; python_version < '3.4' pytz setuptools six==1.10.0 -backports.os +backports.os=0.1.1 diff --git a/setup.py b/setup.py index 0be05f5d..6019dad2 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ extras_require={ "scandir :python_version < '3.5'": ['scandir~=1.5'], ":python_version < '3.4'": ['enum34~=1.1.6'], - ":python_version < '3.0'": ['backports.os'], + ":python_version < '3.2'": ['backports.os~=0.1.1'], }, entry_points={'fs.opener': [ 'ftp = fs.opener.ftpfs:FTPOpener',