Merge pull request #63 from mcmtroffaes/feature/with_name_unicode

Fix some unicode bugs on Python 2.

* Fix unicode bugs in with_name and with_suffix.
* Allow unicode encoding of file paths on systems that support it.
* Add github actions for regression testing.
* Fix mypy warnings.

.github/workflows/python-package.yml

@@ -0,0 +1,43 @@
+name: Python package
+
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ['ubuntu-latest']
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
+        lc-all: ['en_US.utf-8', 'en_US.ascii']
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 check-manifest pytest codecov coverage
+        python -m pip install .
+    - name: Install dependencies (Python 2.7)
+      run: python -m pip install mock
+      if: matrix.python-version == '2.7'
+    - name: Check manifest
+      run: check-manifest
+    - name: Lint with flake8
+      run: |
+        flake8 . --count --select=E9,F63,F7,F82 --show-source
+        flake8 . --count --exit-zero --max-complexity=10
+    - name: Test with pytest
+      run: |
+        LC_ALL=${{ matrix.lc-all}} python -c "import sys; print(sys.getfilesystemencoding())"
+        LC_ALL=${{ matrix.lc-all}} coverage run -m pytest
+        codecov

MANIFEST.in

@@ -4,6 +4,8 @@ recursive-include tests *.py
 include *.rst
 include VERSION
 include requirements.txt
+include mypy.ini
+include pytest.ini
 exclude .travis.yml
 exclude appveyor.yml
 exclude codecov.yml

appveyor.yml

@@ -1,15 +1,13 @@
-environment:
+image: Visual Studio 2019
 
+environment:
   matrix:
     - PYTHON: "C:\\Python27"
-
-    - PYTHON: "C:\\Python34"
-
     - PYTHON: "C:\\Python35"
-
     - PYTHON: "C:\\Python36"
-
     - PYTHON: "C:\\Python37"
+    - PYTHON: "C:\\Python38"
+    - PYTHON: "C:\\Python39"
 
 init:
   - "%PYTHON%/python --version"
@@ -20,11 +18,7 @@ install:
 build: off
 
 test_script:
-  - cd tests
-  - "%PYTHON%/Scripts/py.test --cov-report=xml --cov=pathlib2 ."
+  - "%PYTHON%/Scripts/coverage run -m pytest"
 
 after_test:
-  - ps: |
-      $env:PATH = 'C:\msys64\usr\bin;' + $env:PATH
-      Invoke-WebRequest -Uri 'https://codecov.io/bash' -OutFile codecov.sh
-      bash codecov.sh -f "coverage.xml"
+  - "%PYTHON%/Scripts/codecov"

appveyor/install.ps1

@@ -36,8 +36,10 @@ function InstallRequirements ($python_home, $reqs) {
 function main () {
     InstallPip $env:PYTHON
     InstallRequirements $env:PYTHON -r requirements.txt
-    InstallPackage $env:PYTHON pytest-cov
+    InstallPackage $env:PYTHON pytest
     InstallPackage $env:PYTHON unittest2
+    InstallPackage $env:PYTHON coverage
+    InstallPackage $env:PYTHON codecov
     InstallPackage $env:PYTHON .
 }
 

mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+files = pathlib2/*.py,tests/*.py,setup.py

pathlib2/__init__.py

@@ -20,25 +20,27 @@
     S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO)
 
 try:
-    from collections.abc import Sequence
+    from collections.abc import Sequence  # type: ignore
 except ImportError:
     from collections import Sequence
 
 try:
-    from urllib import quote as urlquote_from_bytes
+    from urllib import quote as urlquote_from_bytes  # type: ignore
 except ImportError:
-    from urllib.parse import quote_from_bytes as urlquote_from_bytes
+    from urllib.parse \
+        import quote_from_bytes as urlquote_from_bytes  # type: ignore
 
 
 try:
-    intern = intern
+    intern = intern  # type: ignore
 except NameError:
-    intern = sys.intern
+    intern = sys.intern  # type: ignore
 
 supports_symlinks = True
 if os.name == 'nt':
-    import nt
-    if sys.getwindowsversion()[:2] >= (6, 0) and sys.version_info >= (3, 2):
+    import nt  # type: ignore
+    if sys.getwindowsversion().major >= 6 \
+            and sys.version_info >= (3, 2):  # type: ignore
         from nt import _getfinalpathname
     else:
         supports_symlinks = False
@@ -47,9 +49,9 @@
     nt = None
 
 try:
-    from os import scandir as os_scandir
+    from os import scandir as os_scandir  # type: ignore
 except ImportError:
-    from scandir import scandir as os_scandir
+    from scandir import scandir as os_scandir  # type: ignore
 
 __all__ = [
     "PurePath", "PurePosixPath", "PureWindowsPath",
@@ -76,15 +78,15 @@ def _ignore_error(exception):
 def _py2_fsencode(parts):
     # py2 => minimal unicode support
     assert six.PY2
-    return [part.encode('ascii') if isinstance(part, six.text_type)
-            else part for part in parts]
+    return [part.encode(sys.getfilesystemencoding() or 'ascii')
+            if isinstance(part, six.text_type) else part for part in parts]
 
 
 def _try_except_fileexistserror(try_func, except_func, else_func=None):
     if sys.version_info >= (3, 3):
         try:
             try_func()
-        except FileExistsError as exc:
+        except FileExistsError as exc:  # noqa: F821
             except_func(exc)
         else:
             if else_func is not None:
@@ -106,7 +108,7 @@ def _try_except_filenotfounderror(try_func, except_func):
     if sys.version_info >= (3, 3):
         try:
             try_func()
-        except FileNotFoundError as exc:
+        except FileNotFoundError as exc:  # noqa: F821
             except_func(exc)
     elif os.name != 'nt':
         try:
@@ -139,7 +141,7 @@ def _try_except_permissionerror_iter(try_iter, except_iter):
         try:
             for x in try_iter():
                 yield x
-        except PermissionError as exc:
+        except PermissionError as exc:  # noqa: F821
             for x in except_iter(exc):
                 yield x
     else:
@@ -203,7 +205,7 @@ class BY_HANDLE_FILE_INFORMATION(Structure):
                        None, OPEN_EXISTING, flags, None)
     if hfile == 0xffffffff:
         if sys.version_info >= (3, 3):
-            raise FileNotFoundError(path)
+            raise FileNotFoundError(path)  # noqa: F821
         else:
             exc = OSError("file not found: path")
             exc.errno = ENOENT
@@ -577,19 +579,21 @@ class _Accessor:
     accessing paths on the filesystem."""
 
 
-class _NormalAccessor(_Accessor):
+def _wrap_strfunc(strfunc):
+    @functools.wraps(strfunc)
+    def wrapped(pathobj, *args):
+        return strfunc(str(pathobj), *args)
+    return staticmethod(wrapped)
+
 
-    def _wrap_strfunc(strfunc):
-        @functools.wraps(strfunc)
-        def wrapped(pathobj, *args):
-            return strfunc(str(pathobj), *args)
-        return staticmethod(wrapped)
+def _wrap_binary_strfunc(strfunc):
+    @functools.wraps(strfunc)
+    def wrapped(pathobjA, pathobjB, *args):
+        return strfunc(str(pathobjA), str(pathobjB), *args)
+    return staticmethod(wrapped)
 
-    def _wrap_binary_strfunc(strfunc):
-        @functools.wraps(strfunc)
-        def wrapped(pathobjA, pathobjB, *args):
-            return strfunc(str(pathobjA), str(pathobjB), *args)
-        return staticmethod(wrapped)
+
+class _NormalAccessor(_Accessor):
 
     stat = _wrap_strfunc(os.stat)
 
@@ -624,6 +628,7 @@ def lchmod(self, pathobj, mode):
         if supports_symlinks:
             symlink = _wrap_binary_strfunc(os.symlink)
         else:
+            @staticmethod
             def symlink(a, b, target_is_directory):
                 raise NotImplementedError(
                     "symlink() not available on this system")
@@ -663,7 +668,7 @@ def _make_selector(pattern_parts):
 
 
 if hasattr(functools, "lru_cache"):
-    _make_selector = functools.lru_cache()(_make_selector)
+    _make_selector = functools.lru_cache()(_make_selector)  # type: ignore
 
 
 class _Selector:
@@ -1077,7 +1082,7 @@ def with_name(self, name):
                 or drv or root or len(parts) != 1):
             raise ValueError("Invalid name %r" % (name))
         return self._from_parsed_parts(self._drv, self._root,
-                                       self._parts[:-1] + [name])
+                                       self._parts[:-1] + parts[-1:])
 
     def with_suffix(self, suffix):
         """Return a new path with the file suffix changed.  If the path
@@ -1090,6 +1095,12 @@ def with_suffix(self, suffix):
             raise ValueError("Invalid suffix %r" % (suffix))
         if suffix and not suffix.startswith('.') or suffix == '.':
             raise ValueError("Invalid suffix %r" % (suffix))
+
+        if (six.PY2 and not isinstance(suffix, str)
+                and isinstance(suffix, six.text_type)):
+            # see _parse_args() above
+            suffix = suffix.encode(sys.getfilesystemencoding() or "ascii")
+
         name = self.name
         if not name:
             raise ValueError("%r has an empty name" % (self,))

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+testpaths =
+  tests

setup.py

@@ -31,12 +31,12 @@ def readfile(filename):
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Software Development :: Libraries',
         'Topic :: System :: Filesystems',
         ],

tests/test_pathlib2.py

@@ -37,15 +37,23 @@
 
 # assertRaisesRegex is missing prior to Python 3.2
 if sys.version_info < (3, 2):
-    unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
+    unittest.TestCase.assertRaisesRegex = \
+        unittest.TestCase.assertRaisesRegexp  # type: ignore
 
 try:
-    from test import support
+    from test import support  # type: ignore
 except ImportError:
-    from test import test_support as support
+    from test import test_support as support  # type: ignore
 
 android_not_root = getattr(support, "android_not_root", False)
 
+try:
+    six.u('\u00e4').encode(sys.getfilesystemencoding() or "ascii")
+except UnicodeEncodeError:
+    fs_ascii_encoding_only = True
+else:
+    fs_ascii_encoding_only = False
+
 TESTFN = support.TESTFN
 
 # work around broken support.rmtree on Python 3.3 on Windows
@@ -58,7 +66,7 @@
     import grp
     import pwd
 except ImportError:
-    grp = pwd = None
+    grp = pwd = None  # type: ignore
 
 # support.can_symlink is missing prior to Python 3
 if six.PY2:
@@ -656,6 +664,13 @@ def test_with_name_common(self):
         self.assertRaises(ValueError, P('a/b').with_name, 'c/')
         self.assertRaises(ValueError, P('a/b').with_name, 'c/d')
 
+    @unittest.skipIf(fs_ascii_encoding_only, "filesystem only supports ascii")
+    def test_with_name_common_unicode(self):
+        P = self.cls
+        self.assertEqual(P('a/b').with_name(six.u('d.xml')), P('a/d.xml'))
+        self.assertEqual(
+             P('a/b').with_name(six.u('\u00e4.xml')), P(six.u('a/\u00e4.xml')))
+
     def test_with_suffix_common(self):
         P = self.cls
         self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz'))
@@ -679,6 +694,13 @@ def test_with_suffix_common(self):
         self.assertRaises(ValueError, P('a/b').with_suffix, './.d')
         self.assertRaises(ValueError, P('a/b').with_suffix, '.d/.')
 
+    @unittest.skipIf(fs_ascii_encoding_only, "filesystem only supports ascii")
+    def test_with_suffix_common_unicode(self):
+        P = self.cls
+        self.assertEqual(P('a/b').with_suffix(six.u('.gz')), P('a/b.gz'))
+        self.assertEqual(
+            P('a/b').with_suffix(six.u('.\u00e4')), P(six.u('a/b.\u00e4')))
+
     def test_relative_to_common(self):
         P = self.cls
         p = P('a/b')
@@ -760,7 +782,7 @@ def test_as_uri(self):
 
     @with_fsencode
     def test_as_uri_non_ascii(self):
-        from urllib.parse import quote_from_bytes
+        from urllib.parse import quote_from_bytes  # type: ignore
         P = self.cls
         try:
             os.fsencode('\xe9')
@@ -1391,7 +1413,7 @@ def assertSame(self, path_a, path_b):
 
     def assertFileNotFound(self, func, *args, **kwargs):
         if sys.version_info >= (3, 3):
-            with self.assertRaises(FileNotFoundError) as cm:
+            with self.assertRaises(FileNotFoundError) as cm:  # noqa: F821
                 func(*args, **kwargs)
         else:
             with self.assertRaises(OSError) as cm:
@@ -1404,7 +1426,7 @@ def assertFileNotFound(self, func, *args, **kwargs):
 
     def assertFileExists(self, func, *args, **kwargs):
         if sys.version_info >= (3, 3):
-            with self.assertRaises(FileExistsError) as cm:
+            with self.assertRaises(FileExistsError) as cm:  # noqa: F821
                 func(*args, **kwargs)
         else:
             with self.assertRaises(OSError) as cm: