Ignore errors copying socket files for source installs.

news/5306.bugfix

@@ -0,0 +1 @@
+Ignore errors copying socket files for local source installs.

src/pip/_internal/download.py

@@ -32,7 +32,7 @@
 # Import ssl from compat so the initial import occurs in only one place.
 from pip._internal.utils.compat import HAS_TLS, ssl
 from pip._internal.utils.encoding import auto_decode
-from pip._internal.utils.filesystem import check_path_owner
+from pip._internal.utils.filesystem import check_path_owner, copytree
 from pip._internal.utils.glibc import libc_ver
 from pip._internal.utils.marker_files import write_delete_marker_file
 from pip._internal.utils.misc import (
@@ -46,6 +46,7 @@
     display_path,
     format_size,
     get_installed_version,
+    path_to_display,
     path_to_url,
     remove_auth_from_url,
     rmtree,
@@ -60,7 +61,7 @@
 
 if MYPY_CHECK_RUNNING:
     from typing import (
-        Optional, Tuple, Dict, IO, Text, Union
+        Dict, IO, Optional, Text, Tuple, Union
     )
     from optparse import Values
     from pip._internal.models.link import Link
@@ -954,6 +955,47 @@ def unpack_http_url(
             os.unlink(from_path)
 
 
+def _copy_source_tree(source, target):
+    # type: (str, str) -> None
+    def ignore(d, names):
+        # Pulling in those directories can potentially be very slow,
+        # exclude the following directories if they appear in the top
+        # level dir (and only it).
+        # See discussion at https://github.com/pypa/pip/pull/6770
+        return ['.tox', '.nox'] if d == source else []
+
+    def ignore_special_file_errors(error_details):
+        # Copying special files is not supported, so we skip errors related to
+        # them. This is a convenience to support users that may have tools
+        # creating e.g. socket files in their source directory.
+        src, dest, error = error_details
+
+        if not isinstance(error, shutil.SpecialFileError):
+            # Then it is some other kind of error that we do want to report.
+            return True
+
+        # SpecialFileError may be raised due to either the source or
+        # destination. If the destination was the cause then we would actually
+        # care, but since the destination directory is deleted prior to
+        # copy we ignore all of them assuming it is caused by the source.
+        logger.warning(
+            "Ignoring special file error '%s' encountered copying %s to %s.",
+            str(error),
+            path_to_display(src),
+            path_to_display(dest),
+        )
+
+    try:
+        copytree(source, target, ignore=ignore, symlinks=True)
+    except shutil.Error as e:
+        errors = e.args[0]
+        normal_file_errors = list(
+            filter(ignore_special_file_errors, errors)
+        )
+        if normal_file_errors:
+            raise shutil.Error(normal_file_errors)
+
+
 def unpack_file_url(
     link,  # type: Link
     location,  # type: str
@@ -969,21 +1011,9 @@ def unpack_file_url(
     link_path = url_to_path(link.url_without_fragment)
     # If it's a url to a local directory
     if is_dir_url(link):
-
-        def ignore(d, names):
-            # Pulling in those directories can potentially be very slow,
-            # exclude the following directories if they appear in the top
-            # level dir (and only it).
-            # See discussion at https://github.com/pypa/pip/pull/6770
-            return ['.tox', '.nox'] if d == link_path else []
-
         if os.path.isdir(location):
             rmtree(location)
-        shutil.copytree(link_path,
-                        location,
-                        symlinks=True,
-                        ignore=ignore)
-
+        _copy_source_tree(link_path, location)
         if download_dir:
             logger.info('Link is a directory, ignoring download_dir')
         return

src/pip/_internal/utils/filesystem.py

@@ -1,5 +1,7 @@
 import os
 import os.path
+import shutil
+import stat
 
 from pip._internal.utils.compat import get_path_uid
 
@@ -28,3 +30,37 @@ def check_path_owner(path):
         else:
             previous, path = path, os.path.dirname(path)
     return False  # assume we don't own the path
+
+
+def copytree(*args, **kwargs):
+    """Wrap shutil.copytree() to map errors copying socket file to
+    SpecialFileError.
+
+    See also https://bugs.python.org/issue37700.
+    """
+    def to_correct_error(src, dest, error):
+        for f in [src, dest]:
+            try:
+                if is_socket(f):
+                    new_error = shutil.SpecialFileError("`%s` is a socket" % f)
+                    return (src, dest, new_error)
+            except OSError:
+                # An error has already occurred. Another error here is not
+                # a problem and we can ignore it.
+                pass
+
+        return (src, dest, error)
+
+    try:
+        shutil.copytree(*args, **kwargs)
+    except shutil.Error as e:
+        errors = e.args[0]
+        new_errors = [
+            to_correct_error(src, dest, error) for src, dest, error in errors
+        ]
+        raise shutil.Error(new_errors)
+
+
+def is_socket(path):
+    # type: (str) -> bool
+    return stat.S_ISSOCK(os.lstat(path).st_mode)

tests/functional/test_install.py

@@ -1,6 +1,7 @@
 import distutils
 import glob
 import os
+import shutil
 import sys
 import textwrap
 from os.path import curdir, join, pardir
@@ -23,6 +24,7 @@
     pyversion_tuple,
     requirements_file,
 )
+from tests.lib.filesystem import make_socket_file
 from tests.lib.local_repos import local_checkout
 from tests.lib.path import Path
 
@@ -488,6 +490,29 @@ def test_install_from_local_directory_with_symlinks_to_directories(
     assert egg_info_folder in result.files_created, str(result)
 
 
+@pytest.mark.skipif("sys.platform == 'win32'")
+def test_install_from_local_directory_with_socket_file(script, data, tmpdir):
+    """
+    Test installing from a local directory containing a socket file.
+    """
+    egg_info_file = (
+        script.site_packages / "FSPkg-0.1.dev0-py%s.egg-info" % pyversion
+    )
+    package_folder = script.site_packages / "fspkg"
+    to_copy = data.packages.joinpath("FSPkg")
+    to_install = tmpdir.joinpath("src")
+
+    shutil.copytree(to_copy, to_install)
+    # Socket file, should be ignored.
+    socket_file_path = os.path.join(to_install, "example")
+    make_socket_file(socket_file_path)
+
+    result = script.pip("install", "--verbose", to_install, expect_error=False)
+    assert package_folder in result.files_created, str(result.stdout)
+    assert egg_info_file in result.files_created, str(result)
+    assert str(socket_file_path) in result.stderr
+
+
 def test_install_from_local_directory_with_no_setup_py(script, data):
     """
     Test installing from a local directory with no 'setup.py'.

tests/lib/filesystem.py

@@ -0,0 +1,47 @@
+"""Helpers for filesystem-dependent tests.
+"""
+import os
+import socket
+import subprocess
+import sys
+from functools import partial
+from itertools import chain
+
+from .path import Path
+
+
+def make_socket_file(path):
+    # Socket paths are limited to 108 characters (sometimes less) so we
+    # chdir before creating it and use a relative path name.
+    cwd = os.getcwd()
+    os.chdir(os.path.dirname(path))
+    try:
+        sock = socket.socket(socket.AF_UNIX)
+        sock.bind(os.path.basename(path))
+    finally:
+        os.chdir(cwd)
+
+
+def make_unreadable_file(path):
+    Path(path).touch()
+    os.chmod(path, 0o000)
+    if sys.platform == "win32":
+        username = os.environ["USERNAME"]
+        # Remove "Read Data/List Directory" permission for current user, but
+        # leave everything else.
+        args = ["icacls", path, "/deny", username + ":(RD)"]
+        subprocess.check_call(args)
+
+
+def get_filelist(base):
+    def join(dirpath, dirnames, filenames):
+        relative_dirpath = os.path.relpath(dirpath, base)
+        join_dirpath = partial(os.path.join, relative_dirpath)
+        return chain(
+            (join_dirpath(p) for p in dirnames),
+            (join_dirpath(p) for p in filenames),
+        )
+
+    return set(chain.from_iterable(
+        join(*dirinfo) for dirinfo in os.walk(base)
+    ))

tests/unit/test_download.py

@@ -1,6 +1,7 @@
 import functools
 import hashlib
 import os
+import shutil
 import sys
 from io import BytesIO
 from shutil import copy, rmtree
@@ -15,6 +16,7 @@
     MultiDomainBasicAuth,
     PipSession,
     SafeFileCache,
+    _copy_source_tree,
     _download_http_url,
     _get_url_scheme,
     parse_content_disposition,
@@ -28,6 +30,12 @@
 from pip._internal.utils.hashes import Hashes
 from pip._internal.utils.misc import path_to_url
 from tests.lib import create_file
+from tests.lib.filesystem import (
+    get_filelist,
+    make_socket_file,
+    make_unreadable_file,
+)
+from tests.lib.path import Path
 
 
 @pytest.fixture(scope="function")
@@ -334,6 +342,85 @@ def test_url_to_path_path_to_url_symmetry_win():
     assert url_to_path(path_to_url(unc_path)) == unc_path
 
 
+@pytest.fixture
+def clean_project(tmpdir_factory, data):
+    tmpdir = Path(str(tmpdir_factory.mktemp("clean_project")))
+    new_project_dir = tmpdir.joinpath("FSPkg")
+    path = data.packages.joinpath("FSPkg")
+    shutil.copytree(path, new_project_dir)
+    return new_project_dir
+
+
+def test_copy_source_tree(clean_project, tmpdir):
+    target = tmpdir.joinpath("target")
+    expected_files = get_filelist(clean_project)
+    assert len(expected_files) == 3
+
+    _copy_source_tree(clean_project, target)
+
+    copied_files = get_filelist(target)
+    assert expected_files == copied_files
+
+
+@pytest.mark.skipif("sys.platform == 'win32'")
+def test_copy_source_tree_with_socket(clean_project, tmpdir, caplog):
+    target = tmpdir.joinpath("target")
+    expected_files = get_filelist(clean_project)
+    socket_path = str(clean_project.joinpath("aaa"))
+    make_socket_file(socket_path)
+
+    _copy_source_tree(clean_project, target)
+
+    copied_files = get_filelist(target)
+    assert expected_files == copied_files
+
+    # Warning should have been logged.
+    assert len(caplog.records) == 1
+    record = caplog.records[0]
+    assert record.levelname == 'WARNING'
+    assert socket_path in record.message
+
+
+@pytest.mark.skipif("sys.platform == 'win32'")
+def test_copy_source_tree_with_socket_fails_with_no_socket_error(
+    clean_project, tmpdir
+):
+    target = tmpdir.joinpath("target")
+    expected_files = get_filelist(clean_project)
+    make_socket_file(clean_project.joinpath("aaa"))
+    unreadable_file = clean_project.joinpath("bbb")
+    make_unreadable_file(unreadable_file)
+
+    with pytest.raises(shutil.Error) as e:
+        _copy_source_tree(clean_project, target)
+
+    errored_files = [err[0] for err in e.value.args[0]]
+    assert len(errored_files) == 1
+    assert unreadable_file in errored_files
+
+    copied_files = get_filelist(target)
+    # All files without errors should have been copied.
+    assert expected_files == copied_files
+
+
+def test_copy_source_tree_with_unreadable_dir_fails(clean_project, tmpdir):
+    target = tmpdir.joinpath("target")
+    expected_files = get_filelist(clean_project)
+    unreadable_file = clean_project.joinpath("bbb")
+    make_unreadable_file(unreadable_file)
+
+    with pytest.raises(shutil.Error) as e:
+        _copy_source_tree(clean_project, target)
+
+    errored_files = [err[0] for err in e.value.args[0]]
+    assert len(errored_files) == 1
+    assert unreadable_file in errored_files
+
+    copied_files = get_filelist(target)
+    # All files without errors should have been copied.
+    assert expected_files == copied_files
+
+
 class Test_unpack_file_url(object):
 
     def prep(self, tmpdir, data):