Fix integration tests on Windows (#4769)

appveyor.yml

@@ -1,16 +1,36 @@
 environment:
   matrix:
+    # Unit and integration tests. 
     - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python33"
-    - PYTHON: "C:\\Python34"
-    - PYTHON: "C:\\Python35"
+      RUN_INTEGRATION_TESTS: "True"
+    - PYTHON: "C:\\Python36-x64"
+      RUN_INTEGRATION_TESTS: "True"
+    # Unit tests only.
     - PYTHON: "C:\\Python27-x64"
+    - PYTHON: "C:\\Python33"
     - PYTHON: "C:\\Python33-x64"
+    - PYTHON: "C:\\Python34"
     - PYTHON: "C:\\Python34-x64"
+    - PYTHON: "C:\\Python35"
     - PYTHON: "C:\\Python35-x64"
+    - PYTHON: "C:\\Python36"
 
 install:
-    cmd: "%PYTHON%\\python.exe -m pip install tox"
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - "python --version"
+  - "pip install certifi tox"
+  # Fix git SSL errors.
+  - "python -m certifi >cacert.txt"
+  - "set /p GIT_SSL_CAINFO=<cacert.txt"
+  - "set GIT_SSL_CAINFO"
+
 build: off
+
 test_script:
-    - "%PYTHON%\\Scripts\\tox.exe -e py -- -m unit -n 8"
+  # Shorten paths, workaround https://bugs.python.org/issue18199
+  - "subst T: %TEMP%"
+  - "set TEMP=T:\\"
+  - "set TMP=T:\\"
+  - "tox -e py -- -m unit -n 3"
+  - "if \"%RUN_INTEGRATION_TESTS%\" == \"True\" (
+    tox -e py -- -m integration -n 3 --duration=5 )"

src/pip/_internal/req/req_install.py

@@ -945,7 +945,7 @@ def move_wheel_files(self, wheeldir, root=None, prefix=None,
 
     def get_dist(self):
         """Return a pkg_resources.Distribution built from self.egg_info_path"""
-        egg_info = self.egg_info_path('').rstrip('/')
+        egg_info = self.egg_info_path('').rstrip(os.path.sep)
         base_dir = os.path.dirname(egg_info)
         metadata = pkg_resources.PathMetadata(base_dir, egg_info)
         dist_name = os.path.splitext(os.path.basename(egg_info))[0]

tests/conftest.py

@@ -77,8 +77,6 @@ def isolate(tmpdir):
     We use an autouse function scoped fixture because we want to ensure that
     every test has it's own isolated home directory.
     """
-    # TODO: Ensure Windows will respect $HOME, including for the cache
-    #       directory
 
     # TODO: Figure out how to isolate from *system* level configuration files
     #       as well as user level configuration files.
@@ -91,21 +89,36 @@ def isolate(tmpdir):
     fake_root = os.path.join(str(tmpdir), "fake-root")
     os.makedirs(fake_root)
 
-    # Set our home directory to our temporary directory, this should force all
-    # of our relative configuration files to be read from here instead of the
-    # user's actual $HOME directory.
-    os.environ["HOME"] = home_dir
-
-    # Isolate ourselves from XDG directories
-    os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share")
-    os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config")
-    os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache")
-    os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime")
-    os.environ["XDG_DATA_DIRS"] = ":".join([
-        os.path.join(fake_root, "usr", "local", "share"),
-        os.path.join(fake_root, "usr", "share"),
-    ])
-    os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg")
+    if sys.platform == 'win32':
+        # Note: this will only take effect in subprocesses...
+        home_drive, home_path = os.path.splitdrive(home_dir)
+        os.environ.update({
+            'USERPROFILE': home_dir,
+            'HOMEDRIVE': home_drive,
+            'HOMEPATH': home_path,
+        })
+        for env_var, sub_path in (
+            ('APPDATA', 'AppData/Roaming'),
+            ('LOCALAPPDATA', 'AppData/Local'),
+        ):
+            path = os.path.join(home_dir, *sub_path.split('/'))
+            os.environ[env_var] = path
+            os.makedirs(path)
+    else:
+        # Set our home directory to our temporary directory, this should force
+        # all of our relative configuration files to be read from here instead
+        # of the user's actual $HOME directory.
+        os.environ["HOME"] = home_dir
+        # Isolate ourselves from XDG directories
+        os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share")
+        os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config")
+        os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache")
+        os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime")
+        os.environ["XDG_DATA_DIRS"] = ":".join([
+            os.path.join(fake_root, "usr", "local", "share"),
+            os.path.join(fake_root, "usr", "share"),
+        ])
+        os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg")
 
     # Configure git, because without an author name/email git will complain
     # and cause test failures.
@@ -116,6 +129,7 @@ def isolate(tmpdir):
     # We want to disable the version check from running in the tests
     os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true"
 
+    # FIXME: Windows...
     os.makedirs(os.path.join(home_dir, ".config", "git"))
     with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp:
         fp.write(
@@ -143,6 +157,15 @@ def virtualenv_template(tmpdir_factory):
         pip_source_dir=pip_src,
         relocatable=True,
     )
+    if sys.platform == 'win32':
+        # Work around setuptools' easy_install.exe
+        # not working properly after relocation.
+        for exe in os.listdir(venv.bin):
+            if exe.startswith('easy_install'):
+                (venv.bin / exe).remove()
+        with open(venv.bin / 'easy_install.bat', 'w') as fp:
+            fp.write('python.exe -m easy_install %*\n')
+
     # Rename original virtualenv directory to make sure
     # it's not reused by mistake from one of the copies.
     venv_template = tmpdir / "venv_template"

tests/functional/test_download.py

@@ -93,7 +93,7 @@ def test_download_wheel_archive(script, data):
     It should download a wheel archive path
     """
     wheel_filename = 'colander-0.9.9-py2.py3-none-any.whl'
-    wheel_path = os.path.join(data.find_links, wheel_filename)
+    wheel_path = '/'.join((data.find_links, wheel_filename))
     result = script.pip(
         'download', wheel_path,
         '-d', '.', '--no-deps'
@@ -107,7 +107,7 @@ def test_download_should_download_wheel_deps(script, data):
     """
     wheel_filename = 'colander-0.9.9-py2.py3-none-any.whl'
     dep_filename = 'translationstring-1.1.tar.gz'
-    wheel_path = os.path.join(data.find_links, wheel_filename)
+    wheel_path = '/'.join((data.find_links, wheel_filename))
     result = script.pip(
         'download', wheel_path,
         '-d', '.', '--find-links', data.find_links, '--no-index'

tests/functional/test_freeze.py

@@ -6,7 +6,10 @@
 
 import pytest
 
-from tests.lib import _create_test_package, _create_test_package_with_srcdir
+from tests.lib import (
+    _create_test_package, _create_test_package_with_srcdir, need_bzr,
+    need_mercurial
+)
 
 distribute_re = re.compile('^distribute==[0-9.]+\n', re.MULTILINE)
 
@@ -319,7 +322,7 @@ def test_freeze_git_remote(script, tmpdir):
     _check_output(result.stdout, expected)
 
 
-@pytest.mark.mercurial
+@need_mercurial
 def test_freeze_mercurial_clone(script, tmpdir):
     """
     Test freezing a Mercurial clone.
@@ -361,7 +364,7 @@ def test_freeze_mercurial_clone(script, tmpdir):
     _check_output(result.stdout, expected)
 
 
-@pytest.mark.bzr
+@need_bzr
 def test_freeze_bazaar_clone(script, tmpdir):
     """
     Test freezing a Bazaar clone.
@@ -478,7 +481,7 @@ def test_freeze_with_requirement_option_multiple(script):
         simple2==1.0
     """)
     expected += "## The following requirements were added by pip freeze:"
-    expected += os.linesep + textwrap.dedent("""\
+    expected += '\n' + textwrap.dedent("""\
         ...meta==1.0...
     """)
     _check_output(result.stdout, expected)

tests/functional/test_install.py

@@ -1,3 +1,4 @@
+import distutils
 import glob
 import os
 import sys
@@ -8,11 +9,11 @@
 
 from pip._internal import pep425tags
 from pip._internal.status_codes import ERROR
-from pip._internal.utils import appdirs
 from pip._internal.utils.misc import rmtree
 from tests.lib import (
     _create_svn_repo, _create_test_package, create_test_package_with_setup,
-    path_to_url, pyversion, pyversion_tuple, requirements_file
+    need_bzr, need_mercurial, path_to_url, pyversion, pyversion_tuple,
+    requirements_file
 )
 from tests.lib.local_repos import local_checkout
 from tests.lib.path import Path
@@ -217,6 +218,7 @@ def test_install_editable_uninstalls_existing_from_path(script, data):
     assert simple_folder in result.files_deleted, str(result.stdout)
 
 
+@need_mercurial
 def test_install_editable_from_hg(script, tmpdir):
     """Test cloning from Mercurial."""
     pkg_path = _create_test_package(script, name='testpackage', vcs='hg')
@@ -225,6 +227,7 @@ def test_install_editable_from_hg(script, tmpdir):
     result.assert_installed('testpackage', with_files=['.hg'])
 
 
+@need_mercurial
 def test_vcs_url_final_slash_normalization(script, tmpdir):
     """
     Test that presence or absence of final slash in VCS URL is normalized.
@@ -235,7 +238,7 @@ def test_vcs_url_final_slash_normalization(script, tmpdir):
     result.assert_installed('testpackage', with_files=['.hg'])
 
 
-@pytest.mark.bzr
+@need_bzr
 def test_install_editable_from_bazaar(script, tmpdir):
     """Test checking out from Bazaar."""
     pkg_path = _create_test_package(script, name='testpackage', vcs='bazaar')
@@ -245,7 +248,7 @@ def test_install_editable_from_bazaar(script, tmpdir):
 
 
 @pytest.mark.network
-@pytest.mark.bzr
+@need_bzr
 def test_vcs_url_urlquote_normalization(script, tmpdir):
     """
     Test that urlquoted characters are normalized for repo URL comparison.
@@ -289,13 +292,14 @@ def test_install_relative_directory(script, data):
 
     # Compute relative install path to FSPkg from scratch path.
     full_rel_path = data.packages.join('FSPkg') - script.scratch_path
+    full_rel_url = (
+        'file:' + full_rel_path.replace(os.path.sep, '/') + '#egg=FSPkg'
+    )
     embedded_rel_path = script.scratch_path.join(full_rel_path)
 
     # For each relative path, install as either editable or not using either
     # URLs with egg links or not.
-    for req_path in (full_rel_path,
-                     'file:' + full_rel_path + '#egg=FSPkg',
-                     embedded_rel_path):
+    for req_path in (full_rel_path, full_rel_url, embedded_rel_path):
         # Regular install.
         result = script.pip('install', req_path,
                             cwd=script.scratch_path)
@@ -498,6 +502,7 @@ def test_install_using_install_option_and_editable(script, tmpdir):
 
 
 @pytest.mark.network
+@need_mercurial
 def test_install_global_option_using_editable(script, tmpdir):
     """
     Test using global distutils options, but in an editable installation
@@ -653,12 +658,10 @@ def test_install_package_with_prefix(script, data):
         '--no-binary', 'simple', '--no-index', 'simple==1.0',
     )
 
-    if hasattr(sys, "pypy_version_info"):
-        path = script.scratch / 'prefix'
-    else:
-        path = script.scratch / 'prefix' / 'lib' / 'python{0}'.format(pyversion)  # noqa
+    rel_prefix_path = script.scratch / 'prefix'
     install_path = (
-        path / 'site-packages' / 'simple-1.0-py{0}.egg-info'.format(pyversion)
+        distutils.sysconfig.get_python_lib(prefix=rel_prefix_path) /
+        'simple-1.0-py{0}.egg-info'.format(pyversion)
     )
     assert install_path in result.files_created, str(result)
 
@@ -673,8 +676,11 @@ def test_install_editable_with_prefix(script):
               version='0.1')
     """))
 
-    site_packages = os.path.join(
-        'prefix', 'lib', 'python{0}'.format(pyversion), 'site-packages')
+    if hasattr(sys, "pypy_version_info"):
+        site_packages = os.path.join(
+            'prefix', 'lib', 'python{0}'.format(pyversion), 'site-packages')
+    else:
+        site_packages = distutils.sysconfig.get_python_lib(prefix='prefix')
 
     # make sure target path is in PYTHONPATH
     pythonpath = script.scratch_path / site_packages
@@ -745,7 +751,7 @@ def test_url_req_case_mismatch_no_index(script, data):
     tests/data/packages contains Upper-1.0.tar.gz and Upper-2.0.tar.gz
     'requiresupper' has install_requires = ['upper']
     """
-    Upper = os.path.join(data.find_links, 'Upper-1.0.tar.gz')
+    Upper = '/'.join((data.find_links, 'Upper-1.0.tar.gz'))
     result = script.pip(
         'install', '--no-index', '-f', data.find_links, Upper, 'requiresupper'
     )
@@ -772,7 +778,7 @@ def test_url_req_case_mismatch_file_index(script, data):
     set of packages as it requires a prepared index.html file and
     subdirectory-per-package structure.
     """
-    Dinner = os.path.join(data.find_links3, 'dinner', 'Dinner-1.0.tar.gz')
+    Dinner = '/'.join((data.find_links3, 'dinner', 'Dinner-1.0.tar.gz'))
     result = script.pip(
         'install', '--index-url', data.find_links3, Dinner, 'requiredinner'
     )
@@ -964,6 +970,12 @@ def test_cleanup_after_failed_wheel(script, data, common_wheels):
 
 @pytest.mark.network
 def test_install_builds_wheels(script, data, common_wheels):
+    # We need to use a subprocess to get the right value on Windows.
+    res = script.run('python', '-c', (
+        'from pip._internal.utils import appdirs; '
+        'print(appdirs.user_cache_dir("pip"))'
+    ))
+    wheels_cache = os.path.join(res.stdout.rstrip('\n'), 'wheels')
     # NB This incidentally tests a local tree + tarball inputs
     # see test_install_editable_from_git_autobuild_wheel for editable
     # vcs coverage.
@@ -976,9 +988,8 @@ def test_install_builds_wheels(script, data, common_wheels):
                 " upper-2.0 wheelbroken-0.1")
     # Must have installed it all
     assert expected in str(res), str(res)
-    root = appdirs.user_cache_dir('pip')
     wheels = []
-    for top, dirs, files in os.walk(os.path.join(root, "wheels")):
+    for top, dirs, files in os.walk(wheels_cache):
         wheels.extend(files)
     # and built wheels for upper and wheelbroken
     assert "Running setup.py bdist_wheel for upper" in str(res), str(res)
@@ -1085,7 +1096,9 @@ def test_double_install(script):
     """
     Test double install passing with two same version requirements
     """
-    result = script.pip('install', 'pip', 'pip', expect_error=False)
+    result = script.pip('install', 'pip', 'pip',
+                        use_module=True,
+                        expect_error=False)
     msg = "Double requirement given: pip (already in pip, name='pip')"
     assert msg not in result.stderr
 
@@ -1115,7 +1128,7 @@ def test_install_incompatible_python_requires(script, common_wheels):
     )
     result = script.pip('install', pkga_path, expect_error=True)
     assert ("pkga requires Python '<1.0' "
-            "but the running Python is ") in result.stderr
+            "but the running Python is ") in result.stderr, str(result)
 
 
 def test_install_incompatible_python_requires_editable(script, common_wheels):
@@ -1134,7 +1147,7 @@ def test_install_incompatible_python_requires_editable(script, common_wheels):
     result = script.pip(
         'install', '--editable=%s' % pkga_path, expect_error=True)
     assert ("pkga requires Python '<1.0' "
-            "but the running Python is ") in result.stderr
+            "but the running Python is ") in result.stderr, str(result)
 
 
 @pytest.mark.network

tests/functional/test_install_cleanup.py

@@ -5,6 +5,7 @@
 
 from pip._internal.locations import write_delete_marker_file
 from pip._internal.status_codes import PREVIOUS_BUILD_DIR_ERROR
+from tests.lib import need_mercurial
 from tests.lib.local_repos import local_checkout
 
 
@@ -36,6 +37,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data):
 
 
 @pytest.mark.network
+@need_mercurial
 def test_cleanup_after_install_editable_from_hg(script, tmpdir):
     """
     Test clean up after cloning from Mercurial.