Merge pull request #12 from sarugaku/fix-windows-normalization

Fix path normalization and location determination

.travis.yml

@@ -11,7 +11,7 @@ env:
     PIP_NO_INPUT=1
 
 install:
-  - "python -m pip install --upgrade pip pytest-timeout"
+  - "python -m pip install --upgrade pip pytest-timeout setuptools"
   - "python -m pip install -e .[tests]"
 script:
   - "python -m pytest -v -n 8 tests/"
@@ -35,7 +35,7 @@ jobs:
     - stage: coverage
       python: "3.6"
       install:
-        - "python -m pip install --upgrade pip pytest-timeout pytest-cov"
+        - "python -m pip install --upgrade pip setuptools pytest-timeout pytest-cov"
         - "python -m pip install --upgrade -e .[tests]"
       script:
         - "python -m pytest -n auto --timeout 300 --cov=mork --cov-report=term-missing --cov-report=xml --cov-report=html tests"

appveyor.yml

@@ -15,8 +15,8 @@ environment:
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "python --version"
-  - "python -m pip install --upgrade pip"
-  - "python -m pip install -e .[tests]"
+  - "python -m pip install --upgrade pip setuptools pytest-timeout pytest-xdist"
+  - "python -m pip install -e .[tests,virtualenv]"
 
 build: off
 

news/11.bugfix.rst

@@ -0,0 +1 @@
+Fixed a bug with path locations which caused cross platform virtualenv operations to fail on windows.

src/mork/virtualenv.py

@@ -11,6 +11,7 @@
 import sys
 import sysconfig
 
+import distlib.scripts
 import distlib.wheel
 import six
 
@@ -26,6 +27,7 @@ def __init__(self, venv_dir):
         self.base_working_set = pkg_resources.WorkingSet(sys_module.path)
         own_dist = pkg_resources.get_distribution(pkg_resources.Requirement("mork"))
         self.extra_dists = self.resolve_dist(own_dist, self.base_working_set)
+        self._modules = {'pkg_resources': pkg_resources}
         try:
             self.recursive_monkey_patch = self.safe_import("recursive_monkey_patch")
         except ImportError:
@@ -151,6 +153,38 @@ def resolve_dist(cls, dist, working_set):
             deps |= cls.resolve_dist(dist, working_set)
         return deps
 
+    @property
+    def pyversion(self):
+        include_dir = self.venv_dir / "include"
+        python_path = next(iter(list(include_dir.iterdir())), None)
+        if python_path and python_path.name.startswith("python"):
+            python_version = python_path.name.replace("python", "")
+            py_version_short, abiflags = python_version[:3], python_version[3:]
+            return {"py_version_short": py_version_short, "abiflags": abiflags}
+        return {}
+
+    @cached_property
+    def base_paths(self):
+        if "sysconfig" not in self._modules:
+            self._modules["sysconfig"] = self.safe_import("sysconfig")
+        sysconfig = self._modules["sysconfig"]
+        prefix = self.venv_dir.as_posix()
+        scheme = sysconfig._get_default_scheme()
+        config = {
+            "base": prefix,
+            "installed_base": prefix,
+            "platbase": prefix,
+            "installed_platbase": prefix
+        }
+        config.update(self.pyversion)
+        paths = {
+            k: v.format(**config)
+            for k, v in sysconfig._INSTALL_SCHEMES[scheme].items()
+        }
+        if "prefix" not in paths:
+            paths["prefix"] = prefix
+        return paths
+
     @cached_property
     def script_basedir(self):
         """Path to the virtualenv scripts dir"""
@@ -193,7 +227,7 @@ def sys_prefix(self):
 
         command = [self.python, "-c" "import sys; print(sys.prefix)"]
         c = vistir.misc.run(command, return_object=True, block=True, nospin=True)
-        sys_prefix = vistir.misc.to_text(c.out).strip()
+        sys_prefix = vistir.compat.Path(vistir.misc.to_text(c.out).strip()).as_posix()
         return sys_prefix
 
     @cached_property
@@ -204,19 +238,23 @@ def paths(self):
             os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8")
             os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1")
             sysconfig = self.safe_import("sysconfig")
-            scheme, _, _ = sysconfig._get_default_scheme().partition('_')
-            scheme = "{0}_user".format(scheme)
-            paths = sysconfig.get_paths(scheme=scheme)
-        if "prefix" not in paths:
-            paths["prefix"] = self.venv_dir.as_posix()
-        if "headers" not in paths:
-            paths["headers"] = paths["include"]
+            self._modules["sysconfig"] = sysconfig
+            paths = self.base_paths
+            if "headers" not in paths:
+                paths["headers"] = paths["include"]
         return paths
 
     @property
     def scripts_dir(self):
         return self.paths["scripts"]
 
+    @property
+    def libdir(self):
+        purelib = self.paths.get("purelib", None)
+        if purelib and os.path.exists(purelib):
+            return "purelib", purelib
+        return "platlib", self.paths["platlib"]
+
     @cached_property
     def initial_working_set(self):
         sysconfig = self.safe_import("sysconfig")
@@ -269,13 +307,13 @@ def get_setup_install_args(self, pkgname, setup_py, develop=False):
         :rtype: list
         """
 
-        headers = vistir.compat.Path(self.sys_prefix) / "include" / "site"
+        headers = self.paths["headers"]
         headers = headers / "python{0}".format(self.python_version) / pkgname
         install_arg = "install" if not develop else "develop"
         return [
             self.python, "-u", "-c", SETUPTOOLS_SHIM % setup_py, install_arg,
             "--single-version-externally-managed",
-            "--install-headers={0}".format(headers.as_posix()),
+            "--install-headers={0}".format(self.paths["headers"]),
             "--install-purelib={0}".format(self.paths["purelib"]),
             "--install-platlib={0}".format(self.paths["platlib"]),
             "--install-scripts={0}".format(self.scripts_dir),
@@ -309,15 +347,14 @@ def install(self, req, editable=False, sources=[]):
         :rtype: int
         """
 
-        with self.activated():
-            try:
-                packagebuilder = self.safe_import("packagebuilder")
-            except ImportError:
-                packagebuilder = None
+        try:
+            packagebuilder = self.safe_import("packagebuilder")
+        except ImportError:
+            packagebuilder = None
+        with self.activated(include_extras=False):
             if not packagebuilder:
                 return 2
             ireq = req.as_ireq()
-            distlib_scripts = self.safe_import("distlib.scripts")
             sources = self.filter_sources(req, sources)
             cache_dir = os.environ.get('PASSA_CACHE_DIR',
                 os.environ.get(
@@ -327,7 +364,8 @@ def install(self, req, editable=False, sources=[]):
             )
             built = packagebuilder.build.build(ireq, sources, cache_dir)
             if isinstance(built, distlib.wheel.Wheel):
-                built.install(self.paths, distlib_scripts.ScriptMaker(None, None))
+                maker = distlib.scripts.ScriptMaker(None, None)
+                built.install(self.paths, maker)
             else:
                 path = vistir.compat.Path(built.path)
                 cd_path = path.parent
@@ -339,7 +377,7 @@ def install(self, req, editable=False, sources=[]):
             return 0
 
     @contextlib.contextmanager
-    def activated(self, extra_dists=[]):
+    def activated(self, include_extras=True, extra_dists=[]):
         """A context manager which activates the virtualenv.
 
         :param list extra_dists: Paths added to the context after the virtualenv is activated.
@@ -369,14 +407,15 @@ def activated(self, extra_dists=[]):
             os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(self.venv_dir.as_posix())
             sys.path = self.sys_path
             sys.prefix = self.sys_prefix
-            site = self.safe_import("site")
-            site.addsitedir(parent_path)
-            extra_dists = list(self.extra_dists) + extra_dists
-            for extra_dist in extra_dists:
-                if extra_dist not in self.get_working_set():
-                    extra_dist.activate(self.sys_path)
-            sys.modules["recursive_monkey_patch"] = self.recursive_monkey_patch
             pkg_resources = self.safe_import("pkg_resources")
+            if include_extras:
+                site = self.safe_import("site")
+                site.addsitedir(parent_path)
+                extra_dists = list(self.extra_dists) + extra_dists
+                for extra_dist in extra_dists:
+                    if extra_dist not in self.get_working_set():
+                        extra_dist.activate(self.sys_path)
+                sys.modules["recursive_monkey_patch"] = self.recursive_monkey_patch
             try:
                 yield
             finally:

tests/conftest.py

@@ -2,6 +2,7 @@
 
 import pytest
 import mork
+import os
 import sys
 import vistir
 
@@ -20,6 +21,10 @@ def virtualenv(tmpdir_factory):
 
 
 @pytest.fixture
-def tmpvenv(virtualenv):
+def tmpvenv(virtualenv, tmpdir):
     venv_path = vistir.compat.Path(virtualenv.strpath).as_posix()
-    return mork.virtualenv.VirtualEnv(venv_path)
+    with vistir.contextmanagers.temp_environ():
+        os.environ["PACKAGEBUILDER_CACHE_DIR"] = tmpdir.strpath
+        yield mork.virtualenv.VirtualEnv(venv_path)
+    if "PACKAGEBUILDER_CACHE_DIR" in os.environ:
+        del os.environ["PACKAGEBUILDER_CACHE_DIR"]

tests/test_virtualenv.py

@@ -42,8 +42,7 @@ def test_normpath(input_path, normalized, monkeypatch):
 
 
 def test_get_dists(tmpvenv):
-    dists = tmpvenv.get_distributions()
-    dist_names = [dist.project_name for dist in dists]
+    dist_names = [dist.project_name for dist in tmpvenv.get_distributions()]
     assert all(pkg in dist_names for pkg in ['setuptools', 'pip', 'wheel']), dist_names
 
 
@@ -69,27 +68,34 @@ def test_properties(tmpvenv):
     assert scripts_dir == vistir.compat.Path(tmpvenv.scripts_dir).as_posix()
     python = "{0}/python".format(tmpvenv.venv_dir.joinpath(script_basedir).as_posix())
     assert python == tmpvenv.python
+    libdir = "Lib" if os.name == "nt" else "lib"
+    libdir = tmpvenv.venv_dir.joinpath(libdir).as_posix().lower()
     assert any(
-        pth.startswith(tmpvenv.venv_dir.joinpath("lib").as_posix())
+        vistir.compat.Path(pth).as_posix().lower().startswith(libdir)
         for pth in tmpvenv.sys_path
-    )
+    ),  list(tmpvenv.sys_path)
     assert tmpvenv.sys_prefix == tmpvenv.venv_dir.as_posix()
 
 
 def test_install(tmpvenv):
     import requirementslib
     requests = requirementslib.Requirement.from_line("requests")
+    pytz = requirementslib.Requirement.from_line("pytz")
     retcode = tmpvenv.install(requests)
     assert retcode == 0
+    retcode = tmpvenv.install(pytz)
+    assert retcode == 0
     assert tmpvenv.is_installed("requests")
+    assert tmpvenv.is_installed("pytz")
     venv_dists = [dist for dist in tmpvenv.get_distributions()]
     venv_workingset = [dist for dist in tmpvenv.get_working_set()]
-    assert "requests" in [dist.project_name for dist in venv_dists]
-    assert "requests" in [dist.project_name for dist in venv_workingset]
+    expected = ["requests", "pytz"]
+    assert all(pkg in [dist.project_name for dist in venv_dists] for pkg in expected)
+    assert all(pkg in [dist.project_name for dist in venv_workingset] for pkg in expected)
     with tmpvenv.activated():
-        tmp_requests = tmpvenv.safe_import("requests")
-        requests_path = vistir.compat.Path(tmp_requests.__path__[0]).as_posix()
-        assert requests_path.startswith(tmpvenv.venv_dir.as_posix())
+        tmp_pytz = tmpvenv.safe_import("pytz")
+        pytz_path = vistir.compat.Path(tmp_pytz.__path__[0]).as_posix()
+        assert pytz_path.startswith(tmpvenv.venv_dir.as_posix()), pytz_path
 
 
 def test_uninstall(tmpvenv):
@@ -109,9 +115,13 @@ def uninstall(pkg):
         dist for dist in tmpvenv.get_distributions()
         if dist.project_name == "requests"), None
     )
-    requests_deps = tmpvenv.resolve_dist(requests_dist, tmpvenv.get_working_set())
+    requests_deps = [
+        dist for dist in tmpvenv.resolve_dist(requests_dist, tmpvenv.get_working_set())
+        if dist is not None
+    ]
     uninstalled_packages = []
     for dep in requests_deps:
-        uninstalled_packages.extend(uninstall(dep.project_name))
+        uninstalled = uninstall(dep.project_name)
+        uninstalled_packages.extend(uninstalled)
     assert uninstalled_packages
     assert all(pkg.project_name in uninstalled_packages for pkg in requests_deps)