Align Windows 3.7 methodology and later with venv (#1976)

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
   hooks:
   - id: pyupgrade
 - repo: https://github.com/PyCQA/isort
-  rev: 5.5.4
+  rev: 5.6.3
   hooks:
   - id: isort
 - repo: https://github.com/ambv/black
@@ -36,11 +36,12 @@ repos:
   hooks:
   - id: rst-backticks
 - repo: https://github.com/tox-dev/tox-ini-fmt
-  rev: "0.2.0"
+  rev: "0.5.0"
   hooks:
   - id: tox-ini-fmt
+    args: ["-p", "fix_lint"]
 - repo: https://github.com/asottile/setup-cfg-fmt
-  rev: v1.11.0
+  rev: v1.15.0
   hooks:
   - id: setup-cfg-fmt
     args: [--min-py3-version, "3.4"]

docs/changelog/1782.bugfix.rst

@@ -0,0 +1,2 @@
+Align with venv module when creating virtual environments with builtin creator on Windows 3.7 and later
+- by :user:`gaborbernat`.

setup.cfg

@@ -5,7 +5,9 @@ long_description = file: README.md
 long_description_content_type = text/markdown
 url = https://virtualenv.pypa.io/
 author = Bernat Gabor
+author_email = gaborjbernat@gmail.com
 maintainer = Bernat Gabor
+maintainer_email = gaborjbernat@gmail.com
 license = MIT
 license_file = LICENSE
 platforms = any
@@ -24,14 +26,13 @@ classifiers =
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
     Topic :: Software Development :: Libraries
     Topic :: Software Development :: Testing
     Topic :: Utilities
-author-email = gaborjbernat@gmail.com
 keywords = virtual, environments, isolated
-maintainer-email = gaborjbernat@gmail.com
 project_urls =
     Source=https://github.com/pypa/virtualenv
     Tracker=https://github.com/pypa/virtualenv/issues

src/virtualenv/create/via_global_ref/builtin/cpython/common.py

@@ -6,6 +6,7 @@
 from six import add_metaclass
 
 from virtualenv.create.describe import PosixSupports, WindowsSupports
+from virtualenv.create.via_global_ref.builtin.ref import RefMust, RefWhen
 from virtualenv.util.path import Path
 
 from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@@ -33,19 +34,26 @@ def _executables(cls, interpreter):
         targets = OrderedDict(
             (i, None) for i in ["python", "python{}".format(major), "python{}.{}".format(major, minor), host_exe.name]
         )
-        yield host_exe, list(targets.keys())
+        must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
+        yield host_exe, list(targets.keys()), must, RefWhen.ANY
 
 
 @add_metaclass(ABCMeta)
 class CPythonWindows(CPython, WindowsSupports):
     @classmethod
     def _executables(cls, interpreter):
-        host = Path(interpreter.system_executable)
+        executables = cls._win_executables(Path(interpreter.system_executable), interpreter, RefWhen.ANY)
+        for src, targets, must, when in executables:
+            yield src, targets, must, when
+
+    @classmethod
+    def _win_executables(cls, host, interpreter, when):
+        must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
         for path in (host.parent / n for n in {"python.exe", host.name}):
-            yield host, [path.name]
+            yield host, [path.name], must, when
         # for more info on pythonw.exe see https://stackoverflow.com/a/30313091
         python_w = host.parent / "pythonw.exe"
-        yield python_w, [python_w.name]
+        yield python_w, [python_w.name], must, when
 
 
 def is_mac_os_framework(interpreter):

src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py

@@ -1,12 +1,13 @@
 from __future__ import absolute_import, unicode_literals
 
 import abc
+from itertools import chain
 from textwrap import dedent
 
 from six import add_metaclass
 
 from virtualenv.create.describe import Python3Supports
-from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen
 from virtualenv.create.via_global_ref.store import is_store_python
 from virtualenv.util.path import Path
 
@@ -55,8 +56,29 @@ def setup_meta(cls, interpreter):
     def sources(cls, interpreter):
         for src in super(CPython3Windows, cls).sources(interpreter):
             yield src
-        for src in cls.include_dll_and_pyd(interpreter):
-            yield src
+        if cls.venv_37p(interpreter):
+            for dll in (i for i in Path(interpreter.system_executable).parent.iterdir() if i.suffix == ".dll"):
+                yield PathRefToDest(dll, cls.to_bin, RefMust.SYMLINK, RefWhen.SYMLINK)
+        else:
+            for src in cls.include_dll_and_pyd(interpreter):
+                yield src
+
+    @classmethod
+    def _executables(cls, interpreter):
+        system_exe = Path(interpreter.system_executable)
+        if cls.venv_37p(interpreter):
+            # starting with CPython 3.7 Windows ships with a venvlauncher.exe that avoids the need for dll/pyd copies
+            launcher = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" / "python.exe"
+            executables = cls._win_executables(launcher, interpreter, RefWhen.COPY)
+            executables = chain(executables, cls._win_executables(system_exe, interpreter, RefWhen.SYMLINK))
+        else:
+            executables = cls._win_executables(system_exe, interpreter, RefWhen.ANY)
+        for src, targets, must, when in executables:
+            yield src, targets, must, when
+
+    @staticmethod
+    def venv_37p(interpreter):
+        return interpreter.version_info.minor > 6
 
     @classmethod
     def include_dll_and_pyd(cls, interpreter):

src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py

@@ -9,7 +9,7 @@
 
 from six import add_metaclass
 
-from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest
+from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest, RefMust
 from virtualenv.util.path import Path
 from virtualenv.util.six import ensure_text
 
@@ -29,7 +29,8 @@ def sources(cls, interpreter):
         for src in super(CPythonmacOsFramework, cls).sources(interpreter):
             yield src
         # add a symlink to the host python image
-        ref = PathRefToDest(cls.image_ref(interpreter), dest=lambda self, _: self.dest / ".Python", must_symlink=True)
+        exe = cls.image_ref(interpreter)
+        ref = PathRefToDest(exe, dest=lambda self, _: self.dest / ".Python", must=RefMust.SYMLINK)
         yield ref
 
     def create(self):
@@ -40,7 +41,7 @@ def create(self):
         current = self.current_mach_o_image_path()
         for src in self._sources:
             if isinstance(src, ExePathRefToDest):
-                if src.must_copy or not self.symlinks:
+                if src.must == RefMust.COPY or not self.symlinks:
                     exes = [self.bin_dir / src.base]
                     if not self.symlinks:
                         exes.extend(self.bin_dir / a for a in src.aliases)
@@ -49,12 +50,12 @@ def create(self):
 
     @classmethod
     def _executables(cls, interpreter):
-        for _, targets in super(CPythonmacOsFramework, cls)._executables(interpreter):
+        for _, targets, must, when in super(CPythonmacOsFramework, cls)._executables(interpreter):
             # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the
             # stub executable in ${sys.prefix}/bin.
             # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951
             fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python"
-            yield fixed_host_exe, targets
+            yield fixed_host_exe, targets, must, when
 
     @abstractmethod
     def current_mach_o_image_path(self):

src/virtualenv/create/via_global_ref/builtin/pypy/common.py

@@ -4,7 +4,7 @@
 
 from six import add_metaclass
 
-from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen
 from virtualenv.util.path import Path
 
 from ..via_global_self_do import ViaGlobalRefVirtualenvBuiltin
@@ -20,7 +20,8 @@ def can_describe(cls, interpreter):
     def _executables(cls, interpreter):
         host = Path(interpreter.system_executable)
         targets = sorted("{}{}".format(name, PyPy.suffix) for name in cls.exe_names(interpreter))
-        yield host, targets
+        must = RefMust.COPY if interpreter.version_info.major == 2 else RefMust.NA
+        yield host, targets, must, RefWhen.ANY
 
     @classmethod
     def exe_names(cls, interpreter):

src/virtualenv/create/via_global_ref/builtin/ref.py

@@ -17,16 +17,28 @@
 from virtualenv.util.six import ensure_text
 
 
+class RefMust(object):
+    NA = "NA"
+    COPY = "copy"
+    SYMLINK = "symlink"
+
+
+class RefWhen(object):
+    ANY = "ANY"
+    COPY = "copy"
+    SYMLINK = "symlink"
+
+
 @add_metaclass(ABCMeta)
 class PathRef(object):
     """Base class that checks if a file reference can be symlink/copied"""
 
     FS_SUPPORTS_SYMLINK = fs_supports_symlink()
     FS_CASE_SENSITIVE = fs_is_case_sensitive()
 
-    def __init__(self, src, must_symlink, must_copy):
-        self.must_symlink = must_symlink
-        self.must_copy = must_copy
+    def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY):
+        self.must = must
+        self.when = when
         self.src = src
         try:
             self.exists = src.exists()
@@ -35,8 +47,6 @@ def __init__(self, src, must_symlink, must_copy):
         self._can_read = None if self.exists else False
         self._can_copy = None if self.exists else False
         self._can_symlink = None if self.exists else False
-        if self.must_copy is True and self.must_symlink is True:
-            raise ValueError("can copy and symlink at the same time")
 
     def __repr__(self):
         return "{}(src={})".format(self.__class__.__name__, self.src)
@@ -57,7 +67,7 @@ def can_read(self):
     @property
     def can_copy(self):
         if self._can_copy is None:
-            if self.must_symlink:
+            if self.must == RefMust.SYMLINK:
                 self._can_copy = self.can_symlink
             else:
                 self._can_copy = self.can_read
@@ -66,7 +76,7 @@ def can_copy(self):
     @property
     def can_symlink(self):
         if self._can_symlink is None:
-            if self.must_copy:
+            if self.must == RefMust.COPY:
                 self._can_symlink = self.can_copy
             else:
                 self._can_symlink = self.FS_SUPPORTS_SYMLINK and self.can_read
@@ -77,9 +87,9 @@ def run(self, creator, symlinks):
         raise NotImplementedError
 
     def method(self, symlinks):
-        if self.must_symlink:
+        if self.must == RefMust.SYMLINK:
             return symlink
-        if self.must_copy:
+        if self.must == RefMust.COPY:
             return copy
         return symlink if symlinks else copy
 
@@ -88,8 +98,8 @@ def method(self, symlinks):
 class ExePathRef(PathRef):
     """Base class that checks if a executable can be references via symlink/copy"""
 
-    def __init__(self, src, must_symlink, must_copy):
-        super(ExePathRef, self).__init__(src, must_symlink, must_copy)
+    def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY):
+        super(ExePathRef, self).__init__(src, must, when)
         self._can_run = None
 
     @property
@@ -114,8 +124,8 @@ def can_run(self):
 class PathRefToDest(PathRef):
     """Link a path on the file system"""
 
-    def __init__(self, src, dest, must_symlink=False, must_copy=False):
-        super(PathRefToDest, self).__init__(src, must_symlink, must_copy)
+    def __init__(self, src, dest, must=RefMust.NA, when=RefWhen.ANY):
+        super(PathRefToDest, self).__init__(src, must, when)
         self.dest = dest
 
     def run(self, creator, symlinks):
@@ -131,15 +141,14 @@ def run(self, creator, symlinks):
 class ExePathRefToDest(PathRefToDest, ExePathRef):
     """Link a exe path on the file system"""
 
-    def __init__(self, src, targets, dest, must_symlink=False, must_copy=False):
-        ExePathRef.__init__(self, src, must_symlink, must_copy)
-        PathRefToDest.__init__(self, src, dest, must_symlink, must_copy)
+    def __init__(self, src, targets, dest, must=RefMust.NA, when=RefWhen.ANY):
+        ExePathRef.__init__(self, src, must, when)
+        PathRefToDest.__init__(self, src, dest, must, when)
         if not self.FS_CASE_SENSITIVE:
             targets = list(OrderedDict((i.lower(), None) for i in targets).keys())
         self.base = targets[0]
         self.aliases = targets[1:]
         self.dest = dest
-        self.must_copy = must_copy
 
     def run(self, creator, symlinks):
         bin_dir = self.dest(creator, self.src).parent

src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py

@@ -4,7 +4,7 @@
 
 from six import add_metaclass
 
-from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest
+from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, RefMust
 from virtualenv.util.path import ensure_dir
 
 from ..api import ViaGlobalRefApi, ViaGlobalRefMeta
@@ -27,37 +27,46 @@ def __init__(self, options, interpreter):
     def can_create(cls, interpreter):
         """By default all built-in methods assume that if we can describe it we can create it"""
         # first we must be able to describe it
-        if cls.can_describe(interpreter):
-            meta = cls.setup_meta(interpreter)
-            if meta is not None and meta:
-                for src in cls.sources(interpreter):
-                    if src.exists:
-                        if meta.can_copy and not src.can_copy:
-                            meta.copy_error = "cannot copy {}".format(src)
-                        if meta.can_symlink and not src.can_symlink:
-                            meta.symlink_error = "cannot symlink {}".format(src)
-                        if not meta.can_copy and not meta.can_symlink:
-                            meta.error = "neither copy or symlink supported, copy: {} symlink: {}".format(
-                                meta.copy_error,
-                                meta.symlink_error,
-                            )
-                    else:
-                        meta.error = "missing required file {}".format(src)
-                    if meta.error:
-                        break
-                    meta.sources.append(src)
-            return meta
-        return None
+        if not cls.can_describe(interpreter):
+            return None
+        meta = cls.setup_meta(interpreter)
+        if meta is not None and meta:
+            cls._sources_can_be_applied(interpreter, meta)
+        return meta
+
+    @classmethod
+    def _sources_can_be_applied(cls, interpreter, meta):
+        for src in cls.sources(interpreter):
+            if src.exists:
+                if meta.can_copy and not src.can_copy:
+                    meta.copy_error = "cannot copy {}".format(src)
+                if meta.can_symlink and not src.can_symlink:
+                    meta.symlink_error = "cannot symlink {}".format(src)
+            else:
+                msg = "missing required file {}".format(src)
+                if src.when == RefMust.NA:
+                    meta.error = msg
+                elif src.when == RefMust.COPY:
+                    meta.copy_error = msg
+                elif src.when == RefMust.SYMLINK:
+                    meta.symlink_error = msg
+            if not meta.can_copy and not meta.can_symlink:
+                meta.error = "neither copy or symlink supported, copy: {} symlink: {}".format(
+                    meta.copy_error,
+                    meta.symlink_error,
+                )
+            if meta.error:
+                break
+            meta.sources.append(src)
 
     @classmethod
     def setup_meta(cls, interpreter):
         return BuiltinViaGlobalRefMeta()
 
     @classmethod
     def sources(cls, interpreter):
-        is_py2 = interpreter.version_info.major == 2
-        for host_exe, targets in cls._executables(interpreter):
-            yield ExePathRefToDest(host_exe, dest=cls.to_bin, targets=targets, must_copy=is_py2)
+        for host_exe, targets, must, when in cls._executables(interpreter):
+            yield ExePathRefToDest(host_exe, dest=cls.to_bin, targets=targets, must=must, when=when)
 
     def to_bin(self, src):
         return self.bin_dir / src.name