Use sys._base_executable

docs/changelog/1345.bugfix.rst

@@ -0,0 +1,2 @@
+Handle running virtualenv from within a virtual environment created
+using the stdlib ``venv`` module. Fixes #1339.

docs/reference.rst

@@ -270,3 +270,62 @@ Here's a more concrete example of how you could use this::
 Another example is available `here`__.
 
 .. __: https://github.com/socialplanning/fassembler/blob/master/fassembler/create-venv-script.py
+
+
+Compatibility with the stdlib venv module
+-----------------------------------------
+
+Starting with Python 3.3, the Python standard library includes a ``venv``
+module that provides similar functionality to ``virtualenv`` - however, the
+mechanisms used by the two modules are very different.
+
+Problems arise when environments get "nested" (a virtual environment is
+created from within another one - for example, running the virtualenv tests
+using tox, where tox creates a virtual environemnt to run the tests, and the
+tests themselves create further virtual environments).
+
+``virtualenv`` supports creating virtual environments from within another one
+(the ``sys.real_prefix`` variable allows ``virtualenv`` to locate the "base"
+environment) but stdlib-style ``venv`` environments don't use that mechanism,
+so explicit support is needed for those environments.
+
+A standard library virtual environment is most easily identified by checking
+``sys.prefix`` and ``sys.base_prefix``. If these differ, the interpreter is
+running in a virtual environment and the base interpreter is located in the
+directory specified by ``sys.base_prefix``. Therefore, when
+``sys.base_prefix`` is set, virtualenv gets the interpreter files from there
+rather than from ``sys.prefix`` (in the same way as ``sys.real_prefix`` is
+used for virtualenv-style environments). In practice, this is sufficient for
+all platforms other than Windows.
+
+On Windows from Python 3.7.2 onwards, a stdlib-style virtual environment does
+not contain an actual Python interpreter executable, but rather a "redirector"
+which launches the actual interpreter from the base environment (this
+redirector is based on the same code as the standard ``py.exe`` launcher). As
+a result, the virtualenv approach of copying the interpreter from the starting
+environment fails. In order to correctly set up the virtualenv, therefore, we
+need to be running from a "full" environment. To ensure that, we re-invoke the
+``virtualenv.py`` script using the "base" interpreter, in the same way as we
+do with the ``--python`` command line option.
+
+The process of identifying the base interpreter is complicated by the fact
+that the implementation changed between different Python versions. The
+logic used is as follows:
+
+1. If the (private) attribute ``sys._base_executable`` is present, this is
+   the base interpreter. This is the long-term solution and should be stable
+   in the future (the attribute may become public, and have the leading
+   underscore removed, in a Python 3.8, but that is not confirmed yet).
+2. In the absence of ``sys._base_executable`` (only the case for Python 3.7.2)
+   we check for the existence of the environment variable
+   ``__PYVENV_LAUNCHER__``. This is used by the redirector, and if it is
+   present, we know that we are in a stdlib-style virtual environment and need
+   to locate the base Python. In most cases, the base environment is located
+   at ``sys.base_prefix`` - however, in the case where the user creates a
+   virtualenv, and then creates a venv from that virtualenv,
+   ``sys.base_prefix`` is not correct - in that case, though, we have
+   ``sys.real_prefix`` (set by virtualenv) which *is* correct.
+
+There is one further complication - as noted above, the environment variable
+``__PYVENV_LAUNCHER__`` affects how the interpreter works, so before we
+re-invoke the virtualenv script, we remove this from the environment.

tests/test_virtualenv.py

@@ -25,6 +25,12 @@
     from pathlib2 import Path
 
 
+try:
+    import venv as std_venv
+except ImportError:
+    std_venv = None
+
+
 def test_version():
     """Should have a version string"""
     assert virtualenv.virtualenv_version, "Should have version"
@@ -607,6 +613,23 @@ def test_create_environment_from_virtual_environment(tmpdir):
     assert not os.path.islink(os.path.join(lib_dir, "distutils"))
 
 
+@pytest.mark.skipif(std_venv is None, reason="needs standard venv module")
+def test_create_environment_from_venv(tmpdir):
+    std_venv_dir = str(tmpdir / "stdvenv")
+    ve_venv_dir = str(tmpdir / "vevenv")
+    home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(ve_venv_dir)
+    builder = std_venv.EnvBuilder()
+    ctx = builder.ensure_directories(std_venv_dir)
+    builder.create_configuration(ctx)
+    builder.setup_python(ctx)
+    builder.setup_scripts(ctx)
+    subprocess.check_call([ctx.env_exe, virtualenv.__file__, "--no-setuptools", "--no-pip", "--no-wheel", ve_venv_dir])
+    ve_exe = os.path.join(bin_dir, "python")
+    out = subprocess.check_output([ve_exe, "-c", "import sys; print(sys.real_prefix)"], universal_newlines=True)
+    # Test against real_prefix if present - we might be running the test from a virtualenv (e.g. tox).
+    assert out.strip() == getattr(sys, "real_prefix", sys.prefix)
+
+
 def test_create_environment_with_old_pip(tmpdir):
     old = Path(__file__).parent / "old-wheels"
     old_pip = old / "pip-9.0.1-py2.py3-none-any.whl"

virtualenv.py

@@ -739,14 +739,50 @@ def main():
     verbosity = options.verbose - options.quiet
     logger = Logger([(Logger.level_for_integer(2 - verbosity), sys.stdout)])
 
-    if options.python and not os.environ.get("VIRTUALENV_INTERPRETER_RUNNING"):
+    def should_reinvoke(options):
+        """Do we need to reinvoke ourself?"""
+        # 1. Did the user specify the --python option?
+        if options.python and not os.environ.get("VIRTUALENV_INTERPRETER_RUNNING"):
+            return options.python
+        # All of the remaining cases are only for Windows
+        if sys.platform == "win32":
+            # 2. Are we running from a venv-style virtual environment with a redirector?
+            if hasattr(sys, "_base_executable"):
+                return sys._base_executable
+            # 3. Special case for Python 3.7.2, where we have a redirector,
+            #    but sys._base_executable does not exist.
+            if sys.version_info[:3] == (3, 7, 2):
+                # We are in a venv if the environment variable __PYVENV_LAUNCHER__ is set.
+                if "__PYVENV_LAUNCHER__" in os.environ:
+                    # The base environment is either sys.real_prefix (if
+                    # we were invoked from a venv built from a virtualenv) or
+                    # sys.base_prefix if real_prefix doesn't exist (a simple venv).
+                    base_prefix = getattr(sys, "real_prefix", sys.base_prefix)
+                    # We assume the Python executable is directly under the prefix
+                    # directory. The only known case where that won't be the case is
+                    # an in-place source build, which we don't support. We don't need
+                    # to consider virtuale environments (where python.exe is in "Scripts"
+                    # because we've just followed the links back to a non-virtual
+                    # environment - we hope!)
+                    base_exe = os.path.join(base_prefix, "python.exe")
+                    if os.path.exists(base_exe):
+                        return base_exe
+        # We don't need to reinvoke
+        return None
+
+    interpreter = should_reinvoke(options)
+    if interpreter:
         env = os.environ.copy()
-        interpreter = resolve_interpreter(options.python)
+        interpreter = resolve_interpreter(interpreter)
         if interpreter == sys.executable:
             logger.warn("Already using interpreter {}".format(interpreter))
         else:
             logger.notify("Running virtualenv with interpreter {}".format(interpreter))
             env["VIRTUALENV_INTERPRETER_RUNNING"] = "true"
+            # Remove the variable __PYVENV_LAUNCHER__ if it's present, as it causes the
+            # interpreter to redirect back to the virtual environment.
+            if "__PYVENV_LAUNCHER__" in env:
+                del env["__PYVENV_LAUNCHER__"]
             file = __file__
             if file.endswith(".pyc"):
                 file = file[:-1]