Support virtual environment

.travis.yml

@@ -1,4 +1,5 @@
 language: julia
+dist: trusty  # update "python3.4-venv" below when updating
 os:
     - linux
     - osx
@@ -11,6 +12,7 @@ addons:
     packages:
       - python-numpy
       - python3-numpy
+      - python3.4-venv  # Ubuntu Trusty 14.04 does not have python3-venv
 env:
   global:
     - PYCALL_DEBUG_BUILD="yes"

deps/build.jl

@@ -12,28 +12,6 @@ struct UseCondaPython <: Exception end
 
 #########################################################################
 
-# Fix the environment for running `python`, and setts IO encoding to UTF-8.
-# If cmd is the Conda python, then additionally removes all PYTHON* and
-# CONDA* environment variables.
-function pythonenv(cmd::Cmd)
-    env = copy(ENV)
-    if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR)
-        pythonvars = String[]
-        for var in keys(env)
-            if startswith(var, "CONDA") || startswith(var, "PYTHON")
-                push!(pythonvars, var)
-            end
-        end
-        for var in pythonvars
-            pop!(env, var)
-        end
-    end
-    # set PYTHONIOENCODING when running python executable, so that
-    # we get UTF-8 encoded text as output (this is not the default on Windows).
-    env["PYTHONIOENCODING"] = "UTF-8"
-    setenv(cmd, env)
-end
-
 pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = chomp(read(pythonenv(`$python -c "import $mod; print($mod.$var)"`), String))
 
 pyconfigvar(python::AbstractString, var::AbstractString) = pyvar(python, "distutils.sysconfig", "get_config_var('$var')")
@@ -194,15 +172,7 @@ try # make sure deps.jl file is removed on error
     # Get PYTHONHOME, either from the environment or from Python
     # itself (if it is not in the environment or if we are using Conda)
     PYTHONHOME = if !haskey(ENV, "PYTHONHOME") || use_conda
-        # PYTHONHOME tells python where to look for both pure python
-        # and binary modules.  When it is set, it replaces both
-        # `prefix` and `exec_prefix` and we thus need to set it to
-        # both in case they differ. This is also what the
-        # documentation recommends.  However, they are documented
-        # to always be the same on Windows, where it causes
-        # problems if we try to include both.
-        exec_prefix = pysys(python, "exec_prefix")
-        Compat.Sys.iswindows() ? exec_prefix : pysys(python, "prefix") * ":" * exec_prefix
+        pythonhome_of(python)
     else
         ENV["PYTHONHOME"]
     end

deps/depsutils.jl

@@ -31,3 +31,86 @@ end
 
 # need to be able to get the version before Python is initialized
 Py_GetVersion(libpy) = unsafe_string(ccall(Libdl.dlsym(libpy, :Py_GetVersion), Ptr{UInt8}, ()))
+
+# Fix the environment for running `python`, and setts IO encoding to UTF-8.
+# If cmd is the Conda python, then additionally removes all PYTHON* and
+# CONDA* environment variables.
+function pythonenv(cmd::Cmd)
+    env = copy(ENV)
+    if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR)
+        pythonvars = String[]
+        for var in keys(env)
+            if startswith(var, "CONDA") || startswith(var, "PYTHON")
+                push!(pythonvars, var)
+            end
+        end
+        for var in pythonvars
+            pop!(env, var)
+        end
+    end
+    # set PYTHONIOENCODING when running python executable, so that
+    # we get UTF-8 encoded text as output (this is not the default on Windows).
+    env["PYTHONIOENCODING"] = "UTF-8"
+    setenv(cmd, env)
+end
+
+
+function pythonhome_of(pyprogramname::AbstractString)
+    if Compat.Sys.iswindows()
+        # PYTHONHOME tells python where to look for both pure python
+        # and binary modules.  When it is set, it replaces both
+        # `prefix` and `exec_prefix` and we thus need to set it to
+        # both in case they differ. This is also what the
+        # documentation recommends.  However, they are documented
+        # to always be the same on Windows, where it causes
+        # problems if we try to include both.
+        script = """
+        import sys
+        if hasattr(sys, "base_exec_prefix"):
+            sys.stdout.write(sys.base_exec_prefix)
+        else:
+            sys.stdout.write(sys.exec_prefix)
+        """
+    else
+        script = """
+        import sys
+        if hasattr(sys, "base_exec_prefix"):
+            sys.stdout.write(sys.base_prefix)
+            sys.stdout.write(":")
+            sys.stdout.write(sys.base_exec_prefix)
+        else:
+            sys.stdout.write(sys.prefix)
+            sys.stdout.write(":")
+            sys.stdout.write(sys.exec_prefix)
+        """
+        # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
+    end
+    return read(pythonenv(`$pyprogramname -c $script`), String)
+end
+# To support `venv` standard library (as well as `virtualenv`), we
+# need to use `sys.base_prefix` and `sys.base_exec_prefix` here.
+# Otherwise, initializing Python in `__init__` below fails with
+# unrecoverable error:
+#
+#   Fatal Python error: initfsencoding: unable to load the file system codec
+#   ModuleNotFoundError: No module named 'encodings'
+#
+# This is because `venv` does not symlink standard libraries like
+# `virtualenv`.  For example, `lib/python3.X/encodings` does not
+# exist.  Rather, `venv` relies on the behavior of Python runtime:
+#
+#   If a file named "pyvenv.cfg" exists one directory above
+#   sys.executable, sys.prefix and sys.exec_prefix are set to that
+#   directory and it is also checked for site-packages
+#   --- https://docs.python.org/3/library/venv.html
+#
+# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and
+# `sys.base_exec_prefix`.  If the virtual environment is created by
+# `virtualenv`, those `sys.base_*` paths point to the virtual
+# environment.  Thus, above code supports both use cases.
+#
+# See also:
+# * https://docs.python.org/3/library/venv.html
+# * https://docs.python.org/3/library/site.html
+# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix
+# * https://github.com/JuliaPy/PyCall.jl/issues/410

src/pyinit.jl

@@ -60,6 +60,76 @@ function pyjlwrap_init()
     end
 end
 
+#########################################################################
+# Virtual environment support
+
+_clength(x::Cstring) = ccall(:strlen, Csize_t, (Cstring,), x) + 1
+_clength(x) = length(x)
+
+function __leak(::Type{T}, x) where T
+    n = _clength(x)
+    ptr = ccall(:malloc, Ptr{T}, (Csize_t,), n * sizeof(T))
+    unsafe_copyto!(ptr, pointer(x), n)
+    return ptr
+end
+
+"""
+    _leak(T::Type, x::AbstractString) :: Ptr
+    _leak(x::Array) :: Ptr
+
+Leak `x` from Julia's GCer.  This is meant to be used only for
+`Py_SetPythonHome` and `Py_SetProgramName` where the Python
+documentation demands that the passed argument must points to "static
+storage whose contents will not change for the duration of the
+program's execution" (although it seems that in newer CPython versions
+the contents are copied internally).
+"""
+_leak(x::Union{Cstring, Array}) = __leak(eltype(x), x)
+_leak(T::Type, x::AbstractString) =
+    _leak(Base.unsafe_convert(T, Base.cconvert(T, x)))
+_leak(::Type{Cwstring}, x::AbstractString) =
+    _leak(Base.cconvert(Cwstring, x))
+
+venv_python(::Nothing) = pyprogramname
+
+function venv_python(venv::AbstractString)
+    # See:
+    # https://github.com/python/cpython/blob/3.7/Lib/venv/__init__.py#L116
+    if Compat.Sys.iswindows()
+        return joinpath(venv, "Scripts", "python.exe")
+    else
+        return joinpath(venv, "bin", "python")
+    end
+end
+
+"""
+    python_cmd(args::Cmd = ``; venv, python) :: Cmd
+
+Create an appropriate `Cmd` for running Python program with command
+line arguments `args`.
+
+# Keyword Arguments
+- `venv::String`: The path of a virtualenv to be used instead of the
+  default environment with which PyCall isconfigured.
+- `python::String`: The path to the Python executable.  `venv` is ignored
+  when this argument is specified.
+"""
+function python_cmd(args::Cmd = ``;
+                    venv::Union{Nothing, AbstractString} = nothing,
+                    python::AbstractString = venv_python(venv))
+    return pythonenv(`$python $args`)
+end
+
+function find_libpython(python::AbstractString)
+    script = joinpath(@__DIR__, "..", "deps", "find_libpython.py")
+    cmd = python_cmd(`$script`; python = python)
+    try
+        return read(cmd, String)
+    catch
+        return nothing
+    end
+end
+
 #########################################################################
 
 function __init__()
@@ -76,7 +146,48 @@ function __init__()
 
     already_inited = 0 != ccall((@pysym :Py_IsInitialized), Cint, ())
 
-    if !already_inited
+    if already_inited
+        # Importing from PyJulia takes this path.
+    elseif isfile(get(ENV, "PYCALL_JL_RUNTIME_PYTHON", ""))
+        venv_python = ENV["PYCALL_JL_RUNTIME_PYTHON"]
+
+        # Check libpython compatibility.
+        venv_libpython = find_libpython(venv_python)
+        if venv_libpython === nothing
+            error("""
+            `libpython` for $venv_python cannot be found.
+            PyCall.jl cannot initialize Python safely.
+            """)
+        elseif venv_libpython != libpython
+            error("""
+            Incompatible `libpython` detected.
+            `libpython` for $venv_python is:
+                $venv_libpython
+            `libpython` for $pyprogramname is:
+                $libpython
+            PyCall.jl only supports loading Python environment using
+            the same `libpython`.
+            """)
+        end
+
+        if haskey(ENV, "PYCALL_JL_RUNTIME_PYTHONHOME")
+            venv_home = ENV["PYCALL_JL_RUNTIME_PYTHONHOME"]
+        else
+            venv_home = pythonhome_of(venv_python)
+        end
+        if pyversion.major < 3
+            ccall((@pysym :Py_SetPythonHome), Cvoid, (Cstring,),
+                  _leak(Cstring, venv_home))
+            ccall((@pysym :Py_SetProgramName), Cvoid, (Cstring,),
+                  _leak(Cstring, venv_python))
+        else
+            ccall((@pysym :Py_SetPythonHome), Cvoid, (Ptr{Cwchar_t},),
+                  _leak(Cwstring, venv_home))
+            ccall((@pysym :Py_SetProgramName), Cvoid, (Ptr{Cwchar_t},),
+                  _leak(Cwstring, venv_python))
+        end
+        ccall((@pysym :Py_InitializeEx), Cvoid, (Cint,), 0)
+    else
         Py_SetPythonHome(libpy_handle, PYTHONHOME, wPYTHONHOME, pyversion)
         if !isempty(pyprogramname)
             if pyversion.major < 3

test/runtests.jl

@@ -667,3 +667,4 @@ def try_call(f):
 end
 
 include("test_pyfncall.jl")
+include("test_venv.jl")