Merge tag 'v1.96.4' into threadsafe_gc

[Diff since v1.96.3](JuliaPy/PyCall.jl@v1.96.3...v1.96.4) **Merged pull requests:** - Rename `distutils.sysconfig` to `sysconfig` (JuliaPy#1068) (@MilesCranmer) - bump to 1.96.4 (JuliaPy#1070) (@stevengj)
MilesCranmer · Jan 8, 2024 · 9e05155 · 9e05155
2 parents c707032 + e54c4ee
commit 9e05155
Show file tree

Hide file tree

Showing 24 changed files with 851 additions and 101 deletions.
diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml
@@ -1,9 +1,12 @@
 name: TagBot
 on:
-  schedule:
-    - cron: 0 * * * *
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
 jobs:
   TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
     runs-on: ubuntu-latest
     steps:
       - uses: JuliaRegistries/TagBot@v1

diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml
@@ -16,8 +16,11 @@ jobs:
         os:
           - ubuntu-latest
         architecture: [x64]
-        python-version: ['3.8']
-        julia-version: ['1.5', 'nightly']
+        python-version: ['3.x']
+        julia-version:
+          - '1'
+          - '1.6'
+          # - 'nightly'  # TODO: reenable
       fail-fast: false
     env:
       PYTHON: python${{ matrix.python-version }}
@@ -34,6 +37,9 @@ jobs:
           architecture: ${{ matrix.architecture }}
       - run: python --version
       - run: python -m pip install --user numpy
+      - name: Install libpython2.7 for `find_libpython` test
+        run: sudo apt-get install python2.7-dev
+        if: ${{ matrix.python-version != '2.7' && matrix.os == 'ubuntu-latest' }}
       - name: Setup julia
         uses: julia-actions/setup-julia@v1
         with:

diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml
@@ -18,11 +18,11 @@ jobs:
           - macos-latest
           - windows-latest
         architecture: [x64]
-        julia-version: ['1.5']
+        julia-version: ['1']
         include:
           - os: windows-latest
             architecture: x86
-            julia-version: '1.5'
+            julia-version: '1'
       fail-fast: false
     env:
       PYTHON: ""
@@ -32,6 +32,9 @@ jobs:
       ${{ matrix.os }} ${{ matrix.architecture }}
     steps:
       - uses: actions/checkout@v1
+      - name: Install libpython2.7 for `find_libpython` test
+        run: sudo apt-get install python2.7-dev
+        if: ${{ matrix.python-version != '2.7' && matrix.os == 'ubuntu-latest' }}
       - name: Setup julia
         uses: julia-actions/setup-julia@v1
         with:

diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml
@@ -18,25 +18,39 @@ jobs:
           - macos-latest
           - windows-latest
         architecture: [x64]
-        python-version: ['2.7', '3.8']
-        julia-version: ['1.0', '1.5', 'nightly']
+        python-version: ['3.x']
+        julia-version:
+          - '1'
+          - 'nightly'
         include:
+          # 32 bit Windows:
           - os: windows-latest
             architecture: x86
-            python-version: '3.8'
-            julia-version: '1.5'
+            python-version: '3.x'
+            julia-version: '1'
+          # Sweep python-version and julia-version only on Ubuntu:
+          - os: ubuntu-latest
+            architecture: x64
+            python-version: '3.9'
+            julia-version: '1'
           - os: ubuntu-latest
             architecture: x64
-            python-version: '3.7'
+            python-version: '3.x'
             julia-version: '1.5'
           - os: ubuntu-latest
             architecture: x64
-            python-version: '3.8'
+            python-version: '3.x'
             julia-version: '1.4'
+          # Test Python 2.7 only with a few combinations (TODO: drop 2.7).
+          # Note that it does not work in macOS:
           - os: ubuntu-latest
             architecture: x64
-            python-version: '3.8'
-            julia-version: '1.3'
+            python-version: '2.7'
+            julia-version: '1'
+          - os: windows-2019
+            architecture: x64
+            python-version: '2.7'
+            julia-version: '1'
       fail-fast: false
     name: Test
       Julia ${{ matrix.julia-version }}
@@ -56,6 +70,9 @@ jobs:
         if: ${{ matrix.python-version != '2.7' }}
       - run: virtualenv --version
         if: ${{ matrix.python-version != '2.7' }}
+      - name: Install libpython2.7 for `find_libpython` test
+        run: sudo apt-get install python2.7-dev
+        if: ${{ matrix.python-version != '2.7' && matrix.os == 'ubuntu-latest' }}
       - name: Setup julia
         uses: julia-actions/setup-julia@v1
         with:
@@ -67,6 +84,12 @@ jobs:
           PYTHON: python
       - run: julia test/check_deps_version.jl ${{ matrix.python-version }}
       - uses: julia-actions/julia-runtest@v1
+        env:
+          # Windows temp directories break the CI:
+          #  https://github.com/actions/runner-images/issues/712
+          #  so, we use
+          #  https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context
+          _TMP_DIR: ${{ runner.temp }}
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
         with:

diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "PyCall"
 uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 authors = ["Steven G. Johnson <stevenj@mit.edu>", "Yichao Yu <yyc1992@gmail.com>", "Takafumi Arakaki <aka.tkf@gmail.com>", "Simon Kornblith <simon@simonster.com>", "Páll Haraldsson <Pall.Haraldsson@gmail.com>", "Jon Malmaud <malmaud@gmail.com>", "Jake Bolewski <jakebolewski@gmail.com>", "Keno Fischer <keno@alumni.harvard.edu>", "Joel Mason <jobba1@hotmail.com>", "Jameson Nash <vtjnash@gmail.com>", "The JuliaPy development team"]
-version = "1.92.2"
+version = "1.96.4"
 
 [deps]
 Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d"
@@ -13,13 +13,19 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289"
 
 [compat]
-Conda = "1.0"
+Conda = "1.9"
 MacroTools = "0.4, 0.5"
 VersionParsing = "1.0"
-julia = "0.7, 1.0"
+
+julia = "1.4"
+Dates = "<0.0.1, 1"
+Libdl = "<0.0.1, 1"
+LinearAlgebra = "<0.0.1, 1"
+Serialization = "<0.0.1, 1"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 
 [targets]
-test = ["Test"]
+test = ["Test", "REPL"]
diff --git a/README.md b/README.md
@@ -20,7 +20,12 @@ install the files. Julia 0.7 or later is required.
 
 The latest development version of PyCall is available from
 <https://github.com/JuliaPy/PyCall.jl>.  If you want to switch to
-this after installing the package, run `Pkg.checkout("PyCall"); Pkg.build("PyCall")`.
+this after installing the package, run:
+
+```julia
+Pkg.add(PackageSpec(name="PyCall", rev="master"))
+Pkg.build("PyCall")
+```
 
 By default on Mac and Windows systems, `Pkg.add("PyCall")`
 or `Pkg.build("PyCall")` will use the
@@ -48,9 +53,9 @@ to the path of the `python` (or `python3` etc.) executable and then re-running `
 In Julia:
 
     ENV["PYTHON"] = "... path of the python executable ..."
-    # ENV["PYTHON"] = raw"C:\Python37-x64\python.exe" # example for Windows, "raw" to not have to escape: "C:\\Python37-x64\\python.exe"
+    # ENV["PYTHON"] = raw"C:\Python310-x64\python.exe" # example for Windows, "raw" to not have to escape: "C:\\Python310-x64\\python.exe"
 
-    # ENV["PYTHON"] = "/usr/bin/python3.7"           # example for *nix
+    # ENV["PYTHON"] = "/usr/bin/python3.10"           # example for *nix
     Pkg.build("PyCall")
 
 Note also that you will need to re-run `Pkg.build("PyCall")` if your
@@ -160,6 +165,8 @@ def sinpi(x):
 py"sinpi"(1)
 ```
 
+You can also execute a whole script `"foo.py"` via `@pyinclude("foo.py")` as if you had pasted it into a `py"""..."""` string.
+
 When creating a Julia module, it is a useful pattern to define Python
 functions or classes in Julia's `__init__` and then use it in Julia
 function with `py"..."`.
@@ -192,7 +199,7 @@ cannot be accessed outside `MyModule`.
 
 Here are solutions to some common problems:
 
-* By default, PyCall [doesn't include the current directory in the Python search path](https://github.com/JuliaPy/PyCall.jl/issues/48).   If you want to do that (in order to load a Python module from the current directory), just run `pushfirst!(PyVector(pyimport("sys")."path"), "")`.
+* By default, PyCall [doesn't include the current directory in the Python search path](https://github.com/JuliaPy/PyCall.jl/issues/48).   If you want to do that (in order to load a Python module from the current directory), just run `pushfirst!(pyimport("sys")."path", "")`.
 
 ## Python object interfaces
 
@@ -298,7 +305,7 @@ use `PyDict` as the return type of a `pycall` returning a dictionary,
 or call `PyDict(o::PyObject)` on a dictionary object `o`.  By
 default, a `PyDict` is an `Any => Any` dictionary (or actually `PyAny
 => PyAny`) that performs runtime type inference, but if your Python
-dictionary has known, fixed types you can insteady use `PyDict{K,V}` given
+dictionary has known, fixed types you can instead use `PyDict{K,V}` given
 the key and value types `K` and `V` respectively.
 
 Currently, passing Julia dictionaries to Python makes a copy of the Julia
@@ -359,7 +366,8 @@ and also by providing more type information to the Julia compiler.
   Python's [`eval`](https://docs.python.org/3/library/functions.html#eval) function, and returns the result
   converted to `PyAny`.  Alternatively, `py"..."o` returns the raw `PyObject`
   (which can then be manually converted if desired).   You can interpolate
-  Julia variables and other expressions into the Python code with `$`,
+  Julia variables and other expressions into the Python code (except for into
+  Python strings contained in Python code), with `$`,
   which interpolates the *value* (converted to `PyObject`) of the given
   expression---data is not passed as a string, so this is different from
   ordinary Julia string interpolation.  e.g. `py"sum($([1,2,3]))"` calls the
@@ -384,10 +392,15 @@ and also by providing more type information to the Julia compiler.
   `__init__`.  Side-effect in Python occurred at top-level Julia scope
   cannot be used at run-time for precompiled modules.
 
+  You can also execute a Python script file `"foo.py"` by running `@pyinclude("foo.py")`, and it will be as if you had pasted the
+  script into a `py"..."` string.  (`@pyinclude` does not support
+  interpolating Julia variables with `$var`, however — the script
+  must be pure Python.)
+
 * `pybuiltin(s)`: Look up `s` (a string or symbol) among the global Python
   builtins.  If `s` is a string it returns a `PyObject`, while if `s` is a
   symbol it returns the builtin converted to `PyAny`.  (You can also use `py"s"`
-  to look up builtins or other Python globas.)
+  to look up builtins or other Python globals.)
 
 Occasionally, you may need to pass a keyword argument to Python that
 is a [reserved word](https://en.wikipedia.org/wiki/Reserved_word) in Julia.

diff --git a/deps/build.jl b/deps/build.jl
@@ -43,7 +43,7 @@ mkpath(dirname(prefsfile))
 try # make sure deps.jl file is removed on error
     python = try
         let py = get(ENV, "PYTHON", isfile(prefsfile) ? readchomp(prefsfile) :
-                     (Sys.isunix() && !Sys.isapple()) || Sys.ARCH ∉ (:i686, :x86_64) ?
+                     (Sys.isunix() && !Sys.isapple()) ?
                      whichfirst("python3", "python") : "Conda"),
             vers = isempty(py) || py == "Conda" ? v"0.0" : vparse(pyconfigvar(py,"VERSION","0.0"))
             if vers < v"2.7"
@@ -64,24 +64,19 @@ try # make sure deps.jl file is removed on error
             py
         end
     catch e1
-        if Sys.ARCH in (:i686, :x86_64)
-            if isa(e1, UseCondaPython)
-                @info string("Using the Python distribution in the Conda package by default.\n",
-                     "To use a different Python version, set ENV[\"PYTHON\"]=\"pythoncommand\" and re-run Pkg.build(\"PyCall\").")
-            else
-                @info string( "No system-wide Python was found; got the following error:\n",
-                      "$e1\nusing the Python distribution in the Conda package")
-            end
-            abspath(Conda.PYTHONDIR, "python" * (Sys.iswindows() ? ".exe" : ""))
+        if isa(e1, UseCondaPython)
+            @info string("Using the Python distribution in the Conda package by default.\n",
+                 "To use a different Python version, set ENV[\"PYTHON\"]=\"pythoncommand\" and re-run Pkg.build(\"PyCall\").")
         else
-            error("No system-wide Python was found; got the following error:\n",
-                  "$e1")
+            @info string("No system-wide Python was found; got the following error:\n",
+                  "$e1\nusing the Python distribution in the Conda package")
         end
+        abspath(Conda.PYTHONDIR, "python" * (Sys.iswindows() ? ".exe" : ""))
     end
 
     use_conda = dirname(python) == abspath(Conda.PYTHONDIR)
     if use_conda
-        Conda.add("numpy")
+        Conda.add("numpy"; satisfied_skip_solve=true)
     end
 
     (libpython, libpy_name) = find_libpython(python)

diff --git a/deps/buildutils.jl b/deps/buildutils.jl
@@ -5,7 +5,15 @@ import Conda, Libdl
 
 pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = chomp(read(pythonenv(`$python -c "import $mod; print($mod.$(var))"`), String))
 
-pyconfigvar(python::AbstractString, var::AbstractString) = pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')")
+function pyconfigvar(python::AbstractString, var::AbstractString)
+    try
+       pyvar(python, "sysconfig", "get_config_var('$(var)')")
+    catch e
+        emsg = sprint(showerror, e)
+        @warn "Encountered error on using `sysconfig`: $emsg. Falling back to `distutils.sysconfig`."
+        pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')")
+    end
+end
 pyconfigvar(python, var, default) = let v = pyconfigvar(python, var)
     v == "None" ? default : v
 end

diff --git a/deps/depsutils.jl b/deps/depsutils.jl
@@ -64,8 +64,10 @@ function _preserveas!(dest::Vector{UInt8}, ::Type{Cstring}, x::AbstractString)
 end
 
 function _preserveas!(dest::Vector{UInt8}, ::Type{Cwstring}, x::AbstractString)
-    s = reinterpret(UInt8, Base.cconvert(Cwstring, x))
-    copyto!(resize!(dest, length(s)), s)
+    s = reinterpret(UInt8, transcode(Cwchar_t, String(x)))
+    len = length(s)
+    copyto!(resize!(dest, len + sizeof(Cwchar_t)), s)
+    dest[len + 1:len + sizeof(Cwchar_t)] .= 0
     return pointer(dest)
 end
 

diff --git a/deps/find_libpython.py b/deps/find_libpython.py
@@ -264,7 +264,7 @@ def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple):
 
     Parameters
     ----------
-    path : str ot None
+    path : str or None
         A candidate path to a shared library.
     """
     if not path: