From fb88f4d0df66fd2ce1bc4dc862611c355be0e50d Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sun, 23 Sep 2018 16:53:07 -0700 Subject: [PATCH] Use find_libpython.py in deps/build.jl (#556) * Copy find_libpython.py from PyJulia * Call find_libpython.py from deps/build.jl * Handle the case find_libpython.py cannot find libpython * Store full path to libpython in deps.jl * Update find_libpython.py * Use --candidate-names in the fallback path * Improve debugging message * Use pythonenv in when invoking find_libpython.py * Add _linked_libpython_windows --- deps/build.jl | 104 ++++------- deps/find_libpython.py | 394 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 433 insertions(+), 65 deletions(-) create mode 100755 deps/find_libpython.py diff --git a/deps/build.jl b/deps/build.jl index e8255e2a..7bd4b393 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -47,91 +47,65 @@ pysys(python::AbstractString, var::AbstractString) = pyvar(python, "sys", var) const dlprefix = Compat.Sys.iswindows() ? "" : "lib" -# return libpython name, libpython pointer -function find_libpython(python::AbstractString) - # it is ridiculous that it is this hard to find the name of libpython - v = pyconfigvar(python,"VERSION","") - libs = [ dlprefix*"python"*v, dlprefix*"python" ] - lib = pyconfigvar(python, "LIBRARY") - lib != "None" && pushfirst!(libs, splitext(lib)[1]) - lib = pyconfigvar(python, "LDLIBRARY") - lib != "None" && pushfirst!(pushfirst!(libs, basename(lib)), lib) - libs = unique(libs) - - # it is ridiculous that it is this hard to find the path of libpython - libpaths = [pyconfigvar(python, "LIBDIR"), - (Compat.Sys.iswindows() ? dirname(pysys(python, "executable")) : joinpath(dirname(dirname(pysys(python, "executable"))), "lib"))] - if Compat.Sys.isapple() - push!(libpaths, pyconfigvar(python, "PYTHONFRAMEWORKPREFIX")) - end +# print out extra info to help with remote debugging +const PYCALL_DEBUG_BUILD = "yes" == get(ENV, "PYCALL_DEBUG_BUILD", "no") - # `prefix` and `exec_prefix` are the path prefixes where python should look for python only and compiled libraries, respectively. - # These are also changed when run in a virtualenv. - exec_prefix = pysys(python, "exec_prefix") - - push!(libpaths, exec_prefix) - push!(libpaths, joinpath(exec_prefix, "lib")) +function exec_find_libpython(python::AbstractString, options) + cmd = `$python $(joinpath(@__DIR__, "find_libpython.py")) $options` + if PYCALL_DEBUG_BUILD + cmd = `$cmd --verbose` + end + return readlines(pythonenv(cmd)) +end - error_strings = String[] +function show_dlopen_error(e) + if PYCALL_DEBUG_BUILD + println(stderr, "dlopen($libpath_lib) ==> ", e) + # Using STDERR since find_libpython.py prints debugging + # messages to STDERR too. + end +end - # TODO: other paths? python-config output? pyconfigvar("LDFLAGS")? +# return libpython name, libpython pointer +function find_libpython(python::AbstractString) + dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL - # find libpython (we hope): - for lib in libs - for libpath in libpaths - libpath_lib = joinpath(libpath, lib) - if isfile(libpath_lib*"."*Libdl.dlext) - try - return (Libdl.dlopen(libpath_lib, - Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL), - libpath_lib) - catch e - push!(error_strings, string("dlopen($libpath_lib) ==> ", e)) - end - end + libpaths = exec_find_libpython(python, `--list-all`) + for lib in libpaths + try + return (Libdl.dlopen(lib, dlopen_flags), lib) + catch e + show_dlopen_error(e) end end + # Try all candidate libpython names and let Libdl find the path. # We do this *last* because the libpython in the system # library path might be the wrong one if multiple python # versions are installed (we prefer the one in LIBDIR): + libs = exec_find_libpython(python, `--candidate-names`) for lib in libs lib = splitext(lib)[1] try - return (Libdl.dlopen(lib, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL), - lib) + libpython = Libdl.dlopen(lib, dlopen_flags) + # Store the fullpath to libpython in deps.jl. This makes + # it easier for users to investigate Python setup + # PyCall.jl trying to use. It also helps PyJulia to + # compare libpython. + return (libpython, Libdl.dlpath(libpython)) catch e - push!(error_strings, string("dlopen($lib) ==> ", e)) - end - end - - if "yes" == get(ENV, "PYCALL_DEBUG_BUILD", "no") # print out extra info to help with remote debugging - println(stderr, "------------------------------------- exceptions -----------------------------------------") - for s in error_strings - print(s, "\n\n") - end - println(stderr, "---------------------------------- get_config_vars ---------------------------------------") - print(stderr, read(`python -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_vars())"`, String)) - println(stderr, "--------------------------------- directory contents -------------------------------------") - for libpath in libpaths - if isdir(libpath) - print(libpath, ":\n") - for file in readdir(libpath) - if occursin("pyth", file) - println(" ", file) - end - end - end + show_dlopen_error(e) end - println(stderr, "------------------------------------------------------------------------------------------") end error(""" Couldn't find libpython; check your PYTHON environment variable. - The python executable we tried was $python (= version $v); - the library names we tried were $libs - and the library paths we tried were $libpaths""") + The python executable we tried was $python (= version $v). + Re-building with + ENV["PYCALL_DEBUG_BUILD"] = "yes" + may provide extra information for why it failed. + """) end ######################################################################### diff --git a/deps/find_libpython.py b/deps/find_libpython.py new file mode 100755 index 00000000..c5156179 --- /dev/null +++ b/deps/find_libpython.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python + +""" +Locate libpython associated with this Python executable. +""" + +# License +# +# Copyright 2018, Takafumi Arakaki +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import print_function, absolute_import + +from logging import getLogger +import ctypes.util +import functools +import os +import sys +import sysconfig + +logger = getLogger("find_libpython") + +is_windows = os.name == "nt" +is_apple = sys.platform == "darwin" + +SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") +if SHLIB_SUFFIX is None: + if is_windows: + SHLIB_SUFFIX = ".dll" + else: + SHLIB_SUFFIX = ".so" +if is_apple: + # sysconfig.get_config_var("SHLIB_SUFFIX") can be ".so" in macOS. + # Let's not use the value from sysconfig. + SHLIB_SUFFIX = ".dylib" + + +def linked_libpython(): + """ + Find the linked libpython using dladdr (in *nix). + + Returns + ------- + path : str or None + A path to linked libpython. Return `None` if statically linked. + """ + if is_windows: + return _linked_libpython_windows() + return _linked_libpython_unix() + + +class Dl_info(ctypes.Structure): + _fields_ = [ + ("dli_fname", ctypes.c_char_p), + ("dli_fbase", ctypes.c_void_p), + ("dli_sname", ctypes.c_char_p), + ("dli_saddr", ctypes.c_void_p), + ] + + +def _linked_libpython_unix(): + libdl = ctypes.CDLL(ctypes.util.find_library("dl")) + libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] + libdl.dladdr.restype = ctypes.c_int + + dlinfo = Dl_info() + retcode = libdl.dladdr( + ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p), + ctypes.pointer(dlinfo)) + if retcode == 0: # means error + return None + path = os.path.realpath(dlinfo.dli_fname.decode()) + if path == os.path.realpath(sys.executable): + return None + return path + + +def _linked_libpython_windows(): + """ + Based on: https://stackoverflow.com/a/16659821 + """ + from ctypes.wintypes import HANDLE, LPWSTR, DWORD + + GetModuleFileName = ctypes.windll.kernel32.GetModuleFileNameW + GetModuleFileName.argtypes = [HANDLE, LPWSTR, DWORD] + GetModuleFileName.restype = DWORD + + MAX_PATH = 260 + try: + buf = ctypes.create_unicode_buffer(MAX_PATH) + GetModuleFileName(ctypes.pythonapi._handle, buf, MAX_PATH) + return buf.value + except (ValueError, OSError): + return None + + + +def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows): + """ + Convert a file basename `name` to a library name (no "lib" and ".so" etc.) + + >>> library_name("libpython3.7m.so") # doctest: +SKIP + 'python3.7m' + >>> library_name("libpython3.7m.so", suffix=".so", is_windows=False) + 'python3.7m' + >>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False) + 'python3.7m' + >>> library_name("python37.dll", suffix=".dll", is_windows=True) + 'python37' + """ + if not is_windows and name.startswith("lib"): + name = name[len("lib"):] + if suffix and name.endswith(suffix): + name = name[:-len(suffix)] + return name + + +def append_truthy(list, item): + if item: + list.append(item) + + +def uniquifying(items): + """ + Yield items while excluding the duplicates and preserving the order. + + >>> list(uniquifying([1, 2, 1, 2, 3])) + [1, 2, 3] + """ + seen = set() + for x in items: + if x not in seen: + yield x + seen.add(x) + + +def uniquified(func): + """ Wrap iterator returned from `func` by `uniquifying`. """ + @functools.wraps(func) + def wrapper(*args, **kwds): + return uniquifying(func(*args, **kwds)) + return wrapper + + +@uniquified +def candidate_names(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate file names of libpython. + + Yields + ------ + name : str + Candidate name libpython. + """ + LDLIBRARY = sysconfig.get_config_var("LDLIBRARY") + if LDLIBRARY: + yield LDLIBRARY + + LIBRARY = sysconfig.get_config_var("LIBRARY") + if LIBRARY: + yield os.path.splitext(LIBRARY)[0] + suffix + + dlprefix = "" if is_windows else "lib" + sysdata = dict( + v=sys.version_info, + # VERSION is X.Y in Linux/macOS and XY in Windows: + VERSION=(sysconfig.get_config_var("VERSION") or + "{v.major}.{v.minor}".format(v=sys.version_info)), + ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or + sysconfig.get_config_var("abiflags") or ""), + ) + + for stem in [ + "python{VERSION}{ABIFLAGS}".format(**sysdata), + "python{VERSION}".format(**sysdata), + "python{v.major}".format(**sysdata), + "python", + ]: + yield dlprefix + stem + suffix + + + +@uniquified +def candidate_paths(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate paths of libpython. + + Yields + ------ + path : str or None + Candidate path to libpython. The path may not be a fullpath + and may not exist. + """ + + yield linked_libpython() + + # List candidates for directories in which libpython may exist + lib_dirs = [] + append_truthy(lib_dirs, sysconfig.get_config_var('LIBPL')) + append_truthy(lib_dirs, sysconfig.get_config_var('srcdir')) + append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR")) + + # LIBPL seems to be the right config_var to use. It is the one + # used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.7.0/Misc/python-config.in#L55-L57 + # + # But we try other places just in case. + + if is_windows: + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + else: + lib_dirs.append(os.path.join( + os.path.dirname(os.path.dirname(sys.executable)), + "lib")) + + # For macOS: + append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")) + + lib_dirs.append(sys.exec_prefix) + lib_dirs.append(os.path.join(sys.exec_prefix, "lib")) + + lib_basenames = list(candidate_names(suffix=suffix)) + + for directory in lib_dirs: + for basename in lib_basenames: + yield os.path.join(directory, basename) + + # In macOS and Windows, ctypes.util.find_library returns a full path: + for basename in lib_basenames: + yield ctypes.util.find_library(library_name(basename)) + +# Possibly useful links: +# * https://packages.ubuntu.com/bionic/amd64/libpython3.6/filelist +# * https://github.com/Valloric/ycmd/issues/518 +# * https://github.com/Valloric/ycmd/pull/519 + + +def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): + """ + Normalize shared library `path` to a real path. + + If `path` is not a full path, `None` is returned. If `path` does + not exists, append `SHLIB_SUFFIX` and check if it exists. + Finally, the path is canonicalized by following the symlinks. + + Parameters + ---------- + path : str ot None + A candidate path to a shared library. + """ + if not path: + return None + if not os.path.isabs(path): + return None + if os.path.exists(path): + return os.path.realpath(path) + if os.path.exists(path + suffix): + return os.path.realpath(path + suffix) + if is_apple: + return normalize_path(_remove_suffix_apple(path), + suffix=".so", is_apple=False) + return None + + +def _remove_suffix_apple(path): + """ + Strip off .so or .dylib. + + >>> _remove_suffix_apple("libpython.so") + 'libpython' + >>> _remove_suffix_apple("libpython.dylib") + 'libpython' + >>> _remove_suffix_apple("libpython3.7") + 'libpython3.7' + """ + if path.endswith(".dylib"): + return path[:-len(".dylib")] + if path.endswith(".so"): + return path[:-len(".so")] + return path + + +@uniquified +def finding_libpython(): + """ + Iterate over existing libpython paths. + + The first item is likely to be the best one. + + Yields + ------ + path : str + Existing path to a libpython. + """ + logger.debug("is_windows = %s", is_windows) + logger.debug("is_apple = %s", is_apple) + for path in candidate_paths(): + logger.debug("Candidate: %s", path) + normalized = normalize_path(path) + if normalized: + logger.debug("Found: %s", normalized) + yield normalized + else: + logger.debug("Not found.") + + +def find_libpython(): + """ + Return a path (`str`) to libpython or `None` if not found. + + Parameters + ---------- + path : str or None + Existing path to the (supposedly) correct libpython. + """ + for path in finding_libpython(): + return os.path.realpath(path) + + +def print_all(items): + for x in items: + print(x) + + +def cli_find_libpython(cli_op, verbose): + import logging + # Importing `logging` module here so that using `logging.debug` + # instead of `logger.debug` outside of this function becomes an + # error. + + if verbose: + logging.basicConfig( + format="%(levelname)s %(message)s", + level=logging.DEBUG) + + if cli_op == "list-all": + print_all(finding_libpython()) + elif cli_op == "candidate-names": + print_all(candidate_names()) + elif cli_op == "candidate-paths": + print_all(p for p in candidate_paths() if p and os.path.isabs(p)) + else: + path = find_libpython() + if path is None: + return 1 + print(path, end="") + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Print debugging information.") + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--list-all", + action="store_const", dest="cli_op", const="list-all", + help="Print list of all paths found.") + group.add_argument( + "--candidate-names", + action="store_const", dest="cli_op", const="candidate-names", + help="Print list of candidate names of libpython.") + group.add_argument( + "--candidate-paths", + action="store_const", dest="cli_op", const="candidate-paths", + help="Print list of candidate paths of libpython.") + + ns = parser.parse_args(args) + parser.exit(cli_find_libpython(**vars(ns))) + + +if __name__ == "__main__": + main()