diff --git a/PyInstaller/hooks/hook-numpy.core.py b/PyInstaller/hooks/hook-numpy.core.py deleted file mode 100644 index 350cba20fa1..00000000000 --- a/PyInstaller/hooks/hook-numpy.core.py +++ /dev/null @@ -1,47 +0,0 @@ -#----------------------------------------------------------------------------- -# Copyright (c) 2013-2020, PyInstaller Development Team. -# -# Distributed under the terms of the GNU General Public License (version 2 -# or later) with exception for distributing the bootloader. -# -# The full license is in the file COPYING.txt, distributed with this software. -# -# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) -#----------------------------------------------------------------------------- -# If numpy is built with MKL support it depends on a set of libraries loaded -# at runtime. Since PyInstaller's static analysis can't find them they must be -# included manually. -# -# See -# https://github.com/pyinstaller/pyinstaller/issues/1881 -# https://github.com/pyinstaller/pyinstaller/issues/1969 -# for more information -import os -import os.path -import re -from PyInstaller.utils.hooks import get_package_paths -from PyInstaller import log as logging -from PyInstaller import compat - -binaries = [] - -# look for libraries in numpy package path -pkg_base, pkg_dir = get_package_paths('numpy.core') -re_anylib = re.compile(r'\w+\.(?:dll|so|dylib)', re.IGNORECASE) -dlls_pkg = [f for f in os.listdir(pkg_dir) if re_anylib.match(f)] -binaries += [(os.path.join(pkg_dir, f), '.') for f in dlls_pkg] - -# look for MKL libraries in pythons lib directory -# TODO: check numpy.__config__ if numpy is actually depending on MKL -# TODO: determine which directories are searched by the os linker -if compat.is_win: - lib_dir = os.path.join(compat.base_prefix, "Library", "bin") -else: - lib_dir = os.path.join(compat.base_prefix, "lib") -if os.path.isdir(lib_dir): - re_mkllib = re.compile(r'^(?:lib)?mkl\w+\.(?:dll|so|dylib)', re.IGNORECASE) - dlls_mkl = [f for f in os.listdir(lib_dir) if re_mkllib.match(f)] - if dlls_mkl: - logger = logging.getLogger(__name__) - logger.info("MKL libraries found when importing numpy. Adding MKL to binaries") - binaries += [(os.path.join(lib_dir, f), '.') for f in dlls_mkl] diff --git a/PyInstaller/hooks/hook-numpy.py b/PyInstaller/hooks/hook-numpy.py index 623164639df..4f3f9f4cc17 100644 --- a/PyInstaller/hooks/hook-numpy.py +++ b/PyInstaller/hooks/hook-numpy.py @@ -1,54 +1,149 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) 2013-2020, PyInstaller Development Team. +#!/usr/bin/env python3 + +# --- Copyright Disclaimer --- +# +# In order to support PyInstaller with numpy<1.20.0 this file will be +# duplicated for a short period inside PyInstaller's repository [1]. However +# this file is the intellectual property of the NumPy team and is under the +# terms and conditions outlined their repository [2]. # -# Distributed under the terms of the GNU General Public License (version 2 -# or later) with exception for distributing the bootloader. +# .. refs: # -# The full license is in the file COPYING.txt, distributed with this software. +# [1] PyInstaller: https://github.com/pyinstaller/pyinstaller/ +# [2] NumPy's license: https://github.com/numpy/numpy/blob/master/LICENSE.txt # -# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception) -# ----------------------------------------------------------------------------- + +""" +This hook should collect all binary files and any hidden modules that numpy +needs. + +Our (some-what inadequate) docs for writing PyInstaller hooks are kept here: +https://pyinstaller.readthedocs.io/en/stable/hooks.html + +PyInstaller has a lot of numpy users so we'd consider maintaining this hook to +be high priority. Feel free to @mention either bwoodsend or Legorooj on Github +for help keeping it working. +""" + import os -import glob -from PyInstaller.compat import is_win, is_venv, base_prefix -from PyInstaller.utils.hooks import get_module_file_attribute - -# numpy.testing is unconditionally imported by numpy, thus we can not exclude -# .testing (which would be preferred). Anyway, this only saves about 7 -# modules. See also https://github.com/numpy/numpy/issues/17183 -#excludedimports = ["numpy.testing"] - -# FIXME check if this workaround is still necessary! -if is_win: - from PyInstaller.utils.win32.winutils import extend_system_path - from distutils.sysconfig import get_python_lib - # SciPy/Numpy Windows builds from http://www.lfd.uci.edu/~gohlke/pythonlibs - # contain some dlls in directory like C:\Python27\Lib\site-packages\numpy\core\ - numpy_core_paths = [os.path.join(get_python_lib(), 'numpy', 'core')] - # In virtualenv numpy might be installed directly in real prefix path. - # Then include this path too. - if is_venv: - numpy_core_paths.append( - os.path.join(base_prefix, 'Lib', 'site-packages', 'numpy', 'core') +import re +from pathlib import Path + +from PyInstaller.utils.hooks import collect_dynamic_libs, exec_statement, logger +from PyInstaller import compat + +# --- Plain official numpy from PyPI --- + +# Supporting regular numpy is actually dead easy. + +# We need to collect all used dll/so/dylibs. As these are all inside of numpy's +# `site-packages/numpy` directory we can just use the following: +binaries = collect_dynamic_libs("numpy", ".") +# Note: this does not collect Python extension modules. They are found using +# Python's native import scheme. + +# PyInstaller can't detect imports from a cython or C extension module. If a +# submodule is *only* ever `import`ed by a cython or C module then it wont be +# found. Numpy currently has none of these. But should one be added, its string +# name should be listed here. e.g. "numpy.core._hidden._guts._module". +# These issues should be very easy to track. You'll get a ModuleNotFoundError +# for that hidden module at runtime if you forget to include one. +hiddenimports = [] + + +# --- Additional support for less official mkl builds --- + +# Check if MKL is being used. +# We avoid using `import numpy` directly in hooks in-case doing so alters either +# sys.path or PATH which could confuse the build. +is_mkl = exec_statement(""" +# XXX: Numpy devs - is this a good way to test if using MKL? +import numpy +print(bool(numpy.__config__.blas_mkl_info)) +""") == "True" + +# The MKL binaries themselves are included inside the numpy folder and will +# therefore already have been found by `collect_dynamic_libs()` above. + +def find_library(name): + """Glob-find and include a dll (like) binary file which is usually found by + searching PATH. + """ + # We'll hopefully include this in later versions of PyInstaller. So that + # NumPy remains compatible with PyInstaller 4.0, I'm copying it here. + + names = set() + binaries = [] + for folder in os.environ["PATH"].split(os.pathsep): + for path in Path(folder).glob(name): + if not path.name in names: + binaries.append((str(path), ".")) + names.add(path.name) + if not binaries: + logger.warning( + "Failed to find '%s' DLL in PATH. Your app will likely crash if run" + " on a different machine that doesn't already have it.", name ) - extend_system_path(numpy_core_paths) - del numpy_core_paths - -# if we bundle the testing module, this will cause -# `scipy` to be pulled in unintentionally but numpy imports -# numpy.testing at the top level for historical reasons. -# excludedimports = collect_submodules('numpy.testing') - -binaries = [] - -# package the DLL bundle that official numpy wheels for Windows ship -# The DLL bundle will either be in extra-dll on windows proper -# and in .libs if installed on a virtualenv created from MinGW (Git-Bash -# for example) -if is_win: - extra_dll_locations = ['extra-dll', '.libs'] - for location in extra_dll_locations: - dll_glob = os.path.join(os.path.dirname( - get_module_file_attribute('numpy')), location, "*.dll") - if glob.glob(dll_glob): - binaries.append((dll_glob, ".")) + return binaries + + +if is_mkl: + # Other dlls that MKL uses which PyInstaller can't detect itself. These are + # determined empirically using dynamic dependency sniffing and will lead + # to issues in the future should they change... + for lib in ["libcrypto*", "libffi*", "libssl*"]: + binaries.extend(find_library(lib)) + + +# --- A vain attempt at Conda's numpy support --- + +# Regular numpy, even with unofficial mkl builds, is pretty trivial to support +# with PyInstaller. Unfortunately Conda's numpy is the opposite. We need +# their help to maintain this because our own attempts have been a disaster. + +if compat.is_conda: + hiddenimports.append("six") + + # There are so many hidden binary dependencies. This list is heavily, OS, + # Python and NumPy versions dependent. Omitting any of these can lead to + # obscure and often traceback-less crashes. + # XXX: As you can see, this is really not a scalable solution. Needs help! + conda_dll_patterns = [re.compile(i) for i in ( + 'apphelp.*', 'crypt32.*', 'imagehlp.*', 'libblas.*', + 'libcblas.*', 'libcrypto.*', 'libffi.*', 'libgcc_.*', + 'libgfortran.*', 'libifcoremd.*', r'libiomp\d+md.*', 'liblapack.*', + 'libmmd.*', 'libomp.*', + 'libopenblas.*', 'libquadmath.*', 'libssl.*', 'libuuid.*', + 'libz.*', 'mkl_avx.*', 'mkl_core.*', + 'mkl_intel_thread.*', 'mkl_rt.*', 'mkl_vml_avx.*', + 'mkl_vml_avx.*', 'msasn.*', 'mswsock.*', 'ole.*', + 'oleaut.*', 'tbbmalloc.*', 'urandom' + )] + + if compat.is_win: + lib_dir = os.path.join(compat.base_prefix, "Library", "bin") + else: + lib_dir = os.path.join(compat.base_prefix, "lib") + + def _is_required(name): + return any(pattern.match(name) for pattern in conda_dll_patterns) + + _to_add = set(filter(_is_required, os.listdir(lib_dir))) + + for name in _to_add: + binaries.append((os.path.join(lib_dir, name), ".")) + + +# --- Remove testing and building code --- + +excludedimports = ["scipy", "pytest", "nose", "distutils", "f2py", "setuptools", + "numpy.f2py", "numpy.distutils"] + +# I would suggest using the following to remove all the `tests` submodules but +# we don't need it. They will be included if any modules that are included +# contain an explicit `import numpy.xxx.tests`. Should you're tests structure +# change so that they start to get sucked in, uncomment the lines below. + +# from PyInstaller.utils.hooks import collect_submodules +# is_tests = lambda x: "tests" in x.split(".") +# excludedimports += collect_submodules("numpy", filter=is_tests)