-
-
Notifications
You must be signed in to change notification settings - Fork 1.9k
/
hook-numpy.py
169 lines (131 loc) · 6.11 KB
/
hook-numpy.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
#!/usr/bin/env python3
# --- Copyright Disclaimer ---
#
# In order to support PyInstaller with numpy<1.20.0 this file will be
# duplicated for a short period inside PyInstaller's repository [1]. However
# this file is the intellectual property of the NumPy team and is under the
# terms and conditions outlined their repository [2].
#
# .. refs:
#
# [1] PyInstaller: https://github.com/pyinstaller/pyinstaller/
# [2] NumPy's license: https://github.com/numpy/numpy/blob/master/LICENSE.txt
#
"""
This hook should collect all binary files and any hidden modules that numpy
needs.
Our (some-what inadequate) docs for writing PyInstaller hooks are kept here:
https://pyinstaller.readthedocs.io/en/stable/hooks.html
PyInstaller has a lot of numpy users so we'd consider maintaining this hook to
be high priority. Feel free to @mention either bwoodsend or Legorooj on Github
for help keeping it working.
"""
import os
import re
from pathlib import Path
from PyInstaller.utils.hooks import collect_dynamic_libs, exec_statement, logger
from PyInstaller import compat
# --- Plain official numpy from PyPI ---
# Supporting regular numpy is actually dead easy.
# We need to collect all used dll/so/dylibs. As these are all inside of numpy's
# `site-packages/numpy` directory we can just use the following:
binaries = collect_dynamic_libs("numpy", ".")
# Note: this does not collect Python extension modules. They are found using
# Python's native import scheme.
# PyInstaller can't detect imports from a cython or C extension module. If a
# submodule is *only* ever `import`ed by a cython or C module then it wont be
# found. Likewise __import__ and importlib are invisible to PyInstaller.
# These issues should be very easy to track. You'll get a ModuleNotFoundError
# for that hidden module at runtime (usually straight away) if you forget to
# include one.
hiddenimports = ['numpy.core._dtype_ctypes']
# --- Additional support for less official mkl builds ---
# Check if MKL is being used.
# We avoid using `import numpy` directly in hooks in-case doing so alters either
# sys.path or PATH which could confuse the build.
is_mkl = exec_statement("""
# XXX: Numpy devs - is this a good way to test if using MKL?
import numpy
print(bool(numpy.__config__.blas_mkl_info))
""") == "True"
# The MKL binaries themselves are included inside the numpy folder and will
# therefore already have been found by `collect_dynamic_libs()` above.
def find_library(name):
"""Glob-find and include a dll (like) binary file which is usually found by
searching PATH.
"""
# We'll hopefully include this in later versions of PyInstaller. So that
# NumPy remains compatible with PyInstaller 4.0, I'm copying it here.
names = set()
binaries = []
for folder in os.environ["PATH"].split(os.pathsep):
for path in Path(folder).glob(name):
if not path.name in names:
binaries.append((str(path), "."))
names.add(path.name)
if not binaries:
logger.warning(
"Failed to find '%s' DLL in PATH. Your app will likely crash if run"
" on a different machine that doesn't already have it.", name
)
return binaries
if is_mkl:
# Other dlls that MKL uses which PyInstaller can't detect itself. These are
# determined empirically using dynamic dependency sniffing and will lead
# to issues in the future should they change...
for lib in ["libcrypto*", "libffi*", "libssl*"]:
binaries.extend(find_library(lib))
# --- A vain attempt at Conda's numpy support ---
# Regular numpy, even with unofficial mkl builds, is pretty trivial to support
# with PyInstaller. Unfortunately Conda's numpy is the opposite. We need
# their help to maintain this because our own attempts have been a disaster.
if compat.is_conda:
hiddenimports.append("six")
# There are so many hidden binary dependencies. This list is heavily, OS,
# Python and NumPy versions dependent. Omitting any of these can lead to
# obscure and often traceback-less crashes.
# XXX: As you can see, this is really not a scalable solution. Needs help!
conda_dll_patterns = [re.compile(i) for i in (
'apphelp.*', 'crypt32.*', 'imagehlp.*', 'libblas.*',
'libcblas.*', 'libcrypto.*', 'libffi.*', 'libgcc_.*',
'libgfortran.*', 'libifcoremd.*', r'libiomp\d+md.*', 'liblapack.*',
'libmmd.*', 'libomp.*',
'libopenblas.*', 'libquadmath.*', 'libssl.*', 'libuuid.*',
'libz.*', 'mkl_avx.*', 'mkl_core.*',
'mkl_intel_thread.*', 'mkl_rt.*', 'mkl_vml_avx.*',
'mkl_vml_avx.*', 'msasn.*', 'mswsock.*', 'ole.*',
'oleaut.*', 'tbbmalloc.*', 'urandom'
)]
if compat.is_win:
lib_dir = os.path.join(compat.base_prefix, "Library", "bin")
else:
lib_dir = os.path.join(compat.base_prefix, "lib")
def _is_required(name):
return any(pattern.match(name) for pattern in conda_dll_patterns)
_to_add = set(filter(_is_required, os.listdir(lib_dir)))
for name in _to_add:
binaries.append((os.path.join(lib_dir, name), "."))
# --- Remove testing and building code ---
excludedimports = ["scipy", "pytest", "nose", "distutils", "f2py", "setuptools",
"numpy.f2py", "numpy.distutils"]
# I would suggest using the following to remove all the `tests` submodules but
# we don't need it. They will be included if any modules that are included
# contain an explicit `import numpy.xxx.tests`. Should you're tests structure
# change so that they start to get sucked in, uncomment the lines below.
# from PyInstaller.utils.hooks import collect_submodules
# is_tests = lambda x: "tests" in x.split(".")
# excludedimports += collect_submodules("numpy", filter=is_tests)
# --- Remove binaries that aren't DLLs ---
import ctypes
def _is_valid(source, dest):
# There really should be a less brute-force way of doing this.
if source_dest[0].endswith(".pdb"):
# Attempting to load a pdb causes a pop-up window. Check for and exclude
# them here.
return False
try:
ctypes.CDLL(source)
return True
except OSError:
return False
binaries = [i for i in binaries if _is_valid(*i)]