-
-
Notifications
You must be signed in to change notification settings - Fork 432
/
find.py
401 lines (300 loc) · 14.5 KB
/
find.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
# sage.doctest: needs SAGE_SRC
"""
Recursive Directory Contents
"""
# ****************************************************************************
# Copyright (C) 2014 Volker Braun <vbraun.name@gmail.com>
# 2014 R. Andrew Ohana
# 2015-2018 Jeroen Demeyer
# 2017 Erik M. Bray
# 2021 Tobias Diez
# 2020-2022 Matthias Koeppe
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
# http://www.gnu.org/licenses/
# ****************************************************************************
import importlib.machinery
import importlib.util
import os
from collections import defaultdict
from sage.misc.package_dir import is_package_or_sage_namespace_package_dir as is_package_or_namespace_package_dir
from sage.misc.package_dir import read_distribution, SourceDistributionFilter
assert read_distribution # unused in this file, re-export for compatibility
def find_python_sources(src_dir, modules=['sage'], distributions=None,
exclude_distributions=None):
"""
Find all Python packages and Python/Cython modules in the sources.
INPUT:
- ``src_dir`` -- root directory for the sources
- ``modules`` -- (default: ``['sage']``) sequence of strings:
the top-level directories in ``src_dir`` to be considered
- ``distributions`` -- (default: ``None``) if not ``None``,
should be a sequence or set of strings: only find modules whose
``distribution`` (from a ``# sage_setup: distribution = PACKAGE``
directive in the module source file) is an element of
``distributions``.
- ``exclude_distributions`` -- (default: ``None``) if not ``None``,
should be a sequence or set of strings: exclude modules whose
``distribution`` (from a ``# sage_setup: distribution = PACKAGE``
directive in the module source file) is in ``exclude_distributions``.
OUTPUT:
Triple consisting of
- the list of package names (corresponding to ordinary packages
or namespace packages, according to
:func:`sage.misc.namespace_package.is_package_or_sage_namespace_package_dir`)
- Python module names (corresponding to ``*.py`` files in ordinary
or namespace packages)
- Cython extensions (corresponding to ``*.pyx`` files).
Both use dot as separator.
EXAMPLES::
sage: from sage.env import SAGE_SRC
sage: from sage_setup.find import find_python_sources
sage: py_packages, py_modules, cy_modules = find_python_sources(SAGE_SRC)
Ordinary package (with ``__init__.py``)::
sage: ['sage.structure' in L for L in (py_packages, py_modules)]
[True, False]
Python module in an ordinary package::
sage: ['sage.structure.formal_sum' in L for L in (py_packages, py_modules)]
[False, True]
Cython module in an ordinary package::
sage: ['sage.structure.sage_object' in L for L in (py_packages, py_modules)]
[False, False]
Subdirectory without any Python files::
sage: ['sage.doctest.tests' in L for L in (py_packages, py_modules)]
[False, False]
Another subdirectory that is neither an ordinary nor a namespace package::
sage: ['sage.extdata' in L for L in (py_packages, py_modules)]
[False, False]
Package designated to become an implicit namespace package (no ``__init__.py``, PEP 420,
but with an ``all.py`` file per Sage library conventions)::
sage: ['sage.graphs.graph_decompositions' in L for L in (py_packages, py_modules)]
[True, False]
Python module in a package designated to become an implicit namespace package::
sage: ['sage.graphs.graph_decompositions.modular_decomposition' in L for L in (py_packages, py_modules)]
[False, True]
Python file (not module) in a directory that is neither an ordinary nor a namespace package::
sage: ['sage.ext_data.nbconvert.postprocess' in L for L in (py_packages, py_modules)]
[False, False]
Filtering by distribution (distribution package)::
sage: find_python_sources(SAGE_SRC, distributions=['sagemath-tdlib'])
([], [], [<setuptools.extension.Extension('sage.graphs.graph_decompositions.tdlib')...>])
Benchmarking::
sage: timeit('find_python_sources(SAGE_SRC)', # random output
....: number=1, repeat=1)
1 loops, best of 1: 30 ms per loop
sage: timeit('find_python_sources(SAGE_SRC, distributions=[""])', # random output
....: number=1, repeat=1)
1 loops, best of 1: 850 ms per loop
sage: find_python_sources(SAGE_SRC, modules=['sage_setup'])
(['sage_setup', ...], [...'sage_setup.find'...], [])
"""
from setuptools import Extension
PYMOD_EXT = get_extensions('source')[0]
python_packages = []
python_modules = []
cython_modules = []
distribution_filter = SourceDistributionFilter(distributions, exclude_distributions)
cwd = os.getcwd()
try:
os.chdir(src_dir)
for module in modules:
for dirpath, dirnames, filenames in os.walk(module):
package = dirpath.replace(os.path.sep, '.')
if not is_package_or_namespace_package_dir(dirpath):
# Skip any subdirectories
dirnames[:] = []
continue
# Ordinary package or namespace package.
if distributions is None or '' in distributions:
python_packages.append(package)
for filename in filenames:
base, ext = os.path.splitext(filename)
filepath = os.path.join(dirpath, filename)
if ext == PYMOD_EXT and base != '__init__':
if filepath in distribution_filter:
python_modules.append(package + '.' + base)
if ext == '.pyx':
if filepath in distribution_filter:
cython_modules.append(Extension(package + '.' + base,
sources=[os.path.join(dirpath, filename)]))
finally:
os.chdir(cwd)
return python_packages, python_modules, cython_modules
def filter_cython_sources(src_dir, distributions, exclude_distributions=None):
"""
Find all Cython modules in the given source directory that belong to the
given distributions.
INPUT:
- ``src_dir`` -- root directory for the sources
- ``distributions`` -- a sequence or set of strings: only find modules whose
``distribution`` (from a ``# sage_setup: distribution = PACKAGE``
directive in the module source file) is an element of
``distributions``.
OUTPUT: list of absolute paths to Cython files (``*.pyx``)
EXAMPLES::
sage: from sage.env import SAGE_SRC
sage: from sage_setup.find import filter_cython_sources
sage: cython_modules = filter_cython_sources(SAGE_SRC, ["sagemath-tdlib"])
Cython module relying on tdlib::
sage: any(f.endswith('sage/graphs/graph_decompositions/tdlib.pyx') for f in cython_modules)
True
Cython module not relying on tdlib::
sage: any(f.endswith('sage/structure/sage_object.pyx') for f in cython_modules)
False
Benchmarking::
sage: timeit('filter_cython_sources(SAGE_SRC, ["sagemath-tdlib"])', # random output
....: number=1, repeat=1)
1 loops, best of 1: 850 ms per loop
"""
files: list[str] = []
distribution_filter = SourceDistributionFilter(distributions, exclude_distributions)
for dirpath, dirnames, filenames in os.walk(src_dir):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
base, ext = os.path.splitext(filename)
if ext == '.pyx' and filepath in distribution_filter:
files.append(filepath)
return files
def _cythonized_dir(src_dir=None, editable_install=None):
"""
Return the path where Cython-generated files are placed by the build system.
INPUT:
- ``src_dir`` -- string or path (default: the value of ``SAGE_SRC``). The
root directory for the sources.
- ``editable_install`` -- boolean (default: determined from the existing
installation). Whether this is an editable install of the Sage library.
EXAMPLES::
sage: from sage_setup.find import _cythonized_dir
sage: from sage.env import SAGE_SRC
sage: _cythonized_dir(SAGE_SRC)
PosixPath('...')
sage: _cythonized_dir(SAGE_SRC, editable_install=False) # optional - sage_spkg
PosixPath('.../build/cythonized')
"""
from importlib import import_module
from pathlib import Path
from sage.env import SAGE_ROOT, SAGE_SRC
if editable_install is None:
if src_dir is None:
src_dir = SAGE_SRC
src_dir = Path(src_dir)
# sage.cpython is an ordinary package, so it has __file__
sage_cpython = import_module('sage.cpython')
d = Path(sage_cpython.__file__).resolve().parent.parent.parent
editable_install = d == src_dir.resolve()
if editable_install:
# Editable install: Cython generates files in the source tree
return src_dir
else:
return Path(SAGE_ROOT) / "build" / "pkgs" / "sagelib" / "src" / "build" / "cythonized"
def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[], *,
distributions=None):
"""
Find all extra files which should be installed.
These are (for each ``module`` in ``modules``):
1. From ``src_dir/module``: all .pyx, .pxd and .pxi files and files
listed in ``special_filenames``.
2. From ``cythonized_dir/module``: all .h files (both the .h files
from the sources, as well as all Cython-generated .h files).
The directories are searched recursively, but only package
directories (containing ``__init__.py`` or a Cython equivalent)
are considered.
INPUT:
- ``src_dir`` -- root directory for the sources
- ``modules`` -- sequence of strings:
the top-level directories in ``src_dir`` to be considered
- ``cythonized_dir`` -- the directory where the Cython-generated
files are
- ``special_filenames`` -- a list of filenames to be installed from
``src_dir``
- ``distributions`` -- (default: ``None``) if not ``None``,
should be a sequence or set of strings: only find files whose
``distribution`` (from a ``# sage_setup: distribution = PACKAGE``
directive in the file) is an element of ``distributions``.
OUTPUT: dict with items ``{dir: files}`` where ``dir`` is a
directory relative to ``src_dir`` and ``files`` is a list of
filenames inside that directory.
EXAMPLES::
sage: from sage_setup.find import find_extra_files, _cythonized_dir
sage: from sage.env import SAGE_SRC, SAGE_ROOT
sage: cythonized_dir = _cythonized_dir(SAGE_SRC)
sage: extras = find_extra_files(SAGE_SRC, ["sage"], cythonized_dir)
sage: extras["sage/libs/mpfr"]
[...sage/libs/mpfr/types.pxd...]
sage: sorted(extras["sage/ext/interpreters"])
['.../sage/ext/interpreters/wrapper_cdf.h', ...wrapper_cdf.pxd...]
"""
data_files = {}
cy_exts = ('.pxd', '.pxi', '.pyx')
cwd = os.getcwd()
try:
os.chdir(src_dir)
for module in modules:
for dir, dirnames, filenames in os.walk(module):
if not is_package_or_namespace_package_dir(dir):
continue
sdir = os.path.join(src_dir, dir)
cydir = os.path.join(cythonized_dir, dir)
files = [os.path.join(sdir, f) for f in filenames
if f.endswith(cy_exts) or f in special_filenames]
if os.path.isdir(cydir): # Not every directory contains Cython files
files += [os.path.join(cydir, f) for f in os.listdir(cydir)
if f.endswith(".h")]
else:
files += [os.path.join(sdir, f) for f in filenames
if f.endswith(".h")]
if distributions is not None:
files = [f for f in files
if read_distribution(f) in distributions]
if files:
data_files[dir] = files
finally:
os.chdir(cwd)
return data_files
def get_extensions(type=None):
"""
Returns the filename extensions for different types of Python module files.
By default returns all extensions, but can be filtered by type. The
possible types are 'source' (for pure Python sources), 'bytecode' (for
compiled bytecode files (i.e. pyc files), or 'extension' for C extension
modules.
INPUT:
- ``type`` -- the module type ('source', 'bytecode', or 'extension') or
None
EXAMPLES::
sage: from sage_setup.find import get_extensions
sage: get_extensions() # random - depends on Python version
['.so', 'module.so', '.py', '.pyc']
sage: get_extensions('source')
['.py']
sage: get_extensions('bytecode')
['.pyc']
sage: get_extensions('extension') # random - depends on Python version
['.so', 'module.so']
"""
if type:
type = type.lower()
if type not in ('source', 'bytecode', 'extension'):
raise ValueError(
"type must by one of 'source' (for Python sources), "
"'bytecode' (for compiled Python bytecoe), or 'extension' "
"(for C extension modules).")
# Note: There is at least one case, for extension modules, where the
# 'extension' does not begin with '.', but rather with 'module', for cases
# in Python's stdlib, for example, where an extension module can be named
# like "<modname>module.so". This breaks things for us if we have a Cython
# module literally named "module".
return [ext for ext in _get_extensions(type) if ext[0] == '.']
def _get_extensions(type):
"""
Python 3.3+ implementation of ``get_extensions()`` using the
`importlib.extensions` module.
"""
if type:
return {'source': importlib.machinery.SOURCE_SUFFIXES,
'bytecode': importlib.machinery.BYTECODE_SUFFIXES,
'extension': importlib.machinery.EXTENSION_SUFFIXES}[type]
return importlib.machinery.all_suffixes()