From cb0842e5d5df3a566045c55aa04b2ad4e3861396 Mon Sep 17 00:00:00 2001 From: Jeroen Demeyer Date: Thu, 15 Mar 2018 14:27:22 +0100 Subject: [PATCH] Fix find_extra_files() to support Cython packages --- src/sage_setup/clean.py | 5 ++- src/sage_setup/find.py | 87 ++++++++++++++++++++++++++--------------- src/setup.py | 5 +-- 3 files changed, 60 insertions(+), 37 deletions(-) diff --git a/src/sage_setup/clean.py b/src/sage_setup/clean.py index aaf99837cb3..3e325d85719 100644 --- a/src/sage_setup/clean.py +++ b/src/sage_setup/clean.py @@ -87,8 +87,9 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module sage: from sage_setup.find import find_python_sources, find_extra_files sage: python_packages, python_modules = find_python_sources( ....: SAGE_SRC, ['sage', 'sage_setup']) - sage: extra_files = find_extra_files(python_packages, SAGE_SRC, - ....: cythonized_dir, ["ntlwrap.cpp"]) + sage: extra_files = list(find_extra_files(SAGE_SRC, + ....: ['sage', 'sage_setup'], cythonized_dir, + ....: ["ntlwrap.cpp"]).items()) sage: from sage_setup.clean import _find_stale_files TODO: move ``module_list.py`` into ``sage_setup`` and also check diff --git a/src/sage_setup/find.py b/src/sage_setup/find.py index 09f8a15b24c..bf166b9ef5f 100644 --- a/src/sage_setup/find.py +++ b/src/sage_setup/find.py @@ -21,22 +21,25 @@ import importlib.util -def find_python_sources(src_dir, modules=('sage',)): +def find_python_sources(src_dir, modules=['sage']): """ - Find all currently installed files + Find all Python packages and Python modules in the sources. INPUT: - - ``src_dir`` -- string. The root directory for the sources. + - ``src_dir`` -- root directory for the sources - - ``module`` -- list/tuple/iterable of strings (default: - ``('sage',)``). The top-level directory name(s) in ``src_dir``. + - ``modules`` -- (default: ``['sage']``) sequence of strings: + the top-level directories in ``src_dir`` to be considered - OUTPUT: + OUTPUT: Pair consisting of + + - the list of package names (corresponding to directories with + ``__init__.py``), + + - module names (corresponding to other ``*.py`` files). - Pair consisting of the list of package names (directories with - ``__init__.py``) and module names (remaining ``*.py`` files). Both - use dot as separator. + Both use dot as separator. EXAMPLES:: @@ -81,22 +84,27 @@ def find_python_sources(src_dir, modules=('sage',)): return python_packages, python_modules -def find_extra_files(packages, src_dir, cythonized_dir, special_filenames=[]): +def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[]): """ Find all extra files which should be installed. - These are: + These are (for each ``module`` in ``modules``): - 1. From ``src_dir``: all .pxd and .pxi files and files listed in - ``special_filenames``. - 2. From ``cythonized_dir``: all .h files (these are both the .h files - from the Sage sources, as well as all Cython-generated .h files). + 1. From ``src_dir/module``: all .pyx, .pxd and .pxi files and files + listed in ``special_filenames``. + 2. From ``cythonized_dir/module``: all .h files (both the .h files + from the sources, as well as all Cython-generated .h files). + + The directories are searched recursively, but only package + directories (containing ``__init__.py`` or a Cython equivalent) + are considered. INPUT: - - ``packages`` -- a list of Python packages to be considered + - ``src_dir`` -- root directory for the sources - - ``src_dir`` -- the directory where to look for source files + - ``modules`` -- sequence of strings: + the top-level directories in ``src_dir`` to be considered - ``cythonized_dir`` -- the directory where the Cython-generated files are @@ -104,31 +112,46 @@ def find_extra_files(packages, src_dir, cythonized_dir, special_filenames=[]): - ``special_filenames`` -- a list of filenames to be installed from ``src_dir`` + OUTPUT: dict with items ``{dir: files}`` where ``dir`` is a + directory relative to ``src_dir`` and ``files`` is a list of + filenames inside that directory. + EXAMPLES:: sage: from sage_setup.find import find_extra_files sage: from sage.env import SAGE_SRC sage: cythonized_dir = os.path.join(SAGE_SRC, "build", "cythonized") - sage: find_extra_files(["sage.ext.interpreters"], SAGE_SRC, cythonized_dir) - [('sage/ext/interpreters', - ['.../src/sage/ext/interpreters/wrapper_cdf.pxd', ...wrapper_cdf.h...])] + sage: extras = find_extra_files(SAGE_SRC, ["sage"], cythonized_dir) + sage: extras["sage/libs/mpfr"] + [...sage/libs/mpfr/types.pxd...] + sage: extras["sage/ext/interpreters"] + ['.../src/sage/ext/interpreters/wrapper_cdf.pxd', ...wrapper_cdf.h...] """ - data_files = [] + from Cython.Utils import is_package_dir + + data_files = {} cy_exts = ('.pxd', '.pxi', '.pyx') - for package in packages: - dir = package.replace('.', os.path.sep) - sdir = os.path.join(src_dir, dir) - cydir = os.path.join(cythonized_dir, dir) + cwd = os.getcwd() + try: + os.chdir(src_dir) + for module in modules: + for dir, dirnames, filenames in os.walk(module): + if not is_package_dir(dir): + continue + sdir = os.path.join(src_dir, dir) + cydir = os.path.join(cythonized_dir, dir) - files = [os.path.join(sdir, f) for f in os.listdir(sdir) - if f.endswith(cy_exts) or f in special_filenames] - if os.path.isdir(cydir): # Not every directory contains Cython files - files += [os.path.join(cydir, f) for f in os.listdir(cydir) - if f.endswith(".h")] + files = [os.path.join(sdir, f) for f in filenames + if f.endswith(cy_exts) or f in special_filenames] + if os.path.isdir(cydir): # Not every directory contains Cython files + files += [os.path.join(cydir, f) for f in os.listdir(cydir) + if f.endswith(".h")] - if files: - data_files.append((dir, files)) + if files: + data_files[dir] = files + finally: + os.chdir(cwd) return data_files diff --git a/src/setup.py b/src/setup.py index 5abd6dc5939..0679f7dac2f 100755 --- a/src/setup.py +++ b/src/setup.py @@ -340,9 +340,8 @@ def get_cythonized_package_files(self): if self.cythonized_files is not None: return self.cythonized_files - dist = self.distribution - self.cythonized_files = find_extra_files(dist.packages, - ".", self.build_dir, ["ntlwrap.cpp"]) + self.cythonized_files = list(find_extra_files( + ".", ["sage"], self.build_dir, ["ntlwrap.cpp"]).items()) return self.cythonized_files