diff --git a/.gitignore b/.gitignore index 4440aee8f9a..d598d2e633f 100644 --- a/.gitignore +++ b/.gitignore @@ -131,9 +131,15 @@ __pycache__/ # Generated Cython files *.so **/*.so +/src/cython_debug +# Most C and C++ files are generated by Cython and should not +# be included in the sdist. /src/sage/**/*.c /src/sage/**/*.cpp +# C header generated by Cython /src/sage/modular/arithgroup/farey_symbol.h +# List of C and C++ files that are actual source files, +# NOT generated by Cython. The same list appears in src/MANIFEST.in !/src/sage/cpython/debugimpl.c !/src/sage/graphs/base/boost_interface.cpp !/src/sage/graphs/cliquer/cl.c @@ -156,7 +162,6 @@ __pycache__/ !/src/sage/stats/distributions/dgs_gauss_dp.c !/src/sage/stats/distributions/dgs_gauss_mp.c !/src/sage/symbolic/ginac/*.cpp -/src/cython_debug # Temporary build files build/temp.*/ diff --git a/Makefile b/Makefile index 004a6750920..414398ddf0d 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ pypi-wheels: rm -f venv/var/lib/sage/installed/$$a-*; \ done for a in $(PYPI_WHEEL_PACKAGES); do \ - $(MAKE) SAGE_EDITABLE=no $$a; \ + $(MAKE) SAGE_EDITABLE=no SAGE_WHEELS=yes $$a; \ done @echo "Built wheels are in venv/var/lib/sage/wheels/" @@ -112,7 +112,7 @@ wheels: rm -f venv/var/lib/sage/installed/$$a-*; \ done for a in $(WHEEL_PACKAGES); do \ - $(MAKE) SAGE_EDITABLE=no $$a; \ + $(MAKE) SAGE_EDITABLE=no SAGE_WHEELS=yes $$a; \ done @echo "Built wheels are in venv/var/lib/sage/wheels/" diff --git a/build/bin/sage-build-env b/build/bin/sage-build-env index ed999b703f0..87cd0fde5f3 100644 --- a/build/bin/sage-build-env +++ b/build/bin/sage-build-env @@ -31,6 +31,10 @@ if [ "x$SAGE_BUILD_ENV_SOURCED" = "x" ]; then if [ "x$SAGE_EDITABLE" = "x" ]; then export SAGE_EDITABLE="$CONFIGURED_SAGE_EDITABLE" fi + # Likewise for SAGE_WHEELS + if [ "x$SAGE_WHEELS" = "x" ]; then + export SAGE_WHEELS="$CONFIGURED_SAGE_WHEELS" + fi # This is usually blank if the system GMP is used, or $SAGE_LOCAL otherwise if [ -n "$SAGE_GMP_PREFIX" ]; then diff --git a/build/bin/sage-build-env-config.in b/build/bin/sage-build-env-config.in index 58d6bd5e5d7..b00fd2a3f99 100644 --- a/build/bin/sage-build-env-config.in +++ b/build/bin/sage-build-env-config.in @@ -58,3 +58,4 @@ export SAGE_SUITESPARSE_PREFIX="@SAGE_SUITESPARSE_PREFIX@" export SAGE_CONFIGURE_FFLAS_FFPACK="@SAGE_CONFIGURE_FFLAS_FFPACK@" export CONFIGURED_SAGE_EDITABLE="@SAGE_EDITABLE@" +export CONFIGURED_SAGE_WHEELS="@SAGE_WHEELS@" diff --git a/build/bin/sage-pip-install b/build/bin/sage-pip-install index 08978fe5717..71f436a4b47 100755 --- a/build/bin/sage-pip-install +++ b/build/bin/sage-pip-install @@ -28,7 +28,7 @@ PIP=pip3 # We should avoid running pip while installing a package because that # is prone to race conditions. Therefore, we use a lockfile while # running pip. This is implemented in the Python script sage-flock -LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" +LOCK="$SAGE_VENV/var/lock/$PIP.lock" # Trac #33155: Pythons installed using the python.org macOS installers # for Python < 3.10 identify macOS Big Sur and newer as "10.16", causing diff --git a/build/bin/sage-pip-uninstall b/build/bin/sage-pip-uninstall index 3017627dbbc..616f86a065b 100755 --- a/build/bin/sage-pip-uninstall +++ b/build/bin/sage-pip-uninstall @@ -14,7 +14,7 @@ PIP=pip3 # We should avoid running pip while uninstalling a package because that # is prone to race conditions. Therefore, we use a lockfile while # running pip. This is implemented in the Python script sage-flock -LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" +LOCK="$SAGE_VENV/var/lock/$PIP.lock" # --disable-pip-version-check: Don't periodically check PyPI to determine whether a new version of pip is available # --no-input: Disable prompting for input. diff --git a/build/pkgs/sage_conf/spkg-install b/build/pkgs/sage_conf/spkg-install index e572d89f80d..a180fb36542 100755 --- a/build/pkgs/sage_conf/spkg-install +++ b/build/pkgs/sage_conf/spkg-install @@ -12,6 +12,9 @@ fi cd src if [ "$SAGE_EDITABLE" = yes ]; then sdh_pip_editable_install . + if [ "$SAGE_WHEELS" = yes ]; then + sdh_setup_bdist_wheel && sdh_store_wheel . + fi else sdh_pip_install . fi diff --git a/build/pkgs/sage_docbuild/spkg-install b/build/pkgs/sage_docbuild/spkg-install index 1bb66bc4a07..cce58b40e22 100755 --- a/build/pkgs/sage_docbuild/spkg-install +++ b/build/pkgs/sage_docbuild/spkg-install @@ -12,6 +12,9 @@ fi cd src if [ "$SAGE_EDITABLE" = yes ]; then sdh_pip_editable_install . + if [ "$SAGE_WHEELS" = yes ]; then + sdh_setup_bdist_wheel && sdh_store_wheel . + fi else sdh_pip_install . fi diff --git a/build/pkgs/sage_setup/spkg-install b/build/pkgs/sage_setup/spkg-install index 1bb66bc4a07..cce58b40e22 100755 --- a/build/pkgs/sage_setup/spkg-install +++ b/build/pkgs/sage_setup/spkg-install @@ -12,6 +12,9 @@ fi cd src if [ "$SAGE_EDITABLE" = yes ]; then sdh_pip_editable_install . + if [ "$SAGE_WHEELS" = yes ]; then + sdh_setup_bdist_wheel && sdh_store_wheel . + fi else sdh_pip_install . fi diff --git a/build/pkgs/sage_sws2rst/spkg-install b/build/pkgs/sage_sws2rst/spkg-install index 12d0d16dff1..47ef9ee6cbf 100755 --- a/build/pkgs/sage_sws2rst/spkg-install +++ b/build/pkgs/sage_sws2rst/spkg-install @@ -3,16 +3,21 @@ # For type=script packages, the build rule in build/make/Makefile sources # sage-env but not sage-dist-helpers. lib="$SAGE_ROOT/build/bin/sage-dist-helpers" -. "$lib" +source "$lib" if [ $? -ne 0 ]; then echo >&2 "Error: failed to source $lib" echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?" exit 1 fi -set -e -# We build the wheel directly with "setup.py bdist_wheel", not with "pip wheel", -# because pip does not handle our symlinks correctly. -(cd src && sdh_setup_bdist_wheel && sdh_store_and_pip_install_wheel .) +cd src +if [ "$SAGE_EDITABLE" = yes ]; then + sdh_pip_editable_install . + if [ "$SAGE_WHEELS" = yes ]; then + sdh_setup_bdist_wheel && sdh_store_wheel . + fi +else + sdh_pip_install . +fi # For type=script packages, spkg-check is not run case "$SAGE_CHECK" in yes) diff --git a/build/pkgs/sagelib/spkg-install b/build/pkgs/sagelib/spkg-install index 8d91b16b3f0..ad8b2ed43fc 100755 --- a/build/pkgs/sagelib/spkg-install +++ b/build/pkgs/sagelib/spkg-install @@ -1,4 +1,15 @@ #!/usr/bin/env bash +# From sage-spkg. +# For type=script packages, the build rule in build/make/Makefile sources +# sage-env but not sage-dist-helpers. +lib="$SAGE_ROOT/build/bin/sage-dist-helpers" +source "$lib" +if [ $? -ne 0 ]; then + echo >&2 "Error: failed to source $lib" + echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?" + exit 1 +fi + if [ "$SAGE_EDITABLE" = yes ]; then cd "$SAGE_SRC" else @@ -27,8 +38,6 @@ export SAGE_SRC_ROOT=/doesnotexist export SAGE_DOC_SRC=/doesnotexist export SAGE_BUILD_DIR=/doesnotexist -export PYTHON="$SAGE_LOCAL/bin/python3" - # We also poison all directories below SAGE_LOCAL. export SAGE_PKGCONFIG=/doesnotexist export SAGE_SPKG_SCRIPTS=/doesnotexist @@ -51,15 +60,27 @@ if [ "$SAGE_EDITABLE" = yes ]; then # under the old distribution name "sage" (before #30912, which switched to setuptools # and renamed the distribution to "sagemath-standard"). There is no clean way to uninstall # them, so we just use rm. - (cd "$SITEPACKAGESDIR" && rm -rf sage sage_setup sage-[1-9]*.egg-info sage-[1-9]*.dist-info) - time python3 -m pip install --verbose --no-deps --no-index --no-build-isolation --isolated --editable . || exit 1 + (cd "$SITEPACKAGESDIR" && rm -rf sage sage-[1-9]*.egg-info sage-[1-9]*.dist-info) + time sdh_pip_editable_install . + + if [ "$SAGE_WHEELS" = yes ]; then + # Additionally build a wheel (for use in other venvs) + cd $SAGE_PKGS/sagelib/src && time sdh_setup_bdist_wheel && sdh_store_wheel . + fi else # Make sure that an installed old version of sagelib in which sage is an ordinary package # does not shadow the namespace package sage during the build. (cd "$SITEPACKAGESDIR" && rm -f sage/__init__.py) # Likewise, we should remove the egg-link that may have been installed previously. (cd "$SITEPACKAGESDIR" && rm -f sagemath-standard.egg-link) - time python3 -u setup.py --no-user-cfg build install || exit 1 + + if [ "$SAGE_WHEELS" = yes ]; then + # Use --no-build-isolation to avoid rebuilds because of dependencies: + # Compiling sage/interfaces/sagespawn.pyx because it depends on /private/var/folders/38/wnh4gf1552g_crsjnv2vmmww0000gp/T/pip-build-env-609n5985/overlay/lib/python3.10/site-packages/Cython/Includes/posix/unistd.pxd + time sdh_pip_install --no-build-isolation . + else + time python3 -u setup.py --no-user-cfg build install || exit 1 + fi fi # Trac #33103: The temp.* directories are large after a full build. diff --git a/configure.ac b/configure.ac index 10351b8c416..1070ac424ab 100644 --- a/configure.ac +++ b/configure.ac @@ -132,6 +132,12 @@ AC_ARG_ENABLE([editable], [AC_SUBST([SAGE_EDITABLE], [$enableval])], [AC_SUBST([SAGE_EDITABLE], [yes])]) +AC_ARG_ENABLE([wheels], + [AS_HELP_STRING([--enable-wheels], + [build wheels for the Sage library and update them on "sage -b"; if disabled, use "make wheels" to build wheels])], + [AC_SUBST([SAGE_WHEELS], [$enableval])], + []) + # Check whether we are on a supported platform AC_CANONICAL_BUILD() AC_CANONICAL_HOST() diff --git a/pkgs/sagemath-standard/setup.py b/pkgs/sagemath-standard/setup.py index b6006188d33..ce9fd0a70aa 100755 --- a/pkgs/sagemath-standard/setup.py +++ b/pkgs/sagemath-standard/setup.py @@ -15,6 +15,15 @@ import multiprocessing multiprocessing.set_start_method('fork', force=True) +# If build isolation is not in use and setuptools_scm is installed, +# then its file_finders entry point is invoked, which we don't need. +# Workaround from ​https://github.com/pypa/setuptools_scm/issues/190#issuecomment-351181286 +try: + import setuptools_scm.integration + setuptools_scm.integration.find_files = lambda _: [] +except ImportError: + pass + ######################################################### ### Set source directory ######################################################### @@ -30,7 +39,7 @@ ### Configuration ######################################################### -if len(sys.argv) > 1 and (sys.argv[1] == "sdist" or sys.argv[1] == "egg_info"): +if len(sys.argv) > 1 and (sys.argv[1] in ["sdist", "egg_info", "dist_info"]): sdist = True else: sdist = False diff --git a/src/MANIFEST.in b/src/MANIFEST.in index f63856e03aa..3a63d07fc32 100644 --- a/src/MANIFEST.in +++ b/src/MANIFEST.in @@ -1,11 +1,47 @@ -global-include *.c *.cc *.cpp *.h *.hh *.hpp *.inc *.py *.pyx *.pxd *.pxi *.rst *.txt *.tex - -include MANIFEST.in -include pyproject.toml - -prune .tox - prune sage/ext/interpreters # In particular, __init__.py must not be present in the distribution; or sage_setup.autogen.interpreters.rebuild will not generate the code prune sage_setup prune sage_docbuild prune doc + +# +# Most C and C++ files are generated by Cython and should not +# be included in the sdist. +# +global-exclude *.c +global-exclude *.cpp + +# +# List of C and C++ files that are actual source files, +# NOT generated by Cython. The same list appears in SAGE_ROOT/.gitignore +# +include sage/cpython/debugimpl.c +include sage/graphs/base/boost_interface.cpp +include sage/graphs/cliquer/cl.c +include sage/graphs/graph_decompositions/sage_tdlib.cpp +include sage/libs/eclib/wrap.cpp +include sage/libs/linkages/padics/relaxed/flint_helper.c +include sage/misc/inherit_comparison_impl.c +include sage/modular/arithgroup/farey.cpp +include sage/modular/arithgroup/sl2z.cpp +include sage/rings/bernmm/bern_modp.cpp +include sage/rings/bernmm/bern_modp_util.cpp +include sage/rings/bernmm/bern_rat.cpp +include sage/rings/bernmm/bernmm-test.cpp +include sage/rings/padics/transcendantal.c +include sage/rings/polynomial/weil/power_sums.c +include sage/schemes/hyperelliptic_curves/hypellfrob/hypellfrob.cpp +include sage/schemes/hyperelliptic_curves/hypellfrob/recurrences_ntl.cpp +include sage/schemes/hyperelliptic_curves/hypellfrob/recurrences_zn_poly.cpp +include sage/stats/distributions/dgs_bern.c +include sage/stats/distributions/dgs_gauss_dp.c +include sage/stats/distributions/dgs_gauss_mp.c +include sage/symbolic/ginac/*.cpp + +global-exclude __pycache__ +global-exclude *.py[co] +global-exclude *.bak +global-exclude *.so +global-exclude *~ +prune .tox +prune build +prune dist diff --git a/src/sage/libs/linkages/__init__.py b/src/sage/libs/linkages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/sage/misc/package_dir.py b/src/sage/misc/package_dir.py index 8b5474972f2..abc64fe7f91 100644 --- a/src/sage/misc/package_dir.py +++ b/src/sage/misc/package_dir.py @@ -1,22 +1,154 @@ +# sage_setup: distribution = sagemath-environment """ Recognizing package directories """ +# **************************************************************************** +# Copyright (C) 2020-2022 Matthias Koeppe +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# https://www.gnu.org/licenses/ +# **************************************************************************** + import os import glob from contextlib import contextmanager -def is_package_or_sage_namespace_package_dir(path): +class SourceDistributionFilter: + r""" + A :class:`collections.abc.Container` for source files in distributions. + + INPUT: + + - ``include_distributions`` -- (default: ``None``) if not ``None``, + should be a sequence or set of strings: include files whose + ``distribution`` (from a ``# sage_setup: distribution = PACKAGE`` + directive in the source file) is an element of ``distributions``. + + - ``exclude_distributions`` -- (default: ``None``) if not ``None``, + should be a sequence or set of strings: exclude files whose + ``distribution`` (from a ``# sage_setup: distribution = PACKAGE`` + directive in the module source file) is in ``exclude_distributions``. + + EXAMPLES:: + + sage: from sage.misc.package_dir import SourceDistributionFilter + sage: F = SourceDistributionFilter() + sage: sage.misc.package_dir.__file__ in F + True + sage: F = SourceDistributionFilter(include_distributions=['sagemath-environment']) + sage: sage.misc.package_dir.__file__ in F + True + sage: F = SourceDistributionFilter(exclude_distributions=['sagemath-environment']) + sage: sage.misc.package_dir.__file__ in F + False + """ + def __init__(self, include_distributions=None, exclude_distributions=None): + r""" + TESTS: + + ``exclude_distributions=None`` is normalized to the empty tuple:: + + sage: from sage.misc.package_dir import SourceDistributionFilter + sage: F = SourceDistributionFilter() + sage: F._exclude_distributions + () + """ + self._include_distributions = include_distributions + if exclude_distributions is None: + exclude_distributions = () + self._exclude_distributions = exclude_distributions + + def __contains__(self, filename): + r""" + TESTS: + + No file access is used when neither ``include_distributions`` nor + ``exclude_distributions`` is given:: + + sage: from sage.misc.package_dir import SourceDistributionFilter + sage: F = SourceDistributionFilter() + sage: '/doesnotexist' in F + True + + ``exclude_distributions`` can also be an empty container:: + + sage: F = SourceDistributionFilter(exclude_distributions=()) + sage: '/doesnotexist' in F + True + """ + if self._include_distributions is None and not self._exclude_distributions: + return True + distribution = read_distribution(filename) + if self._include_distributions is not None: + if distribution not in self._include_distributions: + return False + return distribution not in self._exclude_distributions + + +def read_distribution(src_file): + """ + Parse ``src_file`` for a ``# sage_setup: distribution = PKG`` directive. + + INPUT: + + - ``src_file`` -- file name of a Python or Cython source file + + OUTPUT: + + - a string, the name of the distribution package (``PKG``); or the empty + string if no directive was found. + + EXAMPLES:: + + sage: from sage.env import SAGE_SRC + sage: from sage_setup.find import read_distribution + sage: read_distribution(os.path.join(SAGE_SRC, 'sage', 'graphs', 'graph_decompositions', 'tdlib.pyx')) + 'sagemath-tdlib' + sage: read_distribution(os.path.join(SAGE_SRC, 'sage', 'graphs', 'graph_decompositions', 'modular_decomposition.py')) + '' + """ + from Cython.Utils import open_source_file + with open_source_file(src_file, error_handling='ignore') as fh: + for line in fh: + # Adapted from Cython's Build/Dependencies.py + line = line.lstrip() + if not line: + continue + if line[0] != '#': + break + line = line[1:].lstrip() + kind = "sage_setup:" + if line.startswith(kind): + key, _, value = [s.strip() for s in line[len(kind):].partition('=')] + if key == "distribution": + return value + return '' + + +def is_package_or_sage_namespace_package_dir(path, *, distribution_filter=None): r""" Return whether ``path`` is a directory that contains a Python package. - Ordinary Python packages are recognized by the presence of `__init__.py`. + Ordinary Python packages are recognized by the presence of ``__init__.py``. Implicit namespace packages (PEP 420) are only recognized if they follow the conventions of the Sage library, i.e., the directory contains a file ``all.py`` or a file matching the pattern ``all__*.py`` such as ``all__sagemath_categories.py``. + INPUT: + + - ``path`` -- a directory name. + + - ``distribution_filter`` -- (optional, default: ``None``) + only consider ``all*.py`` files whose distribution (from a + ``# sage_setup: distribution = PACKAGE`` directive in the source file) + is an element of ``distribution_filter``. + EXAMPLES: :mod:`sage.cpython` is an ordinary package:: @@ -49,14 +181,17 @@ def is_package_or_sage_namespace_package_dir(path): sage: is_package_or_sage_namespace_package_dir(directory) False """ - if os.path.exists(os.path.join(path, '__init__.py')): # ordinary package - return True - if os.path.exists(os.path.join(path, '__init__.pxd')): # for consistency with Cython + if os.path.exists(os.path.join(path, '__init__.py')): # ordinary package return True - if os.path.exists(os.path.join(path, 'all.py')): # complete namespace package + if os.path.exists(os.path.join(path, '__init__.pxd')): # for consistency with Cython return True - for _ in glob.iglob(os.path.join(path, 'all__*.py')): - return True # partial namespace package + fname = os.path.join(path, 'all.py') + if os.path.exists(fname): + if distribution_filter is None or fname in distribution_filter: # complete namespace package + return True + for fname in glob.iglob(os.path.join(path, 'all__*.py')): + if distribution_filter is None or fname in distribution_filter: # partial namespace package + return True return False @@ -71,7 +206,7 @@ def cython_namespace_package_support(): import Cython.Build.Cythonize import Cython.Utils orig_is_package_dir = Cython.Utils.is_package_dir - Cython.Utils.is_package_dir = Cython.Build.Cythonize.is_package_dir = Cython.Build.Dependencies.is_package_dir = is_package_or_sage_namespace_package_dir + Cython.Utils.is_package_dir = Cython.Build.Cythonize.is_package_dir = Cython.Build.Dependencies.is_package_dir = Cython.Utils.cached_function(is_package_or_sage_namespace_package_dir) try: yield finally: diff --git a/src/sage_setup/clean.py b/src/sage_setup/clean.py index 4ed286cdac6..3cf0d08a353 100644 --- a/src/sage_setup/clean.py +++ b/src/sage_setup/clean.py @@ -2,7 +2,10 @@ Clean the Install Dir """ #***************************************************************************** -# Copyright (C) 2014 Volker Braun +# Copyright (C) 2014 Volker Braun +# 2015 Jeroen Demeyer +# 2017 Erik M. Bray +# 2020-2022 Matthias Koeppe # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,8 +18,8 @@ import os import importlib.util -from sage_setup.find import installed_files_by_module, get_extensions, read_distribution - +from sage.misc.package_dir import SourceDistributionFilter +from sage_setup.find import installed_files_by_module, get_extensions def _remove(file_set, module_base, to_remove): @@ -83,8 +86,14 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module sage: from sage_setup.find import find_python_sources, find_extra_files sage: python_packages, python_modules, cython_modules = find_python_sources( ....: SAGE_SRC, ['sage', 'sage_setup']) - sage: extra_files = list(find_extra_files(SAGE_SRC, - ....: ['sage', 'sage_setup'], cythonized_dir, []).items()) + sage: extra_files = find_extra_files(SAGE_SRC, + ....: ['sage', 'sage_setup'], cythonized_dir, []) + sage: from importlib.metadata import files + sage: for f in files('sagemath-standard'): + ....: dir = os.path.dirname(str(f)) + ....: extra_files[dir] = extra_files.get(dir, []) + ....: extra_files[dir].append(str(f)) + sage: extra_files = list(extra_files.items()) sage: from sage_setup.clean import _find_stale_files TODO: Also check extension modules:: @@ -141,7 +150,7 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module def clean_install_dir(site_packages, python_packages, python_modules, ext_modules, data_files, nobase_data_files, *, - distributions=None): + distributions=None, exclude_distributions=None): """ Delete all modules that are **not** being installed @@ -178,10 +187,11 @@ def clean_install_dir(site_packages, python_packages, python_modules, ext_module ``distribution`` (from a ``# sage_setup: distribution = PACKAGE`` directive in the file) is an element of ``distributions``. """ + distribution_filter = SourceDistributionFilter(distributions, exclude_distributions) stale_file_iter = _find_stale_files( site_packages, python_packages, python_modules, ext_modules, data_files, nobase_data_files) for f in stale_file_iter: f = os.path.join(site_packages, f) - if distributions is None or read_distribution(f) in distributions: + if f in distribution_filter: print('Cleaning up stale file: {0}'.format(f)) os.unlink(f) diff --git a/src/sage_setup/find.py b/src/sage_setup/find.py index 5cec2c4ee44..61d91abc2eb 100644 --- a/src/sage_setup/find.py +++ b/src/sage_setup/find.py @@ -2,7 +2,12 @@ Recursive Directory Contents """ # **************************************************************************** -# Copyright (C) 2014 Volker Braun +# Copyright (C) 2014 Volker Braun +# 2014 R. Andrew Ohana +# 2015-2018 Jeroen Demeyer +# 2017 Erik M. Bray +# 2021 Tobias Diez +# 2020-2022 Matthias Koeppe # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,49 +24,13 @@ from collections import defaultdict from sage.misc.package_dir import is_package_or_sage_namespace_package_dir as is_package_or_namespace_package_dir +from sage.misc.package_dir import read_distribution, SourceDistributionFilter +assert read_distribution # unused in this file, re-export for compatibility -def read_distribution(src_file): - """ - Parse ``src_file`` for a ``# sage_setup: distribution = PKG`` directive. - - INPUT: - - - ``src_file`` -- file name of a Python or Cython source file - - OUTPUT: - - - a string, the name of the distribution package (``PKG``); or the empty - string if no directive was found. - - EXAMPLES:: - sage: from sage.env import SAGE_SRC - sage: from sage_setup.find import read_distribution - sage: read_distribution(os.path.join(SAGE_SRC, 'sage', 'graphs', 'graph_decompositions', 'tdlib.pyx')) - 'sagemath-tdlib' - sage: read_distribution(os.path.join(SAGE_SRC, 'sage', 'graphs', 'graph_decompositions', 'modular_decomposition.py')) - '' - """ - from Cython.Utils import open_source_file - with open_source_file(src_file, error_handling='ignore') as fh: - for line in fh: - # Adapted from Cython's Build/Dependencies.py - line = line.lstrip() - if not line: - continue - if line[0] != '#': - break - line = line[1:].lstrip() - kind = "sage_setup:" - if line.startswith(kind): - key, _, value = [s.strip() for s in line[len(kind):].partition('=')] - if key == "distribution": - return value - return '' - - -def find_python_sources(src_dir, modules=['sage'], distributions=None): +def find_python_sources(src_dir, modules=['sage'], distributions=None, + exclude_distributions=None): """ Find all Python packages and Python/Cython modules in the sources. @@ -78,6 +47,11 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None): directive in the module source file) is an element of ``distributions``. + - ``exclude_distributions`` -- (default: ``None``) if not ``None``, + should be a sequence or set of strings: exclude modules whose + ``distribution`` (from a ``# sage_setup: distribution = PACKAGE`` + directive in the module source file) is in ``exclude_distributions``. + OUTPUT: Triple consisting of - the list of package names (corresponding to ordinary packages @@ -164,6 +138,8 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None): python_modules = [] cython_modules = [] + distribution_filter = SourceDistributionFilter(distributions, exclude_distributions) + cwd = os.getcwd() try: os.chdir(src_dir) @@ -171,24 +147,21 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None): for dirpath, dirnames, filenames in os.walk(module): package = dirpath.replace(os.path.sep, '.') if not is_package_or_namespace_package_dir(dirpath): + # Skip any subdirectories + dirnames[:] = [] continue # Ordinary package or namespace package. if distributions is None or '' in distributions: python_packages.append(package) - def is_in_distributions(filename): - if distributions is None: - return True - distribution = read_distribution(os.path.join(dirpath, filename)) - return distribution in distributions - for filename in filenames: base, ext = os.path.splitext(filename) + filepath = os.path.join(dirpath, filename) if ext == PYMOD_EXT and base != '__init__': - if is_in_distributions(filename): + if filepath in distribution_filter: python_modules.append(package + '.' + base) if ext == '.pyx': - if is_in_distributions(filename): + if filepath in distribution_filter: cython_modules.append(Extension(package + '.' + base, sources=[os.path.join(dirpath, filename)])) @@ -196,7 +169,7 @@ def is_in_distributions(filename): os.chdir(cwd) return python_packages, python_modules, cython_modules -def filter_cython_sources(src_dir, distributions): +def filter_cython_sources(src_dir, distributions, exclude_distributions=None): """ Find all Cython modules in the given source directory that belong to the given distributions. @@ -235,12 +208,12 @@ def filter_cython_sources(src_dir, distributions): 1 loops, best of 1: 850 ms per loop """ files: list[str] = [] - + distribution_filter = SourceDistributionFilter(distributions, exclude_distributions) for dirpath, dirnames, filenames in os.walk(src_dir): for filename in filenames: filepath = os.path.join(dirpath, filename) base, ext = os.path.splitext(filename) - if ext == '.pyx' and read_distribution(filepath) in distributions: + if ext == '.pyx' and filepath in distribution_filter: files.append(filepath) return files