src/sage_setup/{__init__.py,find.py,command}: Update from sagemath#35095

src/sage_setup/__init__.py

@@ -0,0 +1,156 @@
+def sage_setup(distributions, *,
+               interpreters=(),
+               required_modules=(), optional_modules=(),
+               recurse_packages=(),
+               package_data=None):
+    r"""
+    Replacement for :func:`setuptools.setup` for building distribution packages of the Sage library
+
+    INPUT:
+
+    - ``distributions`` -- (typically one-element) sequence of strings, the distribution names
+      shipped with this distribution package.
+
+    - ``interpreters`` -- sequence of strings, the interpreters to build with :mod:`sage_setup.autogen`.
+
+    - ``required_modules`` -- sequence of strings, pkgconfig modules that are required for the build.
+
+    - ``optional_modules`` -- sequence of strings, pkgconfig modules to checked for the build.
+
+    - ``package_data`` -- ``None`` or a dictionary mapping package names to lists of filename
+      glob patterns, the package data to install.
+
+      * If ``None``, all of ``package_data`` is taken from ``pyproject.toml``.
+
+      * If a dictionary, use it as package data and ignore ``package_data`` in ``pyproject.toml``.
+    """
+    import time
+
+    from setuptools import setup, find_namespace_packages
+    from distutils import log
+    from setuptools.dist import Distribution
+    from sage_setup.command.sage_build_ext_minimal import sage_build_ext_minimal
+    from sage_setup.command.sage_build_py import sage_build_py
+    from sage_setup.cython_options import compiler_directives, compile_time_env_variables
+    from sage_setup.extensions import create_extension
+    from sage_setup.find import find_python_sources, find_extra_files
+
+    # Work around a Cython problem in Python 3.8.x on macOS
+    # https://github.com/cython/cython/issues/3262
+    import os
+    if os.uname().sysname == 'Darwin':
+        import multiprocessing
+        multiprocessing.set_start_method('fork', force=True)
+
+    # setuptools plugins considered harmful:
+    # If build isolation is not in use and setuptools_scm is installed,
+    # then its file_finders entry point is invoked, which we don't need.
+    # Workaround from ​https://github.com/pypa/setuptools_scm/issues/190#issuecomment-351181286
+    try:
+        import setuptools_scm.integration
+        setuptools_scm.integration.find_files = lambda _: []
+    except ImportError:
+        pass
+
+    # And with setuptools_scm 8, we get more trouble:
+    # LookupError: pyproject.toml does not contain a tool.setuptools_scm section
+    # LookupError: setuptools-scm was unable to detect version ...
+    # We just remove all handling of "setuptools.finalize_distribution_options" entry points.
+    Distribution._removed = staticmethod(lambda ep: True)
+
+    # Different workaround: disable `walk_revctrl` in setuptools
+    # This is needed for setuptools_scm >= 8, should work for any version
+    import setuptools.command.egg_info
+    setuptools.command.egg_info.walk_revctrl = lambda: ()
+
+    import sys
+
+    from sage_setup.excepthook import excepthook
+    sys.excepthook = excepthook
+
+    from sage_setup.setenv import setenv
+    setenv()
+
+    import sage.env
+    sage.env.default_required_modules = required_modules
+    sage.env.default_optional_modules = optional_modules
+
+    cmdclass = dict(build_ext=sage_build_ext_minimal,
+                    build_py=sage_build_py)
+
+    sdist = len(sys.argv) > 1 and (sys.argv[1] in ["sdist", "egg_info", "dist_info"])
+
+    # ########################################################
+    # ## Discovering Sources
+    # ########################################################
+    if sdist:
+        extensions = []
+        python_modules = []
+        python_packages = []
+    else:
+        if interpreters:
+            log.info("Generating auto-generated sources")
+            # from sage_setup.autogen import autogen_all
+            # autogen_all()
+            from sage_setup.autogen.interpreters import rebuild
+            rebuild(os.path.join("sage", "ext", "interpreters"),
+                    interpreters=interpreters,
+                    distribution=distributions[0], force=True)
+
+        log.info("Discovering Python/Cython source code....")
+        t = time.time()
+
+        python_packages, python_modules, cython_modules = find_python_sources(
+            '.', ['sage'], distributions=distributions)
+        extra_files = find_extra_files(
+            '.', ['sage'], '/doesnotexist', distributions=distributions)
+
+        python_packages += find_namespace_packages(where='.', include=recurse_packages)
+
+        if package_data is not None:
+            package_data.update({"": [f
+                                      for pkg, files in extra_files.items()
+                                      for f in files]})
+            python_packages += list(package_data)
+
+        log.warn('python_packages = {0}'.format(sorted(python_packages)))
+        log.warn('python_modules = {0}'.format(sorted(m if isinstance(m, str) else m.name for m in python_modules)))
+        log.warn('cython_modules = {0}'.format(sorted(m if isinstance(m, str) else m.name for m in cython_modules)))
+        log.warn('package_data = {0}'.format(package_data))
+
+        log.info(f"Discovered Python/Cython sources, time: {(time.time() - t):.2f} seconds.")
+
+        # from sage_build_cython:
+        import Cython.Compiler.Options
+        Cython.Compiler.Options.embed_pos_in_docstring = True
+        gdb_debug = os.environ.get('SAGE_DEBUG', None) != 'no'
+
+        try:
+            from Cython.Build import cythonize
+            from sage.env import cython_aliases, sage_include_directories
+            from sage.misc.package_dir import cython_namespace_package_support
+            with cython_namespace_package_support():
+                extensions = cythonize(
+                    cython_modules,
+                    include_path=sage_include_directories(use_sources=True) + ['.'],
+                    compile_time_env=compile_time_env_variables(),
+                    compiler_directives=compiler_directives(False),
+                    aliases=cython_aliases(),
+                    create_extension=create_extension,
+                    gdb_debug=gdb_debug,
+                    nthreads=4)
+        except Exception as exception:
+            log.warn(f"Exception while cythonizing source files: {repr(exception)}")
+            raise
+
+    kwds = {}
+
+    if package_data is not None:
+        kwds['package_data'] = package_data
+
+    setup(cmdclass=cmdclass,
+          packages=python_packages,
+          py_modules=python_modules,
+          ext_modules=extensions,
+          **kwds
+    )

src/sage_setup/find.py

@@ -131,7 +131,7 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None,
         sage: find_python_sources(SAGE_SRC, modules=['sage_setup'])
         (['sage_setup', ...], [...'sage_setup.find'...], [])
     """
-    from setuptools import Extension
+    from setuptools.extension import Extension
 
     PYMOD_EXT = get_extensions('source')[0]
 
@@ -147,13 +147,12 @@ def find_python_sources(src_dir, modules=['sage'], distributions=None,
         for module in modules:
             for dirpath, dirnames, filenames in os.walk(module):
                 package = dirpath.replace(os.path.sep, '.')
-                if not is_package_or_namespace_package_dir(dirpath):
+                if not is_package_or_namespace_package_dir(dirpath, distribution_filter=distribution_filter):
                     # Skip any subdirectories
                     dirnames[:] = []
                     continue
                 # Ordinary package or namespace package.
-                if distributions is None or '' in distributions:
-                    python_packages.append(package)
+                python_packages.append(package)
 
                 for filename in filenames:
                     base, ext = os.path.splitext(filename)
@@ -260,7 +259,7 @@ def _cythonized_dir(src_dir=None, editable_install=None):
 
 
 def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[], *,
-                     distributions=None):
+                     distributions=None, exclude_distributions=None):
     """
     Find all extra files which should be installed.
 
@@ -293,6 +292,11 @@ def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[], *,
       ``distribution`` (from a ``# sage_setup: distribution = PACKAGE``
       directive in the file) is an element of ``distributions``.
 
+    - ``exclude_distributions`` -- (default: ``None``) if not ``None``,
+      should be a sequence or set of strings: exclude modules whose
+      ``distribution`` (from a ``# sage_setup: distribution = PACKAGE``
+      directive in the module source file) is in ``exclude_distributions``.
+
     OUTPUT: dict with items ``{dir: files}`` where ``dir`` is a
     directory relative to ``src_dir`` and ``files`` is a list of
     filenames inside that directory.
@@ -311,12 +315,14 @@ def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[], *,
     data_files = {}
     cy_exts = ('.pxd', '.pxi', '.pyx')
 
+    distribution_filter = SourceDistributionFilter(distributions, exclude_distributions)
+
     cwd = os.getcwd()
     try:
         os.chdir(src_dir)
         for module in modules:
             for dir, dirnames, filenames in os.walk(module):
-                if not is_package_or_namespace_package_dir(dir):
+                if not is_package_or_namespace_package_dir(dir, distribution_filter=distribution_filter):
                     continue
                 sdir = os.path.join(src_dir, dir)
                 cydir = os.path.join(cythonized_dir, dir)