Many more namespace packages

src/sage/misc/cython.py

@@ -211,6 +211,28 @@ def cython(filename, verbose=0, compile_message=False,
         sage: cython('''
         ....: cdef size_t foo = 3/2
         ....: ''')
+
+    Check that Cython supports PEP 420 packages::
+
+        sage: cython('''
+        ....: cimport sage.misc.cachefunc
+        ....: ''')
+
+        sage: cython('''
+        ....: from sage.misc.cachefunc cimport cache_key
+        ....: ''')
+
+    In Cython 0.29.33 using `from PACKAGE cimport MODULE` is broken
+    when `PACKAGE` is a namespace package, see :trac:`35322`::
+
+        sage: cython('''
+        ....: from sage.misc cimport cachefunc
+        ....: ''')
+        Traceback (most recent call last):
+        ...
+        RuntimeError: Error compiling Cython file:
+        ...
+        ...: 'sage/misc.pxd' not found
     """
     if not filename.endswith('pyx'):
         print("Warning: file (={}) should have extension .pyx".format(filename), file=sys.stderr)

src/sage/misc/dev_tools.py

@@ -169,7 +169,7 @@ def load_submodules(module=None, exclude_pattern=None):
         load sage.geometry.polyhedron.palp_database... succeeded
         load sage.geometry.polyhedron.ppl_lattice_polygon... succeeded
     """
-    import pkgutil
+    from .package_dir import walk_packages
 
     if module is None:
         import sage
@@ -181,7 +181,7 @@ def load_submodules(module=None, exclude_pattern=None):
     else:
         exclude = None
 
-    for importer, module_name, ispkg in pkgutil.walk_packages(module.__path__, module.__name__ + '.'):
+    for importer, module_name, ispkg in walk_packages(module.__path__, module.__name__ + '.'):
         if ispkg or module_name in sys.modules:
             continue
 

src/sage/misc/package_dir.py

@@ -14,6 +14,7 @@
 
 import os
 import glob
+import sys
 from contextlib import contextmanager
 
 
@@ -154,29 +155,29 @@ def is_package_or_sage_namespace_package_dir(path, *, distribution_filter=None):
     :mod:`sage.cpython` is an ordinary package::
 
         sage: from sage.misc.package_dir import is_package_or_sage_namespace_package_dir
-        sage: directory = os.path.dirname(sage.cpython.__file__); directory
+        sage: directory = sage.cpython.__path__[0]; directory
         '.../sage/cpython'
         sage: is_package_or_sage_namespace_package_dir(directory)
         True
 
     :mod:`sage.libs.mpfr` only has an ``__init__.pxd`` file, but we consider
     it a package directory for consistency with Cython::
 
-        sage: directory = os.path.join(os.path.dirname(sage.libs.all.__file__), 'mpfr'); directory
+        sage: directory = os.path.join(sage.libs.__path__[0], 'mpfr'); directory
         '.../sage/libs/mpfr'
         sage: is_package_or_sage_namespace_package_dir(directory)
         True
 
     :mod:`sage` is designated to become an implicit namespace package::
 
-        sage: directory = os.path.dirname(sage.env.__file__); directory
+        sage: directory = sage.__path__[0]; directory
         '.../sage'
         sage: is_package_or_sage_namespace_package_dir(directory)
         True
 
     Not a package::
 
-        sage: directory = os.path.join(os.path.dirname(sage.symbolic.__file__), 'ginac'); directory
+        sage: directory = os.path.join(sage.symbolic.__path__[0], 'ginac'); directory
         '.../sage/symbolic/ginac'
         sage: is_package_or_sage_namespace_package_dir(directory)
         False
@@ -211,3 +212,124 @@ def cython_namespace_package_support():
         yield
     finally:
         Cython.Utils.is_package_dir = Cython.Build.Cythonize.is_package_dir = Cython.Build.Dependencies.is_package_dir = orig_is_package_dir
+
+
+def walk_packages(path=None, prefix='', onerror=None):
+    r"""
+    Yield :class:`pkgutil.ModuleInfo` for all modules recursively on ``path``.
+
+    This version of the standard library function :func:`pkgutil.walk_packages`
+    addresses https://github.com/python/cpython/issues/73444 by handling
+    the implicit namespace packages in the package layout used by Sage;
+    see :func:`is_package_or_sage_namespace_package_dir`.
+
+    INPUT:
+
+    - ``path`` -- a list of paths to look for modules in or
+      ``None`` (all accessible modules).
+
+    - ``prefix`` -- a string to output on the front of every module name
+      on output.
+
+    - ``onerror`` -- a function which gets called with one argument (the
+      name of the package which was being imported) if any exception
+      occurs while trying to import a package.  If ``None``, ignore
+      :class:`ImportError` but propagate all other exceptions.
+
+    EXAMPLES::
+
+        sage: sorted(sage.misc.package_dir.walk_packages(sage.misc.__path__))  # a namespace package
+        [..., ModuleInfo(module_finder=FileFinder('.../sage/misc'), name='package_dir', ispkg=False), ...]
+    """
+    # Adapted from https://github.com/python/cpython/blob/3.11/Lib/pkgutil.py
+
+    def iter_modules(path=None, prefix=''):
+        """
+        Yield :class:`ModuleInfo` for all submodules on ``path``.
+        """
+        from pkgutil import get_importer, iter_importers, ModuleInfo
+
+        if path is None:
+            importers = iter_importers()
+        elif isinstance(path, str):
+            raise ValueError("path must be None or list of paths to look for modules in")
+        else:
+            importers = map(get_importer, path)
+
+        yielded = {}
+        for i in importers:
+            for name, ispkg in iter_importer_modules(i, prefix):
+                if name not in yielded:
+                    yielded[name] = 1
+                    yield ModuleInfo(i, name, ispkg)
+
+    def iter_importer_modules(importer, prefix=''):
+        r"""
+        Yield :class:`ModuleInfo` for all modules of ``importer``.
+        """
+        from importlib.machinery import FileFinder
+
+        if isinstance(importer, FileFinder):
+            if importer.path is None or not os.path.isdir(importer.path):
+                return
+
+            yielded = {}
+            import inspect
+            try:
+                filenames = os.listdir(importer.path)
+            except OSError:
+                # ignore unreadable directories like import does
+                filenames = []
+            filenames.sort()  # handle packages before same-named modules
+
+            for fn in filenames:
+                modname = inspect.getmodulename(fn)
+                if modname and (modname in ['__init__', 'all']
+                                or modname.startswith('all__')
+                                or modname in yielded):
+                    continue
+
+                path = os.path.join(importer.path, fn)
+                ispkg = False
+
+                if not modname and os.path.isdir(path) and '.' not in fn:
+                    modname = fn
+                    if not (ispkg := is_package_or_sage_namespace_package_dir(path)):
+                        continue
+
+                if modname and '.' not in modname:
+                    yielded[modname] = 1
+                    yield prefix + modname, ispkg
+
+        elif not hasattr(importer, 'iter_modules'):
+            yield from []
+
+        else:
+            yield from importer.iter_modules(prefix)
+
+    def seen(p, m={}):
+        if p in m:
+            return True
+        m[p] = True
+
+    for info in iter_modules(path, prefix):
+        yield info
+
+        if info.ispkg:
+            try:
+                __import__(info.name)
+            except ImportError:
+                if onerror is not None:
+                    onerror(info.name)
+            except Exception:
+                if onerror is not None:
+                    onerror(info.name)
+                else:
+                    raise
+            else:
+                path = getattr(sys.modules[info.name], '__path__', None) or []
+
+                # don't traverse path items we've seen before
+                path = [p for p in path if not seen(p)]
+
+                yield from walk_packages(path, info.name + '.', onerror)

src/sage/symbolic/pynac.pxi

@@ -3,9 +3,9 @@ Declarations for pynac, a Python frontend for ginac
 
 Check that we can externally cimport this (:trac:`18825`)::
 
-    sage: cython(  # long time; random compiler warnings  # optional - sage.misc.cython
+    sage: cython(  # optional - sage.misc.cython
     ....: '''
-    ....: from sage.symbolic cimport expression
+    ....: cimport sage.symbolic.expression
     ....: ''')
 """