Issue #15502: Finish bringing importlib.abc in line with the current

state of the import system. Also make importlib.invalidate_caches()
work with sys.meta_path instead of sys.path_importer_cache to
completely separate the path-based import system from the overall
import system.

Patch by Eric Snow.

Doc/library/importlib.rst

@@ -102,12 +102,11 @@ Functions
 
 .. function:: invalidate_caches()
 
-   Invalidate the internal caches of the finders stored at
-   :data:`sys.path_importer_cache`. If a finder implements
-   :meth:`abc.Finder.invalidate_caches()` then it will be called to perform the
-   invalidation.  This function may be needed if some modules are installed
-   while your program is running and you expect the program to notice the
-   changes.
+   Invalidate the internal caches of finders stored at
+   :data:`sys.meta_path`. If a finder implements ``invalidate_caches()`` then it
+   will be called to perform the invalidation.  This function may be needed if
+   some modules are installed while your program is running and you expect the
+   program to notice the changes.
 
    .. versionadded:: 3.3
 
@@ -129,22 +128,17 @@ are also provided to help in implementing the core ABCs.
    implementations should derive from (or register with) the more specific
    :class:`MetaPathFinder` or :class:`PathEntryFinder` ABCs.
 
-   .. method:: invalidate_caches()
+   .. method:: find_module(fullname, path=None)
 
-      An optional method which, when called, should invalidate any internal
-      cache used by the finder. Used by :func:`invalidate_caches()` when
-      invalidating the caches of all cached finders.
-
-   .. versionchanged:: 3.3
-      The API signatures for meta path finders and path entry finders
-      were separated by PEP 420. Accordingly, the Finder ABC no
-      longer requires implementation of a ``find_module()`` method.
+      An abstact method for finding a :term:`loader` for the specified
+      module.  Originally specified in :pep:`302`, this method was meant
+      for use in :data:`sys.meta_path` and in the path-based import subsystem.
 
 
 .. class:: MetaPathFinder
 
-   An abstract base class representing a :term:`meta path finder` and
-   inheriting from :class:`Finder`.
+   An abstract base class representing a :term:`meta path finder`. For
+   compatibility, this is a subclass of :class:`Finder`.
 
    .. versionadded:: 3.3
 
@@ -156,20 +150,45 @@ are also provided to help in implementing the core ABCs.
       will be the value of :attr:`__path__` from the parent
       package. If a loader cannot be found, ``None`` is returned.
 
+   .. method:: invalidate_caches()
+
+      An optional method which, when called, should invalidate any internal
+      cache used by the finder. Used by :func:`invalidate_caches()` when
+      invalidating the caches of all finders on :data:`sys.meta_path`.
+
 
 .. class:: PathEntryFinder
 
-   An abstract base class representing a :term:`path entry finder` and
-   inheriting from :class:`Finder`.
+   An abstract base class representing a :term:`path entry finder`.  Though
+   it bears some similarities to :class:`MetaPathFinder`, ``PathEntryFinder``
+   is meant for use only within the path-based import subsystem provided
+   by :class:`PathFinder`. This ABC is a subclass of :class:`Finder` for
+   compatibility.
 
    .. versionadded:: 3.3
 
    .. method:: find_loader(fullname):
 
       An abstract method for finding a :term:`loader` for the specified
-      module.  Returns a 2-tuple of ``(loader, portion)`` where portion is a
-      sequence of file system locations contributing to part of a namespace
-      package.  The sequence may be empty.
+      module.  Returns a 2-tuple of ``(loader, portion)`` where ``portion``
+      is a sequence of file system locations contributing to part of a namespace
+      package. The loader may be ``None`` while specifying ``portion`` to
+      signify the contribution of the file system locations to a namespace
+      package. An empty list can be used for ``portion`` to signify the loader
+      is not part of a package. If ``loader`` is ``None`` and ``portion`` is
+      the empty list then no loader or location for a namespace package were
+      found (i.e. failure to find anything for the module).
+
+   .. method:: find_module(fullname):
+
+      A concrete implementation of :meth:`Finder.find_module` which is
+      equivalent to ``self.find_loader(fullname)[0]``.
+
+   .. method:: invalidate_caches()
+
+      An optional method which, when called, should invalidate any internal
+      cache used by the finder. Used by :meth:`PathFinder.invalidate_caches()`
+      when invalidating the caches of all cached finders.
 
 
 .. class:: Loader
@@ -638,6 +657,11 @@ find and load modules.
         module. If no finder is ever found then ``None`` is both stored in
         the cache and returned.
 
+   .. classmethod:: invalidate_caches()
+
+        Call :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all
+        finders stored in :attr:`sys.path_importer_cache`.
+
 
 .. class:: FileFinder(path, \*loader_details)
 

Lib/importlib/__init__.py

@@ -38,9 +38,9 @@
 
 
 def invalidate_caches():
-    """Call the invalidate_caches() method on all finders stored in
-    sys.path_importer_caches (where implemented)."""
-    for finder in sys.path_importer_cache.values():
+    """Call the invalidate_caches() method on all meta path finders stored in
+    sys.meta_path (where implemented)."""
+    for finder in sys.meta_path:
         if hasattr(finder, 'invalidate_caches'):
             finder.invalidate_caches()
 

Lib/importlib/_bootstrap.py

@@ -1185,6 +1185,14 @@ class PathFinder:
 
     """Meta path finder for sys.path and package __path__ attributes."""
 
+    @classmethod
+    def invalidate_caches(cls):
+        """Call the invalidate_caches() method on all path entry finders
+        stored in sys.path_importer_caches (where implemented)."""
+        for finder in sys.path_importer_cache.values():
+            if hasattr(finder, 'invalidate_caches'):
+                finder.invalidate_caches()
+
     @classmethod
     def _path_hooks(cls, path):
         """Search sequence of hooks for a finder for 'path'.
@@ -1235,14 +1243,14 @@ def _get_loader(cls, fullname, path):
                     portions = []
                 if loader is not None:
                     # We found a loader: return it immediately.
-                    return (loader, namespace_path)
+                    return loader, namespace_path
                 # This is possibly part of a namespace package.
                 #  Remember these path entries (if any) for when we
                 #  create a namespace package, and continue iterating
                 #  on path.
                 namespace_path.extend(portions)
         else:
-            return (None, namespace_path)
+            return None, namespace_path
 
     @classmethod
     def find_module(cls, fullname, path=None):

Lib/importlib/abc.py

@@ -25,39 +25,41 @@ def _register(abstract_cls, *classes):
 
 class Finder(metaclass=abc.ABCMeta):
 
-    """Common abstract base class for import finders.
+    """Legacy abstract base class for import finders.
 
-    Finder implementations should derive from the more specific
-    MetaPathFinder or PathEntryFinder ABCs rather than directly from Finder.
+    It may be subclassed for compatibility with legacy third party
+    reimplementations of the import system.  Otherwise, finder
+    implementations should derive from the more specific MetaPathFinder
+    or PathEntryFinder ABCs.
     """
 
+    @abc.abstractmethod
     def find_module(self, fullname, path=None):
-        """An optional legacy method that should find a module.
+        """An abstract method that should find a module.
         The fullname is a str and the optional path is a str or None.
         Returns a Loader object.
-
-        The path finder will use this method only if find_loader() does
-        not exist. It may optionally be implemented for compatibility
-        with legacy third party reimplementations of the import system.
         """
         raise NotImplementedError
 
-    # invalidate_caches() is a completely optional method, so no default
-    # implementation is provided. See the docs for details.
-
 
 class MetaPathFinder(Finder):
 
     """Abstract base class for import finders on sys.meta_path."""
 
     @abc.abstractmethod
     def find_module(self, fullname, path):
-        """Abstract method which when implemented should find a module.
+        """Abstract method which, when implemented, should find a module.
         The fullname is a str and the path is a str or None.
         Returns a Loader object.
         """
         raise NotImplementedError
 
+    def invalidate_caches(self):
+        """An optional method for clearing the finder's cache, if any.
+        This method is used by importlib.invalidate_caches().
+        """
+        return NotImplemented
+
 _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter,
           machinery.PathFinder, machinery.WindowsRegistryFinder)
 
@@ -68,13 +70,25 @@ class PathEntryFinder(Finder):
 
     @abc.abstractmethod
     def find_loader(self, fullname):
-        """Abstract method which when implemented returns a module loader.
+        """Abstract method which, when implemented, returns a module loader.
         The fullname is a str.  Returns a 2-tuple of (Loader, portion) where
         portion is a sequence of file system locations contributing to part of
-        a namespace package.  The sequence may be empty.
+        a namespace package.  The sequence may be empty and the loader may be
+        None.
         """
         raise NotImplementedError
 
+    def find_module(self, fullname):
+        """Compatibility function which is the equivalent of
+        self.find_loader(fullname)[0]."""
+        return self.find_loader(fullname)[0]
+
+    def invalidate_caches(self):
+        """An optional method for clearing the finder's cache, if any.
+        This method is used by PathFinder.invalidate_caches().
+        """
+        return NotImplemented
+
 _register(PathEntryFinder, machinery.FileFinder)
 
 

Lib/test/test_importlib/test_abc.py

@@ -36,11 +36,13 @@ class MetaPathFinder(InheritanceTests, unittest.TestCase):
     subclasses = [machinery.BuiltinImporter, machinery.FrozenImporter,
                     machinery.PathFinder, machinery.WindowsRegistryFinder]
 
+
 class PathEntryFinder(InheritanceTests, unittest.TestCase):
 
     superclasses = [abc.Finder]
     subclasses = [machinery.FileFinder]
 
+
 class Loader(InheritanceTests, unittest.TestCase):
 
     subclasses = [abc.PyLoader]

Lib/test/test_importlib/test_api.py

@@ -148,11 +148,15 @@ def invalidate_caches(self):
                 self.called = True
 
         key = 'gobledeegook'
-        ins = InvalidatingNullFinder()
-        sys.path_importer_cache[key] = ins
+        meta_ins = InvalidatingNullFinder()
+        path_ins = InvalidatingNullFinder()
+        sys.meta_path.insert(0, meta_ins)
         self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
+        sys.path_importer_cache[key] = path_ins
+        self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
         importlib.invalidate_caches()
-        self.assertTrue(ins.called)
+        self.assertTrue(meta_ins.called)
+        self.assertTrue(path_ins.called)
 
     def test_method_lacking(self):
         # There should be no issues if the method is not defined.

Misc/NEWS

@@ -80,6 +80,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #15502: Have importlib.invalidate_caches() work on sys.meta_path
+  instead of sys.path_importer_cache.
+
 - Issue #15163: Pydoc shouldn't list __loader__ as module data.
 
 - Issue #15471: Do not use mutable objects as defaults for