mitogen: Support PEP 451 ModuleSpec API, required for Python 3.12

importlib.machinery.ModuleSpec and find_spec() were introduced in Python 3.4
under PEP 451. They replace the find_module() API of PEP 302, which was
deprecated from Python 3.4. They were removed in Python 3.12 along with the
imp module.

This change adds support for the PEP 451 APIs. Mitogen should no longer import
imp on Python versions that support ModuleSpec. Tests have been added to cover
the new APIs.

CI jobs have been added to cover Python 3.x on macOS.

Refs mitogen-hq#1033
Co-authored-by: Witold Baryluk <witold.baryluk@gmail.com>

.ci/azure-pipelines.yml

@@ -25,21 +25,17 @@ jobs:
         python.version: '3.11'
         tox.env: py311-mode_mitogen
 
-      # TODO: test python3, python3 tests are broken
       Loc_27_210:
         tox.env: py27-mode_localhost-ansible2.10
-      Loc_27_4:
-        tox.env: py27-mode_localhost-ansible4
+      Loc_311_6:
+        python.version: '3.11'
+        tox.env: py311-mode_localhost-ansible6
 
-      # NOTE: this hangs when ran in Ubuntu 18.04
       Van_27_210:
-        tox.env: py27-mode_localhost-ansible2.10
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
-      Van_27_4:
-        tox.env: py27-mode_localhost-ansible4
-        STRATEGY: linear
-        ANSIBLE_SKIP_TAGS: resource_intensive
+        tox.env: py27-mode_localhost-ansible2.10-strategy_linear
+      Van_311_6:
+        python.version: '3.11'
+        tox.env: py311-mode_localhost-ansible6-strategy_linear
 
 - job: Linux
   pool:

.ci/localhost_ansible_tests.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python
 # Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
 
+from __future__ import print_function
+
+import getpass
+import io
 import os
 import subprocess
 import sys
@@ -53,6 +57,38 @@
         os.chdir(IMAGE_PREP_DIR)
         ci_lib.run("ansible-playbook -c local -i localhost, _user_accounts.yml")
 
+    # FIXME Don't hardcode https://github.com/mitogen-hq/mitogen/issues/1022
+    #       and os.environ['USER'] is not populated on Azure macOS runners.
+    os.chdir(HOSTS_DIR)
+    with io.open('default.hosts', 'r+', encoding='utf-8') as f:
+        user = getpass.getuser()
+        content = f.read()
+        content = content.replace("{{ lookup('pipe', 'whoami') }}", user)
+        f.seek(0)
+        f.write(content)
+        f.truncate()
+    ci_lib.dump_file('default.hosts')
+
+    cmd = ';'.join([
+        'from __future__ import print_function',
+        'import os, sys',
+        'print(sys.executable, os.path.realpath(sys.executable))',
+    ])
+    for interpreter in ['/usr/bin/python', '/usr/bin/python2', '/usr/bin/python2.7']:
+        print(interpreter)
+        try:
+            subprocess.call([interpreter, '-c', cmd])
+        except OSError as exc:
+            print(exc)
+
+        print(interpreter, 'with PYTHON_LAUNCHED_FROM_WRAPPER=1')
+        environ = os.environ.copy()
+        environ['PYTHON_LAUNCHED_FROM_WRAPPER'] = '1'
+        try:
+            subprocess.call([interpreter, '-c', cmd], env=environ)
+        except OSError as exc:
+            print(exc)
+
 
 with ci_lib.Fold('ansible'):
     os.chdir(TESTS_DIR)

ansible_mitogen/module_finder.py

@@ -31,12 +31,23 @@
 __metaclass__ = type
 
 import collections
-import imp
+import logging
 import os
+import re
+import sys
+
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
 
 import mitogen.master
 
 
+LOG = logging.getLogger(__name__)
 PREFIX = 'ansible.module_utils.'
 
 
@@ -45,8 +56,6 @@
 #   path    Filesystem path of the module.
 #   kind    One of the constants in `imp`, as returned in `imp.find_module()`
 #   parent  `ansible_mitogen.module_finder.Module` of parent package (if any).
-#
-# FIXME Python 3.12 removed `imp`, leaving no constants for `Module.kind`.
 Module = collections.namedtuple('Module', 'name path kind parent')
 
 
@@ -126,14 +135,121 @@ def find_relative(parent, name, path=()):
 
 
 def scan_fromlist(code):
+    """Return an iterator of (level, name) for explicit imports in a code
+    object.
+
+    Not all names identify a module. `from os import name, path` generates
+    `(0, 'os.name'), (0, 'os.path')`, but `os.name` is usually a string.
+
+    >>> src = 'import a; import b.c; from d.e import f; from g import h, i\\n'
+    >>> code = compile(src, '<str>', 'exec')
+    >>> list(scan_fromlist(code))
+    [(0, 'a'), (0, 'b.c'), (0, 'd.e.f'), (0, 'g.h'), (0, 'g.i')]
+    """
     for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
         for name in fromlist:
-            yield level, '%s.%s' % (modname_s, name)
+            yield level, str('%s.%s' % (modname_s, name))
         if not fromlist:
             yield level, modname_s
 
 
+def walk_imports(code, prefix=None):
+    """Return an iterator of names for implicit parent imports & explicit
+    imports in a code object.
+
+    If a prefix is provided, then only children of that prefix are included.
+    Not all names identify a module. `from os import name, path` generates
+    `'os', 'os.name', 'os.path'`, but `os.name` is usually a string.
+
+    >>> source = 'import a; import b; import b.c; from b.d import e, f\\n'
+    >>> code = compile(source, '<str>', 'exec')
+    >>> list(walk_imports(code))
+    ['a', 'b', 'b', 'b.c', 'b', 'b.d', 'b.d.e', 'b.d.f']
+    >>> list(walk_imports(code, prefix='b'))
+    ['b.c', 'b.d', 'b.d.e', 'b.d.f']
+    """
+    if prefix is None:
+        prefix = ''
+    pattern = re.compile(r'(^|\.)(\w+)')
+    start = len(prefix)
+    for _, name, fromlist in mitogen.master.scan_code_imports(code):
+        if not name.startswith(prefix):
+            continue
+        for match in pattern.finditer(name, start):
+            yield name[:match.end()]
+        for leaf in fromlist:
+            yield str('%s.%s' % (name, leaf))
+
+
 def scan(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    """Return a list of (name, path, is_package) for ansible.module_utils
+    imports used by an Ansible module.
+    """
+    log = LOG.getChild('scan')
+    log.debug('%r, %r, %r', module_name, module_path, search_path)
+
+    if sys.version_info >= (3, 4):
+        result = _scan_importlib_find_spec(
+            module_name, module_path, search_path,
+        )
+        log.debug('_scan_importlib_find_spec %r', result)
+    else:
+        result = _scan_imp_find_module(module_name, module_path, search_path)
+        log.debug('_scan_imp_find_module %r', result)
+    return result
+
+
+def _scan_importlib_find_spec(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
+    module = importlib.machinery.ModuleSpec(
+        module_name, loader=None, origin=module_path,
+    )
+    prefix = importlib.machinery.ModuleSpec(
+        PREFIX.rstrip('.'), loader=None,
+    )
+    prefix.submodule_search_locations = search_path
+    queue = collections.deque([module])
+    specs = {prefix.name: prefix}
+    while queue:
+        spec = queue.popleft()
+        if spec.origin is None:
+            continue
+        try:
+            with open(spec.origin, 'rb') as f:
+                code = compile(f.read(), spec.name, 'exec')
+        except Exception as exc:
+            raise ValueError((exc, module, spec, specs))
+
+        for name in walk_imports(code, prefix.name):
+            if name in specs:
+                continue
+
+            parent_name = name.rpartition('.')[0]
+            parent = specs[parent_name]
+            if parent is None or not parent.submodule_search_locations:
+                specs[name] = None
+                continue
+
+            child = importlib.util._find_spec(
+                name, parent.submodule_search_locations,
+            )
+            if child is None or child.origin is None:
+                specs[name] = None
+                continue
+
+            specs[name] = child
+            queue.append(child)
+
+    del specs[prefix.name]
+    return sorted(
+        (spec.name, spec.origin, spec.submodule_search_locations is not None)
+        for spec in specs.values() if spec is not None
+    )
+
+
+def _scan_imp_find_module(module_name, module_path, search_path):
+    # type: (str, str, list[str]) -> list[(str, str, bool)]
     module = Module(module_name, module_path, imp.PY_SOURCE, None)
     stack = [module]
     seen = set()

ansible_mitogen/runner.py

@@ -40,7 +40,6 @@
 __metaclass__ = type
 
 import atexit
-import imp
 import json
 import os
 import re
@@ -64,6 +63,14 @@
     # Python 2.4
     ctypes = None
 
+try:
+    # Python >= 3.4, PEP 451 ModuleSpec API
+    import importlib.machinery
+    import importlib.util
+except ImportError:
+    # Python < 3.4, PEP 302 Import Hooks
+    import imp
+
 try:
     # Cannot use cStringIO as it does not support Unicode.
     from StringIO import StringIO
@@ -514,14 +521,74 @@ def revert(self):
             sys.modules.pop(fullname, None)
 
     def find_module(self, fullname, path=None):
+        """
+        Return a loader for the module with fullname, if we will load it.
+
+        Implements importlib.abc.MetaPathFinder.find_module().
+        Deprecrated in Python 3.4+, replaced by find_spec().
+        Raises ImportWarning in Python 3.10+. Removed in Python 3.12.
+        """
         if fullname in self._by_fullname:
             return self
 
+    def find_spec(self, fullname, path, target=None):
+        """
+        Return a `ModuleSpec` for module with `fullname` if we will load it.
+        Otherwise return `None`.
+
+        Implements importlib.abc.MetaPathFinder.find_spec(). Python 3.4+.
+        """
+        if fullname.endswith('.'):
+            return None
+
+        try:
+            module_path, is_package = self._by_fullname[fullname]
+        except KeyError:
+            LOG.debug('Skipping %s: not present', fullname)
+            return None
+
+        LOG.debug('Handling %s', fullname)
+        origin = 'master:%s' % (module_path,)
+        return importlib.machinery.ModuleSpec(
+            fullname, loader=self, origin=origin, is_package=is_package,
+        )
+
+    def create_module(self, spec):
+        """
+        Return a module object for the given ModuleSpec.
+
+        Implements PEP-451 importlib.abc.Loader API introduced in Python 3.4.
+        Unlike Loader.load_module() this shouldn't populate sys.modules or
+        set module attributes. Both are done by Python.
+        """
+        module = types.ModuleType(spec.name)
+        # FIXME create_module() shouldn't initialise module attributes
+        module.__file__ = spec.origin
+        return module
+
+    def exec_module(self, module):
+        """
+        Execute the module to initialise it. Don't return anything.
+
+        Implements PEP-451 importlib.abc.Loader API, introduced in Python 3.4.
+        """
+        spec = module.__spec__
+        path, _ = self._by_fullname[spec.name]
+        source = ansible_mitogen.target.get_small_file(self._context, path)
+        code = compile(source, path, 'exec', 0, 1)
+        exec(code, module.__dict__)
+        self._loaded.add(spec.name)
+
     def load_module(self, fullname):
+        """
+        Return the loaded module specified by fullname.
+
+        Implements PEP 302 importlib.abc.Loader.load_module().
+        Deprecated in Python 3.4+, replaced by create_module() & exec_module().
+        """
         path, is_pkg = self._by_fullname[fullname]
         source = ansible_mitogen.target.get_small_file(self._context, path)
         code = compile(source, path, 'exec', 0, 1)
-        # FIXME Python 3.12 removed `imp`
         mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
         mod.__file__ = "master:%s" % (path,)
         mod.__loader__ = self
@@ -819,12 +886,17 @@ def _setup_imports(self):
         synchronization mechanism by importing everything the module will need
         prior to detaching.
         """
+        # I think "custom" means "found in custom module_utils search path",
+        # e.g. playbook relative dir, ~/.ansible/..., Ansible collection.
         for fullname, _, _ in self.module_map['custom']:
             mitogen.core.import_module(fullname)
+
+        # I think "builtin" means "part of ansible/ansible-base/ansible-core",
+        # as opposed to Python builtin modules such as sys.
         for fullname in self.module_map['builtin']:
             try:
                 mitogen.core.import_module(fullname)
-            except ImportError:
+            except ImportError as exc:
                 # #590: Ansible 2.8 module_utils.distro is a package that
                 # replaces itself in sys.modules with a non-package during
                 # import. Prior to replacement, it is a real package containing
@@ -835,8 +907,18 @@ def _setup_imports(self):
                 # loop progresses to the next entry and attempts to preload
                 # 'distro._distro', the import mechanism will fail. So here we
                 # silently ignore any failure for it.
-                if fullname != 'ansible.module_utils.distro._distro':
-                    raise
+                if fullname == 'ansible.module_utils.distro._distro':
+                    continue
+
+                # ansible.module_utils.compat.selinux raises ImportError if it
+                # can't load libselinux.so. The importer would usually catch
+                # this & skip selinux operations. We don't care about selinux,
+                # we're using import to get a copy of the module.
+                if (fullname == 'ansible.module_utils.compat.selinux'
+                    and exc.msg == 'unable to load libselinux.so'):
+                    continue
+
+                raise
 
     def _setup_excepthook(self):
         """

docs/changelog.rst

@@ -23,6 +23,8 @@ Unreleased
 * :gh:issue:`987` Support Python 3.11
 * :gh:issue:`885` Fix :py:exc:`PermissionError` in :py:mod:`importlib` when
   becoming an unprivileged user with Python 3.x
+* :gh:issue:`1033` Support `PEP 451 <https://peps.python.org/pep-0451/>,
+  required by Python 3.12
 
 
 v0.3.4 (2023-07-02)

docs/contributors.rst

@@ -138,4 +138,5 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>randy &mdash; <em>desperate for automation</em></li>
     <li>Michael & Vicky Twomey-Lee</li>
     <li><a href="http://www.wezm.net/">Wesley Moore</a></li>
+    <li><a href="https://github.com/baryluk">Witold Baryluk</a></li>
     </ul>