Migrate pkg_resources usages to importlib.metadata (#9749)

* Migrate entrypoint logic from pkg_resources to importlib.metadata

* Usage of importlib_metadata up to Python 3.9 to align API behavior to Python 3.10

---------

Co-authored-by: Adrien Ferrand <adrien.ferrand@amadeus.com>
Co-authored-by: Adrien Ferrand <adrien.ferrand@arteris.com>

certbot/certbot/_internal/plugins/disco.py

@@ -1,5 +1,4 @@
 """Utilities for plugins discovery and selection."""
-import itertools
 import logging
 import sys
 from typing import Callable
@@ -13,15 +12,18 @@
 from typing import Type
 from typing import Union
 
-import pkg_resources
-
 from certbot import configuration
 from certbot import errors
 from certbot import interfaces
 from certbot._internal import constants
 from certbot.compat import os
 from certbot.errors import Error
 
+if sys.version_info >= (3, 10):  # pragma: no cover
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
 logger = logging.getLogger(__name__)
 
 
@@ -35,7 +37,7 @@ class PluginEntryPoint:
     # this object is mutable, don't allow it to be hashed!
     __hash__ = None  # type: ignore
 
-    def __init__(self, entry_point: pkg_resources.EntryPoint) -> None:
+    def __init__(self, entry_point: importlib_metadata.EntryPoint) -> None:
         self.name = self.entry_point_to_plugin_name(entry_point)
         self.plugin_cls: Type[interfaces.Plugin] = entry_point.load()
         self.entry_point = entry_point
@@ -50,7 +52,7 @@ def check_name(self, name: Optional[str]) -> bool:
         return False
 
     @classmethod
-    def entry_point_to_plugin_name(cls, entry_point: pkg_resources.EntryPoint) -> str:
+    def entry_point_to_plugin_name(cls, entry_point: importlib_metadata.EntryPoint) -> str:
         """Unique plugin name for an ``entry_point``"""
         return entry_point.name
 
@@ -75,7 +77,7 @@ def hidden(self) -> bool:
         return getattr(self.plugin_cls, "hidden", False)
 
     def ifaces(self, *ifaces_groups: Iterable[Type]) -> bool:
-        """Does plugin implements specified interface groups?"""
+        """Does plugin implement specified interface groups?"""
         return not ifaces_groups or any(
             all(issubclass(self.plugin_cls, iface)
                 for iface in ifaces)
@@ -89,7 +91,6 @@ def initialized(self) -> bool:
     def init(self, config: Optional[configuration.NamespaceConfig] = None) -> interfaces.Plugin:
         """Memoized plugin initialization."""
         if not self._initialized:
-            self.entry_point.require()  # fetch extras!
             # For plugins implementing ABCs Plugin, Authenticator or Installer, the following
             # line will raise an exception if some implementations of abstract methods are missing.
             self._initialized = self.plugin_cls(config, self.name)
@@ -181,32 +182,31 @@ def find_all(cls) -> 'PluginsRegistry':
         plugin_paths = plugin_paths_string.split(':') if plugin_paths_string else []
         # XXX should ensure this only happens once
         sys.path.extend(plugin_paths)
-        for plugin_path in plugin_paths:
-            pkg_resources.working_set.add_entry(plugin_path)
-        entry_points = itertools.chain(
-            pkg_resources.iter_entry_points(
-                constants.SETUPTOOLS_PLUGINS_ENTRY_POINT),
-            pkg_resources.iter_entry_points(
-                constants.OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT),)
-        for entry_point in entry_points:
+        entry_points = list(importlib_metadata.entry_points(
+            group=constants.SETUPTOOLS_PLUGINS_ENTRY_POINT))
+        old_entry_points = list(importlib_metadata.entry_points(
+            group=constants.OLD_SETUPTOOLS_PLUGINS_ENTRY_POINT))
+        for entry_point in entry_points + old_entry_points:
             try:
                 cls._load_entry_point(entry_point, plugins)
             except Exception as e:
                 raise errors.PluginError(
-                    f"The '{entry_point.module_name}' plugin errored while loading: {e}. "
-                     "You may need to remove or update this plugin. The Certbot log will "
-                     "contain the full error details and this should be reported to the "
-                     "plugin developer.") from e
+                    f"The '{entry_point.module}' plugin errored while loading: {e}. "
+                    "You may need to remove or update this plugin. The Certbot log will "
+                    "contain the full error details and this should be reported to the "
+                    "plugin developer.") from e
         return cls(plugins)
 
     @classmethod
-    def _load_entry_point(cls, entry_point: pkg_resources.EntryPoint,
+    def _load_entry_point(cls, entry_point: importlib_metadata.EntryPoint,
                           plugins: Dict[str, PluginEntryPoint]) -> None:
         plugin_ep = PluginEntryPoint(entry_point)
         if plugin_ep.name in plugins:
             other_ep = plugins[plugin_ep.name]
-            plugin1 = plugin_ep.entry_point.dist.key if plugin_ep.entry_point.dist else "unknown"
-            plugin2 = other_ep.entry_point.dist.key if other_ep.entry_point.dist else "unknown"
+            plugin1_dist = plugin_ep.entry_point.dist
+            plugin2_dist = other_ep.entry_point.dist
+            plugin1 = plugin1_dist.name.lower() if plugin1_dist else "unknown"
+            plugin2 = plugin2_dist.name.lower() if plugin2_dist else "unknown"
             raise Exception("Duplicate plugin name {0} from {1} and {2}.".format(
                 plugin_ep.name, plugin1, plugin2))
         if issubclass(plugin_ep.plugin_cls, interfaces.Plugin):

certbot/certbot/_internal/tests/plugins/disco_test.py

@@ -6,7 +6,6 @@
 import unittest
 from unittest import mock
 
-import pkg_resources
 import pytest
 
 from certbot import errors
@@ -15,30 +14,55 @@
 from certbot._internal.plugins import standalone
 from certbot._internal.plugins import webroot
 
-EP_SA = pkg_resources.EntryPoint(
-    "sa", "certbot._internal.plugins.standalone",
-    attrs=("Authenticator",),
-    dist=mock.MagicMock(key="certbot"))
-EP_WR = pkg_resources.EntryPoint(
-    "wr", "certbot._internal.plugins.webroot",
-    attrs=("Authenticator",),
-    dist=mock.MagicMock(key="certbot"))
+if sys.version_info >= (3, 10):  # pragma: no cover
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
+
+class _EntryPointLoadFail(importlib_metadata.EntryPoint):
+    def load(self):
+        raise RuntimeError("Loading failure")
+
+
+EP_SA = importlib_metadata.EntryPoint(
+    name="sa",
+    value="certbot._internal.plugins.standalone:Authenticator",
+    group="certbot.plugins")
+
+EP_WR = importlib_metadata.EntryPoint(
+    name="wr",
+    value="certbot._internal.plugins.webroot:Authenticator",
+    group="certbot.plugins")
+
+EP_SA_LOADFAIL = _EntryPointLoadFail(
+    name="sa",
+    value="certbot._internal.plugins.standalone:Authenticator",
+    group="certbot.plugins")
 
 
 class PluginEntryPointTest(unittest.TestCase):
     """Tests for certbot._internal.plugins.disco.PluginEntryPoint."""
 
     def setUp(self):
-        self.ep1 = pkg_resources.EntryPoint(
-            "ep1", "p1.ep1", dist=mock.MagicMock(key="p1"))
-        self.ep1prim = pkg_resources.EntryPoint(
-            "ep1", "p2.ep2", dist=mock.MagicMock(key="p2"))
+        self.ep1 = importlib_metadata.EntryPoint(
+            name="ep1",
+            value="p1.ep1:Authenticator",
+            group="certbot.plugins")
+        self.ep1prim = importlib_metadata.EntryPoint(
+            name="ep1",
+            value="p2.pe2:Authenticator",
+            group="certbot.plugins")
         # nested
-        self.ep2 = pkg_resources.EntryPoint(
-            "ep2", "p2.foo.ep2", dist=mock.MagicMock(key="p2"))
+        self.ep2 = importlib_metadata.EntryPoint(
+            name="ep2",
+            value="p2.foo.ep2:Authenticator",
+            group="certbot.plugins")
         # project name != top-level package name
-        self.ep3 = pkg_resources.EntryPoint(
-            "ep3", "a.ep3", dist=mock.MagicMock(key="p3"))
+        self.ep3 = importlib_metadata.EntryPoint(
+            name="ep3",
+            value="a.ep3:Authenticator",
+            group="certbot.plugins")
 
         from certbot._internal.plugins.disco import PluginEntryPoint
         self.plugin_ep = PluginEntryPoint(EP_SA)
@@ -172,16 +196,18 @@ def setUp(self):
         self.plugin_ep.__hash__.side_effect = TypeError
         self.plugins = {self.plugin_ep.name: self.plugin_ep}
         self.reg = self._create_new_registry(self.plugins)
-        self.ep1 = pkg_resources.EntryPoint(
-            "ep1", "p1.ep1", dist=mock.MagicMock(key="p1"))
+        self.ep1 = importlib_metadata.EntryPoint(
+            name="ep1",
+            value="p1.ep1",
+            group="certbot.plugins")
 
     def test_find_all(self):
         from certbot._internal.plugins.disco import PluginsRegistry
-        with mock.patch("certbot._internal.plugins.disco.pkg_resources") as mock_pkg:
-            mock_pkg.iter_entry_points.side_effect = [
-                iter([EP_SA]), iter([EP_WR, self.ep1])
+        with mock.patch("certbot._internal.plugins.disco.importlib_metadata") as mock_meta:
+            mock_meta.entry_points.side_effect = [
+                [EP_SA], [EP_WR, self.ep1],
             ]
-            with mock.patch.object(pkg_resources.EntryPoint, 'load') as mock_load:
+            with mock.patch.object(importlib_metadata.EntryPoint, 'load') as mock_load:
                 mock_load.side_effect = [
                     standalone.Authenticator, webroot.Authenticator,
                     null.Installer, null.Installer]
@@ -196,10 +222,10 @@ def test_find_all(self):
 
     def test_find_all_error_message(self):
         from certbot._internal.plugins.disco import PluginsRegistry
-        with mock.patch("certbot._internal.plugins.disco.pkg_resources") as mock_pkg:
-            EP_SA.load = None # This triggers a TypeError when the entrypoint loads
-            mock_pkg.iter_entry_points.side_effect = [
-                iter([EP_SA]), iter([EP_WR, self.ep1])
+        with mock.patch("certbot._internal.plugins.disco.importlib_metadata") as mock_meta:
+            #EP_SA.load = None  # This triggers a TypeError when the entrypoint loads
+            mock_meta.entry_points.side_effect = [
+                [EP_SA_LOADFAIL], [EP_WR, self.ep1],
             ]
             with self.assertRaises(errors.PluginError) as cm:
                 PluginsRegistry.find_all()

certbot/setup.py

@@ -36,6 +36,7 @@ def read_file(filename, encoding='utf8'):
     'cryptography>=3.2.1',
     'distro>=1.0.1',
     'importlib_resources>=1.3.1; python_version < "3.9"',
+    'importlib_metadata>=4.6; python_version < "3.10"',
     'josepy>=1.13.0',
     'parsedatetime>=2.4',
     'pyrfc3339',

tools/oldest_constraints.txt

@@ -1,6 +1,7 @@
 # This file was generated by tools/pinning/oldest/repin.sh and can be updated using
 # that script.
 apacheconfig==0.3.2 ; python_version >= "3.7" and python_version < "3.8"
+appdirs==1.4.4 ; python_version >= "3.7" and python_version < "3.8"
 asn1crypto==0.24.0 ; python_version >= "3.7" and python_version < "3.8"
 astroid==2.15.6 ; python_full_version >= "3.7.2" and python_version < "3.8"
 boto3==1.15.15 ; python_version >= "3.7" and python_version < "3.8"
@@ -30,8 +31,8 @@ google-api-python-client==1.6.5 ; python_version >= "3.7" and python_version < "
 google-auth==2.16.0 ; python_version >= "3.7" and python_version < "3.8"
 httplib2==0.9.2 ; python_version >= "3.7" and python_version < "3.8"
 idna==2.6 ; python_version >= "3.7" and python_version < "3.8"
-importlib-metadata==6.7.0 ; python_version >= "3.7" and python_version < "3.8"
-importlib-resources==1.3.1 ; python_version >= "3.7" and python_version < "3.8"
+importlib-metadata==4.6.4 ; python_version >= "3.7" and python_version < "3.8"
+importlib-resources==5.12.0 ; python_version >= "3.7" and python_version < "3.8"
 iniconfig==2.0.0 ; python_version >= "3.7" and python_version < "3.8"
 ipaddress==1.0.16 ; python_version >= "3.7" and python_version < "3.8"
 isort==5.11.5 ; python_full_version >= "3.7.2" and python_version < "3.8"
@@ -48,7 +49,7 @@ packaging==23.1 ; python_version >= "3.7" and python_version < "3.8"
 parsedatetime==2.4 ; python_version >= "3.7" and python_version < "3.8"
 pbr==1.8.0 ; python_version >= "3.7" and python_version < "3.8"
 pip==23.2.1 ; python_version >= "3.7" and python_version < "3.8"
-platformdirs==3.10.0 ; python_version >= "3.7" and python_version < "3.8"
+platformdirs==3.10.0 ; python_full_version >= "3.7.2" and python_version < "3.8"
 pluggy==1.2.0 ; python_version >= "3.7" and python_version < "3.8"
 ply==3.4 ; python_version >= "3.7" and python_version < "3.8"
 py==1.11.0 ; python_version >= "3.7" and python_version < "3.8"
@@ -61,7 +62,7 @@ pyparsing==2.2.1 ; python_version >= "3.7" and python_version < "3.8"
 pyrfc3339==1.0 ; python_version >= "3.7" and python_version < "3.8"
 pytest-cov==4.1.0 ; python_version >= "3.7" and python_version < "3.8"
 pytest-xdist==3.3.1 ; python_version >= "3.7" and python_version < "3.8"
-pytest==7.4.0 ; python_version >= "3.7" and python_version < "3.8"
+pytest==7.4.2 ; python_version >= "3.7" and python_version < "3.8"
 python-augeas==0.5.0 ; python_version >= "3.7" and python_version < "3.8"
 python-dateutil==2.8.2 ; python_version >= "3.7" and python_version < "3.8"
 python-digitalocean==1.11 ; python_version >= "3.7" and python_version < "3.8"
@@ -74,7 +75,7 @@ rsa==4.9 ; python_version >= "3.7" and python_version < "3.8"
 s3transfer==0.3.7 ; python_version >= "3.7" and python_version < "3.8"
 setuptools==41.6.0 ; python_version >= "3.7" and python_version < "3.8"
 six==1.11.0 ; python_version >= "3.7" and python_version < "3.8"
-tldextract==3.4.4 ; python_version >= "3.7" and python_version < "3.8"
+tldextract==3.5.0 ; python_version >= "3.7" and python_version < "3.8"
 tomli==2.0.1 ; python_version >= "3.7" and python_version < "3.8"
 tomlkit==0.12.1 ; python_full_version >= "3.7.2" and python_version < "3.8"
 tox==1.9.2 ; python_version >= "3.7" and python_version < "3.8"
@@ -87,13 +88,13 @@ types-python-dateutil==2.8.19.14 ; python_version >= "3.7" and python_version <
 types-pytz==2023.3.0.1 ; python_version >= "3.7" and python_version < "3.8"
 types-pywin32==306.0.0.4 ; python_version >= "3.7" and python_version < "3.8"
 types-requests==2.31.0.2 ; python_version >= "3.7" and python_version < "3.8"
-types-setuptools==68.1.0.0 ; python_version >= "3.7" and python_version < "3.8"
+types-setuptools==68.2.0.0 ; python_version >= "3.7" and python_version < "3.8"
 types-six==1.16.21.9 ; python_version >= "3.7" and python_version < "3.8"
 types-urllib3==1.26.25.14 ; python_version >= "3.7" and python_version < "3.8"
 typing-extensions==4.7.1 ; python_version >= "3.7" and python_version < "3.8"
 uritemplate==3.0.1 ; python_version >= "3.7" and python_version < "3.8"
 urllib3==1.24.2 ; python_version >= "3.7" and python_version < "3.8"
-virtualenv==20.24.3 ; python_version >= "3.7" and python_version < "3.8"
+virtualenv==20.4.7 ; python_version >= "3.7" and python_version < "3.8"
 wheel==0.33.6 ; python_version >= "3.7" and python_version < "3.8"
 wrapt==1.15.0 ; python_full_version >= "3.7.2" and python_version < "3.8"
 zipp==3.15.0 ; python_version >= "3.7" and python_version < "3.8"

tools/pinning/oldest/pyproject.toml

@@ -61,7 +61,7 @@ google-api-python-client = "1.6.5"
 google-auth = "2.16.0"
 httplib2 = "0.9.2"
 idna = "2.6"
-importlib-resources = "1.3.1"
+importlib-metadata = "4.6.4"
 ipaddress = "1.0.16"
 ndg-httpsclient = "0.3.2"
 parsedatetime = "2.4"