Handle plugin registration failure ContextualVersionConflict with log…

… instead of raising error (#1542)

* ignore loading error

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Move setuptools to pyproject.toml to align with PEP-518

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* revert changes

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* [WIP] - replace pkg_resource with importlib_metadata

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* [WIP] - replace pkg_resource

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Replace entrypoint discovery with importlib

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Tidy up logging message, linting

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* small changes to tidy up the variables

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Add conditional import to fix importlib not available in python 3.7

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Fix test cases

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Update error message

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Switch back to importlib_metadata and fix tests

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* add importlib as requirements

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Fix test with py37

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Fix linting

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Apply changes from review

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Refactor and fix wrong test

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* Fix some tests... should work now

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* I forgot to put the setuptools back...

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* I have gone too far and removed a necessary dependency

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* apply comments after reviews

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

* linting

Signed-off-by: noklam <nok.lam.chan@quantumblack.com>

kedro/framework/cli/cli.py

@@ -3,14 +3,15 @@
 This module implements commands available from the kedro CLI.
 """
 import importlib
+import logging
 import sys
 import webbrowser
 from collections import defaultdict
 from pathlib import Path
 from typing import Sequence
 
 import click
-import pkg_resources
+import importlib_metadata
 
 # pylint: disable=unused-import
 import kedro.config.default_logger  # noqa
@@ -41,6 +42,8 @@
 v{version}
 """
 
+logger = logging.getLogger(__name__)
+
 
 @click.group(context_settings=CONTEXT_SETTINGS, name="Kedro")
 @click.version_option(version, "--version", "-V", help="Show version and exit")
@@ -65,10 +68,9 @@ def info():
     plugin_versions = {}
     plugin_entry_points = defaultdict(set)
     for plugin_entry_point, group in ENTRY_POINT_GROUPS.items():
-        for entry_point in pkg_resources.iter_entry_points(group=group):
+        for entry_point in importlib_metadata.entry_points().select(group=group):
             module_name = entry_point.module_name.split(".")[0]
-            plugin_version = pkg_resources.get_distribution(module_name).version
-            plugin_versions[module_name] = plugin_version
+            plugin_versions[module_name] = entry_point.dist.version
             plugin_entry_points[module_name].add(plugin_entry_point)
 
     click.echo()
@@ -96,14 +98,10 @@ def docs():
     webbrowser.open(index_path)
 
 
-def _init_plugins():
-    group = ENTRY_POINT_GROUPS["init"]
-    for entry_point in pkg_resources.iter_entry_points(group=group):
-        try:
-            init_hook = entry_point.load()
-            init_hook()
-        except Exception as exc:
-            raise KedroCliError(f"Initializing {entry_point}") from exc
+def _init_plugins() -> None:
+    init_hooks = load_entry_points("init")
+    for init_hook in init_hooks:
+        init_hook()
 
 
 class KedroCLI(CommandCollection):

kedro/framework/cli/utils.py

@@ -1,5 +1,6 @@
 """Utilities for use with click."""
 import difflib
+import logging
 import re
 import shlex
 import shutil
@@ -16,7 +17,7 @@
 from typing import Any, Dict, Iterable, List, Mapping, Sequence, Set, Tuple, Union
 
 import click
-import pkg_resources
+import importlib_metadata
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
 MAX_SUGGESTIONS = 3
@@ -33,6 +34,8 @@
     "cli_hooks": "kedro.cli_hooks",
 }
 
+logger = logging.getLogger(__name__)
+
 
 def call(cmd: List[str], **kwargs):  # pragma: no cover
     """Run a subprocess command and raise if it fails.
@@ -328,13 +331,20 @@ def load_entry_points(name: str) -> Sequence[click.MultiCommand]:
         List of entry point commands.
 
     """
-    entry_points = pkg_resources.iter_entry_points(group=ENTRY_POINT_GROUPS[name])
+    entry_points = importlib_metadata.entry_points()
+    entry_points = entry_points.select(group=ENTRY_POINT_GROUPS[name])
+
     entry_point_commands = []
     for entry_point in entry_points:
         try:
             entry_point_commands.append(entry_point.load())
-        except Exception as exc:
-            raise KedroCliError(f"Loading {name} commands from {entry_point}") from exc
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.warning(
+                "Failed to load %s commands from %s. Full exception: %s",
+                name,
+                entry_point,
+                exc,
+            )
     return entry_point_commands
 
 

kedro/framework/session/session.py

@@ -1,4 +1,3 @@
-# pylint: disable=invalid-name,global-statement
 """This module implements Kedro session responsible for project lifecycle."""
 import getpass
 import logging

requirements.txt

@@ -5,6 +5,7 @@ cookiecutter~=1.7.0
 dynaconf>=3.1.2,<4.0.0
 fsspec>=2021.4, <=2022.1
 gitpython~=3.0
+importlib_metadata>=4.4 # Compatible with Python 3.10 importlib.metadata
 jmespath>=0.9.5, <1.0
 pip-tools~=6.5
 pluggy~=1.0.0

tests/framework/cli/conftest.py

@@ -33,13 +33,14 @@
 
 @fixture
 def entry_points(mocker):
-    return mocker.patch("pkg_resources.iter_entry_points")
+    return mocker.patch("importlib_metadata.entry_points")
 
 
 @fixture
 def entry_point(mocker, entry_points):
     ep = mocker.MagicMock()
-    entry_points.return_value = [ep]
+    entry_points.return_value = ep
+    entry_points.return_value.select.return_value = [ep]
     return ep
 
 

tests/framework/cli/test_cli.py

@@ -104,9 +104,8 @@ def test_print_version(self):
         assert result_abr.exit_code == 0
         assert version in result_abr.output
 
-    def test_info_contains_plugin_versions(self, entry_point, mocker):
-        get_distribution = mocker.patch("pkg_resources.get_distribution")
-        get_distribution().version = "1.0.2"
+    def test_info_contains_plugin_versions(self, entry_point):
+        entry_point.dist.version = "1.0.2"
         entry_point.module_name = "bob.fred"
 
         result = CliRunner().invoke(cli, ["info"])
@@ -312,37 +311,44 @@ def test_project_groups(self, entry_points, entry_point):
         entry_point.load.return_value = "groups"
         groups = load_entry_points("project")
         assert groups == ["groups"]
-        entry_points.assert_called_once_with(group="kedro.project_commands")
+        entry_points.return_value.select.assert_called_once_with(
+            group="kedro.project_commands"
+        )
 
-    def test_project_error_is_caught(self, entry_points, entry_point):
+    def test_project_error_is_caught(self, entry_points, entry_point, caplog):
         entry_point.load.side_effect = Exception()
-        with raises(KedroCliError, match="Loading project commands"):
-            load_entry_points("project")
-
-        entry_points.assert_called_once_with(group="kedro.project_commands")
+        load_entry_points("project")
+        assert "Failed to load project commands" in caplog.text
+        entry_points.return_value.select.assert_called_once_with(
+            group="kedro.project_commands"
+        )
 
     def test_global_groups(self, entry_points, entry_point):
         entry_point.load.return_value = "groups"
         groups = load_entry_points("global")
         assert groups == ["groups"]
-        entry_points.assert_called_once_with(group="kedro.global_commands")
+        entry_points.return_value.select.assert_called_once_with(
+            group="kedro.global_commands"
+        )
 
-    def test_global_error_is_caught(self, entry_points, entry_point):
+    def test_global_error_is_caught(self, entry_points, entry_point, caplog):
         entry_point.load.side_effect = Exception()
-        with raises(KedroCliError, match="Loading global commands from"):
-            load_entry_points("global")
-        entry_points.assert_called_once_with(group="kedro.global_commands")
+        load_entry_points("global")
+        assert "Failed to load global commands" in caplog.text
+        entry_points.return_value.select.assert_called_once_with(
+            group="kedro.global_commands"
+        )
 
     def test_init(self, entry_points, entry_point):
         _init_plugins()
-        entry_points.assert_called_once_with(group="kedro.init")
+        entry_points.return_value.select.assert_called_once_with(group="kedro.init")
         entry_point.load().assert_called_once_with()
 
     def test_init_error_is_caught(self, entry_points, entry_point):
-        entry_point.load.side_effect = Exception()
-        with raises(KedroCliError, match="Initializing"):
+        entry_point.load.return_value.side_effect = Exception()
+        with raises(Exception):
             _init_plugins()
-        entry_points.assert_called_once_with(group="kedro.init")
+        entry_points.return_value.select.assert_called_once_with(group="kedro.init")
 
 
 class TestKedroCLI:
@@ -356,6 +362,7 @@ def test_project_commands_no_clipy(self, mocker, fake_metadata):
             "kedro.framework.cli.cli.bootstrap_project", return_value=fake_metadata
         )
         kedro_cli = KedroCLI(fake_metadata.project_path)
+        print(kedro_cli.project_groups)
         assert len(kedro_cli.project_groups) == 6
         assert kedro_cli.project_groups == [
             catalog_cli,

tests/framework/cli/test_cli_hooks.py

@@ -92,6 +92,7 @@ def test_kedro_cli_should_invoke_cli_hooks_from_plugin(
         mocker,
         fake_metadata,
         fake_plugin_distribution,
+        entry_points,  # pylint: disable=unused-argument
     ):
         caplog.set_level(logging.DEBUG, logger="kedro")