From f72639fbfe084c6e39aec329b30bbe672cc72d34 Mon Sep 17 00:00:00 2001 From: wwade Date: Wed, 5 Oct 2022 17:48:06 -0700 Subject: [PATCH] plugins: allow plugins to provide a relative priority value The highest priority for a plugin is 0, the lowest is 1<<31. If there are multiple plugins providing a function at the same level, then it's down to the implementation in Plugins itself. Currently only getResources() combines the output from multiple plugins. workspaceIdentity() and workspaceProject() both return the response from the first plugin at a given level. --- jobrunner/plugins.py | 85 ++++++++++++++++++++------- jobrunner/test/plugin_test.py | 106 ++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 20 deletions(-) create mode 100644 jobrunner/test/plugin_test.py diff --git a/jobrunner/plugins.py b/jobrunner/plugins.py index 922f542..8af97d0 100644 --- a/jobrunner/plugins.py +++ b/jobrunner/plugins.py @@ -1,4 +1,30 @@ +""" +This module implements the plugin contract. + +Plugin modules need to be registered using the wwade.jobrunner entrypoint. Modules +that are registered as such can implement any of the functions: + + def priority(): + return {"getResources": 100, "workspaceProject": 10, "workspaceIdentity": 10} + + def getResources(jobs): + return "some string" + + def workspaceIdentity(): + return "home/tmp-dir" + + def workspaceProject(): + # If the current context has a notion of a "project name", return the project + # name as well as a bool True to indicate that the plugin is authoritative. + return "the current project name", True + +All of these functions are optional. If the plugin cannot provide a sensible value +for the current execution then it should raise NotImplementedError so that the next +plugin at a possibly lower priority will get called instead. +""" import importlib +import logging +from operator import attrgetter import pkgutil import socket from typing import Tuple @@ -8,10 +34,14 @@ from .compat import get_plugins +logger = logging.getLogger(__name__) +PRIO_LOWEST = 1 << 31 +PRIO_HIGHEST = 0 + class Plugins(object): def __init__(self): - self.plugins = {plug.load() for plug in get_plugins("wwade.jobrunner")} + plugins = {plug.load() for plug in get_plugins("wwade.jobrunner")} deprecatedPlugins = { importlib.import_module("jobrunner.plugin.{}".format(name)) for _, name, _ @@ -22,26 +52,43 @@ def __init__(self): "Convert to entry_point 'wwade.jobrunner'" % list( deprecatedPlugins), DeprecationWarning) - self.plugins |= deprecatedPlugins + plugins |= deprecatedPlugins + self.plugins = list(sorted(plugins, key=attrgetter("__name__"))) + logger.debug("all plugins: %r", [p.__name__ for p in self.plugins]) + self._prio = {} + for plugin in self.plugins: + if hasattr(plugin, "priority"): + self._prio[plugin.__name__] = plugin.priority() - def _plugDo(self, which, *args, **kwargs): + def _pluginCalls(self, func, *args, **kwargs): + prio = {} for plugin in self.plugins: - if hasattr(plugin, which): - getattr(plugin, which)(*args, **kwargs) + if hasattr(plugin, func): + pluginPrioMap = self._prio.get(plugin.__name__, {}) + pval = pluginPrioMap.get(func, pluginPrioMap.get("", PRIO_LOWEST)) + prio.setdefault(pval, []).append(plugin) + + if not prio: + return + + for prio, plugins in sorted(prio.items()): + for plugin in plugins: + name = plugin.__name__ + try: + result = getattr(plugin, func)(*args, **kwargs) + logger.debug("%r: yield plugin %s => %r", prio, name, result) + yield result + except NotImplementedError: + logger.debug("%r: plugin %s NotImplementedError", prio, name) + continue def getResources(self, jobs): - ret = "" - for plugin in self.plugins: - if hasattr(plugin, "getResources"): - ret += plugin.getResources(jobs) - return ret + return "".join(self._pluginCalls("getResources", jobs)) def workspaceIdentity(self): - for plugin in self.plugins: - if hasattr(plugin, "workspaceIdentity"): - ret = plugin.workspaceIdentity() - if ret: - return ret + for ret in self._pluginCalls("workspaceIdentity"): + if ret: + return ret return socket.gethostname() def workspaceProject(self) -> Tuple[str, bool]: @@ -49,9 +96,7 @@ def workspaceProject(self) -> Tuple[str, bool]: If the current context has a notion of a "project name", return the project name as well as a bool True to indicate that the plugin is authoritative. """ - for plugin in self.plugins: - if hasattr(plugin, "workspaceProject"): - ret, ok = plugin.workspaceProject() - if ok: - return ret, ok + for (ret, ok) in self._pluginCalls("workspaceProject"): + if ok: + return ret, ok return "", False diff --git a/jobrunner/test/plugin_test.py b/jobrunner/test/plugin_test.py new file mode 100644 index 0000000..f66ab94 --- /dev/null +++ b/jobrunner/test/plugin_test.py @@ -0,0 +1,106 @@ +import logging +from typing import Tuple +from unittest import mock + +import pytest + +from jobrunner.plugins import Plugins + +logger = logging.getLogger(__name__) + + +class Plugin: + @classmethod + def load(cls): + return cls + + +class PluginAAANoPrio(Plugin): + @staticmethod + def workspaceProject() -> Tuple[str, bool]: + return "lowest", True + + @staticmethod + def getResources(jobs): + _ = jobs + return "[no-prio]" + + +class PluginBBBLowPrioResources(Plugin): + @staticmethod + def workspaceProject() -> Tuple[str, bool]: + return "lowest", True + + @staticmethod + def getResources(jobs): + _ = jobs + return "[low-prio]" + + +class PluginZABHighPrioNoProject(Plugin): + @staticmethod + def priority(): + return {"": 0} + + @staticmethod + def workspaceProject() -> Tuple[str, bool]: + raise NotImplementedError + + +class PluginZAAHighestPrio(Plugin): + @staticmethod + def priority(): + return {"": 0} + + @staticmethod + def workspaceProject() -> Tuple[str, bool]: + return "highest", True + + +class PluginMMMLowPrio(Plugin): + @staticmethod + def priority(): + return {"": 1000} + + @staticmethod + def workspaceProject() -> Tuple[str, bool]: + return "low", True + + +@pytest.mark.parametrize( + ["plugins", "workspaceProject", "resources"], + [ + ( + {PluginAAANoPrio, PluginMMMLowPrio, PluginZAAHighestPrio}, + "highest", + "[no-prio]", + ), + ( + {PluginAAANoPrio, PluginMMMLowPrio, PluginZABHighPrioNoProject}, + "low", + "[no-prio]", + ), + ( + {PluginAAANoPrio, PluginBBBLowPrioResources, PluginMMMLowPrio}, + "low", + "[no-prio][low-prio]", + ), + ] +) +def testPluginPriorities(plugins, workspaceProject, resources): + with mock.patch("jobrunner.plugins.get_plugins") as gp, \ + mock.patch("importlib.import_module") as im, \ + mock.patch("socket.gethostname", return_value="xxx"): + im.return_value = [] + gp.return_value = plugins + + p = Plugins() + + logger.info("only PluginNoPrio implements getResources()") + assert resources == p.getResources(None) + + logger.info("no plugins implement workspaceIdentity()") + assert "xxx" == p.workspaceIdentity() + + logger.info("all plugins implement workspaceProject()") + assert (workspaceProject, True) == p.workspaceProject()