From e901b5b3121b136d8726958099dafcb21a122a10 Mon Sep 17 00:00:00 2001 From: stanley karunditu Date: Wed, 20 Dec 2023 14:33:15 -0500 Subject: [PATCH] Add a cache for .py files that stays in sync with the self._paths cache this is based on https://github.com/ansible/ansible/pull/32609 * avoid calling glob.glob() when looking for py files in variable folders for each host listed in the inventory. In addition, I added a global cache so all plugin loaders stay in sync (not sure if this is wanted), and reset the cache when new directories are added to the loader, like the other caches. https://github.com/ansible/ansible/pull/79687 also identifies this as a hot code path. Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com> --- lib/ansible/plugins/__init__.py | 1 + lib/ansible/plugins/loader.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/ansible/plugins/__init__.py b/lib/ansible/plugins/__init__.py index 90452dff3e0d2f..b8b821b41301c5 100644 --- a/lib/ansible/plugins/__init__.py +++ b/lib/ansible/plugins/__init__.py @@ -39,6 +39,7 @@ MODULE_CACHE = {} # type: dict[str, dict[str, types.ModuleType]] PATH_CACHE = {} # type: dict[str, list[PluginPathContext] | None] PLUGIN_PATH_CACHE = {} # type: dict[str, dict[str, dict[str, PluginPathContext]]] +PY_FILES = {} # type: dict[str, list[str]] def get_plugin_class(obj): diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index 90ff17b198a512..ae1ee6af40acbf 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -29,7 +29,7 @@ from ansible.module_utils.six import string_types from ansible.parsing.utils.yaml import from_yaml from ansible.parsing.yaml.loader import AnsibleLoader -from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE +from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE, PY_FILES from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata from ansible.utils.display import Display @@ -229,6 +229,8 @@ def __init__(self, class_name, package, config, subdir, aliases=None, required_b PATH_CACHE[class_name] = None if class_name not in PLUGIN_PATH_CACHE: PLUGIN_PATH_CACHE[class_name] = defaultdict(dict) + if class_name not in PY_FILES: + PY_FILES[class_name] = {} # hold dirs added at runtime outside of config self._extra_dirs = [] @@ -236,6 +238,7 @@ def __init__(self, class_name, package, config, subdir, aliases=None, required_b # caches self._module_cache = MODULE_CACHE[class_name] self._paths = PATH_CACHE[class_name] + self._py_files = PY_FILES[class_name] self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name] try: self._plugin_instance_cache = {} if self.type == 'vars' else None @@ -260,11 +263,13 @@ def _clear_caches(self): MODULE_CACHE[self.class_name] = {} PATH_CACHE[self.class_name] = None PLUGIN_PATH_CACHE[self.class_name] = defaultdict(dict) + PY_FILES[self.class_name] = {} # reset internal caches self._module_cache = MODULE_CACHE[self.class_name] self._paths = PATH_CACHE[self.class_name] self._plugin_path_cache = PLUGIN_PATH_CACHE[self.class_name] + self._py_files = PY_FILES[self.class_name] self._plugin_instance_cache = {} if self.type == 'vars' else None self._searched_paths = set() @@ -282,6 +287,7 @@ def __setstate__(self, data): PATH_CACHE[class_name] = data.get('PATH_CACHE') PLUGIN_PATH_CACHE[class_name] = data.get('PLUGIN_PATH_CACHE') + PY_FILES[class_name] = data.get('PY_FILES') self.__init__(class_name, package, config, subdir, aliases, base_class) self._extra_dirs = data.get('_extra_dirs', []) @@ -303,6 +309,7 @@ def __getstate__(self): _searched_paths=self._searched_paths, PATH_CACHE=PATH_CACHE[self.class_name], PLUGIN_PATH_CACHE=PLUGIN_PATH_CACHE[self.class_name], + PY_FILES=PY_FILES[self.class_name], ) def format_paths(self, paths): @@ -951,6 +958,11 @@ def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_ display.debug(msg) + def _get_py_files(self, path): + if path not in self._py_files: + self._py_files[path] = glob.glob(to_native(os.path.join(path, "*.py"))) + return self._py_files[path] + def all(self, *args, **kwargs): ''' Iterate through all plugins of this type, in configured paths (no collections) @@ -997,7 +1009,7 @@ def all(self, *args, **kwargs): legacy_excluding_builtin = set() for path_with_context in self._get_paths_with_context(): - matches = glob.glob(to_native(os.path.join(path_with_context.path, "*.py"))) + matches = self._get_py_files(path_with_context.path) if not path_with_context.internal: legacy_excluding_builtin.update(matches) # we sort within each path, but keep path precedence from config