Add a cache for .py files that stays in sync with the self._paths cache

this is based on #32609

* avoid calling glob.glob() when looking for py files in variable folders for each host listed in the inventory.

In addition, I added a global cache so all plugin loaders stay in sync
(not sure if this is wanted), and reset the cache when new directories
are added to the loader, like the other caches.

#79687 also identifies this as a hot code path.

Co-authored-by: Sloane Hertel <19572925+s-hertel@users.noreply.github.com>

lib/ansible/plugins/__init__.py

@@ -39,6 +39,7 @@
 MODULE_CACHE = {}  # type: dict[str, dict[str, types.ModuleType]]
 PATH_CACHE = {}  # type: dict[str, list[PluginPathContext] | None]
 PLUGIN_PATH_CACHE = {}  # type: dict[str, dict[str, dict[str, PluginPathContext]]]
+PY_FILES = {}  # type: dict[str, list[str]]
 
 
 def get_plugin_class(obj):

lib/ansible/plugins/loader.py

@@ -29,7 +29,7 @@
 from ansible.module_utils.six import string_types
 from ansible.parsing.utils.yaml import from_yaml
 from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE
+from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE, PY_FILES
 from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef
 from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata
 from ansible.utils.display import Display
@@ -229,13 +229,16 @@ def __init__(self, class_name, package, config, subdir, aliases=None, required_b
             PATH_CACHE[class_name] = None
         if class_name not in PLUGIN_PATH_CACHE:
             PLUGIN_PATH_CACHE[class_name] = defaultdict(dict)
+        if class_name not in PY_FILES:
+            PY_FILES[class_name] = {}
 
         # hold dirs added at runtime outside of config
         self._extra_dirs = []
 
         # caches
         self._module_cache = MODULE_CACHE[class_name]
         self._paths = PATH_CACHE[class_name]
+        self._py_files = PY_FILES[class_name]
         self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name]
         try:
             self._plugin_instance_cache = {} if self.type == 'vars' else None
@@ -260,11 +263,13 @@ def _clear_caches(self):
             MODULE_CACHE[self.class_name] = {}
             PATH_CACHE[self.class_name] = None
             PLUGIN_PATH_CACHE[self.class_name] = defaultdict(dict)
+            PY_FILES[self.class_name] = {}
 
             # reset internal caches
             self._module_cache = MODULE_CACHE[self.class_name]
             self._paths = PATH_CACHE[self.class_name]
             self._plugin_path_cache = PLUGIN_PATH_CACHE[self.class_name]
+            self._py_files = PY_FILES[self.class_name]
             self._plugin_instance_cache = {} if self.type == 'vars' else None
             self._searched_paths = set()
 
@@ -282,6 +287,7 @@ def __setstate__(self, data):
 
         PATH_CACHE[class_name] = data.get('PATH_CACHE')
         PLUGIN_PATH_CACHE[class_name] = data.get('PLUGIN_PATH_CACHE')
+        PY_FILES[class_name] = data.get('PY_FILES')
 
         self.__init__(class_name, package, config, subdir, aliases, base_class)
         self._extra_dirs = data.get('_extra_dirs', [])
@@ -303,6 +309,7 @@ def __getstate__(self):
             _searched_paths=self._searched_paths,
             PATH_CACHE=PATH_CACHE[self.class_name],
             PLUGIN_PATH_CACHE=PLUGIN_PATH_CACHE[self.class_name],
+            PY_FILES=PY_FILES[self.class_name],
         )
 
     def format_paths(self, paths):
@@ -951,6 +958,11 @@ def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_
 
             display.debug(msg)
 
+    def _get_py_files(self, path):
+        if path not in self._py_files:
+            self._py_files[path] = glob.glob(to_native(os.path.join(path, "*.py")))
+        return self._py_files[path]
+
     def all(self, *args, **kwargs):
         '''
         Iterate through all plugins of this type, in configured paths (no collections)
@@ -997,7 +1009,7 @@ def all(self, *args, **kwargs):
 
         legacy_excluding_builtin = set()
         for path_with_context in self._get_paths_with_context():
-            matches = glob.glob(to_native(os.path.join(path_with_context.path, "*.py")))
+            matches = self._get_py_files(path_with_context.path)
             if not path_with_context.internal:
                 legacy_excluding_builtin.update(matches)
             # we sort within each path, but keep path precedence from config