Improve exporter performance (Issue #7040, PR #7151)

# Description

Remove getsiblings

closes #7040

# Self Check:

Strike through any lines that are not applicable (`~~line~~`) then check the box

- [x] Attached issue to pull request
- [x] Changelog entry
- [x] Type annotations are present
- [x] Code is clear and sufficiently documented
- [x] No (preventable) type errors (check using make mypy or make mypy-diff)
- [x] Sufficient test cases (reproduces the bug/tests the requested feature)
- [x] Correct, in line with design
- [ ] End user documentation is included or an issue is created for end-user documentation (add ref to issue here: )
- [ ] If this PR fixes a race condition in the test suite, also push the fix to the relevant stable branche(s) (see [test-fixes](https://internal.inmanta.com/development/core/tasks/build-master.html#test-fixes) for more info)

changelogs/unreleased/7040-export-performance.yml

@@ -0,0 +1,6 @@
+description: Improve exporter performance
+issue-nr: 7040
+change-type: patch
+destination-branches: [master, iso7]
+sections:
+  minor-improvement: "{{description}}"

src/inmanta/loader.py

@@ -31,7 +31,7 @@
 from dataclasses import dataclass
 from importlib.abc import FileLoader, MetaPathFinder
 from importlib.machinery import ModuleSpec, SourcelessFileLoader
-from itertools import chain, starmap
+from itertools import chain
 from typing import TYPE_CHECKING, Optional
 
 from inmanta import const, module
@@ -48,6 +48,17 @@
 LOGGER = logging.getLogger(__name__)
 
 
+def get_inmanta_module_name(python_module_name: str) -> str:
+    """Small utility to convert python module into inmanta module"""
+    module_parts = python_module_name.split(".")
+    if module_parts[0] != const.PLUGINS_PACKAGE:
+        raise Exception(
+            "All instances from which the source is loaded, should be defined in the inmanta plugins package. "
+            "%s does not match" % python_module_name
+        )
+    return module_parts[1]
+
+
 class SourceNotFoundException(Exception):
     """This exception is raised when the source of the provided type is not found"""
 
@@ -86,20 +97,7 @@ def content(self) -> bytes:
 
     def _get_module_name(self) -> str:
         """Get the name of the inmanta module, derived from the python module name"""
-        module_parts = self.module_name.split(".")
-        if module_parts[0] != const.PLUGINS_PACKAGE:
-            raise Exception(
-                "All instances from which the source is loaded, should be defined in the inmanta plugins package. "
-                "%s does not match" % self.module_name
-            )
-
-        return module_parts[1]
-
-    def get_siblings(self) -> Iterator["SourceInfo"]:
-        """
-        Returns an iterator over SourceInfo objects for all plugin source files in this Inmanta module (including this one).
-        """
-        return starmap(SourceInfo, module.Project.get().modules[self._get_module_name()].get_plugin_files())
+        return get_inmanta_module_name(self.module_name)
 
     @property
     def requires(self) -> list[str]:
@@ -120,9 +118,14 @@ class CodeManager:
     """
 
     def __init__(self) -> None:
+        # Old implementation
+        # Use by external code
         self.__type_file: dict[str, set[str]] = {}
         self.__file_info: dict[str, SourceInfo] = {}
 
+        # Cache of module to source info
+        self.__module_to_source_info: dict[str, list[SourceInfo]] = {}
+
     def register_code(self, type_name: str, instance: object) -> None:
         """Register the given type_object under the type_name and register the source associated with this type object.
 
@@ -140,16 +143,29 @@ def register_code(self, type_name: str, instance: object) -> None:
         if file_name in self.__type_file[type_name]:
             return
 
-        # don't just store this file, but all plugin files in its Inmanta module to allow for importing helper modules
-        all_plugin_files: list[SourceInfo] = list(SourceInfo(file_name, instance.__module__).get_siblings())
+        # get the module
+        module_name = get_inmanta_module_name(instance.__module__)
+
+        all_plugin_files: list[SourceInfo] = self._get_source_info_for_module(module_name)
+
         self.__type_file[type_name].update(source_info.path for source_info in all_plugin_files)
 
-        if file_name in self.__file_info:
-            return
+    def _get_source_info_for_module(self, module_name: str) -> list[SourceInfo]:
+        if module_name in self.__module_to_source_info:
+            return self.__module_to_source_info[module_name]
+
+        sources = [
+            SourceInfo(path, module_name) for path, module_name in module.Project.get().modules[module_name].get_plugin_files()
+        ]
 
-        for file_info in all_plugin_files:
+        self.__module_to_source_info[module_name] = sources
+
+        # Register files
+        for file_info in sources:
             self.__file_info[file_info.path] = file_info
 
+        return sources
+
     def get_object_source(self, instance: object) -> Optional[str]:
         """Get the path of the source file in which type_object is defined"""
         try:

src/inmanta/module.py

@@ -2778,6 +2778,7 @@ def __init__(self, project: Optional[Project], path: str) -> None:
         self._ast_cache: dict[str, tuple[list[Statement], BasicBlock]] = {}  # Cache for expensive method calls
         self._import_cache: dict[str, list[DefineImport]] = {}  # Cache for expensive method calls
         self._dir_cache: Dict[str, list[str]] = {}  # Cache containing all the filepaths present in a dir
+        self._plugin_file_cache: Optional[list[tuple[Path, ModuleName]]] = None
 
     @classmethod
     @abstractmethod
@@ -3012,6 +3013,9 @@ def get_plugin_files(self) -> Iterator[tuple[Path, ModuleName]]:
         """
         Returns a tuple (absolute_path, fq_mod_name) of all python files in this module.
         """
+        if self._plugin_file_cache is not None:
+            return iter(self._plugin_file_cache)
+
         plugin_dir: Optional[str] = self.get_plugin_dir()
 
         if plugin_dir is None:
@@ -3022,13 +3026,15 @@ def get_plugin_files(self) -> Iterator[tuple[Path, ModuleName]]:
         ):
             raise InvalidModuleException(f"Directory {plugin_dir} should be a valid python package with a __init__.py file")
 
-        return (
+        self._plugin_file_cache = [
             (
                 Path(file_name),
                 ModuleName(self._get_fq_mod_name_for_py_file(file_name, plugin_dir, self.name)),
             )
             for file_name in self._list_python_files(plugin_dir)
-        )
+        ]
+
+        return iter(self._plugin_file_cache)
 
     def load_plugins(self) -> None:
         """