fix(internal): chained namespace loader [backport #8603 to 2.5] (#8627)

Backport of #8603 to 2.5

We fix the implemention of the support for namespace module imports that
caused issues with some standard library modules, such as `importlib`.
We make sure that the internal module attribute initialisation can
create the correct namespace loader, and then we make sure that we chain
it with the custom loader used to trigger import hooks.

Addresses #8003

## Checklist

- [x] Change(s) are motivated and described in the PR description
- [x] Testing strategy is described if automated tests are not included
in the PR
- [x] Risks are described (performance impact, potential for breakage,
maintainability)
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed or label `changelog/no-changelog` is set
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/))
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
- [x] If this PR changes the public interface, I've notified
`@DataDog/apm-tees`.
- [x] If change touches code that signs or publishes builds or packages,
or handles credentials of any kind, I've requested a review from
`@DataDog/security-design-and-guidance`.

## Reviewer Checklist

- [x] Title is accurate
- [x] All changes are related to the pull request's stated goal
- [x] Description motivates each change
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- [x] Testing strategy adequately addresses listed risks
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] Release note makes sense to a user of the library
- [x] Author has acknowledged and discussed the performance implications
of this PR as reported in the benchmarks PR comment
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Co-authored-by: Federico Mon <federico.mon@datadoghq.com>

ddtrace/internal/module.py

@@ -154,6 +154,21 @@ def __getattr__(self, name):
         # Proxy any other attribute access to the underlying loader.
         return getattr(self.loader, name)
 
+    def namespace_module(self, spec: ModuleSpec) -> ModuleType:
+        module = ModuleType(spec.name)
+        # Pretend that we do not have a loader (this would be self), to
+        # allow _init_module_attrs to create the appropriate NamespaceLoader
+        # for the namespace module.
+        spec.loader = None
+
+        _init_module_attrs(spec, module, override=True)
+
+        # Chain the loaders
+        self.loader = spec.loader
+        module.__loader__ = spec.loader = self  # type: ignore[assignment]
+
+        return module
+
     def add_callback(self, key, callback):
         # type: (Any, Callable[[ModuleType], None]) -> None
         self.callbacks[key] = callback
@@ -173,8 +188,7 @@ def load_module(self, fullname):
         if self.loader is None:
             if self.spec is None:
                 return None
-            sys.modules[self.spec.name] = module = ModuleType(fullname)
-            _init_module_attrs(self.spec, module)
+            sys.modules[self.spec.name] = module = self.namespace_module(self.spec)
         else:
             module = self.loader.load_module(fullname)
 
@@ -187,9 +201,7 @@ def _create_module(self, spec):
             return self.loader.create_module(spec)
 
         if is_namespace_spec(spec):
-            module = ModuleType(spec.name)
-            _init_module_attrs(spec, module)
-            return module
+            return self.namespace_module(spec)
 
         return None
 

docs/spelling_wordlist.txt

@@ -281,3 +281,4 @@ Enablement
 hotspot
 CMake
 libdatadog
+importlib

releasenotes/notes/fix-chained-namespace-loader-1701e040d7b3af9b.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fix an incompatibility between the handling of namespace module imports and
+    parts of the functionalities of the standard library importlib module.

tests/internal/test_module.py

@@ -486,3 +486,19 @@ def test_module_watchdog_pkg_resources_support_already_imported():
     import ddtrace  # noqa
 
     p.resource_listdir("namespace_test.ns_module", ".")
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 10), reason="importlib.resources.files is not available or broken in Python < 3.10"
+)
+@pytest.mark.subprocess(env=dict(NSPATH=str(Path(__file__).parent)))
+def test_module_watchdog_importlib_resources_files():
+    import os
+    import sys
+
+    sys.path.insert(0, os.getenv("NSPATH"))
+
+    from importlib.readers import MultiplexedPath
+    import importlib.resources as r
+
+    assert isinstance(r.files("namespace_test"), MultiplexedPath)