ENH: Delay loading of less likely modules

Using python -X importtime -c "import lazy_loader":

Before
------
import time: self [us] | cumulative | imported package
[...]
import time:       131 |      22995 | lazy_loader

After
-----
import time: self [us] | cumulative | imported package
[...]
import time:       115 |       4248 | lazy_loader

lazy_loader/__init__.py

@@ -7,16 +7,10 @@
 import ast
 import importlib
 import importlib.util
-import inspect
 import os
 import sys
 import types
 
-try:
-    import importlib_metadata
-except ImportError:
-    import importlib.metadata as importlib_metadata
-
 __all__ = ["attach", "load", "attach_stub"]
 
 
@@ -181,9 +175,13 @@ def myfunc():
     if not have_module:
         not_found_message = f"No module named '{fullname}'"
     elif require is not None:
-        # Old style lazy loading to avoid polluting sys.modules
         import packaging.requirements
 
+        try:
+            import importlib_metadata
+        except ImportError:
+            import importlib.metadata as importlib_metadata
+
         req = packaging.requirements.Requirement(require)
         try:
             have_module = req.specifier.contains(
@@ -202,6 +200,8 @@ def myfunc():
     if not have_module:
         if error_on_import:
             raise ModuleNotFoundError(not_found_message)
+        import inspect
+
         try:
             parent = inspect.stack()[1]
             frame_data = {