Conftest.restore_import_state should restore sys.meta_path as well as…

… sys.path (lektor#1140)

* test(conftest.restore_import_state): ensure sys.meta_path is restored

* fix(conftest.restore_import_state): restore sys.meta_path

* fix(test_pluginsystem): preserve sys.meta_path

* refactor(conftest.restore_import_state): cleanup & paranoia

* test(tests): check that tests do not alter sys.path

* fix(tests): fix the tests that were munging sys.path

tests/conftest.py

@@ -72,29 +72,53 @@ def restore_import_state():
     package caches.
 
     """
-    save_path = sys.path
-    sys.path = save_path.copy()
-
-    # Restoring `sys.modules` is an attempt to unload any
-    # modules loaded during the test so that they can be re-loaded for
-    # the next test.  This is not guaranteed to work, since there are
-    # numerous ways that a reference to a loaded module may still be held.
-
-    # NB: some modules (e.g. pickle) appear to hold a reference to sys.modules,
-    # so we have to be careful to manipulate sys.modules in place, rather than
-    # using monkeypatch to swap it out.
-    saved_modules = sys.modules.copy()
-
-    # It's not clear that this is necessary, but it probably won't hurt.
-    importlib.invalidate_caches()
+    path = sys.path.copy()
+    meta_path = sys.meta_path.copy()
+    path_hooks = sys.path_hooks.copy()
+    modules = sys.modules.copy()
+
+    # Importlib_metadata, when it is imported, cripples the stdlib distribution finder
+    # by deleting its find_distributions method.
+    #
+    # https://github.com/python/importlib_metadata/blob/705a7571ec7c5abec4d4b008da3a58df7e5c94e7/importlib_metadata/_compat.py#L31
+    #
+    def clone_class(cls):
+        return type(cls)(cls.__name__, cls.__bases__, cls.__dict__.copy())
+
+    sys.meta_path[:] = [
+        clone_class(finder) if isinstance(finder, type) else finder
+        for finder in meta_path
+    ]
 
     try:
         yield
     finally:
-        for name in set(sys.modules).difference(saved_modules):
-            del sys.modules[name]
-        sys.modules.update(saved_modules)
-        sys.path = save_path
+        importlib.invalidate_caches()
+
+        # NB: Restore sys.modules, sys.path, et. all. in place. (Some modules may hold
+        # references to these — e.g. pickle appears to hold a reference to sys.modules.)
+        for module in set(sys.modules).difference(modules):
+            del sys.modules[module]
+        sys.modules.update(modules)
+        sys.path[:] = path
+        sys.meta_path[:] = meta_path
+        sys.path_hooks[:] = path_hooks
+        sys.path_importer_cache.clear()
+
+
+_initial_path_key = object()
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item):
+    item.stash[_initial_path_key] = sys.path.copy()
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_runtest_teardown(item):
+    # Check that tests don't alter sys.path
+    initial_path = item.stash[_initial_path_key]
+    assert sys.path == initial_path
 
 
 @pytest.fixture
@@ -219,7 +243,7 @@ def scratch_builder(tmp_path, scratch_pad):
 # Builder for child-sources-test-project, a project to test that child sources
 # are built even if they're filtered out by a pagination query.
 @pytest.fixture(scope="function")
-def child_sources_test_project_builder(tmp_path, data_path):
+def child_sources_test_project_builder(tmp_path, data_path, save_sys_path):
     output_path = tmp_path / "output"
     output_path.mkdir()
     project = Project.from_path(data_path / "child-sources-test-project")

tests/test_assets.py

@@ -28,7 +28,7 @@ def project_path(tmp_path_factory, data_path):
 
 
 @pytest.fixture
-def pad(project_path):
+def pad(project_path, save_sys_path):
     return Project.from_path(project_path).make_env().new_pad()
 
 

tests/test_conftest.py

@@ -0,0 +1,39 @@
+import sys
+from contextlib import suppress
+from importlib import import_module
+
+from lektor.compat import importlib_metadata
+
+# pylint: disable-next=wrong-import-order
+from conftest import restore_import_state  # noreorder
+
+
+def test_restore_import_state_restores_meta_path():
+    # Various modules (e.g. setuptools, importlib_metadata), when imported,
+    # add their own finders to sys.meta_path.
+    meta_path = sys.meta_path.copy()
+
+    with restore_import_state(), suppress(ModuleNotFoundError):
+        import_module("importlib_metadata")
+
+    assert sys.meta_path == meta_path
+
+
+def test_restore_import_state_restores_unneutered_PathFinder():
+    # When importlib_metadata is imported, it neuters the stdlib
+    # distribution find, and then adds its own finder to meta_path.
+    #
+    # This tests that restore_import_state manages to unneuter
+    # this find.
+    distributions_pre = [
+        dist.metadata["name"] for dist in importlib_metadata.distributions()
+    ]
+
+    with restore_import_state(), suppress(ModuleNotFoundError):
+        import_module("importlib_metadata")
+
+    distributions_post = [
+        dist.metadata["name"] for dist in importlib_metadata.distributions()
+    ]
+    assert len(distributions_pre) == len(distributions_post)
+    assert set(distributions_pre) == set(distributions_post)

tests/test_pluginsystem.py

@@ -114,7 +114,7 @@ def find_distributions(self, context=metadata.DistributionFinder.Context()):
 
 
 @pytest.fixture
-def dummy_plugin_distribution(save_sys_path, monkeypatch):
+def dummy_plugin_distribution(save_sys_path):
     """Add a dummy plugin distribution to the current working_set."""
     dist = DummyDistribution(
         {
@@ -123,7 +123,8 @@ def dummy_plugin_distribution(save_sys_path, monkeypatch):
         }
     )
     finder = DummyPluginFinder(__name__, dist)
-    monkeypatch.setattr("sys.meta_path", [finder] + sys.meta_path)
+    # The save_sys_path fixture will restore meta_path at end of test
+    sys.meta_path.insert(0, finder)
     return dist
 
 

tests/test_prev_next_sibling.py

@@ -21,7 +21,7 @@ def pntest_project(tmp_path, data_path):
 
 
 @pytest.fixture
-def pntest_env(pntest_project):
+def pntest_env(pntest_project, save_sys_path):
     return Environment(pntest_project)
 
 

tests/test_themes.py

@@ -49,7 +49,7 @@ def theme_project(theme_project_tmpdir, request):
 
 
 @pytest.fixture(scope="function")
-def theme_env(theme_project):
+def theme_env(theme_project, save_sys_path):
 
     return Environment(theme_project)
 

tests/test_tree.py

@@ -9,6 +9,9 @@
 from lektor.db import Tree
 from lektor.project import Project
 
+# pylint: disable-next=wrong-import-order
+from conftest import restore_import_state  # noreorder
+
 
 @pytest.fixture(scope="session")
 def no_alt_pad(tmp_path_factory, data_path):
@@ -29,7 +32,8 @@ def no_alt_pad(tmp_path_factory, data_path):
                 child.unlink()
 
     project = Project.from_path(no_alt_project)
-    return project.make_env().new_pad()
+    with restore_import_state():
+        return project.make_env().new_pad()
 
 
 @pytest.fixture(params=[False, True])

tests/test_unicode.py

@@ -7,7 +7,7 @@
 
 
 @pytest.fixture
-def pad(data_path):
+def pad(data_path, save_sys_path):
     proj = Project.from_path(data_path / "ünicöde-project")
     return proj.make_env().new_pad()