diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index b63b7451e300..102040bd3ba3 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -316,7 +316,7 @@ def _belongs_to_runtime(r: types.ModuleType, attr: str) -> bool:
         return not isinstance(obj, types.ModuleType)
 
     runtime_public_contents = (
-        runtime_all_as_set | {"__all__"}
+        runtime_all_as_set
         if runtime_all_as_set is not None
         else {
             m
diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py
index ad482d26b78f..f1ff9b467d27 100644
--- a/mypy/test/teststubtest.py
+++ b/mypy/test/teststubtest.py
@@ -934,7 +934,7 @@ def test_all_at_runtime_not_stub(self) -> Iterator[Case]:
             runtime="""
             __all__ = []
             Z = 5""",
-            error="__all__",
+            error=None,
         )
 
     @collect_cases
@@ -974,7 +974,7 @@ def h(x: str): ...
             runtime="",
             error="h",
         )
-        yield Case(stub="", runtime="__all__ = []", error="__all__")  # dummy case
+        yield Case(stub="", runtime="__all__ = []", error=None)  # dummy case
         yield Case(stub="", runtime="__all__ += ['y']\ny = 5", error="y")
         yield Case(stub="", runtime="__all__ += ['g']\ndef g(): pass", error="g")
         # Here we should only check that runtime has B, since the stub explicitly re-exports it