diff --git a/pydantic/_internal/_core_utils.py b/pydantic/_internal/_core_utils.py
index 344981a0b9..f53e8ef6c5 100644
--- a/pydantic/_internal/_core_utils.py
+++ b/pydantic/_internal/_core_utils.py
@@ -467,8 +467,12 @@ def count_refs(s: core_schema.CoreSchema, recurse: Recurse) -> core_schema.CoreS
             return recurse(s, count_refs)
         ref = s['schema_ref']
         ref_counts[ref] += 1
-        if current_recursion_ref_count[ref] != 0:
-            involved_in_recursion[ref] = True
+
+        if ref_counts[ref] >= 2:
+            # If this model is involved in a recursion this should be detected
+            # on its second encounter, we can safely stop the walk here.
+            if current_recursion_ref_count[ref] != 0:
+                involved_in_recursion[ref] = True
             return s
 
         current_recursion_ref_count[ref] += 1