From e66354c7d0ff281708700a238be6e91c72c06f92 Mon Sep 17 00:00:00 2001
From: Takumi Shotoku <sinsoku.listy@gmail.com>
Date: Tue, 3 Jun 2025 16:06:22 +0900
Subject: [PATCH] Fix stack overflow error in recursive type alias expansion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit fixes issue #324 where recursive type aliases caused
SystemStackError due to infinite recursion during type expansion.

The fix adds recursion detection in SigTyAliasNode#covariant_vertex0
and #contravariant_vertex0 methods by tracking the expansion stack
using subst[:__expansion_stack__]. When a type alias is already
being expanded, the expansion stops to prevent infinite recursion.

Also adds comprehensive test cases for various recursive type alias
patterns including direct recursion, mutual recursion, and generic
recursive types.

Fixes #324

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 lib/typeprof/core/ast/sig_type.rb    | 30 +++++++++++
 scenario/rbs/recursive-type-alias.rb | 79 ++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 scenario/rbs/recursive-type-alias.rb

diff --git a/lib/typeprof/core/ast/sig_type.rb b/lib/typeprof/core/ast/sig_type.rb
index 263abf1aa..ad3e3a415 100644
--- a/lib/typeprof/core/ast/sig_type.rb
+++ b/lib/typeprof/core/ast/sig_type.rb
@@ -258,9 +258,24 @@ def covariant_vertex0(genv, changes, vtx, subst)
         changes.add_depended_static_read(@static_ret.last)
         tae = @static_ret.last.type_alias_entity
         if tae && tae.exist?
+          # Check for recursive expansion
+          expansion_key = [@cpath, @name]
+          subst[:__expansion_stack__] ||= []
+
+          if subst[:__expansion_stack__].include?(expansion_key)
+            # Recursive expansion detected: this type alias references itself
+            # Stop expansion here to prevent SystemStackError. The type system
+            # will handle the incomplete expansion gracefully, typically by
+            # treating unresolved recursive references as 'untyped', which
+            # maintains type safety while allowing the program to continue.
+            return
+          end
+
           # need to check tae decls are all consistent?
           decl = tae.decls.each {|decl| break decl }
           subst0 = subst.dup
+          subst0[:__expansion_stack__] = subst[:__expansion_stack__].dup + [expansion_key]
+
           # raise if decl.params.size != @args.size # ?
           decl.params.zip(@args) do |param, arg|
             subst0[param] = arg.covariant_vertex(genv, changes, subst0) # passing subst0 is ok?
@@ -273,9 +288,24 @@ def contravariant_vertex0(genv, changes, vtx, subst)
         changes.add_depended_static_read(@static_ret.last)
         tae = @static_ret.last.type_alias_entity
         if tae && tae.exist?
+          # Check for recursive expansion
+          expansion_key = [@cpath, @name]
+          subst[:__expansion_stack__] ||= []
+
+          if subst[:__expansion_stack__].include?(expansion_key)
+            # Recursive expansion detected: this type alias references itself
+            # Stop expansion here to prevent SystemStackError. The type system
+            # will handle the incomplete expansion gracefully, typically by
+            # treating unresolved recursive references as 'untyped', which
+            # maintains type safety while allowing the program to continue.
+            return
+          end
+
           # need to check tae decls are all consistent?
           decl = tae.decls.each {|decl| break decl }
           subst0 = subst.dup
+          subst0[:__expansion_stack__] = subst[:__expansion_stack__].dup + [expansion_key]
+
           # raise if decl.params.size != @args.size # ?
           decl.params.zip(@args) do |param, arg|
             subst0[param] = arg.contravariant_vertex(genv, changes, subst0)
diff --git a/scenario/rbs/recursive-type-alias.rb b/scenario/rbs/recursive-type-alias.rb
new file mode 100644
index 000000000..4f64e407a
--- /dev/null
+++ b/scenario/rbs/recursive-type-alias.rb
@@ -0,0 +1,79 @@
+## update: test.rbs
+# Basic recursive type alias
+type context = nil | [context, bool]
+
+class C
+  def foo: -> context
+end
+
+## update: test.rb
+def test1
+  C.new.foo
+end
+
+## assert: test.rb
+class Object
+  def test1: -> [untyped, bool]?
+end
+
+## diagnostics: test.rb
+
+## update: test.rbs
+# Recursive type alias in class scope
+class D
+  type tree = Integer | [tree, tree]
+  def get_tree: -> tree
+end
+
+## update: test.rb
+def test2
+  D.new.get_tree
+end
+
+## assert: test.rb
+class Object
+  def test2: -> (Integer | [untyped, untyped])
+end
+
+## diagnostics: test.rb
+
+## update: test.rbs
+# Mutually recursive type aliases
+type node = [Integer, nodes]
+type nodes = Array[node]
+
+class E
+  def build_tree: -> node
+end
+
+## update: test.rb
+def test3
+  E.new.build_tree
+end
+
+## assert: test.rb
+class Object
+  def test3: -> [Integer, Array[untyped]]
+end
+
+## diagnostics: test.rb
+
+## update: test.rbs
+# Recursive type alias with generic parameters
+type list[T] = nil | [T, list[T]]
+
+class F
+  def create_list: -> list[String]
+end
+
+## update: test.rb
+def test4
+  F.new.create_list
+end
+
+## assert: test.rb
+class Object
+  def test4: -> [String, untyped]?
+end
+
+## diagnostics: test.rb