Make sure reachable_var not falls into infinite recusion. (#48135)

Author: N5N3

src/subtype.c

@@ -216,6 +216,17 @@ static void restore_env(jl_stenv_t *e, jl_value_t *root, jl_savedenv_t *se) JL_N
         memset(&e->envout[e->envidx], 0, (e->envsz - e->envidx)*sizeof(void*));
 }
 
+static int current_env_length(jl_stenv_t *e)
+{
+    jl_varbinding_t *v = e->vars;
+    int len = 0;
+    while (v) {
+        len++;
+        v = v->prev;
+    }
+    return len;
+}
+
 // type utilities
 
 // quickly test that two types are identical
@@ -2302,16 +2313,40 @@ static int subtype_in_env_existential(jl_value_t *x, jl_value_t *y, jl_stenv_t *
 }
 
 // See if var y is reachable from x via bounds; used to avoid cycles.
-static int reachable_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e)
+static int _reachable_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e)
 {
     if (in_union(x, (jl_value_t*)y))
         return 1;
     if (!jl_is_typevar(x))
         return 0;
     jl_varbinding_t *xv = lookup(e, (jl_tvar_t*)x);
-    if (xv == NULL)
+    if (xv == NULL || xv->right)
         return 0;
-    return reachable_var(xv->ub, y, e) || reachable_var(xv->lb, y, e);
+    xv->right = 1;
+    return _reachable_var(xv->ub, y, e) || _reachable_var(xv->lb, y, e);
+}
+
+static int reachable_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e)
+{
+    int len = current_env_length(e);
+    int8_t *rs = (int8_t*)malloc_s(len);
+    int n = 0;
+    jl_varbinding_t *v = e->vars;
+    while (n < len) {
+        assert(v != NULL);
+        rs[n++] = v->right;
+        v->right = 0;
+        v = v->prev;
+    }
+    int res = _reachable_var(x, y, e);
+    n = 0; v = e->vars;
+    while (n < len) {
+        assert(v != NULL);
+        v->right = rs[n++];
+        v = v->prev;
+    }
+    free(rs);
+    return res;
 }
 
 // check whether setting v == t implies v == SomeType{v}, which is unsatisfiable.

test/subtype.jl

@@ -2314,6 +2314,11 @@ let S1 = Tuple{Int, Any, Union{Val{C1}, C1}} where {R1<:Real, C1<:Union{Complex{
     end
 end
 
+let S = Tuple{T2, V2} where {T2, N2, V2<:(Array{S2, N2} where {S2 <: T2})},
+    T = Tuple{V1, T1} where {T1, N1, V1<:(Array{S1, N1} where {S1 <: T1})}
+    @testintersect(S, T, !Union{})
+end
+
 @testset "known subtype/intersect issue" begin
     #issue 45874
     # Causes a hang due to jl_critical_error calling back into malloc...