Omit circular bounds in upbound to avoid false Union{}

This is not ideal. As some times circular bound seems meaningful. But at least better than `Union{}` ?
JuliaLang · Jan 10, 2023 · 3037342 · 3037342
1 parent 748149e
commit 3037342
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 7 deletions.
diff --git a/src/subtype.c b/src/subtype.c
@@ -2535,6 +2535,39 @@ static int var_occurs_inside(jl_value_t *v, jl_tvar_t *var, int inside, int want
     return 0;
 }
 
+static jl_value_t *omit_bad_union(jl_value_t *u, jl_tvar_t *t)
+{
+    if (!jl_has_typevar(u, t))
+        return u; // return u if possible as many checks use `==`.
+    jl_value_t *res = NULL;
+    if (jl_is_unionall(u)) {
+        jl_tvar_t *var = ((jl_unionall_t *)u)->var;
+        jl_value_t *ub = var->ub, *body = ((jl_unionall_t *)u)->body;
+        JL_GC_PUSH3(&ub, &body, &var);
+        assert(var != t);
+        ub = omit_bad_union(ub, t);
+        body = omit_bad_union(body, t);
+        if (ub != NULL && body != NULL && !jl_has_typevar(var->lb, t)) {
+            if (ub != var->ub)
+                var = jl_new_typevar(var->name, var->lb, ub);
+            res = jl_new_struct(jl_unionall_type, var, body);
+        }
+        JL_GC_POP();
+    }
+    else if (jl_is_uniontype(u)) {
+        jl_value_t *a = ((jl_uniontype_t *)u)->a;
+        jl_value_t *b = ((jl_uniontype_t *)u)->b;
+        JL_GC_PUSH2(&a, &b);
+        a = omit_bad_union(a, t);
+        b = omit_bad_union(b, t);
+        res = a == NULL ? b :
+              b == NULL ? a :
+              jl_new_struct(jl_uniontype_type, a, b);
+        JL_GC_POP();
+    }
+    return res;
+}
+
 // Caller might not have rooted `res`
 static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbinding_t *vb, jl_unionall_t *u, jl_stenv_t *e)
 {
@@ -2618,8 +2651,11 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind
         }
         if (jl_has_typevar(btemp->ub, vb->var)) {
             if (vb->ub == (jl_value_t*)btemp->var) {
-                JL_GC_POP();
-                return jl_bottom_type;
+                btemp->ub = omit_bad_union(btemp->ub, vb->var);
+                if (btemp->ub == NULL) {
+                    JL_GC_POP();
+                    return jl_bottom_type;
+                }
             }
             if (varval) {
                 JL_TRY {
@@ -2739,10 +2775,17 @@ static jl_value_t *intersect_unionall_(jl_value_t *t, jl_unionall_t *u, jl_stenv
             // T=Bottom in covariant position
             res = jl_bottom_type;
         }
-        else if (jl_has_typevar(vb->lb, u->var) || jl_has_typevar(vb->ub, u->var)) {
+        else if (jl_has_typevar(vb->lb, u->var)) {
             // fail on circular constraints
             res = jl_bottom_type;
         }
+        else {
+            JL_GC_PUSH1(&res);
+            vb->ub = omit_bad_union(vb->ub, u->var);
+            JL_GC_POP();
+            if (vb->ub == NULL)
+                res = jl_bottom_type;
+        }
     }
     if (res != jl_bottom_type)
         // res is rooted by callee

diff --git a/test/subtype.jl b/test/subtype.jl
@@ -1893,10 +1893,10 @@ end
 
 let
     # issue #22787
-    # for now check that these don't stack overflow
-    t = typeintersect(Tuple{Type{Q}, Q, Ref{Q}} where Q<:Ref,
-                      Tuple{Type{S}, Union{Ref{S}, Ref{R}}, R} where R where S)
-    @test_broken t != Union{}
+    @testintersect(Tuple{Type{Q}, Q, Ref{Q}} where Q<:Ref,
+                   Tuple{Type{S}, Union{Ref{S}, Ref{R}}, R} where R where S,
+                   !Union{})
+
     t = typeintersect(Tuple{Type{T}, T, Ref{T}} where T,
                       Tuple{Type{S}, Ref{S}, S} where S)
     @test_broken t != Union{}