Issue 11553 - dmd segfault with recursive template

src/template.c

@@ -2309,9 +2309,50 @@ void functionResolve(Match *m, Dsymbol *dstart, Loc loc, Scope *sc,
                 return 0;
 
             Dsymbol *s = ti->inst->toAlias();
-            if (!s->isFuncDeclaration() && !s->isTemplateDeclaration())
+            FuncDeclaration *fd;
+            if (TemplateDeclaration *tdx = s->isTemplateDeclaration())
+            {
+                Objects dedtypesX;  // empty tiargs
+
+                // Bugzilla 11553: Check for recursive instantiation of tdx.
+                for (TemplateDeclaration::Previous *p = tdx->previous; p; p = p->prev)
+                {
+                    if (arrayObjectMatch(p->dedargs, &dedtypesX))
+                    {
+                        //printf("recursive, no match p->sc=%p %p %s\n", p->sc, this, this->toChars());
+                        /* It must be a subscope of p->sc, other scope chains are not recursive
+                         * instantiations.
+                         */
+                        for (Scope *scx = sc; scx; scx = scx->enclosing)
+                        {
+                            if (scx == p->sc)
+                            {
+                                error(loc, "recursive template expansion while looking for %s.%s", ti->toChars(), tdx->toChars());
+                                goto Lerror;
+                            }
+                        }
+                    }
+                    /* BUG: should also check for ref param differences
+                     */
+                }
+
+                TemplateDeclaration::Previous pr;
+                pr.prev = tdx->previous;
+                pr.sc = sc;
+                pr.dedargs = &dedtypesX;
+                tdx->previous = ≺                 // add this to threaded list
+
+                fd = resolveFuncCall(loc, sc, s, NULL, tthis, fargs, 1);
+
+                tdx->previous = pr.prev;             // unlink from threaded list
+            }
+            else if (s->isFuncDeclaration())
+            {
+                fd = resolveFuncCall(loc, sc, s, NULL, tthis, fargs, 1);
+            }
+            else
                 goto Lerror;
-            FuncDeclaration *fd = resolveFuncCall(loc, sc, s, NULL, tthis, fargs, 1);
+
             if (!fd)
                 return 0;
 

test/fail_compilation/ice11553.d

@@ -0,0 +1,22 @@
+/*
+TEST_OUTPUT:
+---
+fail_compilation/ice11553.d(22): Error: recursive template expansion while looking for A!().A()
+fail_compilation/ice11553.d(22): Error: expression template A() of type void does not have a boolean value
+---
+*/
+
+template A(alias T)
+{
+    template A()
+    {
+        alias A = T!();
+    }
+}
+
+template B()
+{
+    alias B = A!(.B);
+}
+
+static if (A!B) {}

test/runnable/template9.d

@@ -2972,6 +2972,61 @@ void test11533c()
     assert(foo.call() == var);
 }
 
+/******************************************/
+// 11553
+
+struct Pack11553(T ...)
+{
+    alias Unpack = T;
+    enum length = T.length;
+}
+
+template isPack11553(TList ...)
+{
+    static if (TList.length == 1 && is(Pack11553!(TList[0].Unpack) == TList[0]))
+    {
+        enum isPack11553 = true;
+    }
+    else
+    {
+        enum isPack11553 = false;
+    }
+}
+
+template PartialApply11553(alias T, uint argLoc, Arg ...)
+    if (Arg.length == 1)
+{
+    template PartialApply11553(L ...)
+    {
+        alias PartialApply11553 = T!(L[0 .. argLoc], Arg, L[argLoc .. $]);
+    }
+}
+
+template _hasLength11553(size_t len, T)
+{
+    static if (T.length == len)
+    {
+        enum _hasLength11553 = true;
+    }
+    else
+    {
+        enum _hasLength11553 = false;
+    }
+}
+
+alias _hasLength11553(size_t len) = PartialApply11553!(._hasLength11553, 0, len);
+
+
+alias hl11553 = _hasLength11553!1;
+
+// this segfaults
+static if (!isPack11553!hl11553) { pragma(msg, "All good 1"); }
+
+// these are fine
+static if ( hl11553!(Pack11553!(5))) { pragma(msg, "All good 2"); }
+
+static if (!hl11553!(Pack11553!( ))) { pragma(msg, "All good 3"); }
+
 /******************************************/
 // 11818