Fix Issue 21591 - missed backrefs in mangled names wrt. unmerged func…

…tion types

TypeFunctions are apparently special and not merged, so make sure to
use the generic merged function type as backref cache key.

Unfortunately, this requires special care to prevent an infinite
recursion, as merging is based on mangling.

src/dmd/dmangle.d

@@ -179,10 +179,12 @@ public:
     AssocArray!(Type, size_t) types;        // Type => (offset+1) in buf
     AssocArray!(Identifier, size_t) idents; // Identifier => (offset+1) in buf
     OutBuffer* buf;
+    Type rootType;
 
-    extern (D) this(OutBuffer* buf)
+    extern (D) this(OutBuffer* buf, Type rootType = null)
     {
         this.buf = buf;
+        this.rootType = rootType;
     }
 
     /**
@@ -229,7 +231,22 @@ public:
     bool backrefType(Type t)
     {
         if (!t.isTypeBasic())
+        {
+            /**
+             * https://issues.dlang.org/show_bug.cgi?id=21591
+             *
+             * Special case for unmerged TypeFunctions: use the generic merged
+             * function type as backref cache key to avoid missed backrefs.
+             *
+             * Merging is based on mangling, so we need to avoid an infinite
+             * recursion by excluding the case where `t` is the function type
+             * passed to `mangleToBuffer()`: `rootType`
+             */
+            if (t != rootType && t.isTypeFunction())
+                t = t.merge2();
+
             return backrefImpl(types, t);
+        }
         return false;
     }
 
@@ -1178,7 +1195,7 @@ extern (C++) void mangleToBuffer(Type t, OutBuffer* buf)
         buf.writestring(t.deco);
     else
     {
-        scope Mangler v = new Mangler(buf);
+        scope Mangler v = new Mangler(buf, t);
         v.visitWithMask(t, 0);
     }
 }

test/compilable/extra-files/test21591a.d

@@ -0,0 +1,9 @@
+struct Test(T)
+{
+    T member;
+    size_t toHash() const { return hashOf(member); }
+}
+
+enum duppable(T) = is(typeof(T.init.dup));
+
+enum test = duppable!(Test!(void delegate())[]);

test/compilable/extra-files/test21591b.d

@@ -0,0 +1,6 @@
+import test21591a;
+
+void main()
+{
+    cast(void) (void delegate()).init;
+}

test/compilable/test21591.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+if [ "${OS}${MODEL}" == "windows32" ]; then
+    echo "Skipping test21591 on Win32 OMF."
+    exit 0
+fi
+
+# Compile both modules separately and verify that the mangled names of the
+# `core.internal.hash.hashOf!(void delegate())` instantiations are equal.
+expected="_D4core8internal4hash__T6hashOfTDFZvZQnFNaNbNiNeMxDQsmZm"
+for i in a b; do
+    obj="${OUTPUT_BASE}${i}${OBJ}"
+    $DMD -c -m${MODEL} -of${obj} -I${EXTRA_FILES} ${EXTRA_FILES}/${TEST_NAME}${i}.d
+    if [ "${OS}" == "windows" ]; then
+        dumpbin /symbols ${obj} | grep ${expected}
+    else
+        nm ${obj} | grep ${expected}
+    fi
+    rm_retry ${obj}
+done