Fix TypeMerging bug with indirectly reachable public types

src/passes/TypeMerging.cpp

@@ -504,11 +504,19 @@ std::vector<HeapType> TypeMerging::getPublicChildren(HeapType type) {
 
 DFA::State<HeapType> TypeMerging::makeDFAState(HeapType type) {
   std::vector<HeapType> succs;
-  for (auto child : type.getHeapTypeChildren()) {
-    // Both private and public heap type children participate in the DFA and are
-    // eligible to be successors.
-    if (!child.isBasic()) {
-      succs.push_back(getMerged(child));
+  // Both private and public heap type children participate in the DFA and are
+  // eligible to be successors, except that public types are terminal states
+  // that do not have successors. This is sufficient because public types are
+  // always in their own singleton partitions, so they already differentiate
+  // types that reach them without needing to consider their children. In the
+  // other direction, including the children is not necessary to differentiate
+  // types reached by the public types because all such reachable types are also
+  // public and not eligible to be merged.
+  if (privateTypes.count(type)) {
+    for (auto child : type.getHeapTypeChildren()) {
+      if (!child.isBasic()) {
+        succs.push_back(getMerged(child));
+      }
     }
   }
   return {type, std::move(succs)};

test/lit/passes/type-merging.wast

@@ -985,6 +985,25 @@
  (global $g2 (ref $C') (struct.new_default $D2'))
 )
 
+;; Regression test for a bug where public types not directly reachable from
+;; private types were included as successors but not given states in the DFA.
+(module
+  ;; CHECK:      (type $public-child (struct))
+  (type $public-child (struct))
+  ;; CHECK:      (type $public (struct (field (ref $public-child))))
+  (type $public (struct (ref $public-child)))
+  ;; CHECK:      (type $private (struct (field (ref $public))))
+  (type $private (struct (ref $public)))
+
+  ;; CHECK:      (global $public (ref null $public) (ref.null none))
+  (global $public (ref null $public) (ref.null none))
+  ;; CHECK:      (global $private (ref null $private) (ref.null none))
+  (global $private (ref null $private) (ref.null none))
+
+  ;; CHECK:      (export "public" (global $public))
+  (export "public" (global $public))
+)
+
 ;; Check that a ref.test inhibits merging (ref.cast is already checked above).
 (module
   ;; CHECK:      (rec