Fix GC memory leak

packages/lexical/src/LexicalGC.ts

@@ -47,6 +47,7 @@ function $garbageCollectDetachedDeepChildNodes(
   let child = node.getFirstChild();
 
   while (child !== null) {
+    const nextChild = child.getNextSibling();
     const childKey = child.__key;
     if (child !== undefined && child.__parent === parentKey) {
       if ($isElementNode(child)) {
@@ -66,7 +67,7 @@ function $garbageCollectDetachedDeepChildNodes(
       }
       nodeMap.delete(childKey);
     }
-    child = child.isAttached() ? child.getNextSibling() : null;
+    child = nextChild;
   }
 }
 
@@ -79,21 +80,8 @@ export function $garbageCollectDetachedNodes(
   const prevNodeMap = prevEditorState._nodeMap;
   const nodeMap = editorState._nodeMap;
 
-  for (const nodeKey of dirtyLeaves) {
-    const node = nodeMap.get(nodeKey);
-
-    if (node !== undefined && !node.isAttached()) {
-      if (!prevNodeMap.has(nodeKey)) {
-        dirtyLeaves.delete(nodeKey);
-      }
-
-      nodeMap.delete(nodeKey);
-    }
-  }
-
   for (const [nodeKey] of dirtyElements) {
     const node = nodeMap.get(nodeKey);
-
     if (node !== undefined) {
       // Garbage collect node and its children if they exist
       if (!node.isAttached()) {
@@ -106,15 +94,23 @@ export function $garbageCollectDetachedNodes(
             dirtyElements,
           );
         }
-
         // If we have created a node and it was dereferenced, then also
         // remove it from out dirty nodes Set.
         if (!prevNodeMap.has(nodeKey)) {
           dirtyElements.delete(nodeKey);
         }
-
         nodeMap.delete(nodeKey);
       }
     }
   }
+
+  for (const nodeKey of dirtyLeaves) {
+    const node = nodeMap.get(nodeKey);
+    if (node !== undefined && !node.isAttached()) {
+      if (!prevNodeMap.has(nodeKey)) {
+        dirtyLeaves.delete(nodeKey);
+      }
+      nodeMap.delete(nodeKey);
+    }
+  }
 }

packages/lexical/src/nodes/__tests__/unit/LexicalGC.test.tsx

@@ -0,0 +1,107 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+
+import {
+  $createParagraphNode,
+  $createTextNode,
+  $getNodeByKey,
+  $getRoot,
+} from 'lexical';
+
+import {
+  $createTestElementNode,
+  initializeUnitTest,
+} from '../../../__tests__/utils';
+
+describe('LexicalGC tests', () => {
+  initializeUnitTest((testEnv) => {
+    test('RootNode.clear() with a child and subchild', async () => {
+      const {editor} = testEnv;
+      await editor.update(() => {
+        $getRoot().append(
+          $createParagraphNode().append($createTextNode('foo')),
+        );
+      });
+      expect(editor.getEditorState()._nodeMap.size).toBe(3);
+      await editor.update(() => {
+        $getRoot().clear();
+      });
+      expect(editor.getEditorState()._nodeMap.size).toBe(1);
+    });
+
+    test('RootNode.clear() with a child and three subchildren', async () => {
+      const {editor} = testEnv;
+      await editor.update(() => {
+        const text1 = $createTextNode('foo');
+        const text2 = $createTextNode('bar').toggleUnmergeable();
+        const text3 = $createTextNode('zzz').toggleUnmergeable();
+        const paragraph = $createParagraphNode();
+        paragraph.append(text1, text2, text3);
+        $getRoot().append(paragraph);
+      });
+      expect(editor.getEditorState()._nodeMap.size).toBe(5);
+      await editor.update(() => {
+        $getRoot().clear();
+      });
+      expect(editor.getEditorState()._nodeMap.size).toBe(1);
+    });
+
+    for (let i = 0; i < 3; i++) {
+      test(`RootNode.clear() with a child and three subchildren, subchild ${i} deleted first`, async () => {
+        const {editor} = testEnv;
+        await editor.update(() => {
+          const text1 = $createTextNode('foo'); // 1
+          const text2 = $createTextNode('bar').toggleUnmergeable(); // 2
+          const text3 = $createTextNode('zzz').toggleUnmergeable(); // 3
+          const paragraph = $createParagraphNode(); // 4
+          paragraph.append(text1, text2, text3);
+          $getRoot().append(paragraph);
+        });
+        expect(editor.getEditorState()._nodeMap.size).toBe(5);
+        await editor.update(() => {
+          const root = $getRoot();
+          const subchild = root.getFirstChild().getChildAtIndex(i);
+          expect(subchild.getTextContent()).toBe(['foo', 'bar', 'zzz'][i]);
+          subchild.remove();
+          root.clear();
+        });
+        expect(editor.getEditorState()._nodeMap.size).toBe(1);
+      });
+    }
+
+    for (let i = 0; i < 7; i++) {
+      /**
+       *          R
+       *          P
+       *     T   TE    T
+       *        T  T
+       */
+      test(`RootNode.clear() with a complex tree, element ${i} node first`, async () => {
+        const {editor} = testEnv;
+        await editor.update(() => {
+          const testElement = $createTestElementNode(); // 1
+          const testElementText1 = $createTextNode('te1').toggleUnmergeable(); // 2
+          const testElementText2 = $createTextNode('te2').toggleUnmergeable(); // 3
+          const text1 = $createTextNode('a').toggleUnmergeable(); // 4
+          const text2 = $createTextNode('b').toggleUnmergeable(); // 5
+          const paragraph = $createParagraphNode(); // 6
+          testElement.append(testElementText1, testElementText2);
+          paragraph.append(text1, testElement, text2);
+          $getRoot().append(paragraph);
+        });
+        expect(editor.getEditorState()._nodeMap.size).toBe(7);
+        await editor.update(() => {
+          const node = $getNodeByKey('1');
+          node.remove();
+          $getRoot().clear();
+        });
+        expect(editor.getEditorState()._nodeMap.size).toBe(1);
+      });
+    }
+  });
+});