[diffshub] stream tree paths incrementally to fix O(N²) publish slowdown

apps/docs/app/(diffshub)/(view)/_components/CodeViewFileTree.tsx

@@ -1,7 +1,10 @@
 'use client';
 
 import { useStableCallback } from '@pierre/diffs/react';
-import type { FileTree as FileTreeModel } from '@pierre/trees';
+import type {
+  FileTreeBatchOperation,
+  FileTree as FileTreeModel,
+} from '@pierre/trees';
 import { FileTree, useFileTree } from '@pierre/trees/react';
 import { type CSSProperties, memo, useEffect, useRef, useState } from 'react';
 
@@ -68,12 +71,41 @@ export const CodeViewFileTree = memo(function CodeViewFileTree({
   });
 
   useEffect(() => {
-    if (previousSourceRef.current === source) {
+    const previousSource = previousSourceRef.current;
+    if (previousSource === source) {
       return;
     }
 
     previousSourceRef.current = source;
-    model.resetPaths(source.paths);
+    // The streaming patch loader links each tree-source snapshot to the prior
+    // one through `previousSource`. When the link matches what this component
+    // last applied, the new paths array is guaranteed to extend the previous
+    // one, so we apply the delta as add() operations instead of asking the
+    // model to throw itself away and rebuild against the full path list. This
+    // turns tree publishes from O(N) each (where N is the total accumulated
+    // path count) into O(delta), which keeps the Diff Stats counter fast as
+    // more files stream in.
+    if (
+      source.previousSource != null &&
+      source.previousSource === previousSource
+    ) {
+      const previousPathCount = previousSource.paths.length;
+      if (source.paths.length > previousPathCount) {
+        const operations: FileTreeBatchOperation[] = [];
+        for (
+          let index = previousPathCount;
+          index < source.paths.length;
+          index++
+        ) {
+          operations.push({ type: 'add', path: source.paths[index] });
+        }
+        if (operations.length > 0) {
+          model.batch(operations);
+        }
+      }
+    } else {
+      model.resetPaths(source.paths);
+    }
     model.setGitStatus(source.gitStatus);
   }, [model, source]);
 

apps/docs/app/(diffshub)/(view)/_components/codeViewDataAccumulator.ts

@@ -11,26 +11,39 @@ import type {
   CodeViewCommentFileByItemId,
   CodeViewCommentSidebarFile,
   CodeViewDiffStats,
+  CodeViewFileTreeSort,
   CodeViewFileTreeSource,
   CommentMetadata,
 } from './types';
 import {
-  createCodeViewFileTreeSource,
+  createPatchOrderSortFromRankMap,
+  extendPatchOrderRanks,
   mapChangeTypeToGitStatus,
 } from './utils';
 
 export interface CodeViewDataAccumulator {
+  diffStats: CodeViewDiffStats;
   fileIndex: number;
   gitStatusByPath: Map<string, GitStatusEntry>;
   itemIdToFile: Map<string, CodeViewCommentSidebarFile>;
   items: CodeViewItem<CommentMetadata>[];
+  // The last tree source emitted by snapshotCodeViewTreeSource for this
+  // accumulator. Each new snapshot links back to this so the consumer can
+  // recognize append-only growth and skip the full PathStore rebuild.
+  lastTreeSource: CodeViewFileTreeSource | undefined;
+  nextCollisionSuffixByBase: Map<string, number>;
   pendingItems: CodeViewItem<CommentMetadata>[];
   pendingItemById: Map<string, CodeViewItem<CommentMetadata>>;
   pathToItemId: Map<string, string>;
   pathStateByTreePath: Map<string, CodeViewPathState>;
   paths: string[];
-  nextCollisionSuffixByBase: Map<string, number>;
-  diffStats: CodeViewDiffStats;
+  // Patch-order ranks for every path and directory ancestor we have ever
+  // appended. Extended incrementally so the sort comparator below stays valid
+  // across publishes without rebuilding the map.
+  rankByPath: Map<string, number>;
+  // Stable comparator that reads from rankByPath. Reused across snapshots so
+  // each publish does not allocate a fresh closure or repopulate a rank map.
+  sort: CodeViewFileTreeSort;
 }
 
 export interface CodeViewItemIdRename {
@@ -53,23 +66,27 @@ export interface LoadedCodeViewData {
 }
 
 export function createCodeViewDataAccumulator(): CodeViewDataAccumulator {
+  const rankByPath = new Map<string, number>();
   return {
+    diffStats: {
+      addedLines: 0,
+      deletedLines: 0,
+      fileCount: 0,
+      totalLinesOfCode: 0,
+    },
     fileIndex: 0,
     gitStatusByPath: new Map(),
     itemIdToFile: new Map(),
     items: [],
+    lastTreeSource: undefined,
+    nextCollisionSuffixByBase: new Map(),
     pendingItems: [],
     pendingItemById: new Map(),
     pathToItemId: new Map(),
     pathStateByTreePath: new Map(),
     paths: [],
-    nextCollisionSuffixByBase: new Map(),
-    diffStats: {
-      addedLines: 0,
-      deletedLines: 0,
-      fileCount: 0,
-      totalLinesOfCode: 0,
-    },
+    rankByPath,
+    sort: createPatchOrderSortFromRankMap(rankByPath),
   };
 }
 
@@ -122,6 +139,11 @@ export function appendFileDiffToCodeViewData(
 
   if (previousPathState == null) {
     accumulator.paths.push(treePath);
+    extendPatchOrderRanks(
+      accumulator.rankByPath,
+      treePath,
+      accumulator.paths.length - 1
+    );
   }
   accumulator.pathToItemId.set(treePath, id);
   updateGitStatusByPath(
@@ -150,14 +172,24 @@ export function takePendingCodeViewItems(
   return pendingItems;
 }
 
+// Produces a tree source snapshot, linking it to the previous snapshot from
+// the same accumulator. The consumer treats that link as a hint that the new
+// paths array is an append-only extension of the prior one and applies the
+// delta with model.batch instead of rebuilding the whole PathStore. Consumers
+// that recreate the accumulator (e.g. a new request) discard the prior link
+// implicitly because lastTreeSource is undefined on a fresh accumulator.
 export function snapshotCodeViewTreeSource(
   accumulator: CodeViewDataAccumulator
 ): CodeViewFileTreeSource {
-  return createCodeViewFileTreeSource(
-    accumulator.paths.slice(),
-    new Map(accumulator.pathToItemId),
-    Array.from(accumulator.gitStatusByPath.values())
-  );
+  const snapshot: CodeViewFileTreeSource = {
+    gitStatus: Array.from(accumulator.gitStatusByPath.values()),
+    paths: accumulator.paths.slice(),
+    pathToItemId: new Map(accumulator.pathToItemId),
+    previousSource: accumulator.lastTreeSource,
+    sort: accumulator.sort,
+  };
+  accumulator.lastTreeSource = snapshot;
+  return snapshot;
 }
 
 // Moves the current CodeView item for a path off the canonical tree id so the

apps/docs/app/(diffshub)/(view)/_components/types.ts

@@ -72,13 +72,21 @@ export interface CodeViewSavedCommentItem {
 }
 
 // The fully pre-computed input this tree needs for a given fetch. It is built
-// once at fetch time by createCodeViewFileTreeSource and stored alongside
-// the viewer items, so later per-item annotation updates do not feed into the
+// once at fetch time by snapshotCodeViewTreeSource and stored alongside the
+// viewer items, so later per-item annotation updates do not feed into the
 // tree and do not cause it to rebuild.
+//
+// Streamed publishes link successive snapshots through `previousSource` so the
+// tree consumer can recognize append-only growth and apply the delta as
+// `model.batch` adds instead of rebuilding the entire path store. The link is
+// present only on snapshots that share the same underlying accumulator; the
+// initial publish and any non-streamed source leave it undefined and force a
+// full reset.
 export interface CodeViewFileTreeSource {
   gitStatus: readonly GitStatusEntry[];
   paths: readonly string[];
   pathToItemId: ReadonlyMap<string, string>;
+  previousSource?: CodeViewFileTreeSource;
   sort: CodeViewFileTreeSort;
 }
 

apps/docs/app/(diffshub)/(view)/_components/utils.ts

@@ -4,13 +4,12 @@ import type {
   CodeViewItem,
   DiffLineAnnotation,
 } from '@pierre/diffs';
-import type { GitStatus, GitStatusEntry } from '@pierre/trees';
+import type { GitStatus } from '@pierre/trees';
 
 import type {
   CodeViewCommentFileByItemId,
   CodeViewDeletedCommentEvent,
   CodeViewFileTreeSort,
-  CodeViewFileTreeSource,
   CodeViewSavedCommentEntry,
   CodeViewSavedCommentEvent,
   CodeViewSavedCommentItem,
@@ -185,28 +184,39 @@ export function mapChangeTypeToGitStatus(type: ChangeTypes): GitStatus {
   }
 }
 
-function createPatchOrderSort(paths: readonly string[]): CodeViewFileTreeSort {
-  const rankByPath = new Map<string, number>();
-  for (let index = 0; index < paths.length; index++) {
-    const path = paths[index];
-    if (path == null || path.length === 0) {
-      continue;
-    }
+// Records the patch-order rank for a path and every directory ancestor inside
+// `rankByPath`. Existing ranks are preserved so callers can fold new paths into
+// the same map without disturbing the ordering established for earlier paths.
+export function extendPatchOrderRanks(
+  rankByPath: Map<string, number>,
+  path: string,
+  index: number
+): void {
+  if (path.length === 0) {
+    return;
+  }
 
-    if (!rankByPath.has(path)) {
-      rankByPath.set(path, index);
-    }
+  if (!rankByPath.has(path)) {
+    rankByPath.set(path, index);
+  }
 
-    let slashIndex = path.lastIndexOf('/');
-    while (slashIndex > 0) {
-      const directory = path.slice(0, slashIndex);
-      if (!rankByPath.has(directory)) {
-        rankByPath.set(directory, index);
-      }
-      slashIndex = directory.lastIndexOf('/');
+  let slashIndex = path.lastIndexOf('/');
+  while (slashIndex > 0) {
+    const directory = path.slice(0, slashIndex);
+    if (!rankByPath.has(directory)) {
+      rankByPath.set(directory, index);
     }
+    slashIndex = directory.lastIndexOf('/');
   }
+}
 
+// Builds a sort comparator that reads from a `rankByPath` map populated by
+// extendPatchOrderRanks. The comparator captures the map by reference so the
+// same comparator instance can be reused across incremental publishes while
+// the map continues to grow.
+export function createPatchOrderSortFromRankMap(
+  rankByPath: ReadonlyMap<string, number>
+): CodeViewFileTreeSort {
   return (left, right) => {
     const leftRank = rankByPath.get(left.path) ?? PATCH_ORDER_FALLBACK_RANK;
     const rightRank = rankByPath.get(right.path) ?? PATCH_ORDER_FALLBACK_RANK;
@@ -230,25 +240,6 @@ function createPatchOrderSort(paths: readonly string[]): CodeViewFileTreeSort {
   };
 }
 
-// Finalizes the stable tree input from a fresh fetch. Callers are expected to
-// populate paths, pathToItemId, and gitStatus in the same pass that builds
-// the viewer items so the tree data structure does not require its own walk
-// over items. Modified-status entries should be excluded from gitStatus before
-// calling here so the tree renders them as the visual default (no tint or badge).
-export function createCodeViewFileTreeSource(
-  paths: readonly string[],
-  pathToItemId: ReadonlyMap<string, string>,
-  gitStatus: readonly GitStatusEntry[]
-): CodeViewFileTreeSource {
-  const sort = createPatchOrderSort(paths);
-  return {
-    gitStatus,
-    paths,
-    pathToItemId,
-    sort,
-  };
-}
-
 function insertCommentInLineOrder(
   comments: readonly CodeViewSavedCommentEntry[],
   entry: CodeViewSavedCommentEntry