fix(resolution): 20+ min PHP/JS indexing wedge at "Resolving refs 97%" — gate closure-collection synthesis to Swift/Kotlin (#1235)

CHANGELOG.md

@@ -11,7 +11,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
-- Indexing very large codebases no longer dies at the end of the "Resolving refs" step. Two failure modes are fixed: on multi-million-symbol projects (e.g. the Linux kernel, ~95,000 files) the final analysis phase ran out of memory and crashed the process outright, and on large projects on slower machines (reported on a 24,000-file Java project on Windows) the same phase could stall long enough that the safety watchdog killed a healthy, still-progressing index at ~98% (#1212). The whole phase now streams its work instead of holding whole-graph snapshots in memory, keeps the process responsive throughout, and skips analysis passes for languages a project doesn't contain — which also makes the tail of indexing noticeably faster on single-language repos. The resulting graph is identical, and a genuinely wedged process is still detected and killed.
+- Fixed `codegraph init` grinding for 20+ minutes (or getting killed by the safety watchdog) near the end of the "Resolving refs" step on large PHP and JavaScript codebases — a regression since 0.9.x reported on a 12,000-file project that used to index in about a minute. One of the dynamic-dispatch analysis passes only ever applies to Swift/Kotlin closure collections, but it was scanning every function in every language, and on `.push(`-heavy JavaScript (or any codebase with big generated functions, especially with non-ASCII text) its per-match bookkeeping went quadratic. The pass now skips languages it can't apply to, does its line accounting in constant time, and stays responsive even inside a single pathological function — the graph produced is identical. Thanks @sniperrenren for the report. (#1235) Two failure modes are fixed: on multi-million-symbol projects (e.g. the Linux kernel, ~95,000 files) the final analysis phase ran out of memory and crashed the process outright, and on large projects on slower machines (reported on a 24,000-file Java project on Windows) the same phase could stall long enough that the safety watchdog killed a healthy, still-progressing index at ~98% (#1212). The whole phase now streams its work instead of holding whole-graph snapshots in memory, keeps the process responsive throughout, and skips analysis passes for languages a project doesn't contain — which also makes the tail of indexing noticeably faster on single-language repos. The resulting graph is identical, and a genuinely wedged process is still detected and killed.
 - Indexing and `codegraph sync` stay responsive through their heaviest internal steps on huge projects: the post-index database maintenance (which on a multi-gigabyte index could stall the process for minutes and get a fully successful index killed by the safety watchdog at the finish line) now runs on a background thread, storing a giant generated file no longer freezes the process mid-extraction, and the reference-resolution bookkeeping between progress updates is broken into small responsive steps. The resulting graph is byte-for-byte identical.
 - Fixed a race that could leave a freshly-attached MCP session permanently silent: when a client's first messages arrived glued together during the daemon's connection handshake (roughly one attach in five on a busy machine), the daemon could drop them and stop reading that connection entirely — every tool call from that session then hung with no reply. The handshake now hands the connection over losslessly, and the fix is validated by hammering the previously-flaky attach test 25× under load.
 - The first tool call after the shared daemon starts no longer waits behind the query workers' cold start (which can take many seconds on a busy machine) — it's served directly until the first worker is warm, so a fresh session answers immediately.

__tests__/closure-collection-synthesizer.test.ts

@@ -106,7 +106,9 @@ describe('closure-collection synthesizer', () => {
     );
     expect(validatorsEdge).toBeTruthy();
     expect(validatorsEdge.source_name).toBe('didCompleteTask');
-    expect(validatorsEdge.registeredAt).toMatch(/DataRequest\.swift:\d+/);
+    // Exact wiring-site line, not just shape: pins the binary-search line
+    // resolver (#1235) to the same answer as the old per-match slice+split.
+    expect(validatorsEdge.registeredAt).toBe('DataRequest.swift:4');
 
     // The handlers flow: runHandlers → onEvent, via the direct `prop.append`
     // form — proves both registrar shapes are covered.
@@ -121,4 +123,37 @@ describe('closure-collection synthesizer', () => {
     expect(rows.some((r: any) => r.field === 'names')).toBe(false);
     expect(rows.some((r: any) => r.target_name === 'addName')).toBe(false);
   });
+
+  it('is a no-op on non-Swift/Kotlin code even when the text patterns occur (#1235)', async () => {
+    // JS that contains every textual trigger the scanner looks for —
+    // `.forEach`, `.push(`, even a `{ $0(` lookalike inside a string — but
+    // `{ $0( ` / `{ it( ` element-invocation is Swift/Kotlin trailing-closure
+    // syntax, so the pass must skip the language entirely (this scan running
+    // ungated over `.push(`-heavy JS/PHP was the #1235 25-minute index).
+    fs.writeFileSync(
+      path.join(dir, 'queue.js'),
+      `class Queue {
+  constructor() { this.tasks = []; }
+  register(fn) { this.tasks.push(fn); }
+  drain() { this.tasks.forEach(function (t) { t(); }); }
+  weird() { const s = "tasks.forEach { $0( } tasks.push("; return s; }
+}
+module.exports = Queue;
+`
+    );
+
+    const cg = await CodeGraph.init(dir, { silent: true });
+    await cg.indexAll();
+
+    const db = (cg as any).db.db;
+    const count = db
+      .prepare(
+        `SELECT count(*) c FROM edges
+         WHERE json_extract(metadata,'$.synthesizedBy') = 'closure-collection'`
+      )
+      .get();
+    cg.close?.();
+
+    expect(count.c).toBe(0);
+  });
 });

src/resolution/callback-synthesizer.ts

@@ -66,6 +66,15 @@ const CC_DISPATCH_RE = /(\w+)\.forEach\s*\{\s*(?:\$0|it)\s*\(/g;
 const CC_APPEND_WRITE_RE = /(\w+)\.write\s*\{\s*\$0(?:\.(\w+))?\.(?:append|add|push|insert)\s*\(/g;
 const CC_APPEND_DIRECT_RE = /(\w+)\.(?:append|add|push|insert)\s*\(/g;
 const CC_FANOUT_CAP = 8; // skip a field name with more dispatchers/registrars than this (too generic to pair confidently)
+// The dispatcher gate — `{ $0( ` / `{ it( ` element-invocation — is Swift/Kotlin
+// trailing-closure syntax, so ONLY those languages can ever contribute a
+// dispatcher, and a cross-language registrar pairing (a JS `.push(` against a
+// Swift dispatcher's field name) would be a wrong edge, not a missed one.
+// Gating both sides here isn't just precision: `.push(`/`.add(` is everywhere
+// in JS/PHP, so an ungated scan slices + regexes nearly every function on repos
+// where the pass cannot emit a single edge — on a 12k-file PHP/JS app that was
+// 20+ minutes of the "Resolving refs" tail and a #850 watchdog kill (#1235).
+const CC_LANGUAGES = new Set(['swift', 'kotlin']);
 
 function kebabToPascal(s: string): string {
   return s.split('-').map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join('');
@@ -100,6 +109,33 @@ function sliceLines(content: string, startLine?: number, endLine?: number): stri
   return content.split('\n').slice(startLine - 1, endLine).join('\n');
 }
 
+/**
+ * Per-match line resolver over `src`, 1-based at `baseLine`. The inline
+ * `src.slice(0, idx).split('\n').length` idiom is O(source-length) PER MATCH,
+ * which goes quadratic on a match-dense source (a generated function full of
+ * `.push(` calls re-scanned tens of thousands of times was most of the #1235
+ * indexing wedge). Builds the newline index once — lazily, since most sources
+ * never produce a match — then answers each call with a binary search.
+ */
+function makeLineAt(src: string, baseLine: number): (idx: number) => number {
+  let nl: number[] | null = null;
+  return (idx: number) => {
+    if (!nl) {
+      nl = [];
+      for (let i = src.indexOf('\n'); i !== -1; i = src.indexOf('\n', i + 1)) nl.push(i);
+    }
+    // Count newlines strictly before idx.
+    let lo = 0;
+    let hi = nl.length;
+    while (lo < hi) {
+      const mid = (lo + hi) >> 1;
+      if (nl[mid]! < idx) lo = mid + 1;
+      else hi = mid;
+    }
+    return baseLine + lo;
+  };
+}
+
 function registrarField(src: string): string | null {
   const m = src.match(/this\.(\w+)\.(?:add|push|set)\(/);
   return m ? m[1]! : null;
@@ -224,24 +260,29 @@ async function closureCollectionEdges(queries: QueryBuilder, ctx: ResolutionCont
     registrars.set(field, arr);
   };
 
-  // Slices EVERY method/function's source (no cheap name-gate), so on a repo
-  // with a huge file this is the heaviest synthesis pass — yield mid-scan so it
-  // can't wedge the #850 watchdog on its own (#1091).
+  // Slices EVERY Swift/Kotlin method/function's source (no cheap name-gate), so
+  // on a repo with a huge file this is the heaviest synthesis pass — yield
+  // mid-scan (and mid-match-loop below: a single generated function dense with
+  // matches must not starve the watchdog either) so it can't wedge the #850
+  // watchdog on its own (#1091, #1235).
   let scanned = 0;
+  let matchTick = 0;
   for (const m of methodAndFunctionNodes(queries)) {
     if ((++scanned & 127) === 0) await onYield();
+    if (!CC_LANGUAGES.has(m.language)) continue;
     const content = ctx.readFile(m.filePath);
     const src = content && sliceLines(content, m.startLine, m.endLine);
     if (!src) continue;
     const hasForEach = src.includes('.forEach');
     const hasAppend = src.includes('.append(') || src.includes('.add(') || src.includes('.push(') || src.includes('.insert(');
     if (!hasForEach && !hasAppend) continue;
-    const lineAt = (idx: number) => (m.startLine ?? 1) + src.slice(0, idx).split('\n').length - 1;
+    const lineAt = makeLineAt(src, m.startLine ?? 1);
 
     if (hasForEach) {
       CC_DISPATCH_RE.lastIndex = 0;
       let d: RegExpExecArray | null;
       while ((d = CC_DISPATCH_RE.exec(src))) {
+        if ((++matchTick & 255) === 0) await onYield();
         const arr = dispatchers.get(d[1]!) ?? [];
         if (!arr.some((n) => n.node.id === m.id)) arr.push({ node: m, line: lineAt(d.index) });
         dispatchers.set(d[1]!, arr);
@@ -250,10 +291,16 @@ async function closureCollectionEdges(queries: QueryBuilder, ctx: ResolutionCont
     if (hasAppend) {
       CC_APPEND_WRITE_RE.lastIndex = 0;
       let w: RegExpExecArray | null;
-      while ((w = CC_APPEND_WRITE_RE.exec(src))) addReg(w[2] || w[1], m, lineAt(w.index)); // nested `$0.streams` else the `.write` receiver
+      while ((w = CC_APPEND_WRITE_RE.exec(src))) {
+        if ((++matchTick & 255) === 0) await onYield();
+        addReg(w[2] || w[1], m, lineAt(w.index)); // nested `$0.streams` else the `.write` receiver
+      }
       CC_APPEND_DIRECT_RE.lastIndex = 0;
       let a: RegExpExecArray | null;
-      while ((a = CC_APPEND_DIRECT_RE.exec(src))) addReg(a[1], m, lineAt(a.index));
+      while ((a = CC_APPEND_DIRECT_RE.exec(src))) {
+        if ((++matchTick & 255) === 0) await onYield();
+        addReg(a[1], m, lineAt(a.index));
+      }
     }
   }
 
@@ -294,7 +341,7 @@ async function eventEmitterEdges(ctx: ResolutionContext, onYield: MaybeYield): P
     const hasOn = content.includes('.on(') || content.includes('.once(') || content.includes('.addListener(');
     if (!hasEmit && !hasOn) continue;
     const nodesInFile = ctx.getNodesInFile(file);
-    const lineOf = (idx: number) => content.slice(0, idx).split('\n').length;
+    const lineOf = makeLineAt(content, 1);
 
     if (hasEmit) {
       EMIT_RE.lastIndex = 0;
@@ -1368,7 +1415,7 @@ async function rnEventEdges(ctx: ResolutionContext, onYield: MaybeYield): Promis
     if (!content) continue;
 
     const nodesInFile = ctx.getNodesInFile(file);
-    const lineOf = (idx: number) => content.slice(0, idx).split('\n').length;
+    const lineOf = makeLineAt(content, 1);
     const addDispatcher = (event: string, line: number) => {
       const disp = enclosingFn(nodesInFile, line);
       if (!disp) return;