diff --git a/CHANGELOG.md b/CHANGELOG.md index 29379d352..3d6b1d028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes -- Indexing very large codebases no longer dies at the end of the "Resolving refs" step. Two failure modes are fixed: on multi-million-symbol projects (e.g. the Linux kernel, ~95,000 files) the final analysis phase ran out of memory and crashed the process outright, and on large projects on slower machines (reported on a 24,000-file Java project on Windows) the same phase could stall long enough that the safety watchdog killed a healthy, still-progressing index at ~98% (#1212). The whole phase now streams its work instead of holding whole-graph snapshots in memory, keeps the process responsive throughout, and skips analysis passes for languages a project doesn't contain — which also makes the tail of indexing noticeably faster on single-language repos. The resulting graph is identical, and a genuinely wedged process is still detected and killed. +- Fixed `codegraph init` grinding for 20+ minutes (or getting killed by the safety watchdog) near the end of the "Resolving refs" step on large PHP and JavaScript codebases — a regression since 0.9.x reported on a 12,000-file project that used to index in about a minute. One of the dynamic-dispatch analysis passes only ever applies to Swift/Kotlin closure collections, but it was scanning every function in every language, and on `.push(`-heavy JavaScript (or any codebase with big generated functions, especially with non-ASCII text) its per-match bookkeeping went quadratic. The pass now skips languages it can't apply to, does its line accounting in constant time, and stays responsive even inside a single pathological function — the graph produced is identical. Thanks @sniperrenren for the report. (#1235) Two failure modes are fixed: on multi-million-symbol projects (e.g. the Linux kernel, ~95,000 files) the final analysis phase ran out of memory and crashed the process outright, and on large projects on slower machines (reported on a 24,000-file Java project on Windows) the same phase could stall long enough that the safety watchdog killed a healthy, still-progressing index at ~98% (#1212). The whole phase now streams its work instead of holding whole-graph snapshots in memory, keeps the process responsive throughout, and skips analysis passes for languages a project doesn't contain — which also makes the tail of indexing noticeably faster on single-language repos. The resulting graph is identical, and a genuinely wedged process is still detected and killed. - Indexing and `codegraph sync` stay responsive through their heaviest internal steps on huge projects: the post-index database maintenance (which on a multi-gigabyte index could stall the process for minutes and get a fully successful index killed by the safety watchdog at the finish line) now runs on a background thread, storing a giant generated file no longer freezes the process mid-extraction, and the reference-resolution bookkeeping between progress updates is broken into small responsive steps. The resulting graph is byte-for-byte identical. - Fixed a race that could leave a freshly-attached MCP session permanently silent: when a client's first messages arrived glued together during the daemon's connection handshake (roughly one attach in five on a busy machine), the daemon could drop them and stop reading that connection entirely — every tool call from that session then hung with no reply. The handshake now hands the connection over losslessly, and the fix is validated by hammering the previously-flaky attach test 25× under load. - The first tool call after the shared daemon starts no longer waits behind the query workers' cold start (which can take many seconds on a busy machine) — it's served directly until the first worker is warm, so a fresh session answers immediately. diff --git a/__tests__/closure-collection-synthesizer.test.ts b/__tests__/closure-collection-synthesizer.test.ts index b516cb496..8b3593956 100644 --- a/__tests__/closure-collection-synthesizer.test.ts +++ b/__tests__/closure-collection-synthesizer.test.ts @@ -106,7 +106,9 @@ describe('closure-collection synthesizer', () => { ); expect(validatorsEdge).toBeTruthy(); expect(validatorsEdge.source_name).toBe('didCompleteTask'); - expect(validatorsEdge.registeredAt).toMatch(/DataRequest\.swift:\d+/); + // Exact wiring-site line, not just shape: pins the binary-search line + // resolver (#1235) to the same answer as the old per-match slice+split. + expect(validatorsEdge.registeredAt).toBe('DataRequest.swift:4'); // The handlers flow: runHandlers → onEvent, via the direct `prop.append` // form — proves both registrar shapes are covered. @@ -121,4 +123,37 @@ describe('closure-collection synthesizer', () => { expect(rows.some((r: any) => r.field === 'names')).toBe(false); expect(rows.some((r: any) => r.target_name === 'addName')).toBe(false); }); + + it('is a no-op on non-Swift/Kotlin code even when the text patterns occur (#1235)', async () => { + // JS that contains every textual trigger the scanner looks for — + // `.forEach`, `.push(`, even a `{ $0(` lookalike inside a string — but + // `{ $0( ` / `{ it( ` element-invocation is Swift/Kotlin trailing-closure + // syntax, so the pass must skip the language entirely (this scan running + // ungated over `.push(`-heavy JS/PHP was the #1235 25-minute index). + fs.writeFileSync( + path.join(dir, 'queue.js'), + `class Queue { + constructor() { this.tasks = []; } + register(fn) { this.tasks.push(fn); } + drain() { this.tasks.forEach(function (t) { t(); }); } + weird() { const s = "tasks.forEach { $0( } tasks.push("; return s; } +} +module.exports = Queue; +` + ); + + const cg = await CodeGraph.init(dir, { silent: true }); + await cg.indexAll(); + + const db = (cg as any).db.db; + const count = db + .prepare( + `SELECT count(*) c FROM edges + WHERE json_extract(metadata,'$.synthesizedBy') = 'closure-collection'` + ) + .get(); + cg.close?.(); + + expect(count.c).toBe(0); + }); }); diff --git a/src/resolution/callback-synthesizer.ts b/src/resolution/callback-synthesizer.ts index 581a7509c..d3beed745 100644 --- a/src/resolution/callback-synthesizer.ts +++ b/src/resolution/callback-synthesizer.ts @@ -66,6 +66,15 @@ const CC_DISPATCH_RE = /(\w+)\.forEach\s*\{\s*(?:\$0|it)\s*\(/g; const CC_APPEND_WRITE_RE = /(\w+)\.write\s*\{\s*\$0(?:\.(\w+))?\.(?:append|add|push|insert)\s*\(/g; const CC_APPEND_DIRECT_RE = /(\w+)\.(?:append|add|push|insert)\s*\(/g; const CC_FANOUT_CAP = 8; // skip a field name with more dispatchers/registrars than this (too generic to pair confidently) +// The dispatcher gate — `{ $0( ` / `{ it( ` element-invocation — is Swift/Kotlin +// trailing-closure syntax, so ONLY those languages can ever contribute a +// dispatcher, and a cross-language registrar pairing (a JS `.push(` against a +// Swift dispatcher's field name) would be a wrong edge, not a missed one. +// Gating both sides here isn't just precision: `.push(`/`.add(` is everywhere +// in JS/PHP, so an ungated scan slices + regexes nearly every function on repos +// where the pass cannot emit a single edge — on a 12k-file PHP/JS app that was +// 20+ minutes of the "Resolving refs" tail and a #850 watchdog kill (#1235). +const CC_LANGUAGES = new Set(['swift', 'kotlin']); function kebabToPascal(s: string): string { return s.split('-').map((p) => p.charAt(0).toUpperCase() + p.slice(1)).join(''); @@ -100,6 +109,33 @@ function sliceLines(content: string, startLine?: number, endLine?: number): stri return content.split('\n').slice(startLine - 1, endLine).join('\n'); } +/** + * Per-match line resolver over `src`, 1-based at `baseLine`. The inline + * `src.slice(0, idx).split('\n').length` idiom is O(source-length) PER MATCH, + * which goes quadratic on a match-dense source (a generated function full of + * `.push(` calls re-scanned tens of thousands of times was most of the #1235 + * indexing wedge). Builds the newline index once — lazily, since most sources + * never produce a match — then answers each call with a binary search. + */ +function makeLineAt(src: string, baseLine: number): (idx: number) => number { + let nl: number[] | null = null; + return (idx: number) => { + if (!nl) { + nl = []; + for (let i = src.indexOf('\n'); i !== -1; i = src.indexOf('\n', i + 1)) nl.push(i); + } + // Count newlines strictly before idx. + let lo = 0; + let hi = nl.length; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (nl[mid]! < idx) lo = mid + 1; + else hi = mid; + } + return baseLine + lo; + }; +} + function registrarField(src: string): string | null { const m = src.match(/this\.(\w+)\.(?:add|push|set)\(/); return m ? m[1]! : null; @@ -224,24 +260,29 @@ async function closureCollectionEdges(queries: QueryBuilder, ctx: ResolutionCont registrars.set(field, arr); }; - // Slices EVERY method/function's source (no cheap name-gate), so on a repo - // with a huge file this is the heaviest synthesis pass — yield mid-scan so it - // can't wedge the #850 watchdog on its own (#1091). + // Slices EVERY Swift/Kotlin method/function's source (no cheap name-gate), so + // on a repo with a huge file this is the heaviest synthesis pass — yield + // mid-scan (and mid-match-loop below: a single generated function dense with + // matches must not starve the watchdog either) so it can't wedge the #850 + // watchdog on its own (#1091, #1235). let scanned = 0; + let matchTick = 0; for (const m of methodAndFunctionNodes(queries)) { if ((++scanned & 127) === 0) await onYield(); + if (!CC_LANGUAGES.has(m.language)) continue; const content = ctx.readFile(m.filePath); const src = content && sliceLines(content, m.startLine, m.endLine); if (!src) continue; const hasForEach = src.includes('.forEach'); const hasAppend = src.includes('.append(') || src.includes('.add(') || src.includes('.push(') || src.includes('.insert('); if (!hasForEach && !hasAppend) continue; - const lineAt = (idx: number) => (m.startLine ?? 1) + src.slice(0, idx).split('\n').length - 1; + const lineAt = makeLineAt(src, m.startLine ?? 1); if (hasForEach) { CC_DISPATCH_RE.lastIndex = 0; let d: RegExpExecArray | null; while ((d = CC_DISPATCH_RE.exec(src))) { + if ((++matchTick & 255) === 0) await onYield(); const arr = dispatchers.get(d[1]!) ?? []; if (!arr.some((n) => n.node.id === m.id)) arr.push({ node: m, line: lineAt(d.index) }); dispatchers.set(d[1]!, arr); @@ -250,10 +291,16 @@ async function closureCollectionEdges(queries: QueryBuilder, ctx: ResolutionCont if (hasAppend) { CC_APPEND_WRITE_RE.lastIndex = 0; let w: RegExpExecArray | null; - while ((w = CC_APPEND_WRITE_RE.exec(src))) addReg(w[2] || w[1], m, lineAt(w.index)); // nested `$0.streams` else the `.write` receiver + while ((w = CC_APPEND_WRITE_RE.exec(src))) { + if ((++matchTick & 255) === 0) await onYield(); + addReg(w[2] || w[1], m, lineAt(w.index)); // nested `$0.streams` else the `.write` receiver + } CC_APPEND_DIRECT_RE.lastIndex = 0; let a: RegExpExecArray | null; - while ((a = CC_APPEND_DIRECT_RE.exec(src))) addReg(a[1], m, lineAt(a.index)); + while ((a = CC_APPEND_DIRECT_RE.exec(src))) { + if ((++matchTick & 255) === 0) await onYield(); + addReg(a[1], m, lineAt(a.index)); + } } } @@ -294,7 +341,7 @@ async function eventEmitterEdges(ctx: ResolutionContext, onYield: MaybeYield): P const hasOn = content.includes('.on(') || content.includes('.once(') || content.includes('.addListener('); if (!hasEmit && !hasOn) continue; const nodesInFile = ctx.getNodesInFile(file); - const lineOf = (idx: number) => content.slice(0, idx).split('\n').length; + const lineOf = makeLineAt(content, 1); if (hasEmit) { EMIT_RE.lastIndex = 0; @@ -1368,7 +1415,7 @@ async function rnEventEdges(ctx: ResolutionContext, onYield: MaybeYield): Promis if (!content) continue; const nodesInFile = ctx.getNodesInFile(file); - const lineOf = (idx: number) => content.slice(0, idx).split('\n').length; + const lineOf = makeLineAt(content, 1); const addDispatcher = (event: string, line: number) => { const disp = enclosingFn(nodesInFile, line); if (!disp) return;