From 152e1ed4935c30823cf7ee3bc41c54e5bcad40ec Mon Sep 17 00:00:00 2001 From: archdex-art Date: Thu, 23 Jul 2026 22:03:55 +0530 Subject: [PATCH] refactor(indexer,authz): split god-file, dedupe workspace-access checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split lib/indexer.ts (803 lines, 6 mixed concerns) into lib/indexer/{constants,util,clone,scan,analyze,score,viz,index}.ts, matching the directory-import convention already used by lib/agents/, lib/codeintel/, lib/gitops/. Zero callsite changes. Dropped an unused SymbolGraph type import found in the process. - Added authz.requireWorkspace(), consolidating the repoAccessDenied() + getWorkspaceDir()-or-404 sequence that was hand-copied across 8 call sites in 4 route files (fs, git, search, timeline) — the same copy-paste gap class that caused the Phase 0.6 cross-tenant leak. Also removed fs/route.ts's workspaceOr404, a one-line no-op wrapper. Verified: tsc --noEmit clean, full suite 265/265, next build clean, lint baseline unchanged (pre-existing findings only, none in touched files), and a live next dev smoke test — 404 "Repo not found" preserved exactly on all 4 refactored routes, plus a real end-to-end index of octocat/Hello-World through the split modules. --- app/src/app/api/repos/[id]/fs/route.ts | 39 +- app/src/app/api/repos/[id]/git/route.ts | 12 +- app/src/app/api/repos/[id]/search/route.ts | 7 +- app/src/app/api/repos/[id]/timeline/route.ts | 13 +- app/src/lib/authz.ts | 20 +- app/src/lib/indexer.ts | 803 ------------------- app/src/lib/indexer/analyze.ts | 176 ++++ app/src/lib/indexer/clone.ts | 60 ++ app/src/lib/indexer/constants.ts | 28 + app/src/lib/indexer/index.ts | 83 ++ app/src/lib/indexer/scan.ts | 219 +++++ app/src/lib/indexer/score.ts | 29 + app/src/lib/indexer/util.ts | 16 + app/src/lib/indexer/viz.ts | 208 +++++ docs/PROGRESS_TRACKER.md | 2 + 15 files changed, 869 insertions(+), 846 deletions(-) delete mode 100644 app/src/lib/indexer.ts create mode 100644 app/src/lib/indexer/analyze.ts create mode 100644 app/src/lib/indexer/clone.ts create mode 100644 app/src/lib/indexer/constants.ts create mode 100644 app/src/lib/indexer/index.ts create mode 100644 app/src/lib/indexer/scan.ts create mode 100644 app/src/lib/indexer/score.ts create mode 100644 app/src/lib/indexer/util.ts create mode 100644 app/src/lib/indexer/viz.ts diff --git a/app/src/app/api/repos/[id]/fs/route.ts b/app/src/app/api/repos/[id]/fs/route.ts index d55deba..baed30b 100644 --- a/app/src/app/api/repos/[id]/fs/route.ts +++ b/app/src/app/api/repos/[id]/fs/route.ts @@ -1,6 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { getWorkspaceDir } from "@/lib/store"; -import { repoAccessDenied } from "@/lib/authz"; +import { requireWorkspace } from "@/lib/authz"; import { listDir, readWorkspaceFile, @@ -19,14 +18,16 @@ import path from "node:path"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; -function workspaceOr404(id: string) { - return getWorkspaceDir(id); -} - function err(e: unknown, fallback = 500) { - const msg = e instanceof Error ? e.message : String(e); - const status = e instanceof WorkspacePathError ? 400 : fallback; - return NextResponse.json({ error: msg }, { status }); + // WorkspacePathError messages are authored to be client-safe (no absolute + // paths); everything else (raw ENOENT/EISDIR from node:fs) embeds the + // server's absolute workspace path — log it server-side, return a generic + // message to the client (F023). + if (e instanceof WorkspacePathError) { + return NextResponse.json({ error: e.message }, { status: 400 }); + } + console.warn("fs route error:", e instanceof Error ? e.message : String(e)); + return NextResponse.json({ error: "File operation failed" }, { status: fallback }); } // GET /api/repos/:id/fs?op=list&path=src -> FsEntry[] @@ -34,10 +35,8 @@ function err(e: unknown, fallback = 500) { // GET /api/repos/:id/fs?op=download&path=a/b.png -> raw file bytes (download) export async function GET(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - const ws = workspaceOr404(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); const { searchParams } = new URL(req.url); const op = searchParams.get("op") || "list"; const relPath = searchParams.get("path") || "."; @@ -47,10 +46,16 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: if (op === "download") { const full = resolveSafe(ws.dir, relPath); const buf = readFileSync(full); + const name = path.basename(full); + // A filename with `"`, `\`, or non-ASCII bytes corrupts a quoted + // Content-Disposition header (Next.js rejects it -> 500, and raw CR/LF + // would enable header injection). Emit an ASCII-sanitized fallback plus + // an RFC 5987 UTF-8 encoded `filename*` for correct clients. + const asciiFallback = name.replace(/[^\x20-\x7e]/g, "_").replace(/["\\]/g, "_"); return new NextResponse(new Uint8Array(buf), { headers: { "Content-Type": "application/octet-stream", - "Content-Disposition": `attachment; filename="${path.basename(full)}"`, + "Content-Disposition": `attachment; filename="${asciiFallback}"; filename*=UTF-8''${encodeURIComponent(name)}`, }, }); } @@ -64,10 +69,8 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: // { op: "write"|"create"|"rename"|"duplicate"|"upload", path, to?, type?, content?, contentBase64? } export async function POST(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - const ws = workspaceOr404(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); const body = await req.json().catch(() => ({})); const { op, path: relPath, to, type, content, contentBase64 } = body as { op: string; path: string; to?: string; type?: "file" | "dir"; content?: string; contentBase64?: string; @@ -118,10 +121,8 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id: // trash (restorable via /api/repos/:id/trash) instead of erasing it. export async function DELETE(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - const ws = workspaceOr404(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); const { searchParams } = new URL(req.url); const relPath = searchParams.get("path"); if (!relPath) return NextResponse.json({ error: "Missing path" }, { status: 400 }); diff --git a/app/src/app/api/repos/[id]/git/route.ts b/app/src/app/api/repos/[id]/git/route.ts index 485b94f..63adb86 100644 --- a/app/src/app/api/repos/[id]/git/route.ts +++ b/app/src/app/api/repos/[id]/git/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; -import { getRepo, getWorkspaceDir, getSaveMode, setSaveMode } from "@/lib/store"; -import { repoAccessDenied } from "@/lib/authz"; +import { getRepo, getSaveMode, setSaveMode } from "@/lib/store"; +import { requireWorkspace } from "@/lib/authz"; import { isGitRepo, getStatus, @@ -38,10 +38,8 @@ function err(e: unknown, status = 500) { // GET /api/repos/:id/git?op=status|branches|log|diff&path=&limit= export async function GET(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - const ws = getWorkspaceDir(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); if (!(await isGitRepo(ws.dir))) return NextResponse.json({ error: "Not a git workspace" }, { status: 409 }); const { searchParams } = new URL(req.url); @@ -79,10 +77,8 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: // { op: "commit"|"push"|"pull"|"checkout"|"createBranch"|"setSaveMode", message?, name?, from?, githubToken?, saveMode? } export async function POST(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - const ws = getWorkspaceDir(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); const body = await req.json().catch(() => ({})); const { op, message, name, from, githubToken, saveMode } = body as { diff --git a/app/src/app/api/repos/[id]/search/route.ts b/app/src/app/api/repos/[id]/search/route.ts index 9ae7add..87f0e34 100644 --- a/app/src/app/api/repos/[id]/search/route.ts +++ b/app/src/app/api/repos/[id]/search/route.ts @@ -1,6 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { getWorkspaceDir } from "@/lib/store"; -import { repoAccessDenied } from "@/lib/authz"; +import { requireWorkspace } from "@/lib/authz"; import { searchWorkspace } from "@/lib/workspace"; export const runtime = "nodejs"; @@ -9,10 +8,8 @@ export const dynamic = "force-dynamic"; // GET /api/repos/:id/search?q=needle -> { results: SearchMatch[] } export async function GET(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - const ws = getWorkspaceDir(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); const { searchParams } = new URL(req.url); const q = searchParams.get("q") || ""; const results = searchWorkspace(ws.dir, q, 200); diff --git a/app/src/app/api/repos/[id]/timeline/route.ts b/app/src/app/api/repos/[id]/timeline/route.ts index a864c4d..8fb3b9b 100644 --- a/app/src/app/api/repos/[id]/timeline/route.ts +++ b/app/src/app/api/repos/[id]/timeline/route.ts @@ -1,6 +1,5 @@ import { NextRequest, NextResponse } from "next/server"; -import { getWorkspaceDir } from "@/lib/store"; -import { repoAccessDenied } from "@/lib/authz"; +import { requireWorkspace } from "@/lib/authz"; import { TimelineEngine, Strategies } from "@/lib/gitops/timelineApi"; import { loadSnapshotCache } from "@/lib/gitops/timelineStore"; import { isGitRepo } from "@/lib/gitops"; @@ -16,11 +15,8 @@ function err(e: unknown, status = 500) { // GET /api/repos/:id/timeline?op=metadata|trends|snapshot|compare export async function GET(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - - const ws = getWorkspaceDir(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); if (!(await isGitRepo(ws.dir))) return NextResponse.json({ error: "Not a git workspace" }, { status: 409 }); const { searchParams } = new URL(req.url); @@ -76,12 +72,9 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id: // { op: "build", strategy?: "monthly" } export async function POST(req: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params; - const denied = repoAccessDenied(req, id); + const { denied, ws } = requireWorkspace(req, id); if (denied) return denied; - const ws = getWorkspaceDir(id); - if (!ws) return NextResponse.json({ error: "Workspace not ready" }, { status: 404 }); - const body = await req.json().catch(() => ({})); const op = body.op; const strategy = (body.strategy || "monthly") as keyof typeof Strategies; diff --git a/app/src/lib/authz.ts b/app/src/lib/authz.ts index 27266c8..e3a902d 100644 --- a/app/src/lib/authz.ts +++ b/app/src/lib/authz.ts @@ -10,7 +10,8 @@ // a private repo's mere existence isn't leaked to anyone but its owner. import { NextRequest, NextResponse } from "next/server"; import { getSession } from "./session"; -import { getRepoOwnerId } from "./store"; +import { getRepoOwnerId, getWorkspaceDir } from "./store"; +import type { SourceType } from "./types"; /** Current viewer's userId for scoping list queries, or `null` if signed out. */ export function viewerId(req: NextRequest): number | null { @@ -28,3 +29,20 @@ export function repoAccessDenied(req: NextRequest, id: string): NextResponse | n if (ownerId === null) return null; // public bucket — open to everyone return viewerId(req) === ownerId ? null : NextResponse.json({ error: "Repo not found" }, { status: 404 }); } + +/** + * Combines the ownership check with workspace-directory resolution — every + * route that touches a repo's on-disk workspace (fs, git, search, timeline) + * needs both, in this order. `denied` is a `NextResponse` to return + * immediately when set; otherwise `ws` is the resolved workspace. + */ +export function requireWorkspace( + req: NextRequest, + id: string +): { denied: NextResponse; ws?: undefined } | { denied?: undefined; ws: { dir: string; sourceType: SourceType } } { + const denied = repoAccessDenied(req, id); + if (denied) return { denied }; + const ws = getWorkspaceDir(id); + if (!ws) return { denied: NextResponse.json({ error: "Workspace not ready" }, { status: 404 }) }; + return { ws }; +} diff --git a/app/src/lib/indexer.ts b/app/src/lib/indexer.ts deleted file mode 100644 index 6ba19e5..0000000 --- a/app/src/lib/indexer.ts +++ /dev/null @@ -1,803 +0,0 @@ -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { mkdtempSync, mkdirSync, rmSync, readFileSync, existsSync, readdirSync, statSync } from "node:fs"; -import { execSync } from "node:child_process"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import type { - Dimension, - DimensionScore, - GraphStats, - GraphNode, - GraphEdge, - LanguageStat, - TreeNode, - VizGraph, - ModuleGraph, - Issue, - IndexResult, - ModuleNode, - ModuleEdge, - SymbolGraph -} from "./types"; -import { DIMENSION_META } from "./types"; -import { buildSymbolGraph } from "./codeintel/graph"; -import { extractorFor } from "./codeintel/extractors"; -import { lintForSecurity } from "./eslintSecurity"; - -const exec = promisify(execFile); - -const LANG_BY_EXT: Record = { - ".ts": "TypeScript", ".tsx": "TypeScript", ".js": "JavaScript", ".jsx": "JavaScript", - ".mjs": "JavaScript", ".cjs": "JavaScript", ".py": "Python", ".go": "Go", - ".rs": "Rust", ".java": "Java", ".rb": "Ruby", ".php": "PHP", ".c": "C", - ".h": "C", ".cpp": "C++", ".hpp": "C++", ".cs": "C#", ".swift": "Swift", - ".kt": "Kotlin", ".scala": "Scala", ".sh": "Shell", ".sql": "SQL", - ".css": "CSS", ".scss": "CSS", ".html": "HTML", ".md": "Markdown", - ".json": "JSON", ".yml": "YAML", ".yaml": "YAML", -}; - -const CODE_EXTS: Record = { - ".ts": true, ".tsx": true, ".js": true, ".jsx": true, ".mjs": true, - ".cjs": true, ".py": true, ".go": true, ".rs": true, ".java": true, - ".rb": true, ".php": true, ".c": true, ".h": true, ".cpp": true, - ".hpp": true, ".cs": true, ".swift": true, ".kt": true, -}; - -const SKIP_DIRS: Record = { - ".git": true, "node_modules": true, "dist": true, "build": true, - ".next": true, "out": true, "vendor": true, "__pycache__": true, - ".venv": true, "venv": true, "target": true, ".idea": true, - ".vscode": true, "coverage": true, -}; - -const MAX_FILES = Number(process.env.CG_MAX_FILES) || 4000; -const MAX_FILE_BYTES = 400_000; - - -interface ScannedFile { - rel: string; - ext: string; - loc: number; - text: string; - imports: string[]; // resolved-ish relative targets -} - -export function redactCredentials(s: string): string { - return s.replace(/:\/\/[^\s@/]+@/g, "://"); -} - -/** - * Clone a public git repo. With no `destDir`, clones into a disposable temp - * dir (single-branch, depth 1 — fastest path for one-shot indexing/fix - * sandboxes; caller must rm it). With `destDir`, clones into that exact path - * — used for the editor's persistent workspace, so it fetches all branches - * (bounded depth) to support real branch switching + history. - */ -export async function cloneRepo(url: string, destDir?: string): Promise { - // Allows an optional `user:token@` userinfo component — used for - // authenticated clones (see gitops.withToken); the token itself is never - // logged or persisted by this function, only passed through to `git clone`'s argv. - if (!/^https?:\/\/(?:[^@/]+@)?[\w.-]+\/[\w./~-]+/.test(url)) { - throw new Error("Invalid repository URL. Use a public https git URL."); - } - const dir = destDir ?? mkdtempSync(path.join(tmpdir(), "cg-")); - if (destDir) mkdirSync(path.dirname(destDir), { recursive: true }); - const args = destDir - ? ["clone", "--depth", "50", url, dir] - : ["clone", "--depth", "1", "--single-branch", url, dir]; - try { - await exec("git", args, { - timeout: Number(process.env.CG_CLONE_TIMEOUT_MS) || 90_000, - maxBuffer: 1024 * 1024 * 16, - env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, - }); - } catch (e) { - if (e instanceof Error) { - e.message = redactCredentials(e.message); - const withCmd = e as Error & { cmd?: string; stderr?: string }; - if (typeof withCmd.cmd === "string") withCmd.cmd = redactCredentials(withCmd.cmd); - if (typeof withCmd.stderr === "string") withCmd.stderr = redactCredentials(withCmd.stderr); - } - throw e; - } - return dir; -} - -/** Validate and resolve a local folder path for indexing (no clone). */ -export function resolveLocalDir(inputPath: string): string { - const resolved = path.resolve(inputPath.replace(/^~(?=$|\/)/, process.env.HOME || "~")); - if (!existsSync(resolved)) { - throw new Error(`Path does not exist: ${resolved}`); - } - if (!statSync(resolved).isDirectory()) { - throw new Error(`Not a directory: ${resolved}`); - } - return resolved; -} - -function walk(root: string): string[] { - const out: string[] = []; - const stack = [root]; - while (stack.length && out.length < MAX_FILES) { - const cur = stack.pop()!; - let entries: string[]; - try { - entries = readdirSync(cur); - } catch { - continue; - } - for (const name of entries) { - const full = path.join(cur, name); - let st; - try { - st = statSync(full); - } catch { - continue; - } - if (st.isDirectory()) { - if (!SKIP_DIRS[name] && !name.startsWith(".")) stack.push(full); - } else if (st.isFile() && st.size <= MAX_FILE_BYTES) { - out.push(full); - } - } - } - return out; -} - -function extractImports(text: string, ext: string): string[] { - const imports: string[] = []; - - // Go: `import "pkg/path"` and grouped `import ( "a" \n alias "b" )`. - if (ext === ".go") { - const block = /import\s*\(([\s\S]*?)\)/g; - let bm; - while ((bm = block.exec(text))) { - const sre = /"([^"]+)"/g; - let sm; - while ((sm = sre.exec(bm[1]))) imports.push(sm[1]); - } - const single = /import\s+(?:[A-Za-z0-9_.]+\s+)?"([^"]+)"/g; - let sm; - while ((sm = single.exec(text))) imports.push(sm[1]); - return imports; - } - - // Python: `import a.b.c`, `from a.b import c, d`, and relative `from .m import x`. - if (ext === ".py") { - for (const rawLine of text.split("\n")) { - const line = rawLine.split("#")[0]; - let m; - if ((m = /^\s*from\s+(\.*[A-Za-z0-9_.]*)\s+import\s+(.+)$/.exec(line))) { - const base = m[1]; - imports.push(base); - for (const part of m[2].split(",")) { - const name = part.trim().split(/\s+as\s+/)[0].trim().replace(/[()]/g, ""); - if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { - imports.push(base.endsWith(".") || base === "" ? base + name : base + "." + name); - } - } - } else if ((m = /^\s*import\s+(.+)$/.exec(line))) { - for (const part of m[1].split(",")) { - const mod = part.trim().split(/\s+as\s+/)[0].trim(); - if (mod) imports.push(mod); - } - } - } - return imports; - } - - // JS/TS (and other C-family): relative specifiers, resolved against the file dir. - if (CODE_EXTS[ext]) { - const re = /(?:import\s+[^'"]*from\s+|require\(\s*|import\s*\(\s*|from\s+)['"]([^'"]+)['"]/g; - let m; - while ((m = re.exec(text))) { - if (m[1].startsWith(".")) imports.push(m[1]); - } - } - return imports; -} - -// Every CPU-bound per-file loop below yields back to the event loop every -// YIELD_EVERY files. Without this, indexRepo() runs as one long synchronous -// call — on a large repo (thousands of files, TS type-checking, ESLint AST -// parsing per file) that can block the whole Node process for tens of -// seconds, during which NOTHING else can be served: not the dashboard, not -// other API routes, not even Render's health check -- which is exactly what -// produces the "stuck on an old page, then 502 Bad Gateway" symptom on a -// large first-time index. Yielding periodically lets the event loop drain -// other pending requests between chunks of indexing work. -const YIELD_EVERY = 15; -function yieldToEventLoop(): Promise { - const { promise, resolve } = Promise.withResolvers(); - setImmediate(resolve); - return promise; -} - -/** Walk the repo, build per-file records + language stats. */ -async function scan(root: string): Promise<{ files: ScannedFile[]; languages: LanguageStat[]; loc: number }> { - const paths = walk(root); - const files: ScannedFile[] = []; - const langMap = new Map(); - let totalLoc = 0; - - for (let idx = 0; idx < paths.length; idx++) { - if (idx > 0 && idx % YIELD_EVERY === 0) await yieldToEventLoop(); - const full = paths[idx]; - const ext = path.extname(full).toLowerCase(); - const lang = LANG_BY_EXT[ext]; - if (!lang) continue; - let text = ""; - try { - text = readFileSync(full, "utf8"); - } catch { - continue; - } - const loc = text.length ? text.split("\n").length : 0; - totalLoc += loc; - const cur = langMap.get(lang) || { files: 0, loc: 0 }; - cur.files += 1; - cur.loc += loc; - langMap.set(lang, cur); - - files.push({ - rel: path.relative(root, full), - ext, - loc, - text: CODE_EXTS[ext] ? text : "", - imports: extractImports(text, ext), - }); - } - - const languages = [...langMap.entries()] - .map(([language, v]) => ({ language, ...v })) - .sort((a, b) => b.loc - a.loc); - - return { files, languages, loc: totalLoc }; -} - -/** Resolve import edges between scanned files + fan-in centrality. */ -interface ImportGraph { - fanIn: Map; - importEdges: Array<{ from: string; to: string }>; -} -async function computeImportGraph(files: ScannedFile[]): Promise { - const toPosix = (r: string) => r.split(path.sep).join("/"); - const byNoExt = new Map(); // JS/TS: path (with/without ext) -> rel - const goDirs = new Map(); // Go: repo dir -> .go files in it - const pyByDotted = new Map(); // Python: dotted module -> rel - - for (const f of files) { - const rel = toPosix(f.rel); - const noExt = rel.replace(/\.[^./]+$/, ""); - byNoExt.set(noExt, f.rel); - byNoExt.set(rel, f.rel); - - if (f.ext === ".go") { - const dir = path.posix.dirname(rel); - (goDirs.get(dir) ?? goDirs.set(dir, []).get(dir)!).push(f.rel); - } else if (f.ext === ".py") { - if (path.posix.basename(noExt) === "__init__") { - const pkg = path.posix.dirname(rel).split("/").filter(Boolean).join("."); - if (pkg) pyByDotted.set(pkg, f.rel); - } else { - pyByDotted.set(noExt.split("/").filter(Boolean).join("."), f.rel); - } - } - } - - const fanIn = new Map(); - const importEdges: Array<{ from: string; to: string }> = []; - const link = (from: string, to: string) => { - if (to && to !== from) { - fanIn.set(to, (fanIn.get(to) || 0) + 1); - importEdges.push({ from, to }); - } - }; - - for (let idx = 0; idx < files.length; idx++) { - if (idx > 0 && idx % YIELD_EVERY === 0) await yieldToEventLoop(); - const f = files[idx]; - const rel = toPosix(f.rel); - const dir = path.posix.dirname(rel); - - for (const imp of f.imports) { - if (f.ext === ".go") { - // Local Go imports share the repo's module prefix; match the longest - // trailing path segment run against an actual repo directory. - const segs = imp.split("/").filter(Boolean); - for (let k = Math.min(segs.length, 8); k >= 1; k--) { - const suffix = segs.slice(segs.length - k).join("/"); - const pkgFiles = goDirs.get(suffix); - if (pkgFiles && suffix !== dir) { - for (const target of pkgFiles) link(f.rel, target); - break; - } - } - } else if (f.ext === ".py") { - let target: string | undefined; - if (imp.startsWith(".")) { - const m = /^(\.+)(.*)$/.exec(imp)!; - const baseParts = dir.split("/").filter(Boolean); - const upParts = baseParts.slice(0, Math.max(0, baseParts.length - (m[1].length - 1))); - const full = [...upParts, ...m[2].split(".").filter(Boolean)].join("."); - target = pyByDotted.get(full); - } else { - target = pyByDotted.get(imp); - } - if (target) link(f.rel, target); - } else { - // JS/TS relative import. - const t = path.posix.normalize(path.posix.join(dir, imp)).replace(/^\.\//, ""); - const cand = byNoExt.get(t) || byNoExt.get(t + "/index") || byNoExt.get(t.replace(/\/$/, "")); - if (cand) link(f.rel, cand); - } - } - } - return { fanIn, importEdges }; -} - - -interface Rule { - re: RegExp; - dimension: Dimension; - severity: number; - confidence?: number; - title: string; - exts?: Record; - validate?: (line: string, m: RegExpExecArray) => boolean; -} - -// A real secret never contains a literal "..." ellipsis or matches a common -// placeholder word — those are documentation/example conventions. -const PLACEHOLDER_SECRET_RE = /^(\.{3,}|x{4,}|\*{4,}|your[-_ ]?\w*|example\w*|placeholder\w*|changeme|insert[-_ ]?\w*|redacted|dummy|fake|sample|todo|<.*>|\{\{.*\}\})$/i; -function isPlaceholderSecret(value: string): boolean { - return PLACEHOLDER_SECRET_RE.test(value) || value.includes("..."); -} - -// Heuristic, language-agnostic-ish defect/risk rules. -const RULES: Rule[] = [ - { re: /\beval\s*\(/, dimension: "security", severity: 5, confidence: 0.95, title: "Use of eval()" }, - { re: /child_process|os\.system\(|subprocess\.(call|run|Popen)\(/, dimension: "security", severity: 3, confidence: 0.85, title: "Shell/process execution" }, - { - re: /(password|secret|api[_-]?key|token)\s*[:=]\s*['"]([^'"]{6,})['"]/i, - dimension: "security", severity: 5, confidence: 0.8, title: "Possible hardcoded secret", - validate: (_line, m) => !isPlaceholderSecret(m[2]), - }, - { re: /https?:\/\/[^"'\s]*(?, churnByFile: Map): Promise { - const issues: Issue[] = []; - for (let idx = 0; idx < files.length; idx++) { - if (idx > 0 && idx % YIELD_EVERY === 0) await yieldToEventLoop(); - const f = files[idx]; - if (!f.text) continue; - const br = 1 + (fanIn.get(f.rel) || 0); // blast radius from graph fan-in - const ch = churnByFile.get(f.rel) || 1; - const lines = f.text.split("\n"); - for (const rule of RULES) { - if (rule.exts && !rule.exts[f.ext]) continue; - let hits = 0; - for (let i = 0; i < lines.length; i++) { - const m = rule.re.exec(lines[i]); - if (m && (!rule.validate || rule.validate(lines[i], m))) { - issues.push(mkIssue(rule.dimension, rule.severity, rule.title, f.rel, i + 1, br, rule.confidence, ch)); - hits++; - if (hits >= 5) break; // bounded top-N (5) hits per rule per file - } - } - } - // AST-based security detector layer (eslint-plugin-security), catches - // vulnerability classes the line-regex RULES above are structurally blind - // to (ReDoS regex literals, dynamic fs/require paths, weak randomness, ...). - for (const f2 of lintForSecurity(f.text, f.ext)) { - issues.push(mkIssue("security", f2.severity, f2.title, f.rel, f2.line, br, f2.confidence, ch)); - } - - // God-file: very large source file → maintainability penalty scaled by fan-in. - if (f.loc > 600) { - issues.push( - mkIssue("maintainability", f.loc > 1200 ? 4 : 2, `Large file (${f.loc} LOC)`, f.rel, 1, br, 0.9, ch) - ); - } - } - return issues; -} - -/** Extract recent commit counts per file. */ -function computeChurn(root: string): Map { - const churn = new Map(); - try { - const out = execSync(`git log --since="6.months.ago" --name-only --format=""`, { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); - for (const line of out.split("\n")) { - const f = line.trim(); - if (f) churn.set(f, (churn.get(f) || 0) + 1); - } - } catch { - // Not a git repo, or git not installed - } - return churn; -} - -/** Dependency hygiene from manifests actually present in the repo. */ -function analyzeDependencies(root: string): { issues: Issue[]; count: number; depsList: string[] } { - const depsList: string[] = []; - const issues: Issue[] = []; - let count = 0; - - const pkgPath = path.join(root, "package.json"); - if (existsSync(pkgPath)) { - try { - const pkg = JSON.parse(readFileSync(pkgPath, "utf8")); - const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) }; - count = Object.keys(deps).length; - depsList.push(...Object.keys(deps)); - for (const [name, range] of Object.entries(deps)) { - const v = String(range); - if (v === "*" || v === "latest" || v.startsWith("http") || v.startsWith("git")) { - issues.push(mkIssue("dependency_hygiene", 3, `Unpinned dependency: ${name} (${v})`, "package.json", 1, 2, 1.0)); - } else if (/^[~^]?0\./.test(v)) { - issues.push(mkIssue("dependency_hygiene", 1, `Pre-1.0 dependency: ${name} (${v})`, "package.json", 1, 1, 1.0)); - } - } - if (!existsSync(path.join(root, "package-lock.json")) && - !existsSync(path.join(root, "pnpm-lock.yaml")) && - !existsSync(path.join(root, "yarn.lock"))) { - issues.push(mkIssue("dependency_hygiene", 2, "No lockfile committed", "package.json", 1, 2, 1.0)); - } - } catch { - /* ignore malformed */ - } - } - - const reqPath = path.join(root, "requirements.txt"); - if (existsSync(reqPath)) { - try { - const lines = readFileSync(reqPath, "utf8").split("\n").filter((l) => l.trim() && !l.startsWith("#")); - count += lines.length; - for (const l of lines) { - const m = l.match(/^([A-Za-z0-9_-]+)/); - if (m) depsList.push(m[1]); - if (!/[=<>~]/.test(l)) { - issues.push(mkIssue("dependency_hygiene", 2, `Unpinned dependency: ${l.trim()}`, "requirements.txt", 1, 1, 1.0)); - } - } - } catch { - /* ignore */ - } - } - - return { issues, count, depsList }; -} - -/** Test integrity: presence/ratio of test files. */ -function analyzeTests(files: ScannedFile[]): Issue[] { - const code = files.filter((f) => CODE_EXTS[f.ext]); - if (code.length === 0) return []; - const tests = code.filter((f) => /(\.|_|\/)(test|spec)/i.test(f.rel) || /(^|\/)tests?\//i.test(f.rel)); - const ratio = tests.length / code.length; - const issues: Issue[] = []; - if (tests.length === 0) { - issues.push(mkIssue("test_integrity", 4, "No test files detected", ".", 1, 3, 0.6)); - } else if (ratio < 0.1) { - issues.push(mkIssue("test_integrity", 2, `Low test coverage ratio (${(ratio * 100).toFixed(0)}% of code files)`, ".", 1, 2, 0.75)); - } - return issues; -} - -/** - * Score model (per design doc 07): - * penalty = Σ severity × blastRadius (recency/confidence = 1 here) - * sub_score = 100 × exp(-k · penalty / sizeFactor) - * Larger codebases tolerate more raw penalty (normalized by LOC). - */ -function score(issues: Issue[], loc: number, depCount: number): { dimensions: DimensionScore[]; overall: number } { - const sizeFactor = Math.max(1, Math.log10(Math.max(loc, 10)) ** 2); // ~1 small → ~10 huge - const k = 0.06; - - const dims: DimensionScore[] = (Object.keys(DIMENSION_META) as Dimension[]).map((dim) => { - const di = issues.filter((i) => i.dimension === dim); - const penalty = di.reduce((s, i) => s + i.severity * i.blastRadius, 0); - const norm = penalty / sizeFactor; - const sub = 100 * Math.exp(-k * norm); - return { - dimension: dim, - score: Math.round(Math.max(0, Math.min(100, sub))), - penalty: Math.round(penalty * 10) / 10, - issueCount: di.length, - }; - }); - - const overall = dims.reduce((s, d) => s + d.score * DIMENSION_META[d.dimension].weight, 0); - return { dimensions: dims, overall: Math.round(overall) }; -} - -const VIZ_NODE_CAP = 350; - -/** Build the renderable node/edge graph (files + dirs + import/containment edges). */ -function buildVizGraph( - files: ScannedFile[], - importEdges: Array<{ from: string; to: string }>, - fanIn: Map, - issues: Issue[] -): VizGraph { - // Per-file issue aggregation. - const issueCount = new Map(); - const worstSev = new Map(); - for (const i of issues) { - issueCount.set(i.file, (issueCount.get(i.file) || 0) + 1); - worstSev.set(i.file, Math.max(worstSev.get(i.file) || 0, i.severity)); - } - - // Choose which files to render; keep highest-impact when over the cap. - let chosen = files; - let truncated = false; - if (files.length > VIZ_NODE_CAP) { - chosen = [...files] - .sort( - (a, b) => - (fanIn.get(b.rel) || 0) * 3 + b.loc / 100 - ((fanIn.get(a.rel) || 0) * 3 + a.loc / 100) - ) - .slice(0, VIZ_NODE_CAP); - truncated = true; - } - const included = new Set(chosen.map((f) => f.rel)); - - const nodes = new Map(); - const edges: GraphEdge[] = []; - - const toPosix = (p: string) => p.split(path.sep).join("/"); - function ensureDir(dir: string): string { - const id = dir === "" || dir === "." ? "." : dir; - if (!nodes.has(id)) { - nodes.set(id, { - id, - label: id === "." ? "/" : path.posix.basename(id), - kind: "dir", - language: null, - loc: 0, - fanIn: 0, - issues: 0, - worstSeverity: 0, - }); - } - return id; - } - // Build the directory chain and containment edges up to root. - function linkChain(relFile: string) { - const posix = toPosix(relFile); - let dir = path.posix.dirname(posix); - let child = posix; - // file's immediate dir -> ... -> root - while (true) { - const dirId = ensureDir(dir); - edges.push({ source: dirId, target: child, kind: "contains" }); - if (dir === "." || dir === "") break; - child = dirId; - dir = path.posix.dirname(dir); - } - } - - for (const f of chosen) { - const posix = toPosix(f.rel); - nodes.set(posix, { - id: posix, - label: path.posix.basename(posix), - kind: "file", - language: LANG_BY_EXT[f.ext] || null, - loc: f.loc, - fanIn: fanIn.get(f.rel) || 0, - issues: issueCount.get(f.rel) || 0, - worstSeverity: worstSev.get(f.rel) || 0, - }); - linkChain(f.rel); - } - - for (const e of importEdges) { - if (included.has(e.from) && included.has(e.to)) { - edges.push({ source: toPosix(e.from), target: toPosix(e.to), kind: "imports" }); - } - } - - return { nodes: [...nodes.values()], edges, truncated }; -} - -/** Build the nested file tree for circle-packing (all files, not capped). */ -function buildTree(files: ScannedFile[], issuesByFile: Map): TreeNode { - const root: TreeNode = { name: "/", path: ".", children: [] }; - const dirCache = new Map([[".", root]]); - - function ensureDir(dirPosix: string): TreeNode { - if (dirCache.has(dirPosix)) return dirCache.get(dirPosix)!; - const parentPath = path.posix.dirname(dirPosix); - const parent = parentPath === dirPosix ? root : ensureDir(parentPath === "" ? "." : parentPath); - const node: TreeNode = { name: path.posix.basename(dirPosix), path: dirPosix, children: [] }; - parent.children!.push(node); - dirCache.set(dirPosix, node); - return node; - } - - for (const f of files) { - const posix = f.rel.split(path.sep).join("/"); - const dirPosix = path.posix.dirname(posix); - const parent = dirPosix === "." || dirPosix === "" ? root : ensureDir(dirPosix); - parent.children!.push({ - name: path.posix.basename(posix), - path: posix, - ext: f.ext, - loc: Math.max(1, f.loc), - issues: issuesByFile.get(f.rel) || 0, - }); - } - return root; -} - -/** Aggregate files into top-level modules + inter-module import edges (flowchart). */ -function buildModuleGraph( - files: ScannedFile[], - importEdges: Array<{ from: string; to: string }>, - issuesByFile: Map -): ModuleGraph { - // Count files per top-level dir; big top dirs get expanded to 2 levels so the - // architecture graph stays meaningful instead of a few giant blobs. - const topCount = new Map(); - for (const f of files) { - const seg = f.rel.split(path.sep).join("/").split("/"); - const top = seg.length > 1 ? seg[0] : "(root)"; - topCount.set(top, (topCount.get(top) || 0) + 1); - } - const EXPAND_THRESHOLD = 12; - const moduleOf = (rel: string): string => { - const seg = rel.split(path.sep).join("/").split("/"); - if (seg.length <= 1) return "(root)"; - const top = seg[0]; - if (seg.length >= 3 && (topCount.get(top) || 0) > EXPAND_THRESHOLD) { - return top + "/" + seg[1]; - } - return top; - }; - - const mods = new Map(); - const langCount = new Map>(); - for (const f of files) { - const id = moduleOf(f.rel); - let m = mods.get(id); - if (!m) { - m = { id, label: id, files: 0, loc: 0, issues: 0, language: null, tier: 0 }; - mods.set(id, m); - langCount.set(id, new Map()); - } - m.files += 1; - m.loc += f.loc; - m.issues += issuesByFile.get(f.rel) || 0; - const lang = LANG_BY_EXT[f.ext]; - if (lang) { - const lc = langCount.get(id)!; - lc.set(lang, (lc.get(lang) || 0) + 1); - } - } - for (const [id, m] of mods) { - const lc = langCount.get(id)!; - let best: string | null = null; - let bestN = 0; - for (const [lang, n] of lc) if (n > bestN) { bestN = n; best = lang; } - m.language = best; - } - - const edgeW = new Map(); - for (const e of importEdges) { - const s = moduleOf(e.from); - const t = moduleOf(e.to); - if (s === t) continue; - const key = s + "→" + t; - const ex = edgeW.get(key); - if (ex) ex.weight += 1; - else edgeW.set(key, { source: s, target: t, weight: 1 }); - } - const edges = [...edgeW.values()]; - - // Assign tiers by longest-path depth (cycles broken by visited guard). - const adj = new Map(); - for (const m of mods.keys()) adj.set(m, []); - for (const e of edges) adj.get(e.source)?.push(e.target); - const tierOf = new Map(); - function depth(node: string, seen: Set): number { - if (tierOf.has(node)) return tierOf.get(node)!; - if (seen.has(node)) return 0; - seen.add(node); - let d = 0; - for (const next of adj.get(node) || []) d = Math.max(d, 1 + depth(next, seen)); - seen.delete(node); - tierOf.set(node, d); - return d; - } - for (const m of mods.keys()) m && (mods.get(m)!.tier = depth(m, new Set())); - - return { nodes: [...mods.values()].sort((a, b) => a.tier - b.tier || b.loc - a.loc), edges }; -} - -/** Full pipeline: scan a repo/folder dir → result (graph + score + viz). */ -export async function indexRepo(root: string): Promise { - _issueSeq = 0; - const churnMap = computeChurn(root); - const { files, languages, loc } = await scan(root); - const { fanIn, importEdges } = await computeImportGraph(files); - - const dep = analyzeDependencies(root); - const codeIssues = await analyzeFiles(files, fanIn, churnMap); - const testIssues = analyzeTests(files); - const issues = [...codeIssues, ...dep.issues, ...testIssues]; - - const dirCount = new Set( - files.map((f) => path.posix.dirname(f.rel.split(path.sep).join("/"))) - ).size; - const graphStats: GraphStats = { - files: files.length, - dirs: dirCount, - dependencies: dep.count, - nodes: files.length + dirCount + dep.count, - edges: importEdges.length + files.length, // imports + containment - }; - - const { dimensions, overall } = score(issues, loc, dep.count); - issues.sort((a, b) => b.severity * b.blastRadius - a.severity * a.blastRadius); - - // Per-file issue counts (shared by viz, tree, modules). - const issuesByFile = new Map(); - for (const i of issues) issuesByFile.set(i.file, (issuesByFile.get(i.file) || 0) + 1); - - const viz = buildVizGraph(files, importEdges, fanIn, issues); - const tree = buildTree(files, issuesByFile); - const modules = buildModuleGraph(files, importEdges, issuesByFile); - - // Symbol-level knowledge graph (code intelligence layer). - const symbolGraph = await buildSymbolGraph( - files - .filter((f) => f.text && extractorFor(f.ext)) - .map((f) => ({ - rel: f.rel.split(path.sep).join("/"), - ext: f.ext, - text: f.text, - language: LANG_BY_EXT[f.ext] || "unknown", - })), - issuesByFile - ); - - return { - loc, - languages, - graphStats, - dimensions, - issues: issues.slice(0, 200), - dependencies: dep.depsList, - churnByFile: Object.fromEntries(churnMap), - score: overall, - viz, - tree, - modules, - symbolGraph, - }; -} - -export function cleanup(dir: string) { - try { - rmSync(dir, { recursive: true, force: true }); - } catch { - /* best effort */ - } -} diff --git a/app/src/lib/indexer/analyze.ts b/app/src/lib/indexer/analyze.ts new file mode 100644 index 0000000..30e655e --- /dev/null +++ b/app/src/lib/indexer/analyze.ts @@ -0,0 +1,176 @@ +import { execSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import type { Dimension, Issue } from "../types"; +import { lintForSecurity } from "../eslintSecurity"; +import { CODE_EXTS } from "./constants"; +import type { ScannedFile } from "./scan"; +import { YIELD_EVERY, yieldToEventLoop } from "./util"; + +interface Rule { + re: RegExp; + dimension: Dimension; + severity: number; + confidence?: number; + title: string; + exts?: Record; + validate?: (line: string, m: RegExpExecArray) => boolean; +} + +// A real secret never contains a literal "..." ellipsis or matches a common +// placeholder word — those are documentation/example conventions. +const PLACEHOLDER_SECRET_RE = /^(\.{3,}|x{4,}|\*{4,}|your[-_ ]?\w*|example\w*|placeholder\w*|changeme|insert[-_ ]?\w*|redacted|dummy|fake|sample|todo|<.*>|\{\{.*\}\})$/i; +function isPlaceholderSecret(value: string): boolean { + return PLACEHOLDER_SECRET_RE.test(value) || value.includes("..."); +} + +// Heuristic, language-agnostic-ish defect/risk rules. +const RULES: Rule[] = [ + { re: /\beval\s*\(/, dimension: "security", severity: 5, confidence: 0.95, title: "Use of eval()" }, + { re: /child_process|os\.system\(|subprocess\.(call|run|Popen)\(/, dimension: "security", severity: 3, confidence: 0.85, title: "Shell/process execution" }, + { + re: /(password|secret|api[_-]?key|token)\s*[:=]\s*['"]([^'"]{6,})['"]/i, + dimension: "security", severity: 5, confidence: 0.8, title: "Possible hardcoded secret", + validate: (_line, m) => !isPlaceholderSecret(m[2]), + }, + { re: /https?:\/\/[^"'\s]*(?, churnByFile: Map): Promise { + const issues: Issue[] = []; + for (let idx = 0; idx < files.length; idx++) { + if (idx > 0 && idx % YIELD_EVERY === 0) await yieldToEventLoop(); + const f = files[idx]; + if (!f.text) continue; + const br = 1 + (fanIn.get(f.rel) || 0); // blast radius from graph fan-in + const ch = churnByFile.get(f.rel) || 1; + const lines = f.text.split("\n"); + for (const rule of RULES) { + if (rule.exts && !rule.exts[f.ext]) continue; + let hits = 0; + for (let i = 0; i < lines.length; i++) { + const m = rule.re.exec(lines[i]); + if (m && (!rule.validate || rule.validate(lines[i], m))) { + issues.push(mkIssue(rule.dimension, rule.severity, rule.title, f.rel, i + 1, br, rule.confidence, ch)); + hits++; + if (hits >= 5) break; // bounded top-N (5) hits per rule per file + } + } + } + // AST-based security detector layer (eslint-plugin-security), catches + // vulnerability classes the line-regex RULES above are structurally blind + // to (ReDoS regex literals, dynamic fs/require paths, weak randomness, ...). + for (const f2 of lintForSecurity(f.text, f.ext)) { + issues.push(mkIssue("security", f2.severity, f2.title, f.rel, f2.line, br, f2.confidence, ch)); + } + + // God-file: very large source file → maintainability penalty scaled by fan-in. + if (f.loc > 600) { + issues.push( + mkIssue("maintainability", f.loc > 1200 ? 4 : 2, `Large file (${f.loc} LOC)`, f.rel, 1, br, 0.9, ch) + ); + } + } + return issues; +} + +/** Extract recent commit counts per file. */ +export function computeChurn(root: string): Map { + const churn = new Map(); + try { + const out = execSync(`git log --since="6.months.ago" --name-only --format=""`, { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); + for (const line of out.split("\n")) { + const f = line.trim(); + if (f) churn.set(f, (churn.get(f) || 0) + 1); + } + } catch { + // Not a git repo, or git not installed + } + return churn; +} + +/** Dependency hygiene from manifests actually present in the repo. */ +export function analyzeDependencies(root: string): { issues: Issue[]; count: number; depsList: string[] } { + const depsList: string[] = []; + const issues: Issue[] = []; + let count = 0; + + const pkgPath = path.join(root, "package.json"); + if (existsSync(pkgPath)) { + try { + const pkg = JSON.parse(readFileSync(pkgPath, "utf8")); + const deps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) }; + count = Object.keys(deps).length; + depsList.push(...Object.keys(deps)); + for (const [name, range] of Object.entries(deps)) { + const v = String(range); + if (v === "*" || v === "latest" || v.startsWith("http") || v.startsWith("git")) { + issues.push(mkIssue("dependency_hygiene", 3, `Unpinned dependency: ${name} (${v})`, "package.json", 1, 2, 1.0)); + } else if (/^[~^]?0\./.test(v)) { + issues.push(mkIssue("dependency_hygiene", 1, `Pre-1.0 dependency: ${name} (${v})`, "package.json", 1, 1, 1.0)); + } + } + if (!existsSync(path.join(root, "package-lock.json")) && + !existsSync(path.join(root, "pnpm-lock.yaml")) && + !existsSync(path.join(root, "yarn.lock"))) { + issues.push(mkIssue("dependency_hygiene", 2, "No lockfile committed", "package.json", 1, 2, 1.0)); + } + } catch { + /* ignore malformed */ + } + } + + const reqPath = path.join(root, "requirements.txt"); + if (existsSync(reqPath)) { + try { + const lines = readFileSync(reqPath, "utf8").split("\n").filter((l) => l.trim() && !l.startsWith("#")); + count += lines.length; + for (const l of lines) { + const m = l.match(/^([A-Za-z0-9_-]+)/); + if (m) depsList.push(m[1]); + if (!/[=<>~]/.test(l)) { + issues.push(mkIssue("dependency_hygiene", 2, `Unpinned dependency: ${l.trim()}`, "requirements.txt", 1, 1, 1.0)); + } + } + } catch { + /* ignore */ + } + } + + return { issues, count, depsList }; +} + +/** Test integrity: presence/ratio of test files. */ +export function analyzeTests(files: ScannedFile[]): Issue[] { + const code = files.filter((f) => CODE_EXTS[f.ext]); + if (code.length === 0) return []; + const tests = code.filter((f) => /(\.|_|\/)(test|spec)/i.test(f.rel) || /(^|\/)tests?\//i.test(f.rel)); + const ratio = tests.length / code.length; + const issues: Issue[] = []; + if (tests.length === 0) { + issues.push(mkIssue("test_integrity", 4, "No test files detected", ".", 1, 3, 0.6)); + } else if (ratio < 0.1) { + issues.push(mkIssue("test_integrity", 2, `Low test coverage ratio (${(ratio * 100).toFixed(0)}% of code files)`, ".", 1, 2, 0.75)); + } + return issues; +} diff --git a/app/src/lib/indexer/clone.ts b/app/src/lib/indexer/clone.ts new file mode 100644 index 0000000..f316684 --- /dev/null +++ b/app/src/lib/indexer/clone.ts @@ -0,0 +1,60 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtempSync, mkdirSync, existsSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +const exec = promisify(execFile); + +export function redactCredentials(s: string): string { + return s.replace(/:\/\/[^\s@/]+@/g, "://"); +} + +/** + * Clone a public git repo. With no `destDir`, clones into a disposable temp + * dir (single-branch, depth 1 — fastest path for one-shot indexing/fix + * sandboxes; caller must rm it). With `destDir`, clones into that exact path + * — used for the editor's persistent workspace, so it fetches all branches + * (bounded depth) to support real branch switching + history. + */ +export async function cloneRepo(url: string, destDir?: string): Promise { + // Allows an optional `user:token@` userinfo component — used for + // authenticated clones (see gitops.withToken); the token itself is never + // logged or persisted by this function, only passed through to `git clone`'s argv. + if (!/^https?:\/\/(?:[^@/]+@)?[\w.-]+\/[\w./~-]+/.test(url)) { + throw new Error("Invalid repository URL. Use a public https git URL."); + } + const dir = destDir ?? mkdtempSync(path.join(tmpdir(), "cg-")); + if (destDir) mkdirSync(path.dirname(destDir), { recursive: true }); + const args = destDir + ? ["clone", "--depth", "50", url, dir] + : ["clone", "--depth", "1", "--single-branch", url, dir]; + try { + await exec("git", args, { + timeout: Number(process.env.CG_CLONE_TIMEOUT_MS) || 90_000, + maxBuffer: 1024 * 1024 * 16, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, + }); + } catch (e) { + if (e instanceof Error) { + e.message = redactCredentials(e.message); + const withCmd = e as Error & { cmd?: string; stderr?: string }; + if (typeof withCmd.cmd === "string") withCmd.cmd = redactCredentials(withCmd.cmd); + if (typeof withCmd.stderr === "string") withCmd.stderr = redactCredentials(withCmd.stderr); + } + throw e; + } + return dir; +} + +/** Validate and resolve a local folder path for indexing (no clone). */ +export function resolveLocalDir(inputPath: string): string { + const resolved = path.resolve(inputPath.replace(/^~(?=$|\/)/, process.env.HOME || "~")); + if (!existsSync(resolved)) { + throw new Error(`Path does not exist: ${resolved}`); + } + if (!statSync(resolved).isDirectory()) { + throw new Error(`Not a directory: ${resolved}`); + } + return resolved; +} diff --git a/app/src/lib/indexer/constants.ts b/app/src/lib/indexer/constants.ts new file mode 100644 index 0000000..a7fb810 --- /dev/null +++ b/app/src/lib/indexer/constants.ts @@ -0,0 +1,28 @@ +// Shared scan/analysis constants for the indexer pipeline. + +export const LANG_BY_EXT: Record = { + ".ts": "TypeScript", ".tsx": "TypeScript", ".js": "JavaScript", ".jsx": "JavaScript", + ".mjs": "JavaScript", ".cjs": "JavaScript", ".py": "Python", ".go": "Go", + ".rs": "Rust", ".java": "Java", ".rb": "Ruby", ".php": "PHP", ".c": "C", + ".h": "C", ".cpp": "C++", ".hpp": "C++", ".cs": "C#", ".swift": "Swift", + ".kt": "Kotlin", ".scala": "Scala", ".sh": "Shell", ".sql": "SQL", + ".css": "CSS", ".scss": "CSS", ".html": "HTML", ".md": "Markdown", + ".json": "JSON", ".yml": "YAML", ".yaml": "YAML", +}; + +export const CODE_EXTS: Record = { + ".ts": true, ".tsx": true, ".js": true, ".jsx": true, ".mjs": true, + ".cjs": true, ".py": true, ".go": true, ".rs": true, ".java": true, + ".rb": true, ".php": true, ".c": true, ".h": true, ".cpp": true, + ".hpp": true, ".cs": true, ".swift": true, ".kt": true, +}; + +export const SKIP_DIRS: Record = { + ".git": true, "node_modules": true, "dist": true, "build": true, + ".next": true, "out": true, "vendor": true, "__pycache__": true, + ".venv": true, "venv": true, "target": true, ".idea": true, + ".vscode": true, "coverage": true, +}; + +export const MAX_FILES = Number(process.env.CG_MAX_FILES) || 4000; +export const MAX_FILE_BYTES = 400_000; diff --git a/app/src/lib/indexer/index.ts b/app/src/lib/indexer/index.ts new file mode 100644 index 0000000..d9deb47 --- /dev/null +++ b/app/src/lib/indexer/index.ts @@ -0,0 +1,83 @@ +import { rmSync } from "node:fs"; +import path from "node:path"; +import type { GraphStats, IndexResult } from "../types"; +import { buildSymbolGraph } from "../codeintel/graph"; +import { extractorFor } from "../codeintel/extractors"; +import { analyzeDependencies, analyzeFiles, analyzeTests, computeChurn, resetIssueSeq } from "./analyze"; +import { LANG_BY_EXT } from "./constants"; +import { computeImportGraph, scan } from "./scan"; +import { score } from "./score"; +import { buildModuleGraph, buildTree, buildVizGraph } from "./viz"; + +export { cloneRepo, redactCredentials, resolveLocalDir } from "./clone"; + +/** Full pipeline: scan a repo/folder dir → result (graph + score + viz). */ +export async function indexRepo(root: string): Promise { + resetIssueSeq(); + const churnMap = computeChurn(root); + const { files, languages, loc } = await scan(root); + const { fanIn, importEdges } = await computeImportGraph(files); + + const dep = analyzeDependencies(root); + const codeIssues = await analyzeFiles(files, fanIn, churnMap); + const testIssues = analyzeTests(files); + const issues = [...codeIssues, ...dep.issues, ...testIssues]; + + const dirCount = new Set( + files.map((f) => path.posix.dirname(f.rel.split(path.sep).join("/"))) + ).size; + const graphStats: GraphStats = { + files: files.length, + dirs: dirCount, + dependencies: dep.count, + nodes: files.length + dirCount + dep.count, + edges: importEdges.length + files.length, // imports + containment + }; + + const { dimensions, overall } = score(issues, loc); + issues.sort((a, b) => b.severity * b.blastRadius - a.severity * a.blastRadius); + + // Per-file issue counts (shared by viz, tree, modules). + const issuesByFile = new Map(); + for (const i of issues) issuesByFile.set(i.file, (issuesByFile.get(i.file) || 0) + 1); + + const viz = buildVizGraph(files, importEdges, fanIn, issues); + const tree = buildTree(files, issuesByFile); + const modules = buildModuleGraph(files, importEdges, issuesByFile); + + // Symbol-level knowledge graph (code intelligence layer). + const symbolGraph = await buildSymbolGraph( + files + .filter((f) => f.text && extractorFor(f.ext)) + .map((f) => ({ + rel: f.rel.split(path.sep).join("/"), + ext: f.ext, + text: f.text, + language: LANG_BY_EXT[f.ext] || "unknown", + })), + issuesByFile + ); + + return { + loc, + languages, + graphStats, + dimensions, + issues: issues.slice(0, 200), + dependencies: dep.depsList, + churnByFile: Object.fromEntries(churnMap), + score: overall, + viz, + tree, + modules, + symbolGraph, + }; +} + +export function cleanup(dir: string) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + /* best effort */ + } +} diff --git a/app/src/lib/indexer/scan.ts b/app/src/lib/indexer/scan.ts new file mode 100644 index 0000000..6124cd7 --- /dev/null +++ b/app/src/lib/indexer/scan.ts @@ -0,0 +1,219 @@ +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import type { LanguageStat } from "../types"; +import { CODE_EXTS, LANG_BY_EXT, MAX_FILES, MAX_FILE_BYTES, SKIP_DIRS } from "./constants"; +import { YIELD_EVERY, yieldToEventLoop } from "./util"; + +export interface ScannedFile { + rel: string; + ext: string; + loc: number; + text: string; + imports: string[]; // resolved-ish relative targets +} + +function walk(root: string): string[] { + const out: string[] = []; + const stack = [root]; + while (stack.length && out.length < MAX_FILES) { + const cur = stack.pop()!; + let entries: string[]; + try { + entries = readdirSync(cur); + } catch { + continue; + } + for (const name of entries) { + const full = path.join(cur, name); + let st; + try { + st = statSync(full); + } catch { + continue; + } + if (st.isDirectory()) { + if (!SKIP_DIRS[name] && !name.startsWith(".")) stack.push(full); + } else if (st.isFile() && st.size <= MAX_FILE_BYTES) { + out.push(full); + } + } + } + return out; +} + +function extractImports(text: string, ext: string): string[] { + const imports: string[] = []; + + // Go: `import "pkg/path"` and grouped `import ( "a" \n alias "b" )`. + if (ext === ".go") { + const block = /import\s*\(([\s\S]*?)\)/g; + let bm; + while ((bm = block.exec(text))) { + const sre = /"([^"]+)"/g; + let sm; + while ((sm = sre.exec(bm[1]))) imports.push(sm[1]); + } + const single = /import\s+(?:[A-Za-z0-9_.]+\s+)?"([^"]+)"/g; + let sm; + while ((sm = single.exec(text))) imports.push(sm[1]); + return imports; + } + + // Python: `import a.b.c`, `from a.b import c, d`, and relative `from .m import x`. + if (ext === ".py") { + for (const rawLine of text.split("\n")) { + const line = rawLine.split("#")[0]; + let m; + if ((m = /^\s*from\s+(\.*[A-Za-z0-9_.]*)\s+import\s+(.+)$/.exec(line))) { + const base = m[1]; + imports.push(base); + for (const part of m[2].split(",")) { + const name = part.trim().split(/\s+as\s+/)[0].trim().replace(/[()]/g, ""); + if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) { + imports.push(base.endsWith(".") || base === "" ? base + name : base + "." + name); + } + } + } else if ((m = /^\s*import\s+(.+)$/.exec(line))) { + for (const part of m[1].split(",")) { + const mod = part.trim().split(/\s+as\s+/)[0].trim(); + if (mod) imports.push(mod); + } + } + } + return imports; + } + + // JS/TS (and other C-family): relative specifiers, resolved against the file dir. + if (CODE_EXTS[ext]) { + const re = /(?:import\s+[^'"]*from\s+|require\(\s*|import\s*\(\s*|from\s+)['"]([^'"]+)['"]/g; + let m; + while ((m = re.exec(text))) { + if (m[1].startsWith(".")) imports.push(m[1]); + } + } + return imports; +} + +/** Walk the repo, build per-file records + language stats. */ +export async function scan(root: string): Promise<{ files: ScannedFile[]; languages: LanguageStat[]; loc: number }> { + const paths = walk(root); + const files: ScannedFile[] = []; + const langMap = new Map(); + let totalLoc = 0; + + for (let idx = 0; idx < paths.length; idx++) { + if (idx > 0 && idx % YIELD_EVERY === 0) await yieldToEventLoop(); + const full = paths[idx]; + const ext = path.extname(full).toLowerCase(); + const lang = LANG_BY_EXT[ext]; + if (!lang) continue; + let text = ""; + try { + text = readFileSync(full, "utf8"); + } catch { + continue; + } + const loc = text.length ? text.split("\n").length : 0; + totalLoc += loc; + const cur = langMap.get(lang) || { files: 0, loc: 0 }; + cur.files += 1; + cur.loc += loc; + langMap.set(lang, cur); + + files.push({ + rel: path.relative(root, full), + ext, + loc, + text: CODE_EXTS[ext] ? text : "", + imports: extractImports(text, ext), + }); + } + + const languages = [...langMap.entries()] + .map(([language, v]) => ({ language, ...v })) + .sort((a, b) => b.loc - a.loc); + + return { files, languages, loc: totalLoc }; +} + +/** Resolve import edges between scanned files + fan-in centrality. */ +export interface ImportGraph { + fanIn: Map; + importEdges: Array<{ from: string; to: string }>; +} + +export async function computeImportGraph(files: ScannedFile[]): Promise { + const toPosix = (r: string) => r.split(path.sep).join("/"); + const byNoExt = new Map(); // JS/TS: path (with/without ext) -> rel + const goDirs = new Map(); // Go: repo dir -> .go files in it + const pyByDotted = new Map(); // Python: dotted module -> rel + + for (const f of files) { + const rel = toPosix(f.rel); + const noExt = rel.replace(/\.[^./]+$/, ""); + byNoExt.set(noExt, f.rel); + byNoExt.set(rel, f.rel); + + if (f.ext === ".go") { + const dir = path.posix.dirname(rel); + (goDirs.get(dir) ?? goDirs.set(dir, []).get(dir)!).push(f.rel); + } else if (f.ext === ".py") { + if (path.posix.basename(noExt) === "__init__") { + const pkg = path.posix.dirname(rel).split("/").filter(Boolean).join("."); + if (pkg) pyByDotted.set(pkg, f.rel); + } else { + pyByDotted.set(noExt.split("/").filter(Boolean).join("."), f.rel); + } + } + } + + const fanIn = new Map(); + const importEdges: Array<{ from: string; to: string }> = []; + const link = (from: string, to: string) => { + if (to && to !== from) { + fanIn.set(to, (fanIn.get(to) || 0) + 1); + importEdges.push({ from, to }); + } + }; + + for (let idx = 0; idx < files.length; idx++) { + if (idx > 0 && idx % YIELD_EVERY === 0) await yieldToEventLoop(); + const f = files[idx]; + const rel = toPosix(f.rel); + const dir = path.posix.dirname(rel); + + for (const imp of f.imports) { + if (f.ext === ".go") { + // Local Go imports share the repo's module prefix; match the longest + // trailing path segment run against an actual repo directory. + const segs = imp.split("/").filter(Boolean); + for (let k = Math.min(segs.length, 8); k >= 1; k--) { + const suffix = segs.slice(segs.length - k).join("/"); + const pkgFiles = goDirs.get(suffix); + if (pkgFiles && suffix !== dir) { + for (const target of pkgFiles) link(f.rel, target); + break; + } + } + } else if (f.ext === ".py") { + let target: string | undefined; + if (imp.startsWith(".")) { + const m = /^(\.+)(.*)$/.exec(imp)!; + const baseParts = dir.split("/").filter(Boolean); + const upParts = baseParts.slice(0, Math.max(0, baseParts.length - (m[1].length - 1))); + const full = [...upParts, ...m[2].split(".").filter(Boolean)].join("."); + target = pyByDotted.get(full); + } else { + target = pyByDotted.get(imp); + } + if (target) link(f.rel, target); + } else { + // JS/TS relative import. + const t = path.posix.normalize(path.posix.join(dir, imp)).replace(/^\.\//, ""); + const cand = byNoExt.get(t) || byNoExt.get(t + "/index") || byNoExt.get(t.replace(/\/$/, "")); + if (cand) link(f.rel, cand); + } + } + } + return { fanIn, importEdges }; +} diff --git a/app/src/lib/indexer/score.ts b/app/src/lib/indexer/score.ts new file mode 100644 index 0000000..1f623d1 --- /dev/null +++ b/app/src/lib/indexer/score.ts @@ -0,0 +1,29 @@ +import type { Dimension, DimensionScore, Issue } from "../types"; +import { DIMENSION_META } from "../types"; + +/** + * Score model (per design doc 07): + * penalty = Σ severity × blastRadius (recency/confidence = 1 here) + * sub_score = 100 × exp(-k · penalty / sizeFactor) + * Larger codebases tolerate more raw penalty (normalized by LOC). + */ +export function score(issues: Issue[], loc: number): { dimensions: DimensionScore[]; overall: number } { + const sizeFactor = Math.max(1, Math.log10(Math.max(loc, 10)) ** 2); // ~1 small → ~10 huge + const k = 0.06; + + const dims: DimensionScore[] = (Object.keys(DIMENSION_META) as Dimension[]).map((dim) => { + const di = issues.filter((i) => i.dimension === dim); + const penalty = di.reduce((s, i) => s + i.severity * i.blastRadius, 0); + const norm = penalty / sizeFactor; + const sub = 100 * Math.exp(-k * norm); + return { + dimension: dim, + score: Math.round(Math.max(0, Math.min(100, sub))), + penalty: Math.round(penalty * 10) / 10, + issueCount: di.length, + }; + }); + + const overall = dims.reduce((s, d) => s + d.score * DIMENSION_META[d.dimension].weight, 0); + return { dimensions: dims, overall: Math.round(overall) }; +} diff --git a/app/src/lib/indexer/util.ts b/app/src/lib/indexer/util.ts new file mode 100644 index 0000000..23c7676 --- /dev/null +++ b/app/src/lib/indexer/util.ts @@ -0,0 +1,16 @@ +// Every CPU-bound per-file loop in the indexer yields back to the event loop +// every YIELD_EVERY files. Without this, indexRepo() runs as one long +// synchronous call — on a large repo (thousands of files, TS type-checking, +// ESLint AST parsing per file) that can block the whole Node process for tens +// of seconds, during which NOTHING else can be served: not the dashboard, not +// other API routes, not even Render's health check -- which is exactly what +// produces the "stuck on an old page, then 502 Bad Gateway" symptom on a +// large first-time index. Yielding periodically lets the event loop drain +// other pending requests between chunks of indexing work. +export const YIELD_EVERY = 15; + +export function yieldToEventLoop(): Promise { + const { promise, resolve } = Promise.withResolvers(); + setImmediate(resolve); + return promise; +} diff --git a/app/src/lib/indexer/viz.ts b/app/src/lib/indexer/viz.ts new file mode 100644 index 0000000..d5f1443 --- /dev/null +++ b/app/src/lib/indexer/viz.ts @@ -0,0 +1,208 @@ +import path from "node:path"; +import type { GraphEdge, GraphNode, Issue, ModuleEdge, ModuleGraph, ModuleNode, TreeNode, VizGraph } from "../types"; +import { LANG_BY_EXT } from "./constants"; +import type { ScannedFile } from "./scan"; + +const VIZ_NODE_CAP = 350; + +/** Build the renderable node/edge graph (files + dirs + import/containment edges). */ +export function buildVizGraph( + files: ScannedFile[], + importEdges: Array<{ from: string; to: string }>, + fanIn: Map, + issues: Issue[] +): VizGraph { + // Per-file issue aggregation. + const issueCount = new Map(); + const worstSev = new Map(); + for (const i of issues) { + issueCount.set(i.file, (issueCount.get(i.file) || 0) + 1); + worstSev.set(i.file, Math.max(worstSev.get(i.file) || 0, i.severity)); + } + + // Choose which files to render; keep highest-impact when over the cap. + let chosen = files; + let truncated = false; + if (files.length > VIZ_NODE_CAP) { + chosen = [...files] + .sort( + (a, b) => + (fanIn.get(b.rel) || 0) * 3 + b.loc / 100 - ((fanIn.get(a.rel) || 0) * 3 + a.loc / 100) + ) + .slice(0, VIZ_NODE_CAP); + truncated = true; + } + const included = new Set(chosen.map((f) => f.rel)); + + const nodes = new Map(); + const edges: GraphEdge[] = []; + + const toPosix = (p: string) => p.split(path.sep).join("/"); + function ensureDir(dir: string): string { + const id = dir === "" || dir === "." ? "." : dir; + if (!nodes.has(id)) { + nodes.set(id, { + id, + label: id === "." ? "/" : path.posix.basename(id), + kind: "dir", + language: null, + loc: 0, + fanIn: 0, + issues: 0, + worstSeverity: 0, + }); + } + return id; + } + // Build the directory chain and containment edges up to root. + function linkChain(relFile: string) { + const posix = toPosix(relFile); + let dir = path.posix.dirname(posix); + let child = posix; + // file's immediate dir -> ... -> root + while (true) { + const dirId = ensureDir(dir); + edges.push({ source: dirId, target: child, kind: "contains" }); + if (dir === "." || dir === "") break; + child = dirId; + dir = path.posix.dirname(dir); + } + } + + for (const f of chosen) { + const posix = toPosix(f.rel); + nodes.set(posix, { + id: posix, + label: path.posix.basename(posix), + kind: "file", + language: LANG_BY_EXT[f.ext] || null, + loc: f.loc, + fanIn: fanIn.get(f.rel) || 0, + issues: issueCount.get(f.rel) || 0, + worstSeverity: worstSev.get(f.rel) || 0, + }); + linkChain(f.rel); + } + + for (const e of importEdges) { + if (included.has(e.from) && included.has(e.to)) { + edges.push({ source: toPosix(e.from), target: toPosix(e.to), kind: "imports" }); + } + } + + return { nodes: [...nodes.values()], edges, truncated }; +} + +/** Build the nested file tree for circle-packing (all files, not capped). */ +export function buildTree(files: ScannedFile[], issuesByFile: Map): TreeNode { + const root: TreeNode = { name: "/", path: ".", children: [] }; + const dirCache = new Map([[".", root]]); + + function ensureDir(dirPosix: string): TreeNode { + if (dirCache.has(dirPosix)) return dirCache.get(dirPosix)!; + const parentPath = path.posix.dirname(dirPosix); + const parent = parentPath === dirPosix ? root : ensureDir(parentPath === "" ? "." : parentPath); + const node: TreeNode = { name: path.posix.basename(dirPosix), path: dirPosix, children: [] }; + parent.children!.push(node); + dirCache.set(dirPosix, node); + return node; + } + + for (const f of files) { + const posix = f.rel.split(path.sep).join("/"); + const dirPosix = path.posix.dirname(posix); + const parent = dirPosix === "." || dirPosix === "" ? root : ensureDir(dirPosix); + parent.children!.push({ + name: path.posix.basename(posix), + path: posix, + ext: f.ext, + loc: Math.max(1, f.loc), + issues: issuesByFile.get(f.rel) || 0, + }); + } + return root; +} + +/** Aggregate files into top-level modules + inter-module import edges (flowchart). */ +export function buildModuleGraph( + files: ScannedFile[], + importEdges: Array<{ from: string; to: string }>, + issuesByFile: Map +): ModuleGraph { + // Count files per top-level dir; big top dirs get expanded to 2 levels so the + // architecture graph stays meaningful instead of a few giant blobs. + const topCount = new Map(); + for (const f of files) { + const seg = f.rel.split(path.sep).join("/").split("/"); + const top = seg.length > 1 ? seg[0] : "(root)"; + topCount.set(top, (topCount.get(top) || 0) + 1); + } + const EXPAND_THRESHOLD = 12; + const moduleOf = (rel: string): string => { + const seg = rel.split(path.sep).join("/").split("/"); + if (seg.length <= 1) return "(root)"; + const top = seg[0]; + if (seg.length >= 3 && (topCount.get(top) || 0) > EXPAND_THRESHOLD) { + return top + "/" + seg[1]; + } + return top; + }; + + const mods = new Map(); + const langCount = new Map>(); + for (const f of files) { + const id = moduleOf(f.rel); + let m = mods.get(id); + if (!m) { + m = { id, label: id, files: 0, loc: 0, issues: 0, language: null, tier: 0 }; + mods.set(id, m); + langCount.set(id, new Map()); + } + m.files += 1; + m.loc += f.loc; + m.issues += issuesByFile.get(f.rel) || 0; + const lang = LANG_BY_EXT[f.ext]; + if (lang) { + const lc = langCount.get(id)!; + lc.set(lang, (lc.get(lang) || 0) + 1); + } + } + for (const [id, m] of mods) { + const lc = langCount.get(id)!; + let best: string | null = null; + let bestN = 0; + for (const [lang, n] of lc) if (n > bestN) { bestN = n; best = lang; } + m.language = best; + } + + const edgeW = new Map(); + for (const e of importEdges) { + const s = moduleOf(e.from); + const t = moduleOf(e.to); + if (s === t) continue; + const key = s + "→" + t; + const ex = edgeW.get(key); + if (ex) ex.weight += 1; + else edgeW.set(key, { source: s, target: t, weight: 1 }); + } + const edges = [...edgeW.values()]; + + // Assign tiers by longest-path depth (cycles broken by visited guard). + const adj = new Map(); + for (const m of mods.keys()) adj.set(m, []); + for (const e of edges) adj.get(e.source)?.push(e.target); + const tierOf = new Map(); + function depth(node: string, seen: Set): number { + if (tierOf.has(node)) return tierOf.get(node)!; + if (seen.has(node)) return 0; + seen.add(node); + let d = 0; + for (const next of adj.get(node) || []) d = Math.max(d, 1 + depth(next, seen)); + seen.delete(node); + tierOf.set(node, d); + return d; + } + for (const m of mods.keys()) m && (mods.get(m)!.tier = depth(m, new Set())); + + return { nodes: [...mods.values()].sort((a, b) => a.tier - b.tier || b.loc - a.loc), edges }; +} diff --git a/docs/PROGRESS_TRACKER.md b/docs/PROGRESS_TRACKER.md index 07cc635..48e9675 100644 --- a/docs/PROGRESS_TRACKER.md +++ b/docs/PROGRESS_TRACKER.md @@ -153,3 +153,5 @@ Added from a direct code-reading assessment of the actual swarm implementation ( | 2026-07-13 | **Executed all 16 tasks of Phase 6** (Agent & Analysis Accuracy) end to end. Root-cause fixes: position-aware caller attribution (`findEnclosingCaller`) + per-file import-aware resolution replace the old "attribute every call to the file's first function" heuristic and global-name-only lookup; iterative Tarjan; bounded per-rule findings. Replaced the RSS-gated (disabled-by-default) tree-sitter path entirely with the TypeScript Compiler API, including a real in-memory `ts.Program` for `checker.getSymbolAtLocation`-based type-aware call resolution. Evidence-derived confidence, locus-based critic dedup, a real fan-in∩untested-files Test agent, AST-computed cyclomatic complexity, and a genuine git-churn hotspot signal threaded into judge scoring. Expanded the Fixer roster from 1 to 3 (found and fixed a real latent line-number-drift bug in fixer chaining along the way; extended the diff builder to handle replacements, not just deletions). Added 37 new tests across 5 new test files (`resolution`, `churn`, `orchestrator`, `specialists`, `eslintSecurity`) plus extended `executor.test.ts` — suite grew from 96 to 133 passing, typecheck clean, `next build` clean, each task verified individually before being marked done. Surfaced the `MAX_SYMBOLS` truncation flag in both the agent summary and a new UI banner. Added a second, AST-based security detector layer (`eslint-plugin-security`, curated 10-rule subset) and shallow graph-verified taint reachability (source-parameter heuristic → sink issue, BFS via a new `reachableCallees()`) to the Security agent. See Phase 6 above for the full per-task breakdown. | | 2026-07-13 | **Executed Phase 7 (Critical Security Remediation)**: closed the 6 audit findings fixable without a product-behavior tradeoff — F006 (github.com host gate before ever attaching a token to a remote, the real severe primitive behind F001/F006), F003 (symlink-aware `resolveSafe`), F002 (symlink-aware executor sandbox walk), F004 (credential redaction in clone-failure errors), F005 (job-ownership check), F010 (OAuth open-redirect guard). Also discovered and fixed a missing `eslint.config.mjs` (`npm run lint` was completely non-functional — no config file existed at all). 19 new regression tests (`tests/security-hardening.test.ts`); full suite 154/154; `tsc --noEmit`, `npm run lint`, and `next build` all clean. Deliberately did NOT apply F001's literal "bind token to session" suggestion — it would have removed the intentional anonymous bring-your-own-PAT push/fix feature; flagged as a residual, lower-severity risk instead. 21 of 27 Security findings and all other categories from the 07-12 audit remain open — see Phase 7 in this file. | 2026-07-17 | **Executed Phase 7's follow-up pass** (tasks 7.9–7.20): closed 11 more of the 21 previously-open Security findings from `docs/AUDIT_2026-07-12.md` — session expiry enforced server-side (F012), Secure-cookie flag derived from the real request scheme (F013), a write-size cap on editor writes/uploads (F011), rate limiting on the OAuth callback/`/api/index`/`/api/browse` (F015, verified live against a running dev server: 429s kick in exactly as expected), optional `/api/browse` root-scoping (F016), credential redaction in executor PR-failure logs (F017), dropped `uptime`/`ts` from the public health payload while deliberately keeping `localAccessAllowed` (F019 — the audit's literal recommendation would have broken a real frontend UX dependency), constant-time OAuth state comparison (F022), sanitized client-facing error messages on 3 routes (F023), and CDN CSP scoping (F021), plus exact-pinning the two tree-sitter packages that parse untrusted repo content (F068). Also fixed, unprompted, a live production bug reported mid-session: the Claude Agent SDK's platform-specific native binary was silently dropped from `.next/standalone`'s output by Next.js's file tracer (confirmed via a real `next build` — this affected every deployment target, not just Render's linux-x64) — isolated onto its own branch/PR (#13) per the user's explicit request, kept separate from the security work. 24 new regression tests (`tests/security-hardening-2.test.ts`); full suite 207/207; `tsc --noEmit`, `npm run lint` (baseline unchanged), and `next build` all clean. Remaining 10 Security findings are either testing-debt (F077/F079/F086/F087) or explicit product/infra tradeoffs (F001/F014/F018/F020/F052/F074) — see Phase 7's closing note for the reasoning behind each deferral. | + +| 2026-07-23 | **Organizational audit + targeted incremental refactor** (scoped down from a generic full-layered-architecture request after grounding against this tracker: security/CI/tests/docs were already done; the real gap was code organization). Audit: no debug leftovers/dead TODOs (the only regex hits were inside the indexer's own rule-definition strings), no genuine circular dependencies (`madge` flagged 3, all resolve to `import type`-only edges erased at compile time — verified by reading every import line by hand), and — per the app's *own* established god-file bar (`indexer.ts`'s own rule flags >600 LOC as a maintainability issue) — exactly one file crossed it: `indexer.ts` itself (803 lines, 6 mixed concerns). `api.ts` (473) and `settings/page.tsx` (582) stayed below the project's own bar and were deliberately left alone rather than split for the sake of a generic template. **Refactor:** split `lib/indexer.ts` into `lib/indexer/{constants,util,clone,scan,analyze,score,viz,index}.ts` (directory-import convention already used by `lib/agents/`, `lib/codeintel/`, `lib/gitops/` — zero callsite changes, all 4 existing importers resolve unchanged) — also dropped an unused `SymbolGraph` type import found in the process. Separately found and fixed real duplication: the exact `repoAccessDenied` + `getWorkspaceDir`-or-404 sequence was hand-copied across 8 call sites in 4 route files (`fs`, `git`, `search`, `timeline`) — the same class of copy-paste gap that caused the Phase 0.6 cross-tenant leak. Consolidated into `requireWorkspace()` in `authz.ts`; also removed `fs/route.ts`'s `workspaceOr404` (a one-line no-op wrapper around `getWorkspaceDir`). Verified: `tsc --noEmit` clean, full suite 265/265, `next build` clean, lint baseline unchanged (22/17, same pre-existing component-file findings as before, none in touched files), and a live `next dev` smoke test — 404 "Repo not found" on all 4 refactored routes for a nonexistent/unowned repo id, then a real end-to-end index of `octocat/Hello-World` through the split indexer modules (score computed, fs/git/search all correctly gated through `requireWorkspace`). |