From 7e561a55cee29b90a94c260cfb5b182306cf6204 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 17 May 2026 01:00:51 +0800 Subject: [PATCH] feat(mapper): add FastAPI route mapping --- CHANGELOG.md | 1 + docs/feature-mapping.md | 15 ++-- src/detect.ts | 141 +++++++++++++++++++++++------ src/mapper.test.ts | 110 +++++++++++++++++++++++ src/mappers/python.ts | 194 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 422 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7cbf81..af9164f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Added FastAPI route feature mapping and kept root/web Python project detection in sync. - Added Flask route feature mapping for Python projects, including `web/` source roots, common root entry files, non-list method literals, and Python framework detection. - Added Next.js route mapping for `src/app` and `src/pages` layouts, thanks @obatried. - Added first-pass Python mapping for project metadata, console scripts, source groups, pytest suites, and conservative validation defaults, thanks @xiamx. diff --git a/docs/feature-mapping.md b/docs/feature-mapping.md index 5a256de..1e97acb 100644 --- a/docs/feature-mapping.md +++ b/docs/feature-mapping.md @@ -33,7 +33,8 @@ Supported deterministic mappers today: - Next.js `app/` and `pages/` routes - Go `cmd/*/main.go` - Go `internal/*` packages -- Python project metadata, console scripts, bounded source groups, pytest suites, and Flask routes +- Python project metadata, console scripts, root app files, bounded source groups, + pytest suites, and Flask/FastAPI routes - Rust Cargo commands, libraries, workspace crates, and integration tests - SwiftPM executable targets, library targets, and test suites - nested SwiftPM packages @@ -54,15 +55,15 @@ discovered below the repo root, Apple projects are grouped by Swift source area, and Gradle modules are grouped from `src/main`, `src/test`, and `src/androidTest`. Python mapping covers `pyproject.toml` metadata, `[project.scripts]` and -`[tool.poetry.scripts]` console scripts, source groups under common Python -source roots including `web/`, pytest files, and Flask `@*.route(...)` -handlers in source roots and common root entry files such as `app.py` and -`wsgi.py`. Flask route methods are read from list, tuple, or set literals. -Framework-specific route mapping for FastAPI and Django is not implemented yet. +`[tool.poetry.scripts]` console scripts, root app files, source groups under +common Python source roots including `web/`, pytest files, Flask `@*.route(...)` +handlers, and FastAPI `@*.get(...)` / `@*.api_route(...)` handlers. Flask and +FastAPI route methods are read from list, tuple, or set literals. FastAPI paths +can be positional strings or literal `path=` keywords. Known gaps: - no Express/Fastify/Hono route mapper yet -- no FastAPI/Django route mapper yet +- no Django route mapper yet - no import graph expansion beyond nearby tests yet - no agent enrichment yet diff --git a/src/detect.ts b/src/detect.ts index 5e2afa4..09ee5c1 100644 --- a/src/detect.ts +++ b/src/detect.ts @@ -682,6 +682,11 @@ async function detectFrameworks(root: string, pkg: PackageJson | null): Promise< frameworks.push(name); } } + for (const name of await pythonImportedFrameworks(root)) { + if (!frameworks.includes(name)) { + frameworks.push(name); + } + } } return frameworks; } @@ -748,14 +753,92 @@ async function isPythonProject(root: string): Promise { } async function containsReviewablePythonFile(root: string): Promise { - for (const prefix of ["src", "app", "apps", "lib", "scripts"]) { - if (await containsFileWithExtension(join(root, prefix), ".py", 4)) { + if (await containsRootReviewablePythonFile(root)) { + return true; + } + for (const prefix of pythonSourceSearchRoots) { + if (await containsFileMatching(join(root, prefix), 4, isReviewablePythonFileName)) { return true; } } return containsFileNamed(root, "__init__.py", 3); } +const pythonSourceSearchRoots = ["src", "app", "apps", "lib", "scripts", "web"] as const; + +async function containsRootReviewablePythonFile(root: string): Promise { + return (await readdir(root, { withFileTypes: true }).catch(() => [])).some( + (entry) => entry.isFile() && isReviewablePythonFileName(entry.name), + ); +} + +function isReviewablePythonFileName(entry: string): boolean { + return ( + entry.endsWith(".py") && + !/^test_[^/]+\.py$/u.test(entry) && + !entry.endsWith("_test.py") && + !/(?:generated|_pb2|_pb2_grpc|\.gen)\.py$/iu.test(entry) + ); +} + +async function pythonImportedFrameworks(root: string): Promise { + const frameworks = new Set(); + for (const path of await pythonFrameworkScanFiles(root)) { + const source = await readFile(path, "utf8").catch(() => ""); + for (const name of ["flask", "fastapi", "django"] as const) { + const importPattern = new RegExp( + `^\\s*(?:from\\s+${name}\\s+import\\s+|import\\s+${name}\\b)`, + "mu", + ); + if (importPattern.test(source)) { + frameworks.add(name); + } + } + } + return [...frameworks]; +} + +async function pythonFrameworkScanFiles(root: string): Promise { + const files: string[] = []; + for (const entry of await readdir(root, { withFileTypes: true }).catch(() => [])) { + if (entry.isFile() && isReviewablePythonFileName(entry.name)) { + files.push(join(root, entry.name)); + } + } + for (const prefix of pythonSourceSearchRoots) { + await collectPythonFrameworkScanFiles(join(root, prefix), 4, files); + } + return [...new Set(files)].slice(0, 200); +} + +async function collectPythonFrameworkScanFiles( + dir: string, + remainingDepth: number, + files: string[], +): Promise { + if (remainingDepth < 0 || !(await pathExists(dir))) { + return; + } + const dirInfo = await lstat(dir); + if (!dirInfo.isDirectory() || dirInfo.isSymbolicLink()) { + return; + } + for (const entry of await readdir(dir, { withFileTypes: true })) { + if (shouldSkipSearchEntry(entry.name)) { + continue; + } + const full = join(dir, entry.name); + if (entry.isSymbolicLink()) { + continue; + } + if (entry.isFile() && isReviewablePythonFileName(entry.name)) { + files.push(full); + } else if (entry.isDirectory()) { + await collectPythonFrameworkScanFiles(full, remainingDepth - 1, files); + } + } +} + async function containsFileNamed(root: string, name: string, maxDepth: number): Promise { return containsFileMatching(root, maxDepth, (entry) => entry === name); } @@ -781,32 +864,7 @@ async function containsFileMatching( return false; } for (const entry of await readdir(dir)) { - if ( - [ - "node_modules", - "dist", - "build", - "target", - ".build", - ".swiftpm", - ".git", - ".clawpatch", - ".worktrees", - ".venv", - "venv", - "__pycache__", - ".mypy_cache", - ".ruff_cache", - ".pytest_cache", - "fixtures", - "__fixtures__", - "testdata", - "Pods", - "Carthage", - "SourcePackages", - "DerivedData", - ].includes(entry) - ) { + if (shouldSkipSearchEntry(entry)) { continue; } const full = join(dir, entry); @@ -824,6 +882,33 @@ async function containsFileMatching( return false; } +function shouldSkipSearchEntry(entry: string): boolean { + return [ + "node_modules", + "dist", + "build", + "target", + ".build", + ".swiftpm", + ".git", + ".clawpatch", + ".worktrees", + ".venv", + "venv", + "__pycache__", + ".mypy_cache", + ".ruff_cache", + ".pytest_cache", + "fixtures", + "__fixtures__", + "testdata", + "Pods", + "Carthage", + "SourcePackages", + "DerivedData", + ].includes(entry); +} + function stripLineComments(source: string, marker: "//"): string { return source .split("\n") diff --git a/src/mapper.test.ts b/src/mapper.test.ts index ea6687e..5f87c22 100644 --- a/src/mapper.test.ts +++ b/src/mapper.test.ts @@ -1419,6 +1419,116 @@ let package = Package(name: "HybridApp", targets: [.target(name: "HybridApp")]) expect(result.features.some((feature) => feature.source === "python-flask-route")).toBe(false); }); + it("maps FastAPI routes in root and web source files", async () => { + const root = await fixtureRoot("clawpatch-python-fastapi-routes-"); + await writeFixture(root, "requirements.txt", "fastapi\npytest\n"); + await writeFixture( + root, + "app.py", + [ + "from fastapi import FastAPI", + "", + "app = FastAPI()", + "", + "@app.get('/health')", + "async def health():", + " return {'ok': True}", + "", + "@app.api_route('/webhook/{token}', methods=['GET', 'HEAD'])", + "def webhook(token: str):", + " return token", + "", + "@app.api_route('/submit', methods=('POST',))", + "def submit():", + " return {'ok': True}", + "", + ].join("\n"), + ); + await writeFixture( + root, + "web/api.py", + [ + "from fastapi import APIRouter", + "", + "router = APIRouter()", + "", + "@router.post(", + " path='/admin/jobs',", + ")", + "def create_job():", + " return {'queued': True}", + "", + ].join("\n"), + ); + await writeFixture(root, "tests/test_app.py", "def test_health():\n pass\n"); + + const project = await detectProject(root); + const result = await mapFeatures(root, project, []); + const health = result.features.find((feature) => feature.title === "FastAPI route GET /health"); + const webhook = result.features.find( + (feature) => feature.title === "FastAPI route GET,HEAD /webhook/{token}", + ); + const submit = result.features.find( + (feature) => feature.title === "FastAPI route POST /submit", + ); + const admin = result.features.find( + (feature) => feature.title === "FastAPI route POST /admin/jobs", + ); + + expect(project.detected.frameworks).toContain("fastapi"); + expect(health?.source).toBe("python-fastapi-route"); + expect(health?.entrypoints[0]).toMatchObject({ + path: "app.py", + symbol: "health", + route: "GET /health", + }); + expect(health?.tests).toEqual([{ path: "tests/test_app.py", command: "pytest" }]); + expect(webhook?.entrypoints[0]?.route).toBe("GET,HEAD /webhook/{token}"); + expect(submit?.entrypoints[0]?.route).toBe("POST /submit"); + expect(admin?.entrypoints[0]).toMatchObject({ + path: "web/api.py", + symbol: "create_job", + route: "POST /admin/jobs", + }); + expect(admin?.trustBoundaries).toContain("auth"); + }); + + it("detects metadata-free root and web Python sources", async () => { + const root = await fixtureRoot("clawpatch-python-root-web-detect-"); + await writeFixture(root, "app.py", "def app():\n pass\n"); + await writeFixture( + root, + "web/api.py", + [ + "from fastapi import APIRouter", + "", + "router = APIRouter()", + "", + "@router.get(path='/health')", + "def health():", + " return {'ok': True}", + "", + ].join("\n"), + ); + + const project = await detectProject(root); + const result = await mapFeatures(root, project, []); + const rootSource = result.features.find((feature) => feature.title === "Python source root"); + const webRoute = result.features.find( + (feature) => feature.title === "FastAPI route GET /health", + ); + + expect(project.detected.languages).toContain("python"); + expect(project.detected.packageManagers).toContain("python"); + expect(project.detected.frameworks).toContain("fastapi"); + expect(rootSource?.ownedFiles).toEqual([{ path: "app.py", reason: "source group root" }]); + expect(webRoute?.entrypoints[0]).toMatchObject({ + path: "web/api.py", + symbol: "health", + route: "GET /health", + }); + }); + it("uses Hatch pytest commands in mapped Python features", async () => { const root = await fixtureRoot("clawpatch-python-hatch-map-"); await writeFixture( diff --git a/src/mappers/python.ts b/src/mappers/python.ts index 8ef8bbc..d04ed2a 100644 --- a/src/mappers/python.ts +++ b/src/mappers/python.ts @@ -24,6 +24,13 @@ type FlaskRoute = { methods: string[]; }; +type FastApiRoute = { + filePath: string; + functionName: string; + routePath: string; + methods: string[]; +}; + type SourceGroup = { label: string; files: string[]; @@ -36,6 +43,19 @@ type PyprojectInfo = { }; const sourceRoots = ["src", "app", "apps", "lib", "scripts", "web"] as const; +const fastApiRouteTargetPattern = [ + "(?:[A-Za-z_][A-Za-z0-9_]*\\.)*", + "(?:app|application|api|router|[A-Za-z_][A-Za-z0-9_]*(?:app|api|router))", +].join(""); +const fastApiRouteMethods = "api_route|get|post|put|patch|delete|options|head|trace"; +const fastApiRouteDecoratorStartPattern = new RegExp( + `^@${fastApiRouteTargetPattern}\\.(?:${fastApiRouteMethods})\\(`, + "u", +); +const fastApiRouteDecoratorPattern = new RegExp( + `^\\s*@${fastApiRouteTargetPattern}\\.(${fastApiRouteMethods})\\((.*)\\)\\s*(?:#.*)?$`, + "u", +); const projectMetadataFiles = [ "pyproject.toml", "setup.py", @@ -117,6 +137,10 @@ export async function pythonSeeds(root: string): Promise { seeds.push(route); } + for (const route of await fastApiRouteSeeds(root, testFiles, testCommand)) { + seeds.push(route); + } + for (const group of await pythonSourceGroups(root)) { const tests = associatedTests(group.files, testFiles, testCommand); seeds.push({ @@ -243,6 +267,7 @@ async function dependencyFileHas(root: string, dependency: string): Promise { const groups: SourceGroup[] = []; + groups.push(...(await rootPythonSourceGroups(root))); const seenRoots = new Set(); for (const sourceRoot of await pythonSourceRoots(root)) { if (seenRoots.has(sourceRoot)) { @@ -257,6 +282,17 @@ async function pythonSourceGroups(root: string): Promise { return groups; } +async function rootPythonSourceGroups(root: string): Promise { + return chunkFiles("root", await rootPythonSourceFiles(root), sourceGroupMaxOwnedFiles); +} + +async function rootPythonSourceFiles(root: string): Promise { + return (await readdir(root, { withFileTypes: true }).catch(() => [])) + .filter((entry) => entry.isFile() && isReviewablePythonSourceFile(entry.name)) + .map((entry) => entry.name) + .toSorted(); +} + async function pythonSourceRoots(root: string): Promise { const roots: string[] = []; for (const sourceRoot of sourceRoots) { @@ -330,6 +366,142 @@ async function resolvePythonScript( return { entryPath: "pyproject.toml", symbol }; } +async function fastApiRouteSeeds( + root: string, + testFiles: string[], + testCommand: string | null, +): Promise { + const routeFiles = uniquePaths([ + ...(await rootPythonSourceFiles(root)), + ...(await walk(root, await pythonSourceRoots(root))).filter(isReviewablePythonSourceFile), + ]); + const seeds: FeatureSeed[] = []; + for (const filePath of routeFiles) { + const source = await readFile(join(root, filePath), "utf8"); + if (!sourceLooksFastApi(source)) { + continue; + } + const routes = parseFastApiRoutes(filePath, source); + for (const route of routes) { + const methodLabel = route.methods.join(","); + const tests = associatedTests([route.filePath], testFiles, testCommand); + seeds.push({ + title: `FastAPI route ${methodLabel} ${route.routePath}`, + summary: + `FastAPI route ${methodLabel} ${route.routePath} handled by ` + + `${route.functionName} in ${route.filePath}.`, + kind: "route", + source: "python-fastapi-route", + confidence: "high", + entryPath: route.filePath, + symbol: route.functionName, + route: `${methodLabel} ${route.routePath}`, + command: null, + ownedFiles: [ + { path: route.filePath, reason: `FastAPI route handler ${route.functionName}` }, + ], + contextFiles: tests.map((test) => ({ path: test.path, reason: "associated test" })), + tests, + tags: ["python", "fastapi", "route"], + trustBoundaries: fastApiRouteTrustBoundaries(route), + testCommand, + skipNearbyTests: true, + }); + } + } + return seeds; +} + +function sourceLooksFastApi(source: string): boolean { + return /^\s*(?:from\s+fastapi\s+import\s+|import\s+fastapi\b)/mu.test(source); +} + +function parseFastApiRoutes(filePath: string, source: string): FastApiRoute[] { + const routes: FastApiRoute[] = []; + let pending: Array<{ routePath: string; methods: string[] }> = []; + let decoratorSource: string | null = null; + let decoratorDepth = 0; + for (const line of source.split("\n")) { + const trimmed = line.trim(); + if (decoratorSource !== null) { + decoratorSource = `${decoratorSource} ${trimmed}`; + decoratorDepth += parenDelta(trimmed); + if (decoratorDepth <= 0) { + const route = parseFastApiRouteDecorator(decoratorSource); + if (route !== null) { + pending.push(route); + } + decoratorSource = null; + decoratorDepth = 0; + } + continue; + } + + if (startsFastApiRouteDecorator(trimmed)) { + decoratorSource = trimmed; + decoratorDepth = parenDelta(trimmed); + if (decoratorDepth <= 0) { + const route = parseFastApiRouteDecorator(decoratorSource); + if (route !== null) { + pending.push(route); + } + decoratorSource = null; + decoratorDepth = 0; + } + continue; + } + + const functionName = /^\s*(?:async\s+)?def\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/u.exec(line)?.[1]; + if (functionName !== undefined && pending.length > 0) { + for (const item of pending) { + routes.push({ filePath, functionName, ...item }); + } + pending = []; + continue; + } + + if ( + pending.length > 0 && + trimmed !== "" && + !trimmed.startsWith("@") && + !trimmed.startsWith("#") + ) { + pending = []; + } + } + return routes; +} + +function startsFastApiRouteDecorator(line: string): boolean { + return fastApiRouteDecoratorStartPattern.test(line); +} + +function parseFastApiRouteDecorator(line: string): { routePath: string; methods: string[] } | null { + const match = fastApiRouteDecoratorPattern.exec(line); + const method = match?.[1]; + const args = match?.[2]; + if (method === undefined || args === undefined) { + return null; + } + const routePath = parseFastApiPath(args); + if (routePath === null) { + return null; + } + const methods = method === "api_route" ? parsePythonRouteMethods(args) : [method.toUpperCase()]; + if (methods === null) { + return null; + } + return { routePath, methods }; +} + +function parseFastApiPath(args: string): string | null { + const positional = /^\s*(["'])(.*?)\1/u.exec(args)?.[2]; + if (positional !== undefined) { + return positional; + } + return /\bpath\s*=\s*(["'])(.*?)\1/u.exec(args)?.[2] ?? null; +} + async function flaskRouteSeeds( root: string, testFiles: string[], @@ -464,7 +636,7 @@ function parseFlaskRouteDecorator(line: string): { routePath: string; methods: s if (match?.[2] === undefined) { return null; } - const methods = parseFlaskMethods(match[3] ?? ""); + const methods = parsePythonRouteMethods(match[3] ?? ""); if (methods === null) { return null; } @@ -474,12 +646,12 @@ function parseFlaskRouteDecorator(line: string): { routePath: string; methods: s }; } -function parseFlaskMethods(args: string): string[] | null { +function parsePythonRouteMethods(args: string): string[] | null { const methodsIndex = args.search(/\bmethods\s*=/u); if (methodsIndex === -1) { return ["GET"]; } - const literal = flaskMethodsLiteral(args.slice(methodsIndex)); + const literal = pythonRouteMethodsLiteral(args.slice(methodsIndex)); if (literal === null) { return null; } @@ -489,7 +661,7 @@ function parseFlaskMethods(args: string): string[] | null { return methods.length > 0 ? [...new Set(methods)] : null; } -function flaskMethodsLiteral(source: string): string | null { +function pythonRouteMethodsLiteral(source: string): string | null { const match = /^\s*methods\s*=\s*([[({])/u.exec(source); if (match === null) { return null; @@ -573,6 +745,17 @@ function flaskRouteTrustBoundaries(route: FlaskRoute): FeatureSeed["trustBoundar return boundaries; } +function fastApiRouteTrustBoundaries(route: FastApiRoute): FeatureSeed["trustBoundaries"] { + const boundaries: FeatureSeed["trustBoundaries"] = ["network", "user-input", "serialization"]; + if ( + route.methods.some((method) => method !== "GET") || + /(^|\/)(admin|auth|login|token)(\/|$)/iu.test(route.routePath) + ) { + boundaries.push("auth"); + } + return boundaries; +} + function standaloneTestSuites(testFiles: string[], command: string | null): FeatureSeed[] { if (testFiles.length === 0) { return []; @@ -777,6 +960,9 @@ function pythonShouldSkip(path: string): boolean { } async function containsReviewablePythonSource(root: string): Promise { + if ((await rootPythonSourceFiles(root)).length > 0) { + return true; + } for (const sourceRoot of sourceRoots) { if (await containsPythonSourceInDirectory(root, sourceRoot, 4)) { return true;