Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Unreleased

- Added FastAPI route feature mapping and kept root/web Python project detection in sync.
- Added Flask route feature mapping for Python projects, including `web/` source roots, common root entry files, non-list method literals, and Python framework detection.
- Added Next.js route mapping for `src/app` and `src/pages` layouts, thanks @obatried.
- Added first-pass Python mapping for project metadata, console scripts, source groups, pytest suites, and conservative validation defaults, thanks @xiamx.
Expand Down
15 changes: 8 additions & 7 deletions docs/feature-mapping.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ Supported deterministic mappers today:
- Next.js `app/` and `pages/` routes
- Go `cmd/*/main.go`
- Go `internal/*` packages
- Python project metadata, console scripts, bounded source groups, pytest suites, and Flask routes
- Python project metadata, console scripts, root app files, bounded source groups,
pytest suites, and Flask/FastAPI routes
- Rust Cargo commands, libraries, workspace crates, and integration tests
- SwiftPM executable targets, library targets, and test suites
- nested SwiftPM packages
Expand All @@ -54,15 +55,15 @@ discovered below the repo root, Apple projects are grouped by Swift source area,
and Gradle modules are grouped from `src/main`, `src/test`, and `src/androidTest`.

Python mapping covers `pyproject.toml` metadata, `[project.scripts]` and
`[tool.poetry.scripts]` console scripts, source groups under common Python
source roots including `web/`, pytest files, and Flask `@*.route(...)`
handlers in source roots and common root entry files such as `app.py` and
`wsgi.py`. Flask route methods are read from list, tuple, or set literals.
Framework-specific route mapping for FastAPI and Django is not implemented yet.
`[tool.poetry.scripts]` console scripts, root app files, source groups under
common Python source roots including `web/`, pytest files, Flask `@*.route(...)`
handlers, and FastAPI `@*.get(...)` / `@*.api_route(...)` handlers. Flask and
FastAPI route methods are read from list, tuple, or set literals. FastAPI paths
can be positional strings or literal `path=` keywords.

Known gaps:

- no Express/Fastify/Hono route mapper yet
- no FastAPI/Django route mapper yet
- no Django route mapper yet
- no import graph expansion beyond nearby tests yet
- no agent enrichment yet
141 changes: 113 additions & 28 deletions src/detect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,11 @@ async function detectFrameworks(root: string, pkg: PackageJson | null): Promise<
frameworks.push(name);
}
}
for (const name of await pythonImportedFrameworks(root)) {
if (!frameworks.includes(name)) {
frameworks.push(name);
}
}
}
return frameworks;
}
Expand Down Expand Up @@ -748,14 +753,92 @@ async function isPythonProject(root: string): Promise<boolean> {
}

async function containsReviewablePythonFile(root: string): Promise<boolean> {
for (const prefix of ["src", "app", "apps", "lib", "scripts"]) {
if (await containsFileWithExtension(join(root, prefix), ".py", 4)) {
if (await containsRootReviewablePythonFile(root)) {
return true;
}
for (const prefix of pythonSourceSearchRoots) {
if (await containsFileMatching(join(root, prefix), 4, isReviewablePythonFileName)) {
return true;
}
}
return containsFileNamed(root, "__init__.py", 3);
}

const pythonSourceSearchRoots = ["src", "app", "apps", "lib", "scripts", "web"] as const;

async function containsRootReviewablePythonFile(root: string): Promise<boolean> {
return (await readdir(root, { withFileTypes: true }).catch(() => [])).some(
(entry) => entry.isFile() && isReviewablePythonFileName(entry.name),
);
}

function isReviewablePythonFileName(entry: string): boolean {
return (
entry.endsWith(".py") &&
!/^test_[^/]+\.py$/u.test(entry) &&
!entry.endsWith("_test.py") &&
!/(?:generated|_pb2|_pb2_grpc|\.gen)\.py$/iu.test(entry)
);
}

async function pythonImportedFrameworks(root: string): Promise<string[]> {
const frameworks = new Set<string>();
for (const path of await pythonFrameworkScanFiles(root)) {
const source = await readFile(path, "utf8").catch(() => "");
for (const name of ["flask", "fastapi", "django"] as const) {
const importPattern = new RegExp(
`^\\s*(?:from\\s+${name}\\s+import\\s+|import\\s+${name}\\b)`,
"mu",
);
if (importPattern.test(source)) {
frameworks.add(name);
}
}
}
return [...frameworks];
}

async function pythonFrameworkScanFiles(root: string): Promise<string[]> {
const files: string[] = [];
for (const entry of await readdir(root, { withFileTypes: true }).catch(() => [])) {
if (entry.isFile() && isReviewablePythonFileName(entry.name)) {
files.push(join(root, entry.name));
}
}
for (const prefix of pythonSourceSearchRoots) {
await collectPythonFrameworkScanFiles(join(root, prefix), 4, files);
}
return [...new Set(files)].slice(0, 200);
}

async function collectPythonFrameworkScanFiles(
dir: string,
remainingDepth: number,
files: string[],
): Promise<void> {
if (remainingDepth < 0 || !(await pathExists(dir))) {
return;
}
const dirInfo = await lstat(dir);
if (!dirInfo.isDirectory() || dirInfo.isSymbolicLink()) {
return;
}
for (const entry of await readdir(dir, { withFileTypes: true })) {
if (shouldSkipSearchEntry(entry.name)) {
continue;
}
const full = join(dir, entry.name);
if (entry.isSymbolicLink()) {
continue;
}
if (entry.isFile() && isReviewablePythonFileName(entry.name)) {
files.push(full);
} else if (entry.isDirectory()) {
await collectPythonFrameworkScanFiles(full, remainingDepth - 1, files);
}
}
}

async function containsFileNamed(root: string, name: string, maxDepth: number): Promise<boolean> {
return containsFileMatching(root, maxDepth, (entry) => entry === name);
}
Expand All @@ -781,32 +864,7 @@ async function containsFileMatching(
return false;
}
for (const entry of await readdir(dir)) {
if (
[
"node_modules",
"dist",
"build",
"target",
".build",
".swiftpm",
".git",
".clawpatch",
".worktrees",
".venv",
"venv",
"__pycache__",
".mypy_cache",
".ruff_cache",
".pytest_cache",
"fixtures",
"__fixtures__",
"testdata",
"Pods",
"Carthage",
"SourcePackages",
"DerivedData",
].includes(entry)
) {
if (shouldSkipSearchEntry(entry)) {
continue;
}
const full = join(dir, entry);
Expand All @@ -824,6 +882,33 @@ async function containsFileMatching(
return false;
}

function shouldSkipSearchEntry(entry: string): boolean {
return [
"node_modules",
"dist",
"build",
"target",
".build",
".swiftpm",
".git",
".clawpatch",
".worktrees",
".venv",
"venv",
"__pycache__",
".mypy_cache",
".ruff_cache",
".pytest_cache",
"fixtures",
"__fixtures__",
"testdata",
"Pods",
"Carthage",
"SourcePackages",
"DerivedData",
].includes(entry);
}

function stripLineComments(source: string, marker: "//"): string {
return source
.split("\n")
Expand Down
110 changes: 110 additions & 0 deletions src/mapper.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1419,6 +1419,116 @@ let package = Package(name: "HybridApp", targets: [.target(name: "HybridApp")])
expect(result.features.some((feature) => feature.source === "python-flask-route")).toBe(false);
});

it("maps FastAPI routes in root and web source files", async () => {
const root = await fixtureRoot("clawpatch-python-fastapi-routes-");
await writeFixture(root, "requirements.txt", "fastapi\npytest\n");
await writeFixture(
root,
"app.py",
[
"from fastapi import FastAPI",
"",
"app = FastAPI()",
"",
"@app.get('/health')",
"async def health():",
" return {'ok': True}",
"",
"@app.api_route('/webhook/{token}', methods=['GET', 'HEAD'])",
"def webhook(token: str):",
" return token",
"",
"@app.api_route('/submit', methods=('POST',))",
"def submit():",
" return {'ok': True}",
"",
].join("\n"),
);
await writeFixture(
root,
"web/api.py",
[
"from fastapi import APIRouter",
"",
"router = APIRouter()",
"",
"@router.post(",
" path='/admin/jobs',",
")",
"def create_job():",
" return {'queued': True}",
"",
].join("\n"),
);
await writeFixture(root, "tests/test_app.py", "def test_health():\n pass\n");

const project = await detectProject(root);
const result = await mapFeatures(root, project, []);
const health = result.features.find((feature) => feature.title === "FastAPI route GET /health");
const webhook = result.features.find(
(feature) => feature.title === "FastAPI route GET,HEAD /webhook/{token}",
);
const submit = result.features.find(
(feature) => feature.title === "FastAPI route POST /submit",
);
const admin = result.features.find(
(feature) => feature.title === "FastAPI route POST /admin/jobs",
);

expect(project.detected.frameworks).toContain("fastapi");
expect(health?.source).toBe("python-fastapi-route");
expect(health?.entrypoints[0]).toMatchObject({
path: "app.py",
symbol: "health",
route: "GET /health",
});
expect(health?.tests).toEqual([{ path: "tests/test_app.py", command: "pytest" }]);
expect(webhook?.entrypoints[0]?.route).toBe("GET,HEAD /webhook/{token}");
expect(submit?.entrypoints[0]?.route).toBe("POST /submit");
expect(admin?.entrypoints[0]).toMatchObject({
path: "web/api.py",
symbol: "create_job",
route: "POST /admin/jobs",
});
expect(admin?.trustBoundaries).toContain("auth");
});

it("detects metadata-free root and web Python sources", async () => {
const root = await fixtureRoot("clawpatch-python-root-web-detect-");
await writeFixture(root, "app.py", "def app():\n pass\n");
await writeFixture(
root,
"web/api.py",
[
"from fastapi import APIRouter",
"",
"router = APIRouter()",
"",
"@router.get(path='/health')",
"def health():",
" return {'ok': True}",
"",
].join("\n"),
);

const project = await detectProject(root);
const result = await mapFeatures(root, project, []);
const rootSource = result.features.find((feature) => feature.title === "Python source root");
const webRoute = result.features.find(
(feature) => feature.title === "FastAPI route GET /health",
);

expect(project.detected.languages).toContain("python");
expect(project.detected.packageManagers).toContain("python");
expect(project.detected.frameworks).toContain("fastapi");
expect(rootSource?.ownedFiles).toEqual([{ path: "app.py", reason: "source group root" }]);
expect(webRoute?.entrypoints[0]).toMatchObject({
path: "web/api.py",
symbol: "health",
route: "GET /health",
});
});

it("uses Hatch pytest commands in mapped Python features", async () => {
const root = await fixtureRoot("clawpatch-python-hatch-map-");
await writeFixture(
Expand Down
Loading