diff --git a/CHANGELOG.md b/CHANGELOG.md index 86dc8f127..64318eeb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,146 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.6.9 (2026-05-03) + +- Fix: `source_file` path separators normalized to forward slashes at graph ingestion — same physical file emitted with backslashes (Windows AST extractor) and forward slashes (semantic subagents) now merges into one node instead of splitting into two disconnected components (#683) +- Fix: two-phase cohesion re-clustering — communities with cohesion < 0.05 and ≥ 50 nodes are re-split, preventing doc-hub nodes (e.g. `CLAUDE.md`) from merging unrelated subsystems into one giant community (#683) +- Fix: VS Code Copilot instructions rewritten to be prescriptive — agent's first tool call must read `GRAPH_REPORT.md`, explicit trigger list, narrow allowlist for raw source reads (#688) +- Feat: `GRAPHIFY_OUT` env var overrides the output directory — accepts a relative name or absolute path, wires through `cache.py`, `watch.py`, and the CLI; useful for sharing one graph across multiple git worktrees (#686) +- Fix: `graphify antigravity install` now auto-updates stale rules and workflow files on re-run instead of silently skipping them (#652) +- Docs: README simplified — less dense, plain language; technical pipeline details moved to `docs/how-it-works.md` + +## 0.6.8 (2026-05-03) + +- Fix: `.graphifyignore` negation patterns (`!src/**`) now work correctly — when any `!` pattern is present, directory pruning is deferred to per-file checks so negated files inside ignored directories are reached (#676) +- Fix: Antigravity slash command `/graphify` now appears in the command dropdown — workflow file now includes YAML frontmatter with `name: graphify` required for Antigravity discovery (#678) +- Fix: Gemini CLI BeforeTool hook replaced `[ -f ... ] && echo` (bash-only) with cross-platform `python -c` using `json.dumps` — fixes hook failure on Windows CMD and Git Bash (#681) +- Fix: Codex hook-check exits silently — resolves `additionalContext` rejection on Codex Desktop PreToolUse (#651) +- Fix: `graphify install --platform codex` now writes absolute path to `graphify` executable — fixes PATH resolution in VS Code extension on Windows (#651) +- Fix: thin communities (fewer than 3 concept nodes) are now omitted from the Communities section in `GRAPH_REPORT.md` by default; report header shows `(N total, M thin omitted)` and Knowledge Gaps collapses thin communities to one summary line (#664) + +## 0.6.7 (2026-05-02) + +- Feat: `graphify tree` — self-contained D3 v7 collapsible-tree HTML view of `graph.json`; expand/collapse controls, depth-based colours, hover inspector; XSS-safe (#557) +- Feat: token-aware chunking with split-and-retry on truncation (#625) +- Feat: cross-language edge context filters in MCP `query_graph` tool (#573) +- Feat: dynamic `import()` extraction for JS/TS (#579) +- Fix: `save_semantic_cache` crashed with `IsADirectoryError` when a node's `source_file` was a directory path — `p.exists()` → `p.is_file()` (#655) +- Fix: `sanitize_label(None)` raised `TypeError` crashing `to_html` on graphs with null `source_file` rationale nodes — return `""` early (#656) +- Fix: chunk-extraction prompt omitted `rationale` from valid `file_type` values — model hallucinated `concept` on every doc/paper run; explicit merge step added to all skill variants (#657) +- Fix: `cost.json` always reported 0 tokens — chunk JSONs have placeholder zeros; orchestrator now globs and sums real token counts before merging (#658) +## 0.6.6 (2026-05-02) + +- Fix: `skill-windows.md` rewritten from PowerShell to bash — Claude Code on Windows uses git-bash so PowerShell syntax (`$null`, `$LASTEXITCODE`, `Select-Object`, `& (Get-Content ...)`, `Remove-Item`) caused exit code 49 failures; now mirrors `skill.md` structure with `python` added as fallback after `python3` for Windows Conda (#39) +- Fix: wiki `to_wiki()` now clears stale articles before regenerating, preventing orphan .md accumulation (#558) +- Fix: `_safe_filename()` in wiki.py now strips Windows-reserved characters (`< > : " / \ | ? *`) and caps length at 200 chars (#594) +- Fix: rationale-node leakage in cross-file INFERRED call resolution — rationale nodes now excluded from name lookup; edge direction (`calls`, `rationale_for`) preserved correctly at JSON export (#576) +- Feat: `.graphifyinclude` hidden path allowlist — opt specific hidden dirs into traversal (e.g. `.hermes/plans/**/*.md`) (#583) +- Feat: `--no-viz` flag wired in `cluster-only`; `GRAPHIFY_VIZ_NODE_LIMIT` env var overrides 5000-node HTML threshold (#565) +- Fix: stray colon SyntaxError in `skill-trae.md` `--cluster-only` block (#603) +- Docs: skill INFERRED confidence score guidance changed to discrete rubric (0.55/0.65/0.75/0.85/0.95) backed by calibration data (#546) +- Docs: skill `--update` prune output clarified — splits no-drift vs drift cases (#544) +- Docs: skill `--update` merge step now calls `save_manifest` to prevent deleted files reappearing (#545) +- Feat: `graphify tree` — self-contained D3 v7 collapsible-tree HTML view of `graph.json`; expand/collapse controls, depth-based colours, hover inspector; XSS-safe via `html.escape()` and `_js_safe()` (#557) + +## 0.6.5 (2026-05-02) + +- Fix: Kotlin call-walker now accepts both `simple_identifier` and `identifier` node types — PyPI's `tree_sitter_kotlin` grammar uses `identifier` while older forks use `simple_identifier`, causing zero `calls` edges to be emitted (#659) +- Feat: community sidebar now uses checkbox-based multi-select instead of show/hide buttons — supports indeterminate "select all" state (#647) +- Feat: `graphify update --force` and `GRAPHIFY_FORCE=1` env var — bypass the node-count safety check after refactors that legitimately shrink the graph (#639) +- Fix: Codex PreToolUse hook on Windows — replaced `python3 -c "..."` inline command (fails on Conda where only `python` exists, and breaks PowerShell JSON parsing) with `graphify hook-check`, a new shell-agnostic subcommand. Re-run `graphify codex install` to regenerate the hook (#651, #522) + +## 0.6.4 (2026-05-02) + +- Fix: Codex PreToolUse hook failed on Windows — `[ -f ]` is bash-only and crashes on `cmd.exe`; replaced with a cross-platform Python one-liner (`pathlib.Path.exists()`) (#651) + +## 0.6.3 (2026-05-02) + +- Fix: incremental rebuild (`graphify update`, post-commit hook) dropped INFERRED/AMBIGUOUS semantic nodes extracted from code files — node preservation now filters by ID membership in the new AST output instead of `file_type`, so LLM-extracted call/data-flow edges survive code-only rebuilds (#653) +- Fix: post-commit and post-checkout hooks blocked `git commit` for the full rebuild duration (hours on large repos) — rebuilds now detach via `nohup & disown`, git returns in ~100ms, log written to `~/.cache/graphify-rebuild.log` (#650) +- Fix: cross-file INFERRED `calls` resolution used a last-write-wins name map, causing common short names (`log`, `execute`, `find`) to accumulate hundreds of spurious edges and dominate god_nodes ranking — resolution now skips any callee name that matches 2+ candidates (ambiguous, no import evidence to pick the right target) (#543) +- Fix: `cluster-only` command crashed on graphs with >5000 nodes due to unguarded `to_html` call — now wrapped in try/except ValueError matching the watch/hook path (#541) + +## 0.6.2 (2026-05-01) + +- Fix: Kimi K2.6 reasoning mode consumed entire token budget leaving `content` empty — thinking now disabled on Moonshot calls so graphs actually populate (#623) +- Fix: `graphify update` / `graphify watch` never persisted the manifest, so every subsequent `--update` re-extracted all files — manifest now saved after each rebuild (#621) +- Fix: inline comments in `.graphifyignore` (e.g. `vendor/ # legacy`) now stripped correctly — whitespace + `#` suffix is treated as a comment, `path#hash.py` preserved (#605) +- Fix: `graphify query "FunctionName"` now returns the exact matching node first instead of high-degree hub modules hijacking the output — 100-point exact-match bonus + seeds render before BFS expansion (#638) +- Fix: concurrent AST extractors raced on a shared `.tmp` cache file — each writer now gets a unique tempfile via `mkstemp`, eliminating cache corruption under parallel extraction (#589) +- Fix: `_clone_repo` branch names starting with `-` could be interpreted as git flags — validation added, `--` separator inserted before positional args (#589) +- Fix: replaced `html2text` (GPL-3.0) with `markdownify` (MIT) — removes the only copyleft dependency from a MIT project (#586) +- Fix: `--update` re-extracted files whose mtime was bumped by sync tools (Obsidian, Nextcloud) without content changes — manifest now stores content hash alongside mtime; mtime bump triggers an MD5 check before re-extraction (#593) +- Feat: R language support — `.r` files classified as code and processed via LLM semantic extraction (#617) +- Feat: extensionless shell scripts now detected via shebang (`#!/bin/bash`, `#!/usr/bin/env python3`, etc.) and included as code (#619) +- Fix: cross-language INFERRED `calls` edges (e.g. Python→TypeScript name collision) no longer appear as top surprising connections in GRAPH_REPORT.md (#630) +- Fix: `cluster-only` CLI silently flipped directed graphs to undirected — `directed` flag now read from graph.json and preserved through re-clustering (#590) +- Fix: Windows UNC / extended-length paths (`\\?\C:\...`) now normalize to consistent cache keys (#629) +- Fix: `.graphifyignore` negation patterns (`!src/lib/secrets.ts`) now work — full last-match-wins evaluation with `!` un-ignore support (#628) + +## 0.6.1 (2026-05-01) + +- Fix: `.graphifyignore` discovery now uses correct gitignore semantics — outer rules are loaded first so inner (closer) rules always win via last-match-wins, matching standard gitignore behavior (#643) +- Fix: without a VCS root, `.graphifyignore` discovery is now hermetic to the scan folder — no leakage across sibling projects in a shared workspace (#643) +- Fix: anchored patterns (leading `/`) in a parent `.graphifyignore` now correctly apply only relative to their own directory, not the scan root (#643) +- Fix: trailing spaces in patterns are now handled per gitignore spec — unescaped trailing spaces are stripped, `vendor\ ` (escaped) is preserved (#643) + +## 0.6.0 (2026-05-01) + +- Feat: SQL AST extractor — `.sql` files now processed deterministically via tree-sitter. Extracts tables, views, functions/procedures, foreign key references, and FROM/JOIN reads_from edges. No LLM needed. Requires `pip install 'graphifyy[sql]'` (#349) +- Feat: `xlsx_extract_structure()` utility — extracts sheet names, named tables, and column headers from .xlsx files as structural nodes + +## 0.5.7 (2026-04-30) + +- Feat: YAML/YML files now indexed for semantic extraction — Kubernetes, Kustomize, Helm, and any YAML corpus now picked up automatically (#633) + +## 0.5.6 (2026-04-30) + +- Fix: `NameError: name '_os' is not defined` crash after `graphify update` — this was fixed in v5 branch but not released to PyPI (#618, #612) + +## 0.5.5 (2026-04-29) + +- Feat: Kimi K2.6 backend — `pip install 'graphifyy[kimi]'` + `MOONSHOT_API_KEY` routes semantic extraction through Kimi K2.6. 3-6x richer relation extraction at ~3x lower cost. Claude remains default; Kimi is opt-in. +- Fix: phantom god nodes (#598) — member-call callees (`this.logger.log()` → `log`) no longer cross-file resolved. Go package-qualified calls (`pkg.Func()`) correctly preserved. Affects JS/TS, Go, Rust, Swift, Kotlin, Scala, PHP, C++, C#, Zig, Elixir. +- Fix: `concept` file_type no longer triggers validation warnings (#601) +- Fix: `graphify update` remembers scan root via `graphify-out/.graphify_root` — no path argument needed on subsequent runs +- Fix: Kimi K2.6 temperature 400 error — temperature param is now skipped for Kimi backends (model enforces its own fixed value) (#610) +- Fix: community labels deleted in Step 9 cleanup — `.graphify_labels.json` is now preserved so wiki/obsidian/HTML retain human-readable names after re-cluster (#608) +- Fix: `NameError: name '_os' is not defined` in `graphify update` Kimi tip (#612) +- Fix: `SyntaxWarning` in `__main__.py` for shell glob pattern with backslash escapes +- Fix: Python upper bound removed — `requires-python = ">=3.10"` now supports Python 3.14+ (#607) + +## 0.5.4 (2026-04-28) + +- Fix: SSRF DNS rebinding — `safe_fetch` now patches `socket.getaddrinfo` for the full request duration (#591) +- Fix: yt-dlp SSRF bypass — `download_audio` now calls `validate_url` before handing URL to yt-dlp (#592) + +## 0.5.3 (2026-04-27) + +- Fix: cache namespace — AST and semantic entries now live in `cache/ast/` and `cache/semantic/` subdirectories; flat entries read as migration fallback + +## 0.5.2 (2026-04-26) + +- Fix: PreToolUse hook now matches on `Bash` instead of `Glob|Grep` for Claude Code v2.1.117+ + +## 0.5.1 (2026-04-25) + +- Fix: node ID collision for same-named files in different directories +- Fix: `source_file` paths relativized before return so `graph.json` is portable +- Fix: desync guard — `to_json()` returns bool; report only written on successful JSON write +- Feat: TypeScript `@/` path aliases resolved via `tsconfig.json` +- Feat: Show All / Hide All buttons in HTML community panel + +## 0.5.0 (2026-04-24) + +- Feat: `graphify clone ` — clone and graph any public repo +- Feat: `graphify merge-graphs` — combine multiple `graph.json` outputs into one cross-repo graph +- Feat: `CLAUDE_CONFIG_DIR` support in `graphify install` +- Feat: shrink guard — `to_json()` refuses to overwrite with a smaller graph +- Feat: `build_merge()` for safe incremental updates +- Feat: duplicate node deduplication via `deduplicate_by_label()` +- Fix: `graphify-out/` excluded from source scanning + ## 0.4.23 (2026-04-18) - Fix: stale skill version warning persists after running `graphify install` when multiple platforms were previously installed — `graphify install` now refreshes `.graphify_version` in all other known skill directories so the warning clears across the board (#178) diff --git a/README.md b/README.md index efcca4de1..2849c2f82 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@

+ The Memory Layer CI PyPI Downloads @@ -20,67 +21,45 @@

-**An AI coding assistant skill.** Type `/graphify` in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, VS Code Copilot Chat, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, or Google Antigravity - it reads your files, builds a knowledge graph, and gives you back structure you didn't know was there. Understand a codebase faster. Find the "why" behind architectural decisions. +Type `/graphify` in your AI coding assistant and it maps your entire project — code, docs, PDFs, images, videos — into a knowledge graph you can query instead of grepping through files. -Fully multimodal. Drop in code, PDFs, markdown, screenshots, diagrams, whiteboard photos, images in other languages, or video and audio files - graphify extracts concepts and relationships from all of it and connects them into one graph. Videos are transcribed with Whisper using a domain-aware prompt derived from your corpus. 25 languages supported via tree-sitter AST (Python, JS, TS, Go, Rust, Java, C, C++, Ruby, C#, Kotlin, Scala, PHP, Swift, Lua, Zig, PowerShell, Elixir, Objective-C, Julia, Verilog, SystemVerilog, Vue, Svelte, Dart). - -> Andrej Karpathy keeps a `/raw` folder where he drops papers, tweets, screenshots, and notes. graphify is the answer to that problem - 71.5x fewer tokens per query vs reading the raw files, persistent across sessions, honest about what it found vs guessed. - -``` -/graphify . # works on any folder - your codebase, notes, papers, anything -``` +Works in Claude Code, Codex, OpenCode, Cursor, Gemini CLI, GitHub Copilot CLI, VS Code Copilot Chat, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, and Google Antigravity. ``` -graphify-out/ -├── graph.html interactive graph - open in any browser, click nodes, search, filter by community -├── GRAPH_REPORT.md god nodes, surprising connections, suggested questions -├── graph.json persistent graph - query weeks later without re-reading -└── cache/ SHA256 cache - re-runs only process changed files +/graphify . ``` -Add a `.graphifyignore` file to exclude folders you don't want in the graph: +That's it. You get three files: ``` -# .graphifyignore -vendor/ -node_modules/ -dist/ -*.generated.py +graphify-out/ +├── graph.html open in any browser — click nodes, filter, search +├── GRAPH_REPORT.md the highlights: key concepts, surprising connections, suggested questions +└── graph.json the full graph — query it anytime without re-reading your files ``` -Same syntax as `.gitignore`. You can keep a single `.graphifyignore` at your repo root — patterns work correctly even when graphify is run on a subfolder. - -## How it works - -graphify runs in three passes. First, a deterministic AST pass extracts structure from code files (classes, functions, imports, call graphs, docstrings, rationale comments) with no LLM needed. Second, video and audio files are transcribed locally with faster-whisper using a domain-aware prompt derived from corpus god nodes — transcripts are cached so re-runs are instant. Third, Claude subagents run in parallel over docs, papers, images, and transcripts to extract concepts, relationships, and design rationale. The results are merged into a NetworkX graph, clustered with Leiden community detection, and exported as interactive HTML, queryable JSON, and a plain-language audit report. - -**Clustering is graph-topology-based — no embeddings.** Leiden finds communities by edge density. The semantic similarity edges that Claude extracts (`semantically_similar_to`, marked INFERRED) are already in the graph, so they influence community detection directly. The graph structure is the similarity signal — no separate embedding step or vector database needed. - -Every relationship is tagged `EXTRACTED` (found directly in source), `INFERRED` (reasonable inference, with a confidence score), or `AMBIGUOUS` (flagged for review). You always know what was found vs guessed. +--- ## Install -**Requires:** Python 3.10+ and one of: [Claude Code](https://claude.ai/code), [Codex](https://openai.com/codex), [OpenCode](https://opencode.ai), [Cursor](https://cursor.com), [Gemini CLI](https://github.com/google-gemini/gemini-cli), [GitHub Copilot CLI](https://docs.github.com/en/copilot/how-tos/copilot-cli), [VS Code Copilot Chat](https://code.visualstudio.com/docs/copilot/overview), [Aider](https://aider.chat), [OpenClaw](https://openclaw.ai), [Factory Droid](https://factory.ai), [Trae](https://trae.ai), [Kiro](https://kiro.dev), Hermes, or [Google Antigravity](https://antigravity.google) +**Requires Python 3.10+** ```bash -# Recommended — works on Mac and Linux with no PATH setup needed uv tool install graphifyy && graphify install -# or with pipx -pipx install graphifyy && graphify install -# or plain pip -pip install graphifyy && graphify install +# or: pipx install graphifyy && graphify install +# or: pip install graphifyy && graphify install ``` -> **Official package:** The PyPI package is named `graphifyy` (install with `pip install graphifyy`). Other packages named `graphify*` on PyPI are not affiliated with this project. The only official repository is [safishamsi/graphify](https://github.com/safishamsi/graphify). The CLI and skill command are still `graphify`. +> **Official package:** The PyPI package is `graphifyy` (double-y). Other `graphify*` packages on PyPI are not affiliated. The CLI command is still `graphify`. -> **`graphify: command not found`?** Use `uv tool install graphifyy` (recommended) or `pipx install graphifyy` — both put the CLI in a managed location that's automatically on PATH. With plain `pip`, you may need to add `~/.local/bin` (Linux) or `~/Library/Python/3.x/bin` (Mac) to your PATH, or run `python -m graphify` instead. On Windows, pip scripts land in `%APPDATA%\Python\PythonXY\Scripts`. +> **`graphify: command not found`?** Use `uv tool install graphifyy` or `pipx install graphifyy` — both put the CLI on PATH automatically. With plain `pip`, add `~/.local/bin` (Linux) or `~/Library/Python/3.x/bin` (Mac) to your PATH, or run `python -m graphify`. -### Platform support +### Pick your platform | Platform | Install command | |----------|----------------| | Claude Code (Linux/Mac) | `graphify install` | -| Claude Code (Windows) | `graphify install` (auto-detected) or `graphify install --platform windows` | +| Claude Code (Windows) | `graphify install --platform windows` | | Codex | `graphify install --platform codex` | | OpenCode | `graphify install --platform opencode` | | GitHub Copilot CLI | `graphify install --platform copilot` | @@ -93,22 +72,18 @@ pip install graphifyy && graphify install | Gemini CLI | `graphify install --platform gemini` | | Hermes | `graphify install --platform hermes` | | Kiro IDE/CLI | `graphify kiro install` | +| Pi coding agent | `graphify install --platform pi` | | Cursor | `graphify cursor install` | | Google Antigravity | `graphify antigravity install` | -Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw and Aider use sequential extraction (parallel agent support is still early on those platforms). Trae uses the Agent tool for parallel subagent dispatch and does **not** support PreToolUse hooks — AGENTS.md is the always-on mechanism. Codex supports PreToolUse hooks — `graphify codex install` installs one in `.codex/hooks.json` in addition to writing AGENTS.md. - -Then open your AI coding assistant and type: - -``` -/graphify . -``` +> Codex users: also add `multi_agent = true` under `[features]` in `~/.codex/config.toml`. +> Codex uses `$graphify` instead of `/graphify`. -Note: Codex uses `$` instead of `/` for skill calling, so type `$graphify .` instead. +--- -### Make your assistant always use the graph (recommended) +## Make your assistant always use the graph -After building a graph, run this once in your project: +Run this once in your project after building a graph: | Platform | Command | |----------|---------| @@ -126,319 +101,212 @@ After building a graph, run this once in your project: | Gemini CLI | `graphify gemini install` | | Hermes | `graphify hermes install` | | Kiro IDE/CLI | `graphify kiro install` | +| Pi coding agent | `graphify pi install` | | Google Antigravity | `graphify antigravity install` | -**Claude Code** does two things: writes a `CLAUDE.md` section telling Claude to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs a **PreToolUse hook** (`settings.json`) that fires before every Glob and Grep call. If a knowledge graph exists, Claude sees: _"graphify: Knowledge graph exists. Read GRAPH_REPORT.md for god nodes and community structure before searching raw files."_ — so Claude navigates via the graph instead of grepping through every file. +This writes a small config file that tells your assistant to read `GRAPH_REPORT.md` before answering questions about your codebase. On platforms that support hooks (Claude Code, Codex, Gemini CLI), a hook fires automatically before every file-read call — your assistant navigates by the graph instead of grepping through everything. -**Codex** writes to `AGENTS.md` and also installs a **PreToolUse hook** in `.codex/hooks.json` that fires before every Bash tool call — same always-on mechanism as Claude Code. +Uninstall with the matching command (e.g. `graphify claude uninstall`). -**OpenCode** writes to `AGENTS.md` and also installs a **`tool.execute.before` plugin** (`.opencode/plugins/graphify.js` + `opencode.json` registration) that fires before bash tool calls and injects the graph reminder into tool output when the graph exists. +--- -**Cursor** writes `.cursor/rules/graphify.mdc` with `alwaysApply: true` — Cursor includes it in every conversation automatically, no hook needed. +## What's in the report -**Gemini CLI** copies the skill to `~/.gemini/skills/graphify/SKILL.md`, writes a `GEMINI.md` section, and installs a `BeforeTool` hook in `.gemini/settings.json` that fires before file-read tool calls — same always-on mechanism as Claude Code. +- **God nodes** — the most-connected concepts in your project. Everything flows through these. +- **Surprising connections** — links between things that live in different files or modules. Ranked by how unexpected they are. +- **The "why"** — inline comments (`# NOTE:`, `# WHY:`, `# HACK:`), docstrings, and design rationale from docs are extracted as separate nodes linked to the code they explain. +- **Suggested questions** — 4–5 questions the graph is uniquely positioned to answer. +- **Confidence tags** — every inferred relationship is marked `EXTRACTED`, `INFERRED`, or `AMBIGUOUS`. You always know what was found vs guessed. -**Aider, OpenClaw, Factory Droid, Trae, and Hermes** write the same rules to `AGENTS.md` in your project root and copy the skill to the platform's global skill directory. These platforms don't support tool hooks, so AGENTS.md is the always-on mechanism. +--- -**Kiro IDE/CLI** writes the skill to `.kiro/skills/graphify/SKILL.md` (invoked via `/graphify`) and a steering file to `.kiro/steering/graphify.md` with `inclusion: always` — Kiro injects this into every conversation automatically, no hook needed. +## What files it handles -**Google Antigravity** writes `.agents/rules/graphify.md` (always-on rules) and `.agents/workflows/graphify.md` (registers `/graphify` as a slash command). No hook equivalent exists in Antigravity — rules are the always-on mechanism. +| Type | Extensions | +|------|-----------| +| Code (25 languages) | `.py .ts .js .jsx .tsx .go .rs .java .c .cpp .rb .cs .kt .scala .php .swift .lua .zig .ps1 .ex .exs .m .jl .vue .svelte .sql` | +| Docs | `.md .mdx .html .txt .rst .yaml .yml` | +| Office | `.docx .xlsx` (requires `pip install graphifyy[office]`) | +| PDFs | `.pdf` | +| Images | `.png .jpg .webp .gif` | +| Video / Audio | `.mp4 .mov .mp3 .wav` and more (requires `pip install graphifyy[video]`) | +| YouTube / URLs | any video URL (requires `pip install graphifyy[video]`) | -**GitHub Copilot CLI** copies the skill to `~/.copilot/skills/graphify/SKILL.md`. Run `graphify copilot install` to set it up. +Code is extracted locally with no API calls (AST via tree-sitter). Everything else goes through your AI assistant's model API. -**VS Code Copilot Chat** installs a Python-only skill (works on Windows PowerShell and macOS/Linux alike) and writes `.github/copilot-instructions.md` in your project root — VS Code reads this automatically every session, making graph context always-on without any hook mechanism. Run `graphify vscode install`. Note: this configures the chat panel in VS Code, not the Copilot CLI terminal tool. +--- -Uninstall with the matching uninstall command (e.g. `graphify claude uninstall`). +## Common commands -**Always-on vs explicit trigger — what's the difference?** +```bash +/graphify . # build graph for current folder +/graphify ./docs --update # re-extract only changed files +/graphify . --cluster-only # rerun clustering without re-extracting +/graphify . --no-viz # skip the HTML, just the report + JSON +/graphify . --wiki # build a markdown wiki from the graph -The always-on hook surfaces `GRAPH_REPORT.md` — a one-page summary of god nodes, communities, and surprising connections. Your assistant reads this before searching files, so it navigates by structure instead of keyword matching. That covers most everyday questions. +/graphify query "what connects auth to the database?" +/graphify path "UserService" "DatabasePool" +/graphify explain "RateLimiter" -`/graphify query`, `/graphify path`, and `/graphify explain` go deeper: they traverse the raw `graph.json` hop by hop, trace exact paths between nodes, and surface edge-level detail (relation type, confidence score, source location). Use them when you want a specific question answered from the graph rather than a general orientation. +/graphify add https://arxiv.org/abs/1706.03762 # fetch a paper and add it +/graphify add # transcribe and add a video -Think of it this way: the always-on hook gives your assistant a map. The `/graphify` commands let it navigate the map precisely. +graphify hook install # auto-rebuild on git commit +graphify merge-graphs a.json b.json # combine two graphs +``` -### Team workflows +See the [full command reference](#full-command-reference) below. -`graphify-out/` is designed to be committed to git so every teammate starts with a fresh map. +--- -**Recommended `.gitignore` additions:** -``` -# keep graph outputs, skip heavy/local-only files -graphify-out/cache/ # optional: commit for shared extraction speed, skip to keep repo small -graphify-out/manifest.json # mtime-based, invalid after git clone — always gitignore this -graphify-out/cost.json # local token tracking, not useful to share -``` - -**Shared setup:** -1. One person runs `/graphify .` to build the initial graph and commits `graphify-out/`. -2. Everyone else pulls — their assistant reads `GRAPH_REPORT.md` immediately with no extra steps. -3. Install the post-commit hook (`graphify hook install`) so the graph rebuilds automatically after code changes — no LLM calls needed for code-only updates. -4. For doc/paper changes, whoever edits the files runs `/graphify --update` to refresh semantic nodes. +## Ignoring files -**Excluding paths** — create `.graphifyignore` in your project root (same syntax as `.gitignore`). Files matching those patterns are skipped during detection and extraction. +Create a `.graphifyignore` in your project root — same syntax as `.gitignore`, including `!` negation: ``` -# .graphifyignore example -AGENTS.md # graphify install files — don't extract your own instructions as knowledge -CLAUDE.md -GEMINI.md -.gemini/ -.opencode/ -docs/translations/ # generated content you don't want in the graph -``` +# .graphifyignore +node_modules/ +dist/ +*.generated.py -## Using `graph.json` with an LLM +# only index src/, ignore everything else +* +!src/ +!src/** +``` -`graph.json` is not meant to be pasted into a prompt all at once. The useful -workflow is: +--- -1. Start with `graphify-out/GRAPH_REPORT.md` for the high-level overview. -2. Use `graphify query` to pull a smaller subgraph for the specific question - you want to answer. -3. Give that focused output to your assistant instead of dumping the full raw - corpus. +## Team setup -For example, after running graphify on a project: +`graphify-out/` is meant to be committed to git so everyone on the team starts with a map. -```bash -graphify query "show the auth flow" --graph graphify-out/graph.json -graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json +**Recommended `.gitignore` additions:** +``` +graphify-out/manifest.json # mtime-based, breaks after git clone +graphify-out/cost.json # local only +# graphify-out/cache/ # optional: commit for speed, skip to keep repo small ``` -The output includes node labels, edge types, confidence tags, source files, and -source locations. That makes it a good intermediate context block for an LLM: +**Workflow:** +1. One person runs `/graphify .` and commits `graphify-out/`. +2. Everyone pulls — their assistant reads the graph immediately. +3. Run `graphify hook install` to auto-rebuild after each commit (AST only, no API cost). +4. When docs or papers change, run `/graphify --update` to refresh those nodes. -```text -Use this graph query output to answer the question. Prefer the graph structure -over guessing, and cite the source files when possible. -``` +--- -If your assistant supports tool calling or MCP, use the graph directly instead -of pasting text. graphify can expose `graph.json` as an MCP server: +## Using the graph directly ```bash +# query the graph from the terminal +graphify query "show the auth flow" +graphify query "what connects DigestAuth to Response?" --graph graphify-out/graph.json + +# expose the graph as an MCP server (for repeated tool-call access) python -m graphify.serve graphify-out/graph.json ``` -That gives the assistant structured graph access for repeated queries such as -`query_graph`, `get_node`, `get_neighbors`, and `shortest_path`. +The MCP server gives your assistant structured access: `query_graph`, `get_node`, `get_neighbors`, `shortest_path`. -> **WSL / Linux note:** Ubuntu ships `python3`, not `python`. Install into a project venv to avoid PEP 668 conflicts, and use the full venv path in your `.mcp.json`: +> **WSL / Linux note:** Ubuntu ships `python3`, not `python`. Use a venv to avoid conflicts: > ```bash > python3 -m venv .venv && .venv/bin/pip install "graphifyy[mcp]" > ``` -> ```json -> { "mcpServers": { "graphify": { "type": "stdio", "command": ".venv/bin/python3", "args": ["-m", "graphify.serve", "graphify-out/graph.json"] } } } -> ``` -> Also note: the PyPI package is `graphifyy` (double-y) — `pip install graphify` installs an unrelated package. -
-Manual install (curl) +--- -```bash -mkdir -p ~/.claude/skills/graphify -curl -fsSL https://raw.githubusercontent.com/safishamsi/graphify/v4/graphify/skill.md \ - > ~/.claude/skills/graphify/SKILL.md -``` +## Privacy -Add to `~/.claude/CLAUDE.md`: +- **Code files** — processed locally via tree-sitter. Nothing leaves your machine. +- **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine. +- **Docs, PDFs, images** — sent to your AI assistant's model API (Anthropic, OpenAI, etc.) using your own API key. +- No telemetry, no usage tracking, no analytics. -``` -- **graphify** (`~/.claude/skills/graphify/SKILL.md`) - any input to knowledge graph. Trigger: `/graphify` -When the user types `/graphify`, invoke the Skill tool with `skill: "graphify"` before doing anything else. -``` +--- -
- -## Usage +## Full command reference ``` /graphify # run on current directory /graphify ./raw # run on a specific folder -/graphify ./raw --mode deep # more aggressive INFERRED edge extraction -/graphify ./raw --update # re-extract only changed files, merge into existing graph -/graphify ./raw --directed # build directed graph (preserves edge direction: source→target) -/graphify ./raw --cluster-only # rerun clustering on existing graph, no re-extraction -/graphify ./raw --no-viz # skip HTML, just produce report + JSON -/graphify ./raw --obsidian # also generate Obsidian vault (opt-in) -/graphify ./raw --obsidian --obsidian-dir ~/vaults/myproject # write vault to a specific directory - -/graphify add https://arxiv.org/abs/1706.03762 # fetch a paper, save, update graph -/graphify add https://x.com/karpathy/status/... # fetch a tweet -/graphify add # download audio, transcribe, add to graph -/graphify add https://... --author "Name" # tag the original author -/graphify add https://... --contributor "Name" # tag who added it to the corpus - -/graphify query "what connects attention to the optimizer?" -/graphify query "what connects attention to the optimizer?" --dfs # trace a specific path -/graphify query "what connects attention to the optimizer?" --budget 1500 # cap at N tokens -/graphify path "DigestAuth" "Response" -/graphify explain "SwinTransformer" - -/graphify ./raw --watch # auto-sync graph as files change (code: instant, docs: notifies you) -/graphify ./raw --wiki # build agent-crawlable wiki (index.md + article per community) +/graphify ./raw --mode deep # more aggressive relationship extraction +/graphify ./raw --update # re-extract only changed files +/graphify ./raw --directed # preserve edge direction +/graphify ./raw --cluster-only # rerun clustering on existing graph +/graphify ./raw --no-viz # skip HTML visualization +/graphify ./raw --obsidian # generate Obsidian vault +/graphify ./raw --wiki # build agent-crawlable markdown wiki /graphify ./raw --svg # export graph.svg -/graphify ./raw --graphml # export graph.graphml (Gephi, yEd) +/graphify ./raw --graphml # export for Gephi / yEd /graphify ./raw --neo4j # generate cypher.txt for Neo4j -/graphify ./raw --neo4j-push bolt://localhost:7687 # push directly to a running Neo4j instance +/graphify ./raw --neo4j-push bolt://localhost:7687 +/graphify ./raw --watch # auto-sync as files change /graphify ./raw --mcp # start MCP stdio server -# git hooks - platform-agnostic, rebuild graph on commit and branch switch -graphify hook install -graphify hook uninstall -graphify hook status - -# always-on assistant instructions - platform-specific -graphify claude install # CLAUDE.md + PreToolUse hook (Claude Code) -graphify claude uninstall -graphify codex install # AGENTS.md + PreToolUse hook in .codex/hooks.json (Codex) -graphify opencode install # AGENTS.md + tool.execute.before plugin (OpenCode) -graphify cursor install # .cursor/rules/graphify.mdc (Cursor) -graphify cursor uninstall -graphify gemini install # GEMINI.md + BeforeTool hook (Gemini CLI) -graphify gemini uninstall -graphify copilot install # skill file (GitHub Copilot CLI) -graphify copilot uninstall -graphify aider install # AGENTS.md (Aider) -graphify aider uninstall -graphify claw install # AGENTS.md (OpenClaw) -graphify droid install # AGENTS.md (Factory Droid) -graphify trae install # AGENTS.md (Trae) -graphify trae uninstall -graphify trae-cn install # AGENTS.md (Trae CN) -graphify trae-cn uninstall -graphify hermes install # AGENTS.md + ~/.hermes/skills/ (Hermes) -graphify hermes uninstall -graphify kiro install # .kiro/skills/ + .kiro/steering/graphify.md (Kiro IDE/CLI) -graphify kiro uninstall -graphify antigravity install # .agents/rules + .agents/workflows (Google Antigravity) -graphify antigravity uninstall - -# query and navigate the graph directly from the terminal (no AI assistant needed) -graphify query "what connects attention to the optimizer?" -graphify query "show the auth flow" --dfs -graphify query "what is CfgNode?" --budget 500 -graphify query "..." --graph path/to/graph.json -graphify path "DigestAuth" "Response" # shortest path between two nodes -graphify explain "SwinTransformer" # plain-language explanation of a node - -# add content and update the graph from the terminal -graphify add https://arxiv.org/abs/1706.03762 # fetch paper, save to ./raw, update graph -graphify add https://... --author "Name" --contributor "Name" - -# incremental update and maintenance -graphify watch ./src # auto-rebuild on code changes -graphify check-update ./src # check if semantic re-extraction is pending (cron-safe) -graphify update ./src # re-extract code files, no LLM needed -graphify cluster-only ./my-project # rerun clustering on existing graph.json -``` - -Works with any mix of file types: - -| Type | Extensions | Extraction | -|------|-----------|------------| -| Code | `.py .ts .js .jsx .tsx .mjs .go .rs .java .c .cpp .rb .cs .kt .scala .php .swift .lua .zig .ps1 .ex .exs .m .mm .jl .vue .svelte` | AST via tree-sitter + call-graph (cross-file for all languages) + Java extends/implements + docstring/comment rationale | -| Docs | `.md .mdx .html .txt .rst` | Concepts + relationships + design rationale via Claude | -| Office | `.docx .xlsx` | Converted to markdown then extracted via Claude (requires `pip install graphifyy[office]`) | -| Papers | `.pdf` | Citation mining + concept extraction | -| Images | `.png .jpg .webp .gif` | Claude vision - screenshots, diagrams, any language | -| Video / Audio | `.mp4 .mov .mkv .webm .avi .m4v .mp3 .wav .m4a .ogg` | Transcribed locally with faster-whisper, transcript fed into Claude extraction (requires `pip install graphifyy[video]`) | -| YouTube / URLs | any video URL | Audio downloaded via yt-dlp, then same Whisper pipeline (requires `pip install graphifyy[video]`) | - -## Video and audio corpus - -Drop video or audio files into your corpus folder alongside your code and docs — graphify picks them up automatically: - -```bash -pip install 'graphifyy[video]' # one-time setup -/graphify ./my-corpus # transcribes any video/audio files it finds -``` - -Add a YouTube video (or any public video URL) directly: - -```bash +/graphify add https://arxiv.org/abs/1706.03762 /graphify add -``` +/graphify add https://... --author "Name" --contributor "Name" -yt-dlp downloads audio-only (fast, small), Whisper transcribes it locally, and the transcript is fed into the same extraction pipeline as your other docs. Transcripts are cached in `graphify-out/transcripts/` so re-runs skip already-transcribed files. +/graphify query "what connects attention to the optimizer?" +/graphify query "..." --dfs --budget 1500 +/graphify path "DigestAuth" "Response" +/graphify explain "SwinTransformer" -For better accuracy on technical content, use a larger model: +graphify hook install # post-commit + post-checkout hooks +graphify hook uninstall +graphify hook status -```bash -/graphify ./my-corpus --whisper-model medium +graphify claude install / uninstall +graphify codex install / uninstall +graphify opencode install +graphify cursor install / uninstall +graphify gemini install / uninstall +graphify copilot install / uninstall +graphify aider install / uninstall +graphify claw install / uninstall +graphify droid install / uninstall +graphify trae install / uninstall +graphify trae-cn install / uninstall +graphify hermes install / uninstall +graphify kiro install / uninstall +graphify antigravity install / uninstall + +graphify clone https://github.com/karpathy/nanoGPT +graphify merge-graphs a.json b.json --out merged.json +graphify watch ./src +graphify check-update ./src +graphify update ./src +graphify cluster-only ./my-project ``` -Audio never leaves your machine. All transcription runs locally. - -## What you get - -**God nodes** - highest-degree concepts (what everything connects through) - -**Surprising connections** - ranked by composite score. Code-paper edges rank higher than code-code. Each result includes a plain-English why. - -**Suggested questions** - 4-5 questions the graph is uniquely positioned to answer - -**The "why"** - docstrings, inline comments (`# NOTE:`, `# IMPORTANT:`, `# HACK:`, `# WHY:`), and design rationale from docs are extracted as `rationale_for` nodes. Not just what the code does - why it was written that way. - -**Confidence scores** - every INFERRED edge has a `confidence_score` (0.0-1.0). You know not just what was guessed but how confident the model was. EXTRACTED edges are always 1.0. - -**Semantic similarity edges** - cross-file conceptual links with no structural connection. Two functions solving the same problem without calling each other, a class in code and a concept in a paper describing the same algorithm. - -**Hyperedges** - group relationships connecting 3+ nodes that pairwise edges can't express. All classes implementing a shared protocol, all functions in an auth flow, all concepts from a paper section forming one idea. +--- -**Token benchmark** - printed automatically after every run. On a mixed corpus (Karpathy repos + papers + images): **71.5x** fewer tokens per query vs reading raw files. The first run extracts and builds the graph (this costs tokens). Every subsequent query reads the compact graph instead of raw files — that's where the savings compound. The SHA256 cache means re-runs only re-process changed files. +## Learn more -**Auto-sync** (`--watch`) - run in a background terminal and the graph updates itself as your codebase changes. Code file saves trigger an instant rebuild (AST only, no LLM). Doc/image changes notify you to run `--update` for the LLM re-pass. +- [How it works](docs/how-it-works.md) — the extraction pipeline, community detection, confidence scoring, benchmarks +- [ARCHITECTURE.md](ARCHITECTURE.md) — module breakdown, how to add a language +- [Optional integrations](docs/docker-mcp-sqlite.md) — Docker MCP Toolkit + SQLite -**Git hooks** (`graphify hook install`) - installs post-commit and post-checkout hooks. Graph rebuilds automatically after every commit and every branch switch. If a rebuild fails, the hook exits with a non-zero code so git surfaces the error instead of silently continuing. No background process needed. - -**Wiki** (`--wiki`) - Wikipedia-style markdown articles per community and god node, with an `index.md` entry point. Point any agent at `index.md` and it can navigate the knowledge base by reading files instead of parsing JSON. - -## Worked examples - -| Corpus | Files | Reduction | Output | -|--------|-------|-----------|--------| -| Karpathy repos + 5 papers + 4 images | 52 | **71.5x** | [`worked/karpathy-repos/`](worked/karpathy-repos/) | -| graphify source + Transformer paper | 4 | **5.4x** | [`worked/mixed-corpus/`](worked/mixed-corpus/) | -| httpx (synthetic Python library) | 6 | ~1x | [`worked/httpx/`](worked/httpx/) | - -Token reduction scales with corpus size. 6 files fits in a context window anyway, so graph value there is structural clarity, not compression. At 52 files (code + papers + images) you get 71x+. Each `worked/` folder has the raw input files and the actual output (`GRAPH_REPORT.md`, `graph.json`) so you can run it yourself and verify the numbers. - -## Privacy - -graphify sends file contents to your AI coding assistant's underlying model API for semantic extraction of docs, papers, and images — Anthropic (Claude Code), OpenAI (Codex), or whichever provider your platform uses. Code files are processed locally via tree-sitter AST — no file contents leave your machine for code. Video and audio files are transcribed locally with faster-whisper — audio never leaves your machine. No telemetry, usage tracking, or analytics of any kind. The only network calls are to your platform's model API during extraction, using your own API key. - -## Tech stack - -NetworkX + Leiden (graspologic) + tree-sitter + vis.js. Semantic extraction via Claude (Claude Code), GPT-4 (Codex), or whichever model your platform runs. Video transcription via faster-whisper + yt-dlp (optional, `pip install graphifyy[video]`). No Neo4j required, no server, runs entirely locally. +--- ## Built on graphify — Penpax -[**Penpax**](https://safishamsi.github.io/penpax.ai) is the enterprise layer on top of graphify. Where graphify turns a folder of files into a knowledge graph, Penpax applies the same graph to your entire working life — continuously. - -| | graphify | Penpax | -|---|---|---| -| Input | A folder of files | Browser history, meetings, emails, files, code — everything | -| Runs | On demand | Continuously in the background | -| Scope | A project | Your entire working life | -| Query | CLI / MCP / AI skill | Natural language, always on | -| Privacy | Local by default | Fully on-device, no cloud | - -Built for lawyers, consultants, executives, doctors, researchers — anyone whose work lives across hundreds of conversations and documents they can never fully reconstruct. +[**Penpax**](https://graphifylabs.ai) is the always-on layer built on top of graphify — it applies the same graph approach to your entire working life: meetings, browser history, emails, files, and code, updating continuously in the background. -**Free trial launching soon.** [Join the waitlist →](https://safishamsi.github.io/penpax.ai) +Built for people whose work lives across hundreds of conversations and documents they can never fully reconstruct. No cloud, fully on-device. -## What we are building next +**Free trial launching soon.** [Join the waitlist →](https://graphifylabs.ai) -graphify is the graph layer. Penpax is the always-on layer on top of it — an on-device digital twin that connects your meetings, browser history, files, emails, and code into one continuously updating knowledge graph. No cloud, no training on your data. [Join the waitlist.](https://safishamsi.github.io/penpax.ai) +---
Contributing -**Worked examples** are the most trust-building contribution. Run `/graphify` on a real corpus, save output to `worked/{slug}/`, write an honest `review.md` evaluating what the graph got right and wrong, submit a PR. +**Worked examples** are the most useful contribution. Run `/graphify` on a real corpus, save the output to `worked/{slug}/`, write an honest `review.md` covering what the graph got right and wrong, and open a PR. -**Extraction bugs** - open an issue with the input file, the cache entry (`graphify-out/cache/`), and what was missed or invented. +**Extraction bugs** — open an issue with the input file, the cache entry (`graphify-out/cache/`), and what was missed or wrong. See [ARCHITECTURE.md](ARCHITECTURE.md) for module responsibilities and how to add a language. diff --git a/docs/docker-mcp-sqlite.md b/docs/docker-mcp-sqlite.md new file mode 100644 index 000000000..6cdf52b06 --- /dev/null +++ b/docs/docker-mcp-sqlite.md @@ -0,0 +1,138 @@ +# Docker MCP Toolkit + SQLite MCP server + +A reproducible runbook for installing the **SQLite MCP server** into the +[Docker MCP Toolkit](https://docs.docker.com/desktop/features/mcp/) so any +connected MCP client (Claude Code, Claude Desktop, Cursor, VS Code, etc.) gains +six SQLite tools: `read_query`, `write_query`, `create_table`, `list_tables`, +`describe_table`, and `append_insight`. + +This document is *not* required to use graphify — it lives here as a known-good +recipe for users who want a lightweight, persistent SQL workspace exposed to +their AI clients alongside graphify's knowledge-graph tools. + +## Why SQLite (and not `sqlite-mcp-server`) +At time of writing the catalog ships two SQLite MCP images: + +| Catalog name | Image | Status | +| ------------------- | ---------------------- | ------ | +| `SQLite` | `mcp/sqlite` | Marked "Archived" in catalog metadata, but **boots and serves correctly** | +| `sqlite-mcp-server` | `mcp/sqlite-mcp-server`| **Broken**: entrypoint `/app/.venv/bin/mcp-server-sqlite` does not exist in the published layer | + +Use `SQLite` (`mcp/sqlite`) until the newer image is fixed upstream. + +## Prerequisites +- Docker Desktop running and healthy + - `docker info` returns a `Server Version` + - Public socket present at `/var/run/docker.sock` (or its symlink to + `~/.docker/run/docker.sock`) +- Docker MCP Toolkit CLI plugin (`docker mcp`) + - Bundled with recent Docker Desktop releases; `docker mcp --version` should + print a version string + +## Install +```bash +# Add the working SQLite server to the default MCP profile +docker mcp profile server add default \ + --server catalog://mcp/docker-mcp-catalog/SQLite + +# Pre-pull the image so the first tool call is fast +docker pull mcp/sqlite:latest +``` + +Verify the profile now contains both `fetch` (built-in) and `SQLite`: +```bash +docker mcp profile show default | grep -E '^[[:space:]]+name:' +``` + +Expected output: +``` + name: fetch + name: SQLite +``` + +The Docker MCP gateway should now expose 6 additional tools: +```bash +docker mcp tools count +# → 15 tools (was 9 before adding SQLite) +``` + +## Smoke test +The CLI can call MCP tools directly (each call boots a fresh gateway, ~5s +overhead per call): +```bash +docker mcp tools call list_tables +docker mcp tools call create_table \ + query='CREATE TABLE IF NOT EXISTS notes (id INTEGER PRIMARY KEY AUTOINCREMENT, body TEXT NOT NULL, created_at TEXT DEFAULT CURRENT_TIMESTAMP)' +docker mcp tools call write_query \ + query="INSERT INTO notes(body) VALUES ('first row'), ('second row')" +docker mcp tools call read_query \ + query='SELECT * FROM notes ORDER BY id' +docker mcp tools call describe_table table_name=notes +docker mcp tools call append_insight insight='3 rows inserted; aggregates work.' +``` + +`read_query` should return the inserted rows with timestamps. + +## Storage layout +Database file lives in a Docker named volume `mcp-sqlite`, mounted at `/mcp` +inside containers: +``` +mcp-sqlite (named volume) → /mcp/db.sqlite +``` + +Inspect from the host: +```bash +docker volume inspect mcp-sqlite +docker run --rm -v mcp-sqlite:/mcp:ro alpine ls -la /mcp +docker run --rm -v mcp-sqlite:/mcp:ro keinos/sqlite3 \ + sqlite3 /mcp/db.sqlite '.schema' +``` + +The volume persists across `docker run --rm` invocations of the SQLite MCP +container, so writes from one MCP tool call are visible to the next. + +## Wiring into MCP clients +Connect once per client; the gateway exposes every server in the active profile: +```bash +docker mcp client connect claude-code # already connected for many users +docker mcp client connect cursor +docker mcp client connect vscode +docker mcp client connect claude-desktop +# Supported: claude-code, claude-desktop, cline, codex, continue, crush, +# cursor, gemini, goose, gordon, kiro, lmstudio, opencode, sema4, +# vscode, zed +``` + +Verify wiring: +```bash +docker mcp client ls +``` + +## Uninstall / reset +```bash +# Remove server from the profile +docker mcp profile server remove default SQLite + +# Drop the database volume (irreversible) +docker volume rm mcp-sqlite + +# Remove the image +docker rmi mcp/sqlite:latest +``` + +## Troubleshooting +- **`starting client: calling "initialize": EOF`** — the requested server + failed its MCP handshake. Run the image directly to see the error: + ```bash + printf '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"smoke","version":"0.0"}}}\n' \ + | docker run --rm -i -v mcp-sqlite:/mcp --db-path /mcp/db.sqlite + ``` + Common causes: missing entrypoint binary in the image (the + `sqlite-mcp-server` failure mode) or missing required env/secrets. +- **`cannot use --enable-all-servers with --servers flag`** — these gateway + args are mutually exclusive; pick one. +- **No new tools appear in `docker mcp tools count` after install** — the + gateway may be running with `dynamic-tools` enabled, exposing only meta-tools + (`mcp-add`, `mcp-find`, …) until a profile is activated mid-session. Either + invoke `docker mcp tools` (which spins up an ephemeral gateway against the + default profile) or call `mcp-activate-profile` from inside an MCP session. diff --git a/docs/how-it-works.md b/docs/how-it-works.md new file mode 100644 index 000000000..c79ab5cfc --- /dev/null +++ b/docs/how-it-works.md @@ -0,0 +1,90 @@ +# How graphify works + +## The three passes + +graphify processes your files in three passes: + +**Pass 1 — Code structure (free, no API calls)** +Tree-sitter parses your code files and extracts classes, functions, imports, call graphs, and inline comments. This runs locally with no LLM involved. 25 languages supported. SQL files get special treatment: tables, views, foreign keys, and JOIN relationships are extracted deterministically. + +**Pass 2 — Video and audio (local, no API calls)** +Video and audio files are transcribed with faster-whisper. To focus the transcript on your domain, the transcription prompt is seeded with your top god nodes (the most-connected concepts in your code graph so far). Transcripts are cached — re-runs skip already-processed files. + +**Pass 3 — Docs, papers, images (Claude subagents, costs tokens)** +Claude runs in parallel over markdown, PDFs, images, and transcripts. Each subagent reads a batch of files and outputs a JSON fragment: nodes, edges, and any group relationships. The fragments are merged into a single graph. + +--- + +## How community detection works + +Communities are found using the [Leiden algorithm](https://www.nature.com/articles/s41598-019-41695-z) — a graph-clustering method that groups nodes by edge density. Nodes with many connections between them end up in the same community. + +**No embeddings needed.** The semantic similarity edges that Claude extracts (`semantically_similar_to`) are already in the graph, so they influence community shape directly. The graph structure is the similarity signal — there's no separate embedding step or vector database. + +--- + +## Confidence tagging + +Every relationship is tagged with one of three labels: + +| Tag | Meaning | +|-----|---------| +| `EXTRACTED` | Found directly in the source (e.g. a function call, an import) | +| `INFERRED` | A reasonable inference Claude made, with a `confidence_score` (0.0–1.0) | +| `AMBIGUOUS` | Uncertain — flagged in the report for manual review | + +EXTRACTED edges always have confidence 1.0. INFERRED edges use a discrete rubric: +- **0.95** — near-certain (explicit cross-file reference, one plausible target) +- **0.85** — strong evidence (naming + context align) +- **0.75** — reasonable (contextual but not explicit) +- **0.65** — weak (naming similarity only) +- **0.55** — speculative + +--- + +## Token benchmark + +The first run extracts and builds the graph — this costs tokens. Every subsequent query reads the compact graph instead of raw files. That's where the savings compound. + +On a mixed corpus (Karpathy repos + 5 papers + 4 images, 52 files): **71.5x fewer tokens per query** vs reading the raw files directly. + +| Corpus | Files | Reduction | +|--------|-------|-----------| +| Karpathy repos + papers + images | 52 | **71.5x** | +| graphify source + Transformer paper | 4 | **5.4x** | +| httpx (synthetic Python library) | 6 | ~1x | + +Token reduction scales with corpus size. Six files already fits in a context window — the graph value there is structural clarity, not compression. At 52 files the savings compound quickly. + +Each `worked/` folder in the repo has the raw input files and actual output (`GRAPH_REPORT.md`, `graph.json`) so you can run it yourself and verify. + +--- + +## Parallel extraction + +Code files are extracted in parallel using `ProcessPoolExecutor` — bypasses Python's GIL for genuine multiprocessing. Doc/paper/image batches are dispatched as parallel Claude subagents. On a corpus of 84 code files, parallel AST extraction runs in about 1.66x less time than sequential. + +--- + +## SHA256 cache + +Every extracted file is fingerprinted by content hash. Re-runs skip unchanged files entirely — only new or modified files go through extraction again. The cache lives in `graphify-out/cache/`. + +--- + +## The graph format + +The output `graph.json` uses NetworkX's node-link format. Each node has: +- `id` — stable identifier +- `label` — human-readable name +- `file_type` — `code`, `document`, `paper`, `image`, `rationale` +- `source_file` — where it came from + +Each edge has: +- `source`, `target` — node IDs +- `relation` — verb phrase (e.g. `calls`, `imports`, `implements`, `semantically_similar_to`) +- `confidence` — `EXTRACTED`, `INFERRED`, or `AMBIGUOUS` +- `confidence_score` — float (INFERRED only) +- `source_file` — where the relationship was found + +Hyperedges (group relationships connecting 3+ nodes) live in `G.graph["hyperedges"]`. diff --git a/graphify/__main__.py b/graphify/__main__.py index 9abaf5a55..2724d8c6d 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -1,6 +1,7 @@ """graphify CLI - `graphify install` sets up the Claude Code skill.""" from __future__ import annotations import json +import os import platform import re import shutil @@ -13,6 +14,10 @@ except Exception: __version__ = "unknown" +# Output directory — override with GRAPHIFY_OUT env var for worktrees or shared-output setups. +# Accepts a relative name ("graphify-out-feature") or an absolute path ("/shared/graphify-out"). +_GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") + def _check_skill_version(skill_dst: Path) -> None: """Warn if the installed skill is from an older graphify version.""" @@ -37,14 +42,22 @@ def _refresh_all_version_stamps() -> None: vf.write_text(__version__, encoding="utf-8") _SETTINGS_HOOK = { - "matcher": "Glob|Grep", + # Claude Code v2.1.117+ removed dedicated Grep/Glob tools; searches now go through Bash. + # We match on Bash and inspect the command string to avoid firing on every shell call. + "matcher": "Bash", "hooks": [ { "type": "command", "command": ( - "[ -f graphify-out/graph.json ] && " - r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}}' """ - "|| true" + "CMD=$(python3 -c \"" + "import json,sys; d=json.load(sys.stdin); " + "print(d.get('tool_input',d).get('command',''))\" 2>/dev/null || true); " + "case \"$CMD\" in " + r"*grep*|*rg\ *|*ripgrep*|*find\ *|*fd\ *|*ack\ *|*ag\ *) " + " [ -f graphify-out/graph.json ] && " + r""" echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}}' """ + " || true ;; " + "esac" ), } ], @@ -115,9 +128,14 @@ def _refresh_all_version_stamps() -> None: "skill_dst": Path(".kiro") / "skills" / "graphify" / "SKILL.md", "claude_md": False, }, + "pi": { + "skill_file": "skill-pi.md", + "skill_dst": Path(".pi") / "agent" / "skills" / "graphify" / "SKILL.md", + "claude_md": False, + }, "antigravity": { "skill_file": "skill.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", + "skill_dst": Path(".agent") / "skills" / "graphify" / "SKILL.md", "claude_md": False, }, "windows": { @@ -148,7 +166,12 @@ def install(platform: str = "claude") -> None: print(f"error: {cfg['skill_file']} not found in package - reinstall graphify", file=sys.stderr) sys.exit(1) - skill_dst = Path.home() / cfg["skill_dst"] + import os as _os + if platform in ("claude", "windows") and _os.environ.get("CLAUDE_CONFIG_DIR"): + _claude_base = Path(_os.environ["CLAUDE_CONFIG_DIR"]) + skill_dst = _claude_base / "skills" / "graphify" / "SKILL.md" + else: + skill_dst = Path.home() / cfg["skill_dst"] skill_dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy(skill_src, skill_dst) (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") @@ -233,9 +256,13 @@ def install(platform: str = "claude") -> None: { "type": "command", "command": ( - "[ -f graphify-out/graph.json ] && " - r"""echo '{"decision":"allow","additionalContext":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}' """ - r"""|| echo '{"decision":"allow"}'""" + 'python -c "' + "import sys,pathlib,json;" + "e=pathlib.Path('graphify-out/graph.json').exists();" + "d={'decision':'allow'};" + "e and d.update({'additionalContext':'graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files.'});" + "sys.stdout.write(json.dumps(d))" + '"' ), } ], @@ -347,9 +374,17 @@ def gemini_uninstall(project_dir: Path | None = None) -> None: _VSCODE_INSTRUCTIONS_SECTION = """\ ## graphify -Before answering architecture or codebase questions, read `graphify-out/GRAPH_REPORT.md` if it exists. -If `graphify-out/wiki/index.md` exists, navigate it for deep questions. -Type `/graphify` in Copilot Chat to build or update the knowledge graph. +For any question about this repo's architecture, structure, components, or how to add/modify/find +code, your **first tool call must be** to read `graphify-out/GRAPH_REPORT.md` (if it exists). + +Triggers: "how do I…", "where is…", "what does … do", "add/modify a ", +"explain the architecture", or anything that depends on how files or classes relate. + +After reading the report (and `graphify-out/wiki/index.md` for deep questions), answer from the +graph. Only read source files when (a) modifying/debugging specific code, (b) the graph lacks +the needed detail, or (c) the graph is missing or stale. + +Type `/graphify` in Copilot Chat to build or update the graph. """ @@ -412,8 +447,8 @@ def vscode_uninstall(project_dir: Path | None = None) -> None: print(f" {instructions} -> deleted (was empty after removal)") -_ANTIGRAVITY_RULES_PATH = Path(".agents") / "rules" / "graphify.md" -_ANTIGRAVITY_WORKFLOW_PATH = Path(".agents") / "workflows" / "graphify.md" +_ANTIGRAVITY_RULES_PATH = Path(".agent") / "rules" / "graphify.md" +_ANTIGRAVITY_WORKFLOW_PATH = Path(".agent") / "workflows" / "graphify.md" _ANTIGRAVITY_RULES = """\ ## graphify @@ -429,12 +464,14 @@ def vscode_uninstall(project_dir: Path | None = None) -> None: """ _ANTIGRAVITY_WORKFLOW = """\ +--- +name: graphify +description: Turn any folder of files into a navigable knowledge graph +--- + # Workflow: graphify -**Command:** /graphify -**Description:** Turn any folder of files into a navigable knowledge graph -## Steps -Follow the graphify skill installed at ~/.agents/skills/graphify/SKILL.md to run the full pipeline. +Follow the graphify skill installed at ~/.agent/skills/graphify/SKILL.md to run the full pipeline. If no path argument is given, use `.` (current directory). """ @@ -504,8 +541,8 @@ def _kiro_uninstall(project_dir: Path) -> None: def _antigravity_install(project_dir: Path) -> None: - """Install graphify for Google Antigravity: skill + .agents/rules + .agents/workflows.""" - # 1. Copy skill file to ~/.agents/skills/graphify/SKILL.md + """Install graphify for Google Antigravity: skill + .agent/rules + .agent/workflows.""" + # 1. Copy skill file to ~/.agent/skills/graphify/SKILL.md install(platform="antigravity") # 1.5. Inject YAML frontmatter for native Antigravity tool discovery @@ -516,20 +553,30 @@ def _antigravity_install(project_dir: Path) -> None: frontmatter = "---\nname: graphify-manager\ndescription: Rebuild the code graph or perform manual CLI queries when MCP server is offline.\n---\n\n" skill_dst.write_text(frontmatter + content, encoding="utf-8") - # 2. Write .agents/rules/graphify.md + # 2. Write .agent/rules/graphify.md rules_path = project_dir / _ANTIGRAVITY_RULES_PATH rules_path.parent.mkdir(parents=True, exist_ok=True) if rules_path.exists(): - print(f"graphify rule already exists at {rules_path} (no change)") + existing = rules_path.read_text(encoding="utf-8") + if _ANTIGRAVITY_RULES.strip() != existing.strip(): + rules_path.write_text(_ANTIGRAVITY_RULES, encoding="utf-8") + print(f"graphify rule updated at {rules_path.resolve()}") + else: + print(f"graphify rule already up to date at {rules_path.resolve()}") else: rules_path.write_text(_ANTIGRAVITY_RULES, encoding="utf-8") print(f"graphify rule written to {rules_path.resolve()}") - # 3. Write .agents/workflows/graphify.md + # 3. Write .agent/workflows/graphify.md wf_path = project_dir / _ANTIGRAVITY_WORKFLOW_PATH wf_path.parent.mkdir(parents=True, exist_ok=True) if wf_path.exists(): - print(f"graphify workflow already exists at {wf_path} (no change)") + existing = wf_path.read_text(encoding="utf-8") + if _ANTIGRAVITY_WORKFLOW.strip() != existing.strip(): + wf_path.write_text(_ANTIGRAVITY_WORKFLOW, encoding="utf-8") + print(f"graphify workflow updated at {wf_path.resolve()}") + else: + print(f"graphify workflow already up to date at {wf_path.resolve()}") else: wf_path.write_text(_ANTIGRAVITY_WORKFLOW, encoding="utf-8") print(f"graphify workflow written to {wf_path.resolve()}") @@ -704,11 +751,11 @@ def _uninstall_opencode_plugin(project_dir: Path) -> None: "hooks": [ { "type": "command", - "command": ( - "[ -f graphify-out/graph.json ] && " - r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files."}}' """ - "|| true" - ), + # Use the graphify CLI itself so the hook is shell-agnostic: + # no [ -f ] bash syntax, no python3 vs python Conda issue, + # no JSON escaping inside PowerShell strings. Works on + # Windows (PowerShell/cmd.exe), macOS, and Linux. + "command": "graphify hook-check", } ], } @@ -717,6 +764,26 @@ def _uninstall_opencode_plugin(project_dir: Path) -> None: } +def _resolve_graphify_exe() -> str: + """Return the absolute path to the graphify executable. + + Falls back to bare 'graphify' if resolution fails. Using an absolute path + ensures the hook works in environments where the venv Scripts/ directory is + not on PATH (e.g. VS Code Codex extension on Windows). + """ + import shutil + found = shutil.which("graphify") + if found: + return found + # Derive from sys.executable: same Scripts/ (Windows) or bin/ (Unix) dir + scripts_dir = Path(sys.executable).parent + for name in ("graphify.exe", "graphify"): + candidate = scripts_dir / name + if candidate.exists(): + return str(candidate) + return "graphify" + + def _install_codex_hook(project_dir: Path) -> None: """Add graphify PreToolUse hook to .codex/hooks.json.""" hooks_path = project_dir / ".codex" / "hooks.json" @@ -730,11 +797,23 @@ def _install_codex_hook(project_dir: Path) -> None: else: existing = {} + graphify_exe = _resolve_graphify_exe() + hook_entry = { + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [{"type": "command", "command": f"{graphify_exe} hook-check"}], + } + ] + } + } + pre_tool = existing.setdefault("hooks", {}).setdefault("PreToolUse", []) existing["hooks"]["PreToolUse"] = [h for h in pre_tool if "graphify" not in str(h)] - existing["hooks"]["PreToolUse"].extend(_CODEX_HOOK["hooks"]["PreToolUse"]) + existing["hooks"]["PreToolUse"].extend(hook_entry["hooks"]["PreToolUse"]) hooks_path.write_text(json.dumps(existing, indent=2), encoding="utf-8") - print(f" .codex/hooks.json -> PreToolUse hook registered") + print(f" .codex/hooks.json -> PreToolUse hook registered ({graphify_exe} hook-check)") def _uninstall_codex_hook(project_dir: Path) -> None: @@ -852,7 +931,7 @@ def _install_claude_hook(project_dir: Path) -> None: hooks = settings.setdefault("hooks", {}) pre_tool = hooks.setdefault("PreToolUse", []) - hooks["PreToolUse"] = [h for h in pre_tool if not (h.get("matcher") == "Glob|Grep" and "graphify" in str(h))] + hooks["PreToolUse"] = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash") and "graphify" in str(h))] hooks["PreToolUse"].append(_SETTINGS_HOOK) settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") print(f" .claude/settings.json -> PreToolUse hook registered") @@ -868,7 +947,7 @@ def _uninstall_claude_hook(project_dir: Path) -> None: except json.JSONDecodeError: return pre_tool = settings.get("hooks", {}).get("PreToolUse", []) - filtered = [h for h in pre_tool if not (h.get("matcher") == "Glob|Grep" and "graphify" in str(h))] + filtered = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash") and "graphify" in str(h))] if len(filtered) == len(pre_tool): return settings["hooks"]["PreToolUse"] = filtered @@ -906,6 +985,63 @@ def claude_uninstall(project_dir: Path | None = None) -> None: _uninstall_claude_hook(project_dir or Path(".")) +def _clone_repo(url: str, branch: str | None = None, out_dir: Path | None = None) -> Path: + """Clone a GitHub repo to a local cache dir and return the path. + + Clones into ~/.graphify/repos// by default so repeated + runs on the same URL reuse the existing clone (git pull instead of clone). + """ + import subprocess as _sp + import re as _re + + # Normalise URL — strip trailing .git if present + url = url.rstrip("/") + if not url.endswith(".git"): + git_url = url + ".git" + else: + git_url = url + url = url[:-4] + + # Extract owner/repo from URL + m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) + if not m: + print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) + sys.exit(1) + owner, repo = m.group(1), m.group(2) + + if out_dir: + dest = out_dir + else: + dest = Path.home() / ".graphify" / "repos" / owner / repo + + if branch and branch.startswith("-"): + print(f"error: invalid branch name: {branch!r}", file=sys.stderr) + sys.exit(1) + + if dest.exists(): + print(f"Repo already cloned at {dest} — pulling latest...", flush=True) + cmd = ["git", "-C", str(dest), "pull"] + if branch: + cmd += ["origin", "--", branch] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) + else: + dest.parent.mkdir(parents=True, exist_ok=True) + print(f"Cloning {url} → {dest} ...", flush=True) + cmd = ["git", "clone", "--depth", "1"] + if branch: + cmd += ["--branch", branch] + cmd += ["--", git_url, str(dest)] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + + print(f"Ready at: {dest}", flush=True) + return dest + + def main() -> None: # Check all known skill install locations for a stale version stamp. # Skip during install/uninstall (hook writes trigger a fresh check anyway). @@ -918,20 +1054,29 @@ def main() -> None: print("Usage: graphify ") print() print("Commands:") - print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|aider|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro)") + print(" install [--platform P] copy skill to platform config dir (claude|windows|codex|opencode|aider|claw|droid|trae|trae-cn|gemini|cursor|antigravity|hermes|kiro|pi)") print(" path \"A\" \"B\" shortest path between two nodes in graph.json") print(" --graph path to graph.json (default graphify-out/graph.json)") print(" explain \"X\" plain-language explanation of a node and its neighbors") print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" clone clone a GitHub repo locally and print its path for /graphify") + print(" merge-graphs merge two or more graph.json files into one cross-repo graph") + print(" --out output path (default: graphify-out/merged-graph.json)") + print(" --branch checkout a specific branch (default: repo default)") + print(" --out clone to a custom directory (default: ~/.graphify/repos//)") print(" add fetch a URL and save it to ./raw, then update the graph") print(" --author \"Name\" tag the author of the content") print(" --contributor \"Name\" tag who added it to the corpus") print(" --dir target directory (default: ./raw)") print(" watch watch a folder and rebuild the graph on code changes") print(" update re-extract code files and update the graph (no LLM needed)") + print(" --force overwrite graph.json even if the rebuild has fewer nodes") + print(" (also: GRAPHIFY_FORCE=1 env var; use after refactors that delete code)") print(" cluster-only rerun clustering on an existing graph.json and regenerate report") + print(" --no-viz skip graph.html generation (useful for >5000 node graphs / CI)") print(" query \"\" BFS traversal of graph.json for a question") print(" --dfs use depth-first instead of breadth-first") + print(" --context C explicit edge-context filter (repeatable)") print(" --budget N cap output at N tokens (default 2000)") print(" --graph path to graph.json (default graphify-out/graph.json)") print(" save-result save a Q&A result to graphify-out/memory/ for graph feedback loop") @@ -941,6 +1086,13 @@ def main() -> None: print(" --nodes N1 N2 ... source node labels cited in the answer") print(" --memory-dir DIR memory directory (default: graphify-out/memory)") print(" check-update check needs_update flag and notify if semantic re-extraction is pending (cron-safe)") + print(" tree emit a D3 v7 collapsible-tree HTML for graph.json") + print(" --graph PATH path to graph.json (default graphify-out/graph.json)") + print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") + print(" --root PATH filesystem root for the hierarchy") + print(" --max-children N cap children per node (default 200)") + print(" --top-k-edges N per-symbol outbound edges in inspector (default 12)") + print(" --label NAME project label in header") print(" benchmark [graph.json] measure token reduction vs naive full-corpus approach") print(" hook install install post-commit/post-checkout git hooks (all platforms)") print(" hook uninstall remove git hooks") @@ -969,12 +1121,14 @@ def main() -> None: print(" trae uninstall remove graphify section from AGENTS.md") print(" trae-cn install write graphify section to AGENTS.md (Trae CN)") print(" trae-cn uninstall remove graphify section from AGENTS.md") - print(" antigravity install write .agents/rules + .agents/workflows + skill (Google Antigravity)") - print(" antigravity uninstall remove .agents/rules, .agents/workflows, and skill") + print(" antigravity install write .agent/rules + .agent/workflows + skill (Google Antigravity)") + print(" antigravity uninstall remove .agent/rules, .agent/workflows, and skill") print(" hermes install write skill to ~/.hermes/skills/graphify/ (Hermes)") print(" hermes uninstall remove skill from ~/.hermes/skills/graphify/") print(" kiro install write skill to .kiro/skills/graphify/ + steering file (Kiro IDE/CLI)") print(" kiro uninstall remove skill + steering file") + print(" pi install write skill to ~/.pi/agent/skills/graphify/ (Pi coding agent)") + print(" pi uninstall remove skill from ~/.pi/agent/skills/graphify/") print() return @@ -1062,6 +1216,26 @@ def main() -> None: else: print("Usage: graphify kiro [install|uninstall]", file=sys.stderr) sys.exit(1) + elif cmd == "pi": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + install("pi") + elif subcmd == "uninstall": + skill_dst = Path.home() / ".pi" / "agent" / "skills" / "graphify" / "SKILL.md" + if skill_dst.exists(): + skill_dst.unlink() + print(f" skill removed -> {skill_dst}") + version_file = skill_dst.parent / ".graphify_version" + if version_file.exists(): + version_file.unlink() + for d in (skill_dst.parent, skill_dst.parent.parent, skill_dst.parent.parent.parent): + try: + d.rmdir() + except OSError: + break + else: + print("Usage: graphify pi [install|uninstall]", file=sys.stderr) + sys.exit(1) elif cmd in ("aider", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): subcmd = sys.argv[2] if len(sys.argv) > 2 else "" if subcmd == "install": @@ -1096,15 +1270,16 @@ def main() -> None: sys.exit(1) elif cmd == "query": if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--budget N] [--graph path]", file=sys.stderr) + print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) sys.exit(1) - from graphify.serve import _score_nodes, _bfs, _dfs, _subgraph_to_text + from graphify.serve import _query_graph_text from graphify.security import sanitize_label from networkx.readwrite import json_graph question = sys.argv[2] use_dfs = "--dfs" in sys.argv budget = 2000 graph_path = "graphify-out/graph.json" + context_filters: list[str] = [] args = sys.argv[3:] i = 0 while i < len(args): @@ -1122,6 +1297,12 @@ def main() -> None: print(f"error: --budget must be an integer", file=sys.stderr) sys.exit(1) i += 1 + elif args[i] == "--context" and i + 1 < len(args): + context_filters.append(args[i + 1]) + i += 2 + elif args[i].startswith("--context="): + context_filters.append(args[i].split("=", 1)[1]) + i += 1 elif args[i] == "--graph" and i + 1 < len(args): graph_path = args[i + 1]; i += 2 else: @@ -1144,14 +1325,16 @@ def main() -> None: except Exception as exc: print(f"error: could not load graph: {exc}", file=sys.stderr) sys.exit(1) - terms = [t.lower() for t in question.split() if len(t) > 2] - scored = _score_nodes(G, terms) - if not scored: - print("No matching nodes found.") - sys.exit(0) - start = [nid for _, nid in scored[:5]] - nodes, edges = (_dfs if use_dfs else _bfs)(G, start, depth=2) - print(_subgraph_to_text(G, nodes, edges, token_budget=budget)) + print( + _query_graph_text( + G, + question, + mode="dfs" if use_dfs else "bfs", + depth=2, + token_budget=budget, + context_filters=context_filters, + ) + ) elif cmd == "save-result": # graphify save-result --question Q --answer A --type T [--nodes N1 N2 ...] import argparse as _ap @@ -1307,6 +1490,9 @@ def main() -> None: elif cmd == "cluster-only": watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") + no_viz = "--no-viz" in sys.argv + _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) + min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 graph_json = watch_path / "graphify-out" / "graph.json" if not graph_json.exists(): print(f"error: no graph found at {graph_json} — run /graphify first", file=sys.stderr) @@ -1319,7 +1505,8 @@ def main() -> None: from graphify.export import to_json, to_html print("Loading existing graph...") _raw = json.loads(graph_json.read_text(encoding="utf-8")) - G = build_from_json(_raw) + _directed = bool(_raw.get("directed", False)) + G = build_from_json(_raw, directed=_directed) print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") print("Re-clustering...") communities = cluster(G) @@ -1331,27 +1518,65 @@ def main() -> None: tokens = {"input": 0, "output": 0} report = generate(G, communities, cohesion, labels, gods, surprises, {"warning": "cluster-only mode — file stats not available"}, - tokens, str(watch_path), suggested_questions=questions) + tokens, str(watch_path), suggested_questions=questions, + min_community_size=min_community_size) out = watch_path / "graphify-out" (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") to_json(G, communities, str(out / "graph.json")) - to_html(G, communities, str(out / "graph.html"), community_labels=labels or None) - print(f"Done — {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") + + # Mirror watch.py pattern: gate to_html so core outputs (graph.json + + # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise + # fall back to ValueError handling so an oversized graph doesn't crash + # the CLI mid-write and leave a stale graph.html on disk. + html_target = out / "graph.html" + if no_viz: + if html_target.exists(): + html_target.unlink() + print(f"Done — {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") + else: + try: + to_html(G, communities, str(html_target), community_labels=labels or None) + print(f"Done — {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") + except ValueError as viz_err: + if html_target.exists(): + html_target.unlink() + print(f"Skipped graph.html: {viz_err}") + print(f"Done — {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") elif cmd == "update": - watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") + force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") + argv = list(sys.argv) + if "--force" in argv[2:]: + force = True + argv = [a for a in argv if a != "--force"] + if len(argv) > 2: + watch_path = Path(argv[2]) + else: + # Try to recover the scan root saved by the last full build + saved = Path(_GRAPHIFY_OUT) / ".graphify_root" + if saved.exists(): + watch_path = Path(saved.read_text(encoding="utf-8").strip()) + else: + watch_path = Path(".") if not watch_path.exists(): print(f"error: path not found: {watch_path}", file=sys.stderr) sys.exit(1) from graphify.watch import _rebuild_code print(f"Re-extracting code files in {watch_path} (no LLM needed)...") - ok = _rebuild_code(watch_path) + ok = _rebuild_code(watch_path, force=force) if ok: print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") + if not os.environ.get("MOONSHOT_API_KEY") and not os.environ.get("GRAPHIFY_NO_TIPS"): + print("Tip: set MOONSHOT_API_KEY to use Kimi K2.6 for semantic extraction — 3x cheaper, richer graphs. pip install 'graphifyy[kimi]'") else: print("Nothing to update or rebuild failed — check output above.", file=sys.stderr) sys.exit(1) + elif cmd == "hook-check": + # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. + # Keep this as a cross-platform no-op so installed hooks never break Bash + # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. + sys.exit(0) elif cmd == "check-update": if len(sys.argv) < 3: print("Usage: graphify check-update ", file=sys.stderr) @@ -1359,6 +1584,122 @@ def main() -> None: from graphify.watch import check_update check_update(Path(sys.argv[2]).resolve()) sys.exit(0) + elif cmd == "tree": + # Emit a D3 v7 collapsible-tree HTML view of graph.json: + # expand-all / collapse-all / reset-view buttons, multi-line + # wrapText labels with separately-coloured name + count, + # depth-based palette, click-to-toggle subtree, hover inspector + # showing top-K outbound edges per symbol. + from typing import Optional as _Opt + from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN + graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + output_path: "_Opt[Path]" = None + root: "_Opt[str]" = None + max_children = DEFAULT_MAX_CHILDREN + top_k_edges = 0 + project_label: "_Opt[str]" = None + args = sys.argv[2:] + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--graph" and i_arg + 1 < len(args): + graph_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--output" and i_arg + 1 < len(args): + output_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--root" and i_arg + 1 < len(args): + root = args[i_arg + 1]; i_arg += 2 + elif a == "--max-children" and i_arg + 1 < len(args): + max_children = int(args[i_arg + 1]); i_arg += 2 + elif a == "--top-k-edges" and i_arg + 1 < len(args): + top_k_edges = int(args[i_arg + 1]); i_arg += 2 + elif a == "--label" and i_arg + 1 < len(args): + project_label = args[i_arg + 1]; i_arg += 2 + elif a in ("-h", "--help"): + print("Usage: graphify tree [--graph PATH] [--output HTML]") + print(" --graph PATH path to graph.json (default graphify-out/graph.json)") + print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") + print(" --root PATH filesystem root (default: longest common dir of all source_files)") + print(" --max-children N cap visible children per node (default 200)") + print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") + print(" --label NAME project label shown in the page header") + return + else: + i_arg += 1 + if not graph_path.is_file(): + print(f"error: graph.json not found at {graph_path}", file=sys.stderr) + sys.exit(1) + if output_path is None: + output_path = graph_path.parent / "GRAPH_TREE.html" + out = write_tree_html( + graph_path=graph_path, output_path=output_path, + root=root, max_children=max_children, + top_k_edges=top_k_edges, project_label=project_label, + ) + size_kb = out.stat().st_size / 1024 + print(f"wrote {out} ({size_kb:.1f} KB)") + print(f"open with: xdg-open {out} (or file://{out.resolve()})") + sys.exit(0) + + elif cmd == "merge-graphs": + # graphify merge-graphs graph1.json graph2.json ... --out merged.json + args = sys.argv[2:] + graph_paths: list[Path] = [] + out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" + i = 0 + while i < len(args): + if args[i] == "--out" and i + 1 < len(args): + out_path = Path(args[i + 1]); i += 2 + else: + graph_paths.append(Path(args[i])); i += 1 + if len(graph_paths) < 2: + print("Usage: graphify merge-graphs [...] [--out merged.json]", file=sys.stderr) + sys.exit(1) + import networkx as _nx + from networkx.readwrite import json_graph as _jg + graphs = [] + for gp in graph_paths: + if not gp.exists(): + print(f"error: not found: {gp}", file=sys.stderr) + sys.exit(1) + data = json.loads(gp.read_text(encoding="utf-8")) + try: + G = _jg.node_link_graph(data, edges="links") + except TypeError: + G = _jg.node_link_graph(data) + # Tag every node with which repo it came from + repo_tag = gp.parent.parent.name # graphify-out/../ → repo dir name + for node in G.nodes: + G.nodes[node].setdefault("repo", repo_tag) + graphs.append(G) + merged = _nx.compose_all(graphs) + try: + out_data = _jg.node_link_data(merged, edges="links") + except TypeError: + out_data = _jg.node_link_data(merged) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") + print(f"Merged {len(graphs)} graphs → {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") + print(f"Written to: {out_path}") + + elif cmd == "clone": + if len(sys.argv) < 3: + print("Usage: graphify clone [--branch ] [--out ]", file=sys.stderr) + sys.exit(1) + url = sys.argv[2] + branch: str | None = None + out_dir: Path | None = None + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--branch" and i + 1 < len(args): + branch = args[i + 1]; i += 2 + elif args[i] == "--out" and i + 1 < len(args): + out_dir = Path(args[i + 1]); i += 2 + else: + i += 1 + local_path = _clone_repo(url, branch=branch, out_dir=out_dir) + print(local_path) + elif cmd == "benchmark": from graphify.benchmark import run_benchmark, print_benchmark graph_path = sys.argv[2] if len(sys.argv) > 2 else "graphify-out/graph.json" diff --git a/graphify/analyze.py b/graphify/analyze.py index 4f480c5bc..de07cc132 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -1,7 +1,34 @@ """Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions.""" from __future__ import annotations +from pathlib import Path import networkx as nx +# Language families — extensions sharing a runtime can legitimately call each other +_LANG_FAMILY: dict[str, str] = { + **{e: "python" for e in (".py", ".pyw")}, + **{e: "js" for e in (".js", ".jsx", ".mjs", ".ejs", ".ts", ".tsx", ".vue", ".svelte")}, + **{e: "go" for e in (".go",)}, + **{e: "rust" for e in (".rs",)}, + **{e: "jvm" for e in (".java", ".kt", ".kts", ".scala")}, + **{e: "c" for e in (".c", ".h", ".cpp", ".cc", ".cxx", ".hpp")}, + **{e: "ruby" for e in (".rb",)}, + **{e: "swift" for e in (".swift",)}, + **{e: "dotnet" for e in (".cs",)}, + **{e: "php" for e in (".php",)}, + **{e: "r" for e in (".r",)}, +} + + +def _cross_language(src_a: str, src_b: str) -> bool: + """Return True if two source files belong to different language families.""" + ext_a = Path(src_a).suffix.lower() + ext_b = Path(src_b).suffix.lower() + fam_a = _LANG_FAMILY.get(ext_a) + fam_b = _LANG_FAMILY.get(ext_b) + if fam_a is None or fam_b is None: + return False + return fam_a != fam_b + def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: """Invert communities dict: node_id -> community_id.""" @@ -143,7 +170,13 @@ def _surprise_score( # 1. Confidence weight - uncertain connections are more noteworthy conf = data.get("confidence", "EXTRACTED") + relation = data.get("relation", "") conf_bonus = {"AMBIGUOUS": 3, "INFERRED": 2, "EXTRACTED": 1}.get(conf, 1) + + # Cross-language INFERRED calls are likely resolver pollution, not real surprises + if conf == "INFERRED" and relation == "calls" and _cross_language(u_source, v_source): + conf_bonus = 0 # downgrade: don't promote likely false positives + score += conf_bonus if conf in ("AMBIGUOUS", "INFERRED"): reasons.append(f"{conf.lower()} connection - not explicitly stated in source") diff --git a/graphify/build.py b/graphify/build.py index 19b5b1867..373662712 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -21,8 +21,10 @@ # before any graph construction happens. # from __future__ import annotations +import json import re import sys +from pathlib import Path import networkx as nx from .validate import validate_extraction @@ -37,6 +39,12 @@ def _normalize_id(s: str) -> str: return cleaned.strip("_").lower() +def _norm_source_file(p: str | None) -> str | None: + """Normalize path separators to forward slashes so Windows backslash paths + and POSIX paths from semantic subagents resolve to the same node identity.""" + return p.replace("\\", "/") if p else p + + def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: """Build a NetworkX graph from an extraction dict. @@ -50,6 +58,18 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: # Canonicalize legacy node/edge schema before validation. for node in extraction.get("nodes", []): if isinstance(node, dict) and "source" in node and "source_file" not in node: + # Count edges that reference this node so the warning is actionable (#479) + node_id = node.get("id", "?") + affected_edges = sum( + 1 for e in extraction.get("edges", []) + if e.get("source") == node_id or e.get("target") == node_id + ) + print( + f"[graphify] WARNING: node '{node_id}' uses field 'source' instead of " + f"'source_file' — {affected_edges} edge(s) may be misrouted. " + f"Rename the field to 'source_file' to silence this warning.", + file=sys.stderr, + ) node["source_file"] = node.pop("source") errors = validate_extraction(extraction) @@ -59,6 +79,8 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr) G: nx.Graph = nx.DiGraph() if directed else nx.Graph() for node in extraction.get("nodes", []): + if "source_file" in node: + node["source_file"] = _norm_source_file(node["source_file"]) G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) node_set = set(G.nodes()) # Normalized ID map: lets edges survive when the LLM generates IDs with @@ -81,6 +103,8 @@ def build_from_json(extraction: dict, *, directed: bool = False) -> nx.Graph: if src not in node_set or tgt not in node_set: continue # skip edges to external/stdlib nodes - expected, not an error attrs = {k: v for k, v in edge.items() if k not in ("source", "target")} + if "source_file" in attrs: + attrs["source_file"] = _norm_source_file(attrs["source_file"]) # Preserve original edge direction - undirected graphs lose it otherwise, # causing display functions to show edges backwards. attrs["_src"] = src @@ -111,3 +135,110 @@ def build(extractions: list[dict], *, directed: bool = False) -> nx.Graph: combined["input_tokens"] += ext.get("input_tokens", 0) combined["output_tokens"] += ext.get("output_tokens", 0) return build_from_json(combined, directed=directed) + + +def _norm_label(label: str) -> str: + """Canonical dedup key — lowercase, alphanumeric only.""" + return re.sub(r"[^a-z0-9 ]", "", label.lower()).strip() + + +def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dict], list[dict]]: + """Merge nodes that share a normalised label, rewriting edge references. + + Prefers IDs without chunk suffixes (_c\\d+) and shorter IDs when tied. + Drops self-loops created by the merge. Called in build() automatically. + """ + _CHUNK_SUFFIX = re.compile(r"_c\d+$") + canonical: dict[str, dict] = {} # norm_label -> surviving node + remap: dict[str, str] = {} # old_id -> surviving_id + + for node in nodes: + key = _norm_label(node.get("label", node.get("id", ""))) + if not key: + continue + existing = canonical.get(key) + if existing is None: + canonical[key] = node + else: + has_suffix = bool(_CHUNK_SUFFIX.search(node["id"])) + existing_has_suffix = bool(_CHUNK_SUFFIX.search(existing["id"])) + if has_suffix and not existing_has_suffix: + remap[node["id"]] = existing["id"] + elif existing_has_suffix and not has_suffix: + remap[existing["id"]] = node["id"] + canonical[key] = node + elif len(node["id"]) < len(existing["id"]): + remap[existing["id"]] = node["id"] + canonical[key] = node + else: + remap[node["id"]] = existing["id"] + + if not remap: + return nodes, edges + + print(f"[graphify] Deduplicated {len(remap)} duplicate node(s) by label.", file=sys.stderr) + deduped_nodes = list(canonical.values()) + deduped_edges = [] + for edge in edges: + e = dict(edge) + e["source"] = remap.get(e["source"], e["source"]) + e["target"] = remap.get(e["target"], e["target"]) + if e["source"] != e["target"]: + deduped_edges.append(e) + return deduped_nodes, deduped_edges + + +def build_merge( + new_chunks: list[dict], + graph_path: str | Path = "graphify-out/graph.json", + prune_sources: list[str] | None = None, + *, + directed: bool = False, +) -> nx.Graph: + """Load existing graph.json, merge new chunks into it, and save back. + + Never replaces — only grows (or prunes deleted-file nodes via prune_sources). + Safe to call repeatedly: existing nodes and edges are preserved. + """ + from networkx.readwrite import json_graph as _jg + + graph_path = Path(graph_path) + if graph_path.exists(): + data = json.loads(graph_path.read_text(encoding="utf-8")) + try: + existing_G = _jg.node_link_graph(data, edges="links") + except TypeError: + existing_G = _jg.node_link_graph(data) + # Reconstruct as a plain extraction dict so build() can merge it + existing_nodes = [{"id": n, **existing_G.nodes[n]} for n in existing_G.nodes] + existing_edges = [ + {"source": u, "target": v, **d} for u, v, d in existing_G.edges(data=True) + ] + base = [{"nodes": existing_nodes, "edges": existing_edges}] + else: + base = [] + + all_chunks = base + list(new_chunks) + G = build(all_chunks, directed=directed) + + # Prune nodes from deleted source files + if prune_sources: + to_remove = [ + n for n, d in G.nodes(data=True) + if d.get("source_file") in prune_sources + ] + G.remove_nodes_from(to_remove) + if to_remove: + print(f"[graphify] Pruned {len(to_remove)} node(s) from deleted sources.", file=sys.stderr) + + # Safety check: refuse to shrink the graph silently (#479) + if graph_path.exists(): + existing_n = len(existing_nodes) + new_n = G.number_of_nodes() + if new_n < existing_n: + raise ValueError( + f"graphify: build_merge would shrink graph from {existing_n} → {new_n} nodes. " + f"Pass prune_sources explicitly if you intend to remove nodes." + ) + + return G diff --git a/graphify/cache.py b/graphify/cache.py index af153d93d..ef7582d03 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -4,8 +4,14 @@ import hashlib import json import os +import tempfile from pathlib import Path +# Output directory name — override with GRAPHIFY_OUT env var for worktrees or +# shared-output setups. Accepts a relative name ("graphify-out-feature") or an +# absolute path ("/shared/graphify-out"). +_GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") + def _body_content(content: bytes) -> bytes: """Strip YAML frontmatter from Markdown content, returning only the body.""" @@ -17,6 +23,17 @@ def _body_content(content: bytes) -> bytes: return content +def _normalize_path(path: Path) -> Path: + """Normalize path for consistent cache keys across Windows path spellings.""" + import sys + if sys.platform != "win32": + return path + s = str(path) + if s.startswith("\\\\?\\"): + s = s[4:] # strip extended-length prefix \\?\ + return Path(os.path.normcase(s)) + + def file_hash(path: Path, root: Path = Path(".")) -> str: """SHA256 of file contents + path relative to root. @@ -27,7 +44,8 @@ def file_hash(path: Path, root: Path = Path(".")) -> str: For Markdown files (.md), only the body below the YAML frontmatter is hashed, so metadata-only changes (e.g. reviewed, status, tags) do not invalidate the cache. """ - p = Path(path) + p = _normalize_path(Path(path)) + root = _normalize_path(Path(root)) if not p.is_file(): raise IsADirectoryError(f"file_hash requires a file, got: {p}") raw = p.read_bytes() @@ -43,37 +61,54 @@ def file_hash(path: Path, root: Path = Path(".")) -> str: return h.hexdigest() -def cache_dir(root: Path = Path(".")) -> Path: - """Returns graphify-out/cache/ - creates it if needed.""" - d = Path(root).resolve() / "graphify-out" / "cache" +def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: + """Returns graphify-out/cache/{kind}/ - creates it if needed. + + kind is "ast" or "semantic". Separate subdirectories prevent semantic cache + entries from overwriting AST cache entries for the same source_file (#582). + """ + _out = Path(_GRAPHIFY_OUT) + base = _out if _out.is_absolute() else Path(root).resolve() / _out + d = base / "cache" / kind d.mkdir(parents=True, exist_ok=True) return d -def load_cached(path: Path, root: Path = Path(".")) -> dict | None: +def load_cached(path: Path, root: Path = Path("."), kind: str = "ast") -> dict | None: """Return cached extraction for this file if hash matches, else None. Cache key: SHA256 of file contents. - Cache value: stored as graphify-out/cache/{hash}.json + Cache value: stored as graphify-out/cache/{kind}/{hash}.json + + For kind="ast", also checks the legacy flat cache/ directory so users + upgrading from pre-0.5.3 don't lose their existing AST cache entries. Returns None if no cache entry or file has changed. """ try: h = file_hash(path, root) except OSError: return None - entry = cache_dir(root) / f"{h}.json" - if not entry.exists(): - return None - try: - return json.loads(entry.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return None - - -def save_cached(path: Path, result: dict, root: Path = Path(".")) -> None: + entry = cache_dir(root, kind) / f"{h}.json" + if entry.exists(): + try: + return json.loads(entry.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + # Migration fallback: check legacy flat cache/ dir for AST entries + if kind == "ast": + legacy = Path(root).resolve() / _GRAPHIFY_OUT / "cache" / f"{h}.json" + if legacy.exists(): + try: + return json.loads(legacy.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + return None + + +def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "ast") -> None: """Save extraction result for this file. - Stores as graphify-out/cache/{hash}.json where hash = SHA256 of current file contents. + Stores as graphify-out/cache/{kind}/{hash}.json where hash = SHA256 of current file contents. result should be a dict with 'nodes' and 'edges' lists. No-ops if `path` is not a regular file. Subagent-produced semantic fragments @@ -84,34 +119,60 @@ def save_cached(path: Path, result: dict, root: Path = Path(".")) -> None: if not p.is_file(): return h = file_hash(p, root) - entry = cache_dir(root) / f"{h}.json" - tmp = entry.with_suffix(".tmp") + target_dir = cache_dir(root, kind) + entry = target_dir / f"{h}.json" + fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") try: - tmp.write_text(json.dumps(result), encoding="utf-8") + os.write(fd, json.dumps(result).encode()) + os.close(fd) try: - os.replace(tmp, entry) + os.replace(tmp_path, entry) except PermissionError: # Windows: os.replace can fail with WinError 5 if the target is # briefly locked. Fall back to copy-then-delete. import shutil - shutil.copy2(tmp, entry) - tmp.unlink(missing_ok=True) + shutil.copy2(tmp_path, entry) + os.unlink(tmp_path) except Exception: - tmp.unlink(missing_ok=True) + try: + os.close(fd) + except OSError: + pass + try: + os.unlink(tmp_path) + except OSError: + pass raise def cached_files(root: Path = Path(".")) -> set[str]: - """Return set of file paths that have a valid cache entry (hash still matches).""" - d = cache_dir(root) - return {p.stem for p in d.glob("*.json")} + """Return set of file hashes that have a valid cache entry (any kind).""" + base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" + hashes: set[str] = set() + # Legacy flat entries + if base.is_dir(): + hashes.update(p.stem for p in base.glob("*.json")) + # Namespaced entries + for kind in ("ast", "semantic"): + d = base / kind + if d.is_dir(): + hashes.update(p.stem for p in d.glob("*.json")) + return hashes def clear_cache(root: Path = Path(".")) -> None: - """Delete all graphify-out/cache/*.json files.""" - d = cache_dir(root) - for f in d.glob("*.json"): - f.unlink() + """Delete all cache entries (ast/, semantic/, and legacy flat entries).""" + base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" + # Legacy flat entries + if base.is_dir(): + for f in base.glob("*.json"): + f.unlink() + # Namespaced entries + for kind in ("ast", "semantic"): + d = base / kind + if d.is_dir(): + for f in d.glob("*.json"): + f.unlink() def check_semantic_cache( @@ -129,7 +190,7 @@ def check_semantic_cache( uncached: list[str] = [] for fpath in files: - result = load_cached(Path(fpath), root) + result = load_cached(Path(fpath), root, kind="semantic") if result is not None: cached_nodes.extend(result.get("nodes", [])) cached_edges.extend(result.get("edges", [])) @@ -148,7 +209,9 @@ def save_semantic_cache( ) -> int: """Save semantic extraction results to cache, keyed by source_file. - Groups nodes and edges by source_file, then saves one cache entry per file. + Groups nodes and edges by source_file, then saves one cache entry per file + under cache/semantic/ (separate from AST entries in cache/ast/) to prevent + hash-key collisions (#582). Returns the number of files cached. """ from collections import defaultdict @@ -173,6 +236,6 @@ def save_semantic_cache( if not p.is_absolute(): p = Path(root) / p if p.is_file(): - save_cached(p, result, root) + save_cached(p, result, root, kind="semantic") saved += 1 return saved diff --git a/graphify/cluster.py b/graphify/cluster.py index 9227c0d13..b5555a85b 100644 --- a/graphify/cluster.py +++ b/graphify/cluster.py @@ -54,6 +54,8 @@ def _partition(G: nx.Graph) -> dict[str, int]: _MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split _MIN_SPLIT_SIZE = 10 # only split if community has at least this many nodes +_COHESION_SPLIT_THRESHOLD = 0.05 # re-split communities with cohesion below this +_COHESION_SPLIT_MIN_SIZE = 50 # only cohesion-split if community has at least this many nodes def cluster(G: nx.Graph) -> dict[int, list[str]]: @@ -99,6 +101,17 @@ def cluster(G: nx.Graph) -> dict[int, list[str]]: else: final_communities.append(nodes) + # Second pass: re-split low-cohesion communities caused by doc-hub nodes + # that bridge otherwise-unrelated subsystems (e.g. CLAUDE.md connected to everything). + second_pass: list[list[str]] = [] + for nodes in final_communities: + if len(nodes) >= _COHESION_SPLIT_MIN_SIZE and cohesion_score(G, nodes) < _COHESION_SPLIT_THRESHOLD: + splits = _split_community(G, nodes) + second_pass.extend(splits if len(splits) > 1 else [nodes]) + else: + second_pass.append(nodes) + final_communities = second_pass + # Re-index by size descending for deterministic ordering final_communities.sort(key=len, reverse=True) return {i: sorted(nodes) for i, nodes in enumerate(final_communities)} diff --git a/graphify/detect.py b/graphify/detect.py index 392625877..e1df09de1 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -18,8 +18,8 @@ class FileType(str, Enum): _MANIFEST_PATH = "graphify-out/manifest.json" -CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.dart', '.v', '.sv'} -DOC_EXTENSIONS = {'.md', '.mdx', '.txt', '.rst', '.html'} +CODE_EXTENSIONS = {'.py', '.ts', '.js', '.jsx', '.tsx', '.mjs', '.ejs', '.go', '.rs', '.java', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.toc', '.zig', '.ps1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.dart', '.v', '.sv', '.sql', '.r'} +DOC_EXTENSIONS = {'.md', '.mdx', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} @@ -78,11 +78,42 @@ def _looks_like_paper(path: Path) -> bool: _ASSET_DIR_MARKERS = {".imageset", ".xcassets", ".appiconset", ".colorset", ".launchimage"} +_SHEBANG_CODE_INTERPRETERS = { + "python", "python3", "python2", + "ruby", "perl", "node", "nodejs", + "bash", "sh", "dash", "zsh", "fish", "ksh", "tcsh", + "lua", "php", "julia", "Rscript", +} + + +def _shebang_file_type(path: Path) -> FileType | None: + """Peek at the first line of an extensionless file for a shebang.""" + try: + with path.open("rb") as f: + first = f.read(128) + if not first.startswith(b"#!"): + return None + line = first.split(b"\n")[0].decode(errors="replace") + parts = line[2:].strip().split() + if not parts: + return None + interp = parts[0].split("/")[-1] # /usr/bin/env → env + if interp == "env" and len(parts) > 1: + interp = parts[1].split("/")[-1] + if interp in _SHEBANG_CODE_INTERPRETERS: + return FileType.CODE + except OSError: + pass + return None + + def classify_file(path: Path) -> FileType | None: # Compound extensions must be checked before simple suffix lookup if path.name.lower().endswith(".blade.php"): return FileType.CODE ext = path.suffix.lower() + if not ext: + return _shebang_file_type(path) if ext in CODE_EXTENSIONS: return FileType.CODE if ext in PAPER_EXTENSIONS: @@ -169,7 +200,6 @@ def xlsx_to_markdown(path: Path) -> str: ws = wb[sheet_name] rows = [] for row in ws.iter_rows(values_only=True): - # Skip entirely empty rows if all(cell is None for cell in row): continue rows.append([str(cell) if cell is not None else "" for cell in row]) @@ -190,6 +220,91 @@ def xlsx_to_markdown(path: Path) -> str: return "" +def xlsx_extract_structure(path: Path) -> dict: + """Extract structural nodes (sheets, named tables, column headers) from an .xlsx file. + + Returns a nodes/edges dict compatible with the graphify extract pipeline. + Used in addition to xlsx_to_markdown so Claude sees both structure and content. + """ + def _nid(*parts: str) -> str: + return re.sub(r"[^a-z0-9_]", "_", "_".join(p.lower() for p in parts).strip("_")) + + try: + import openpyxl + except ImportError: + return {"nodes": [], "edges": []} + + try: + wb = openpyxl.load_workbook(str(path), read_only=False, data_only=True) + except Exception: + return {"nodes": [], "edges": []} + + stem = _re.sub(r"[^a-z0-9]", "_", path.stem.lower()) + str_path = str(path) + file_nid = _nid(str_path) + nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "document", + "source_file": str_path, "source_location": None}] + edges: list[dict] = [] + seen: set[str] = {file_nid} + + def _add(nid: str, label: str) -> None: + if nid not in seen: + seen.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "document", + "source_file": str_path, "source_location": None}) + + def _edge(src: str, tgt: str, relation: str) -> None: + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": None, "weight": 1.0}) + + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + sheet_nid = _nid(stem, sheet_name) + _add(sheet_nid, f"{sheet_name} (sheet)") + _edge(file_nid, sheet_nid, "contains") + + # Named Excel Tables (ListObjects) + if hasattr(ws, "tables"): + for tbl in ws.tables.values(): + tbl_nid = _nid(stem, sheet_name, tbl.name) + _add(tbl_nid, tbl.name) + _edge(sheet_nid, tbl_nid, "contains") + # Column headers from table header row + ref = tbl.ref # e.g. "A1:D10" + if ref: + try: + from openpyxl.utils import range_boundaries + min_col, min_row, max_col, _ = range_boundaries(ref) + header_row = list(ws.iter_rows(min_row=min_row, max_row=min_row, + min_col=min_col, max_col=max_col, + values_only=True)) + if header_row: + for col_name in header_row[0]: + if col_name: + col_nid = _nid(stem, tbl.name, str(col_name)) + _add(col_nid, str(col_name)) + _edge(tbl_nid, col_nid, "contains") + except Exception: + pass + else: + # Fallback: first non-empty row as column headers + for row in ws.iter_rows(max_row=1, values_only=True): + for cell in row: + if cell: + col_nid = _nid(stem, sheet_name, str(cell)) + _add(col_nid, str(cell)) + _edge(sheet_nid, col_nid, "contains") + break + + try: + wb.close() + except Exception: + pass + + return {"nodes": nodes, "edges": edges} + + def convert_office_file(path: Path, out_dir: Path) -> Path | None: """Convert a .docx or .xlsx to a markdown sidecar in out_dir. @@ -241,6 +356,7 @@ def count_words(path: Path) -> int: "site-packages", "lib64", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".tox", ".eggs", "*.egg-info", + "graphify-out", # never treat own output as source input (#524) } # Large generated files that are never useful to extract @@ -262,38 +378,87 @@ def _is_noise_dir(part: str) -> bool: return False -def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: - """Read .graphifyignore from root **and ancestor directories**. +_VCS_MARKERS = (".git", ".hg", ".svn", "_darcs", ".fossil") + - Returns a list of (anchor_dir, pattern) pairs. Each pattern is matched - against paths relative to both the scan root and the anchor_dir where - the .graphifyignore file was found — so patterns written relative to a - parent directory still work when graphify is run on a subfolder. +def _parse_gitignore_line(raw: str) -> str: + """Parse one raw line from a .graphifyignore file per gitignore spec. - Walks upward from *root* towards the filesystem root, stopping at a - ``.git`` boundary. Lines starting with # are comments; blank lines ignored. + - Strip newline chars + - Strip inline comments (whitespace + # suffix), but only when # is + preceded by whitespace — so path#with#hash.py is preserved + - Unescape \\# to literal # + - Remove trailing spaces unless escaped with backslash + - Strip leading whitespace + - Return empty string for blank lines and full-line comments """ - patterns: list[tuple[Path, str]] = [] - current = root.resolve() + line = raw.rstrip("\n\r") + line = line.lstrip() + if not line or line.startswith("#"): + return "" + # Strip inline comments: require whitespace before # (gitignore extension) + line = re.sub(r"\s+#+[^\\].*$", "", line) + # Unescape \# → literal # + line = line.replace("\\#", "#") + # Remove unescaped trailing spaces (per gitignore spec) + line = re.sub(r"(? Path | None: + """Walk upward from start; return the first directory containing a VCS marker.""" + current = start.resolve() + home = Path.home() while True: - ignore_file = current / ".graphifyignore" - if ignore_file.exists(): - for line in ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines(): - line = line.strip() - if line and not line.startswith("#"): - patterns.append((current, line)) - # Stop climbing once we've processed the git repo root - if (current / ".git").exists(): - break + if any((current / m).exists() for m in _VCS_MARKERS): + return current parent = current.parent - if parent == current: - break # filesystem root + if parent == current or current == home: + return None current = parent + + +def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: + """Read .graphifyignore files and return (anchor_dir, pattern) pairs. + + Patterns are returned outer-first so that inner (closer) rules are + appended last and win via last-match-wins semantics — matching gitignore + behavior exactly. + + Walk ceiling: the nearest VCS root if inside a repo, otherwise the scan + root itself (hermetic — no leakage across unrelated sibling projects). + """ + root = root.resolve() + ceiling = _find_vcs_root(root) or root + + # Collect ancestor dirs from ceiling down to root (outer → inner) + dirs: list[Path] = [] + current = root + while True: + dirs.append(current) + if current == ceiling: + break + current = current.parent + dirs.reverse() # ceiling first, scan root last + + patterns: list[tuple[Path, str]] = [] + for d in dirs: + ignore_file = d / ".graphifyignore" + if ignore_file.exists(): + for raw in ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = _parse_gitignore_line(raw) + if line: + patterns.append((d, line)) return patterns def _is_ignored(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool: - """Return True if path matches any .graphifyignore pattern.""" + """Return True if the path should be ignored per .graphifyignore patterns. + + Uses gitignore last-match-wins semantics: all patterns are evaluated in + order; the final matching pattern determines the result. Negation patterns + (starting with !) un-ignore a previously ignored path. + """ if not patterns: return False @@ -310,26 +475,146 @@ def _matches(rel: str, p: str) -> bool: return True return False + result = False for anchor, pattern in patterns: - p = pattern.strip("/") + negated = pattern.startswith("!") + raw = pattern[1:] if negated else pattern + anchored = raw.startswith("/") + p = raw.strip("/") if not p: continue - # Try path relative to the scan root - try: - rel = str(path.relative_to(root)).replace(os.sep, "/") - if _matches(rel, p): + + matched = False + if anchored: + try: + rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") + matched = _matches(rel_anchor, p) + except ValueError: + pass + else: + try: + rel = str(path.relative_to(root)).replace(os.sep, "/") + matched = _matches(rel, p) + except ValueError: + pass + if not matched and anchor != root: + try: + rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") + matched = _matches(rel_anchor, p) + except ValueError: + pass + + if matched: + result = not negated # last match wins; ! flips to un-ignore + return result + + +def _load_graphifyinclude(root: Path) -> list[tuple[Path, str]]: + """Read .graphifyinclude allowlist patterns from root and ancestors. + + Include patterns opt matching hidden files/dirs into traversal. Sensitive + files and hard-skipped noise directories are still excluded later. + Uses the same VCS-root ceiling logic as _load_graphifyignore. + """ + root = root.resolve() + ceiling = _find_vcs_root(root) or root + + dirs: list[Path] = [] + current = root + while True: + dirs.append(current) + if current == ceiling: + break + current = current.parent + dirs.reverse() + + patterns: list[tuple[Path, str]] = [] + for d in dirs: + include_file = d / ".graphifyinclude" + if include_file.exists(): + for raw in include_file.read_text(encoding="utf-8", errors="ignore").splitlines(): + line = _parse_gitignore_line(raw) + if line: + patterns.append((d, line)) + return patterns + + +def _is_included(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool: + """Return True if path matches any .graphifyinclude allowlist pattern.""" + if not patterns: + return False + + def _matches(rel: str, p: str) -> bool: + parts = rel.split("/") + if fnmatch.fnmatch(rel, p): + return True + if fnmatch.fnmatch(path.name, p): + return True + for i, part in enumerate(parts): + if fnmatch.fnmatch(part, p): return True - except ValueError: - pass - # Also try relative to the anchor dir (the .graphifyignore's location), - # so patterns written at a parent level still fire when running on a subfolder - if anchor != root: + if fnmatch.fnmatch("/".join(parts[:i + 1]), p): + return True + return False + + for anchor, pattern in patterns: + anchored = pattern.startswith("/") + p = pattern.strip("/") + if not p: + continue + if anchored: try: rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") if _matches(rel_anchor, p): return True except ValueError: pass + else: + try: + rel = str(path.relative_to(root)).replace(os.sep, "/") + if _matches(rel, p): + return True + except ValueError: + pass + if anchor != root: + try: + rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") + if _matches(rel_anchor, p): + return True + except ValueError: + pass + return False + + +def _could_contain_included_path(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool: + """Return True if a directory may contain files matched by .graphifyinclude.""" + if not patterns: + return False + + rels: list[str] = [] + try: + rels.append(str(path.relative_to(root)).replace(os.sep, "/")) + except ValueError: + pass + for anchor, _ in patterns: + if anchor != root: + try: + rels.append(str(path.relative_to(anchor)).replace(os.sep, "/")) + except ValueError: + pass + + for rel in rels: + rel = rel.strip("/") + if not rel: + return True + for _, pattern in patterns: + p = pattern.strip("/") + if not p: + continue + if p == rel or p.startswith(rel + "/"): + return True + if fnmatch.fnmatch(rel, p): + return True return False @@ -346,6 +631,7 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: skipped_sensitive: list[str] = [] ignore_patterns = _load_graphifyignore(root) + include_patterns = _load_graphifyinclude(root) # Always include graphify-out/memory/ - query results filed back into the graph memory_dir = root / "graphify-out" / "memory" @@ -367,12 +653,17 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: dirnames.clear() continue if not in_memory_tree: - # Prune noise dirs in-place so os.walk never descends into them + # Prune noise dirs in-place so os.walk never descends into them. + # Hidden dirs are allowed through if they could contain an + # explicitly included path (.graphifyinclude allowlist). + # When negation patterns (!) exist, skip directory-level ignore + # pruning so negated files inside can still be reached. + has_negation = any(p.startswith("!") for _, p in ignore_patterns) dirnames[:] = [ d for d in dirnames - if not d.startswith(".") + if (not d.startswith(".") or _could_contain_included_path(dp / d, root, include_patterns)) and not _is_noise_dir(d) - and not _is_ignored(dp / d, root, ignore_patterns) + and (has_negation or not _is_ignored(dp / d, root, ignore_patterns)) ] for fname in filenames: if fname in _SKIP_FILES: @@ -389,8 +680,9 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: in_memory = memory_dir.exists() and str(p).startswith(str(memory_dir)) if not in_memory: # Hidden files are already excluded via dir pruning above, - # but catch hidden files at the root level - if p.name.startswith("."): + # but catch hidden files at the root level. A .graphifyinclude + # entry can opt a specific hidden file back in. + if p.name.startswith(".") and not _is_included(p, root, include_patterns): continue # Skip files inside our own converted/ dir (avoid re-processing sidecars) if str(p).startswith(str(converted_dir)): @@ -444,8 +736,21 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: } -def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]: - """Load the file modification time manifest from a previous run.""" +def _md5_file(path: Path) -> str: + """MD5 of file contents streamed in 64KB chunks — for change detection only.""" + import hashlib as _hl + h = _hl.md5(usedforsecurity=False) + try: + with path.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + except OSError: + return "" + return h.hexdigest() + + +def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict: + """Load the manifest from a previous run. Returns {} on any error.""" try: return json.loads(Path(manifest_path).read_text(encoding="utf-8")) except Exception: @@ -453,12 +758,13 @@ def load_manifest(manifest_path: str = _MANIFEST_PATH) -> dict[str, float]: def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PATH) -> None: - """Save current file mtimes so the next --update run can diff against them.""" - manifest: dict[str, float] = {} + """Save current file mtimes + content hashes for change detection on --update.""" + manifest: dict[str, dict] = {} for file_list in files.values(): for f in file_list: try: - manifest[f] = Path(f).stat().st_mtime + p = Path(f) + manifest[f] = {"mtime": p.stat().st_mtime, "hash": _md5_file(p)} except OSError: pass # file deleted between detect() and manifest write - skip it Path(manifest_path).parent.mkdir(parents=True, exist_ok=True) @@ -468,8 +774,11 @@ def save_manifest(files: dict[str, list[str]], manifest_path: str = _MANIFEST_PA def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict: """Like detect(), but returns only new or modified files since the last run. - Compares current file mtimes against the stored manifest. - Use for --update mode: re-extract only what changed, merge into existing graph. + Fast path: mtime unchanged → unchanged (free, no hash). + Slow path: mtime bumped → compare MD5. Same hash = sync tool touched mtime, + treat as unchanged. Different hash = actually changed, re-extract. + + Backwards compatible with legacy manifests storing plain float mtime values. """ full = detect(root) manifest = load_manifest(manifest_path) @@ -487,12 +796,26 @@ def detect_incremental(root: Path, manifest_path: str = _MANIFEST_PATH) -> dict: for ftype, file_list in full["files"].items(): for f in file_list: - stored_mtime = manifest.get(f) + stored = manifest.get(f) try: current_mtime = Path(f).stat().st_mtime except Exception: current_mtime = 0 - if stored_mtime is None or current_mtime > stored_mtime: + + # Legacy manifest: plain float value + if isinstance(stored, (int, float)): + changed = stored is None or current_mtime > stored + elif isinstance(stored, dict): + stored_mtime = stored.get("mtime") + if stored_mtime is None or current_mtime != stored_mtime: + # mtime bumped — verify with content hash before re-extracting + changed = _md5_file(Path(f)) != stored.get("hash", "") + else: + changed = False + else: + changed = True # unknown format, re-extract to be safe + + if changed: new_files[ftype].append(f) else: unchanged_files[ftype].append(f) diff --git a/graphify/export.py b/graphify/export.py index 12cafb874..14587addb 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -25,6 +25,22 @@ def _strip_diacritics(text: str) -> str: MAX_NODES_FOR_VIZ = 5_000 +def _viz_node_limit() -> int: + """Return the effective viz node limit, honoring GRAPHIFY_VIZ_NODE_LIMIT env var. + + Falls back to MAX_NODES_FOR_VIZ when the env var is unset, empty, or non-integer. + Set to 0 to disable HTML viz unconditionally (useful for CI runners). + """ + import os + raw = os.environ.get("GRAPHIFY_VIZ_NODE_LIMIT") + if raw is None or not raw.strip(): + return MAX_NODES_FOR_VIZ + try: + return int(raw) + except ValueError: + return MAX_NODES_FOR_VIZ + + def _html_styles() -> str: return """""" @@ -240,15 +264,42 @@ def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: }}); const hiddenCommunities = new Set(); + +const selectAllCb = document.getElementById('select-all-cb'); + +function updateSelectAllState() {{ + const total = LEGEND.length; + const hidden = hiddenCommunities.size; + selectAllCb.checked = hidden === 0; + selectAllCb.indeterminate = hidden > 0 && hidden < total; +}} + +function toggleAllCommunities(hide) {{ + document.querySelectorAll('.legend-item').forEach(item => {{ + hide ? item.classList.add('dimmed') : item.classList.remove('dimmed'); + }}); + document.querySelectorAll('.legend-cb').forEach(cb => {{ + cb.checked = !hide; + }}); + LEGEND.forEach(c => {{ + if (hide) hiddenCommunities.add(c.cid); else hiddenCommunities.delete(c.cid); + }}); + const updates = RAW_NODES.map(n => ({{ id: n.id, hidden: hide }})); + nodesDS.update(updates); + updateSelectAllState(); +}} + const legendEl = document.getElementById('legend'); LEGEND.forEach(c => {{ const item = document.createElement('div'); item.className = 'legend-item'; - item.innerHTML = `
- ${{c.label}} - ${{c.count}}`; - item.onclick = () => {{ - if (hiddenCommunities.has(c.cid)) {{ + const cb = document.createElement('input'); + cb.type = 'checkbox'; + cb.className = 'legend-cb'; + cb.checked = true; + cb.addEventListener('change', (e) => {{ + e.stopPropagation(); + if (cb.checked) {{ hiddenCommunities.delete(c.cid); item.classList.remove('dimmed'); }} else {{ @@ -257,8 +308,18 @@ def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: }} const updates = RAW_NODES .filter(n => n.community === c.cid) - .map(n => ({{ id: n.id, hidden: hiddenCommunities.has(c.cid) }})); + .map(n => ({{ id: n.id, hidden: !cb.checked }})); nodesDS.update(updates); + updateSelectAllState(); + }}); + item.innerHTML = `
+ ${{c.label}} + ${{c.count}}`; + item.prepend(cb); + item.onclick = (e) => {{ + if (e.target === cb) return; + cb.checked = !cb.checked; + cb.dispatchEvent(new Event('change')); }}; legendEl.appendChild(item); }}); @@ -279,7 +340,27 @@ def attach_hyperedges(G: nx.Graph, hyperedges: list) -> None: G.graph["hyperedges"] = existing -def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> None: +def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, force: bool = False) -> bool: + # Safety check: refuse to silently shrink an existing graph (#479) + existing_path = Path(output_path) + if not force and existing_path.exists(): + try: + existing_data = json.loads(existing_path.read_text(encoding="utf-8")) + existing_n = len(existing_data.get("nodes", [])) + new_n = G.number_of_nodes() + if new_n < existing_n: + import sys as _sys + print( + f"[graphify] WARNING: new graph has {new_n} nodes but existing " + f"graph.json has {existing_n}. Refusing to overwrite — you may be " + f"missing chunk files from a previous session. " + f"Pass force=True to override.", + file=_sys.stderr, + ) + return False + except Exception: + pass # unreadable existing file — proceed with write + node_community = _node_community_map(communities) try: data = json_graph.node_link_data(G, edges="links") @@ -292,9 +373,19 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str) -> if "confidence_score" not in link: conf = link.get("confidence", "EXTRACTED") link["confidence_score"] = _CONFIDENCE_SCORE_DEFAULTS.get(conf, 1.0) + # Restore original edge direction. Undirected NetworkX storage may + # canonicalize endpoint order, flipping `calls` and other directional + # edges in graph.json. The build path stashes the true endpoints in + # _src/_tgt for exactly this purpose (#563). + true_src = link.pop("_src", None) + true_tgt = link.pop("_tgt", None) + if true_src is not None and true_tgt is not None: + link["source"] = true_src + link["target"] = true_tgt data["hyperedges"] = getattr(G, "graph", {}).get("hyperedges", []) - with open(output_path, "w", encoding="utf-8") as f: + with open(output_path, "w", encoding="utf-8") as f: # nosec json.dump(data, f, indent=2) + return True def prune_dangling_edges(graph_data: dict) -> tuple[dict, int]: @@ -335,7 +426,7 @@ def to_cypher(G: nx.Graph, output_path: str) -> None: f"MATCH (a {{id: '{u_esc}'}}), (b {{id: '{v_esc}'}}) " f"MERGE (a)-[:{rel} {{confidence: '{conf}'}}]->(b);" ) - with open(output_path, "w", encoding="utf-8") as f: + with open(output_path, "w", encoding="utf-8") as f: # nosec f.write("\n".join(lines)) @@ -355,10 +446,12 @@ def to_html( If member_counts is provided (aggregated community view), node sizes are based on community member counts rather than graph degree. """ - if G.number_of_nodes() > MAX_NODES_FOR_VIZ: + limit = _viz_node_limit() + if G.number_of_nodes() > limit: raise ValueError( - f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz. " - f"Use --no-viz or reduce input size." + f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " + f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " + f"or reduce input size." ) node_community = _node_community_map(communities) @@ -395,14 +488,19 @@ def to_html( "degree": deg, }) - # Build edges list + # Build edges list. Restore original edge direction from _src/_tgt + # (stashed by build.py for exactly this reason): undirected NetworkX + # canonicalizes endpoint order, which would otherwise flip the arrow + # for `calls` and `rationale_for` in the rendered graph (#563). vis_edges = [] for u, v, data in G.edges(data=True): confidence = data.get("confidence", "EXTRACTED") relation = data.get("relation", "") + true_src = data.get("_src", u) + true_tgt = data.get("_tgt", v) vis_edges.append({ - "from": u, - "to": v, + "from": true_src, + "to": true_tgt, "label": relation, "title": _html.escape(f"{relation} [{confidence}]"), "dashes": confidence != "EXTRACTED", @@ -451,6 +549,9 @@ def _js_safe(obj) -> str:

Communities

+
+ +
{stats}
@@ -460,7 +561,7 @@ def _js_safe(obj) -> str: """ - Path(output_path).write_text(html, encoding="utf-8") + Path(output_path).write_text(html, encoding="utf-8") # nosec # Keep backward-compatible alias - skill.md calls generate_html @@ -575,7 +676,7 @@ def _dominant_confidence(node_id: str) -> str: lines.append(inline_tags) fname = node_filename[node_id] + ".md" - (out / fname).write_text("\n".join(lines), encoding="utf-8") + (out / fname).write_text("\n".join(lines), encoding="utf-8") # nosec # Write one _COMMUNITY_name.md overview note per community # Build inter-community edge counts for "Connections to other communities" @@ -692,7 +793,7 @@ def _community_reach(node_id: str) -> int: community_safe = safe_name(community_name) fname = f"_COMMUNITY_{community_safe}.md" - (out / fname).write_text("\n".join(lines), encoding="utf-8") + (out / fname).write_text("\n".join(lines), encoding="utf-8") # nosec community_notes_written += 1 # Improvement 4: write .obsidian/graph.json to color nodes by community in graph view @@ -707,7 +808,7 @@ def _community_reach(node_id: str) -> int: for cid, label in sorted((community_labels or {}).items()) ] } - (obsidian_dir / "graph.json").write_text(json.dumps(graph_config, indent=2), encoding="utf-8") + (obsidian_dir / "graph.json").write_text(json.dumps(graph_config, indent=2), encoding="utf-8") # nosec return G.number_of_nodes() + community_notes_written @@ -868,7 +969,7 @@ def safe_name(label: str) -> str: }) canvas_data = {"nodes": canvas_nodes, "edges": canvas_edges} - Path(output_path).write_text(json.dumps(canvas_data, indent=2), encoding="utf-8") + Path(output_path).write_text(json.dumps(canvas_data, indent=2), encoding="utf-8") # nosec def push_to_neo4j( diff --git a/graphify/extract.py b/graphify/extract.py index dbd441c6c..3c47ff1df 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -18,6 +18,48 @@ def _make_id(*parts: str) -> str: return cleaned.strip("_").lower() +def _file_stem(path: Path) -> str: + """Return a stem qualified with the parent directory name to avoid ID collisions + when multiple files share the same filename in different directories (#550).""" + parent = path.parent.name + if parent and parent not in (".", ""): + return f"{parent}.{path.stem}" + return path.stem + + +_TSCONFIG_ALIAS_CACHE: dict[str, dict[str, str]] = {} + + +def _load_tsconfig_aliases(start_dir: Path) -> dict[str, str]: + """Walk up from start_dir to find tsconfig.json and return compilerOptions.paths aliases. + + Returns a dict mapping alias prefix (e.g. "@/") to resolved base dir (e.g. "src/"). + Result is cached by tsconfig path string. + """ + current = start_dir.resolve() + for candidate in [current, *current.parents]: + tsconfig = candidate / "tsconfig.json" + if tsconfig.exists(): + key = str(tsconfig) + if key not in _TSCONFIG_ALIAS_CACHE: + try: + data = json.loads(tsconfig.read_text(encoding="utf-8")) + paths = data.get("compilerOptions", {}).get("paths", {}) + aliases: dict[str, str] = {} + for alias, targets in paths.items(): + if not targets: + continue + # Strip trailing /* from alias and target + alias_prefix = alias.rstrip("/*") + target_base = targets[0].rstrip("/*") + aliases[alias_prefix] = str(candidate / target_base) + _TSCONFIG_ALIAS_CACHE[key] = aliases + except Exception: + _TSCONFIG_ALIAS_CACHE[key] = {} + return _TSCONFIG_ALIAS_CACHE[key] + return {} + + # ── LanguageConfig dataclass ───────────────────────────────────────────────── @dataclass @@ -108,6 +150,7 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -132,6 +175,7 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s "source": file_nid, "target": tgt_nid, "relation": "imports_from", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -140,6 +184,7 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None: + resolved_path: "Path | None" = None for child in node.children: if child.type == "string": raw = _read_text(child, source).strip("'\"` ") @@ -155,14 +200,133 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p elif resolved.suffix == ".jsx": resolved = resolved.with_suffix(".tsx") tgt_nid = _make_id(str(resolved)) + resolved_path = resolved + else: + # Check tsconfig.json path aliases (e.g. "@/" → "src/") before treating as external (#575) + aliases = _load_tsconfig_aliases(Path(str_path).parent) + resolved_alias = None + for alias_prefix, alias_base in aliases.items(): + if raw == alias_prefix or raw.startswith(alias_prefix + "/"): + rest = raw[len(alias_prefix):].lstrip("/") + resolved_alias = Path(os.path.normpath(Path(alias_base) / rest)) + break + if resolved_alias is not None: + tgt_nid = _make_id(str(resolved_alias)) + resolved_path = resolved_alias + else: + # Bare/scoped import (node_modules) - use last segment; dropped as external + module_name = raw.split("/")[-1] + if not module_name: + break + tgt_nid = _make_id(module_name) + edges.append({ + "source": file_nid, + "target": tgt_nid, + "relation": "imports_from", + "context": "import", + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{node.start_point[0] + 1}", + "weight": 1.0, + }) + break + + # Emit symbol-level edges for named imports from local/aliased files. + # e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED) + # Uses the same _make_id(target_stem, name) key that _extract_generic emits when + # defining the symbol, so these edges wire importers directly to existing symbol nodes. + if resolved_path is not None: + target_stem = _file_stem(resolved_path) + line = node.start_point[0] + 1 + for child in node.children: + if child.type == "import_clause": + for sub in child.children: + if sub.type == "named_imports": + for spec in sub.children: + if spec.type == "import_specifier": + name_node = spec.child_by_field_name("name") + if name_node: + sym = _read_text(name_node, source) + edges.append({ + "source": file_nid, + "target": _make_id(target_stem, sym), + "relation": "imports", + "context": "import", + "confidence": "EXTRACTED", + "source_file": str_path, + "source_location": f"L{line}", + "weight": 1.0, + }) + + +def _dynamic_import_js(node, source: bytes, caller_nid: str, str_path: str, edges: list, + seen_dyn_pairs: set) -> bool: + """Detect dynamic import() calls in JS/TS and emit imports_from edges. + + Handles patterns like: + await import('./foo.js') + import('./foo.js').then(...) + const m = await import(`./foo`) + + Returns True if the node was a dynamic import (caller should skip normal call handling). + """ + # Dynamic import is a call_expression whose function child is the keyword "import". + # tree-sitter-typescript parses `import('...')` as call_expression with first child + # being an "import" token (type="import"). + func_node = node.child_by_field_name("function") + if func_node is None: + # Fallback: check first child directly (some TS versions) + if node.children and _read_text(node.children[0], source) == "import": + func_node = node.children[0] + else: + return False + if _read_text(func_node, source) != "import": + return False + + # Extract the module path from the arguments + args = node.child_by_field_name("arguments") + if args is None: + return True # It's an import() but no args — skip + for arg in args.children: + if arg.type == "template_string": + # Skip dynamic template literals — path can't be statically resolved + if any(c.type == "template_substitution" for c in arg.children): + break + raw = _read_text(arg, source).strip("`") + elif arg.type == "string": + raw = _read_text(arg, source).strip("'\" ") + else: + continue + if not raw: + break + # Resolve path using the same logic as static imports + if raw.startswith("."): + resolved = Path(os.path.normpath(Path(str_path).parent / raw)) + if resolved.suffix == ".js": + resolved = resolved.with_suffix(".ts") + elif resolved.suffix == ".jsx": + resolved = resolved.with_suffix(".tsx") + tgt_nid = _make_id(str(resolved)) + else: + aliases = _load_tsconfig_aliases(Path(str_path).parent) + resolved_alias = None + for alias_prefix, alias_base in aliases.items(): + if raw == alias_prefix or raw.startswith(alias_prefix + "/"): + rest = raw[len(alias_prefix):].lstrip("/") + resolved_alias = Path(os.path.normpath(Path(alias_base) / rest)) + break + if resolved_alias is not None: + tgt_nid = _make_id(str(resolved_alias)) else: - # Bare/scoped import (node_modules) - use last segment; dropped as external module_name = raw.split("/")[-1] if not module_name: break tgt_nid = _make_id(module_name) + pair = (caller_nid, tgt_nid) + if pair not in seen_dyn_pairs: + seen_dyn_pairs.add(pair) edges.append({ - "source": file_nid, + "source": caller_nid, "target": tgt_nid, "relation": "imports_from", "confidence": "EXTRACTED", @@ -170,7 +334,8 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, }) - break + break + return True def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> None: @@ -203,6 +368,7 @@ def _walk_scoped(n) -> str: "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -222,6 +388,7 @@ def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_pa "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -241,6 +408,7 @@ def _import_csharp(node, source: bytes, file_nid: str, stem: str, edges: list, s "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -260,6 +428,7 @@ def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, s "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -275,6 +444,7 @@ def _import_kotlin(node, source: bytes, file_nid: str, stem: str, edges: list, s "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -294,6 +464,7 @@ def _import_scala(node, source: bytes, file_nid: str, stem: str, edges: list, st "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -313,6 +484,7 @@ def _import_php(node, source: bytes, file_nid: str, stem: str, edges: list, str_ "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -537,7 +709,11 @@ def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: s call_function_field="", call_accessor_node_types=frozenset({"navigation_expression"}), call_accessor_field="", - name_fallback_child_types=("simple_identifier",), + # Different tree-sitter-kotlin grammar versions name plain identifier + # nodes differently: PyPI's `tree_sitter_kotlin` uses `identifier`, + # older forks use `simple_identifier`. Accept both so the extractor + # works across grammar generations. + name_fallback_child_types=("simple_identifier", "identifier"), body_fallback_child_types=("function_body", "class_body"), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_kotlin, @@ -591,6 +767,7 @@ def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_ "source": file_nid, "target": module_name, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "confidence_score": 1.0, "source_file": str_path, @@ -625,6 +802,7 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st "source": file_nid, "target": tgt_nid, "relation": "imports", + "context": "import", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", @@ -633,6 +811,27 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st break +def _read_csharp_type_name(node, source: bytes) -> str | None: + """Resolve a readable C# type name from a field/type node.""" + if node is None: + return None + if node.type in ("identifier", "predefined_type"): + return _read_text(node, source) + if node.type == "qualified_name": + return _read_text(node, source).split(".")[-1] + if node.type == "generic_name": + name_node = node.child_by_field_name("name") + if name_node is not None: + return _read_text(name_node, source) + for child in node.children: + if not child.is_named: + continue + name = _read_csharp_type_name(child, source) + if name: + return name + return None + + _SWIFT_CONFIG = LanguageConfig( ts_module="tree_sitter_swift", class_types=frozenset({"class_declaration", "protocol_declaration"}), @@ -648,7 +847,6 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st import_handler=_import_swift, ) - # ── Generic extractor ───────────────────────────────────────────────────────── def _extract_generic(path: Path, config: LanguageConfig) -> dict: @@ -676,7 +874,7 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -696,8 +894,9 @@ def add_node(nid: str, label: str, line: int) -> None: }) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({ + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = { "source": src, "target": tgt, "relation": relation, @@ -705,7 +904,19 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "source_file": str_path, "source_location": f"L{line}", "weight": weight, - }) + } + if context: + edge["context"] = context + edges.append(edge) + + def ensure_named_node(name: str, line: int) -> str: + nid = _make_id(stem, name) + if nid in seen_ids: + return nid + nid = _make_id(name) + if nid not in seen_ids: + add_node(nid, name, line) + return nid file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -904,6 +1115,23 @@ def _emit_java_parent(base_name: str, rel: str, at_line: int) -> None: break return + if (config.ts_module == "tree_sitter_c_sharp" + and t == "field_declaration" + and parent_class_nid): + type_node = node.child_by_field_name("type") + if type_node is None: + for child in node.children: + if child.type == "variable_declaration": + type_node = child.child_by_field_name("type") + if type_node is not None: + break + type_name = _read_csharp_type_name(type_node, source) + if type_name: + line = node.start_point[0] + 1 + add_edge(parent_class_nid, ensure_named_node(type_name, line), + "references", line, context="field") + return + # Function types if t in config.function_types: # Swift deinit/subscript have no name field — resolve before generic fallback @@ -977,6 +1205,7 @@ def _emit_java_parent(base_name: str, rel: str, at_line: int) -> None: label_to_nid[normalised.lower()] = n["id"] seen_call_pairs: set[tuple[str, str]] = set() + seen_dyn_import_pairs: set[tuple[str, str]] = set() seen_static_ref_pairs: set[tuple[str, str, str]] = set() seen_helper_ref_pairs: set[tuple[str, str, str]] = set() seen_bind_pairs: set[tuple[str, str, str]] = set() @@ -998,7 +1227,17 @@ def walk_calls(node, caller_nid: str) -> None: return if node.type in config.call_types: + # JS/TS dynamic imports: await import('./foo.js') + if config.ts_module in ("tree_sitter_javascript", "tree_sitter_typescript"): + if _dynamic_import_js(node, source, caller_nid, str_path, + edges, seen_dyn_import_pairs): + # Still recurse into children (import().then(...) may have calls) + for child in node.children: + walk_calls(child, caller_nid) + return + callee_name: str | None = None + is_member_call: bool = False # Special handling per language if config.ts_module == "tree_sitter_swift": @@ -1008,20 +1247,26 @@ def walk_calls(node, caller_nid: str) -> None: if first.type == "simple_identifier": callee_name = _read_text(first, source) elif first.type == "navigation_expression": + is_member_call = True for child in first.children: if child.type == "navigation_suffix": for sc in child.children: if sc.type == "simple_identifier": callee_name = _read_text(sc, source) elif config.ts_module == "tree_sitter_kotlin": - # Kotlin: first child may be simple_identifier or navigation_expression + # Kotlin: first child may be simple_identifier/identifier or + # navigation_expression. PyPI's `tree_sitter_kotlin` produces + # `identifier` for plain identifier nodes; older grammar + # versions (including the JVM `io.github.bonede:tree-sitter-kotlin` + # binding) produce `simple_identifier`. Accept both. first = node.children[0] if node.children else None if first: - if first.type == "simple_identifier": + if first.type in ("simple_identifier", "identifier"): callee_name = _read_text(first, source) elif first.type == "navigation_expression": + is_member_call = True for child in reversed(first.children): - if child.type == "simple_identifier": + if child.type in ("simple_identifier", "identifier"): callee_name = _read_text(child, source) break elif config.ts_module == "tree_sitter_scala": @@ -1031,6 +1276,7 @@ def walk_calls(node, caller_nid: str) -> None: if first.type == "identifier": callee_name = _read_text(first, source) elif first.type == "field_expression": + is_member_call = True field = first.child_by_field_name("field") if field: callee_name = _read_text(field, source) @@ -1050,6 +1296,7 @@ def walk_calls(node, caller_nid: str) -> None: raw = _read_text(child, source) if "." in raw: callee_name = raw.split(".")[-1] + is_member_call = True else: callee_name = raw break @@ -1065,6 +1312,8 @@ def walk_calls(node, caller_nid: str) -> None: if scope_node: callee_name = _read_text(scope_node, source) else: + # member_call_expression: $obj->method() + is_member_call = True name_node = node.child_by_field_name("name") if name_node: callee_name = _read_text(name_node, source) @@ -1075,6 +1324,7 @@ def walk_calls(node, caller_nid: str) -> None: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type in ("field_expression", "qualified_identifier"): + is_member_call = True name = func_node.child_by_field_name("field") or func_node.child_by_field_name("name") if name: callee_name = _read_text(name, source) @@ -1085,6 +1335,7 @@ def walk_calls(node, caller_nid: str) -> None: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type in config.call_accessor_node_types: + is_member_call = True if config.call_accessor_field: attr = func_node.child_by_field_name(config.call_accessor_field) if attr: @@ -1104,6 +1355,7 @@ def walk_calls(node, caller_nid: str) -> None: "source": caller_nid, "target": tgt_nid, "relation": "calls", + "context": "call", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", @@ -1114,6 +1366,7 @@ def walk_calls(node, caller_nid: str) -> None: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -1301,7 +1554,7 @@ def _extract_python_rationale(path: Path, result: dict) -> None: except Exception: return - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes = result["nodes"] edges = result["edges"] @@ -1560,7 +1813,7 @@ def extract_verilog(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -1647,6 +1900,137 @@ def walk(node, module_nid: str | None = None) -> None: return {"nodes": nodes, "edges": edges} +def extract_sql(path: Path) -> dict: + """Extract tables, views, functions, and relationships from .sql files via tree-sitter.""" + try: + import tree_sitter_sql as tssql + from tree_sitter import Language, Parser + except ImportError: + return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"} + + try: + language = Language(tssql.language()) + parser = Parser(language) + source = path.read_bytes() + tree = parser.parse(source) + root = tree.root_node + except Exception as e: + return {"nodes": [], "edges": [], "error": str(e)} + + stem = re.sub(r"[^a-z0-9]", "_", path.stem.lower()) + str_path = str(path) + file_nid = _make_id(str_path) + nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code", + "source_file": str_path, "source_location": None}] + edges: list[dict] = [] + seen_ids: set[str] = {file_nid} + table_nids: dict[str, str] = {} # name → nid for reference resolution + + def _read(n) -> str: + return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + + def _obj_name(n) -> str | None: + for c in n.children: + if c.type == "object_reference": + for cc in c.children: + if cc.type == "identifier": + return _read(cc) + return None + + def _add_node(nid: str, label: str, line: int) -> None: + if nid not in seen_ids: + seen_ids.add(nid) + nodes.append({"id": nid, "label": label, "file_type": "code", + "source_file": str_path, "source_location": f"L{line}"}) + edges.append({"source": file_nid, "target": nid, "relation": "contains", + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0}) + + def _add_edge(src: str, tgt: str, relation: str, line: int) -> None: + edges.append({"source": src, "target": tgt, "relation": relation, + "confidence": "EXTRACTED", "source_file": str_path, + "source_location": f"L{line}", "weight": 1.0}) + + def walk(node) -> None: + t = node.type + line = node.start_point[0] + 1 + + if t == "create_table": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, name, line) + table_nids[name.lower()] = nid + # Foreign key REFERENCES + for col in node.children: + if col.type == "column_definitions": + for cd in col.children: + if cd.type != "column_definition": + continue + ref_name: str | None = None + found_ref = False + for cc in cd.children: + if cc.type == "keyword_references": + found_ref = True + elif found_ref and cc.type == "object_reference": + for ccc in cc.children: + if ccc.type == "identifier": + ref_name = _read(ccc) + break + if ref_name: + ref_nid = _make_id(stem, ref_name) + _add_edge(nid, ref_nid, "references", line) + + elif t == "create_view": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, name, line) + table_nids[name.lower()] = nid + # FROM/JOIN table references inside view body + _walk_from_refs(node, nid, line) + + elif t == "create_function": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, f"{name}()", line) + _walk_from_refs(node, nid, line) + + elif t == "create_procedure": + name = _obj_name(node) + if name: + nid = _make_id(stem, name) + _add_node(nid, f"{name}()", line) + _walk_from_refs(node, nid, line) + + for child in node.children: + walk(child) + + def _walk_from_refs(node, caller_nid: str, line: int) -> None: + """Recursively find FROM/JOIN table references inside a node.""" + if node.type in ("from", "join"): + for c in node.children: + if c.type == "relation": + for cc in c.children: + if cc.type == "object_reference": + for ccc in cc.children: + if ccc.type == "identifier": + tbl = _read(ccc) + tbl_nid = _make_id(stem, tbl) + _add_edge(caller_nid, tbl_nid, "reads_from", + c.start_point[0] + 1) + for child in node.children: + _walk_from_refs(child, caller_nid, line) + + for stmt in root.children: + if stmt.type == "statement": + for child in stmt.children: + walk(child) + + return {"nodes": nodes, "edges": edges} + + def extract_lua(path: Path) -> dict: """Extract functions, methods, require() imports, and calls from a .lua file.""" return _extract_generic(path, _LUA_CONFIG) @@ -1676,7 +2060,7 @@ def extract_julia(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -1695,8 +2079,9 @@ def add_node(nid: str, label: str, line: int) -> None: }) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({ + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = { "source": src, "target": tgt, "relation": relation, @@ -1704,7 +2089,10 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "source_file": str_path, "source_location": f"L{line}", "weight": weight, - }) + } + if context: + edge["context"] = context + edges.append(edge) file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -1731,14 +2119,14 @@ def walk_calls(body_node, func_nid: str) -> None: callee_name = _read_text(callee, source) target_nid = _make_id(stem, callee_name) add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, - confidence="EXTRACTED") + confidence="EXTRACTED", context="call") # Method call: obj.method(...) elif callee.type == "field_expression" and len(callee.children) >= 3: method_node = callee.children[-1] method_name = _read_text(method_node, source) target_nid = _make_id(stem, method_name) add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, - confidence="EXTRACTED") + confidence="EXTRACTED", context="call") for child in body_node.children: walk_calls(child, func_nid) @@ -1839,14 +2227,14 @@ def walk(node, scope_nid: str) -> None: mod_name = _read_text(child, source) imp_nid = _make_id(mod_name) add_node(imp_nid, mod_name, line) - add_edge(scope_nid, imp_nid, "imports", line) + add_edge(scope_nid, imp_nid, "imports", line, context="import") elif child.type == "selected_import": identifiers = [c for c in child.children if c.type == "identifier"] if identifiers: pkg_name = _read_text(identifiers[0], source) pkg_nid = _make_id(pkg_name) add_node(pkg_nid, pkg_name, line) - add_edge(scope_nid, pkg_nid, "imports", line) + add_edge(scope_nid, pkg_nid, "imports", line, context="import") return for child in node.children: @@ -1888,7 +2276,7 @@ def extract_go(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) # Use directory name as package scope so methods on the same type across # multiple files in a package share one canonical type node. pkg_scope = path.parent.name or stem @@ -1897,6 +2285,7 @@ def extract_go(path: Path) -> dict: edges: list[dict] = [] seen_ids: set[str] = set() function_bodies: list[tuple[str, object]] = [] + go_imported_pkgs: set[str] = set() # local names of imported packages def add_node(nid: str, label: str, line: int) -> None: if nid not in seen_ids: @@ -1910,8 +2299,9 @@ def add_node(nid: str, label: str, line: int) -> None: }) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({ + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = { "source": src, "target": tgt, "relation": relation, @@ -1919,7 +2309,10 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "source_file": str_path, "source_location": f"L{line}", "weight": weight, - }) + } + if context: + edge["context"] = context + edges.append(edge) file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -1993,13 +2386,22 @@ def walk(node) -> None: # Prefix with go_pkg_ so stdlib names (e.g. "context") # don't collide with local files of the same basename. tgt_nid = _make_id("go", "pkg", raw) - add_edge(file_nid, tgt_nid, "imports_from", spec.start_point[0] + 1) + add_edge(file_nid, tgt_nid, "imports_from", spec.start_point[0] + 1, context="import") + # Track local name (alias or last path segment) + alias = spec.child_by_field_name("name") + local_name = _read_text(alias, source) if alias else raw.split("/")[-1] + if local_name and local_name != "_" and local_name != ".": + go_imported_pkgs.add(local_name) elif child.type == "import_spec": path_node = child.child_by_field_name("path") if path_node: raw = _read_text(path_node, source).strip('"') tgt_nid = _make_id("go", "pkg", raw) - add_edge(file_nid, tgt_nid, "imports_from", child.start_point[0] + 1) + add_edge(file_nid, tgt_nid, "imports_from", child.start_point[0] + 1, context="import") + alias = child.child_by_field_name("name") + local_name = _read_text(alias, source) if alias else raw.split("/")[-1] + if local_name and local_name != "_" and local_name != ".": + go_imported_pkgs.add(local_name) return for child in node.children: @@ -2022,11 +2424,17 @@ def walk_calls(node, caller_nid: str) -> None: if node.type == "call_expression": func_node = node.child_by_field_name("function") callee_name: str | None = None + is_member_call: bool = False if func_node: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type == "selector_expression": field = func_node.child_by_field_name("field") + operand = func_node.child_by_field_name("operand") + receiver_name = _read_text(operand, source) if operand else "" + # Package-qualified call (e.g. fmt.Println) → allow cross-file resolution. + # Receiver method call (e.g. s.logger.Log) → skip, no import evidence. + is_member_call = receiver_name not in go_imported_pkgs if field: callee_name = _read_text(field, source) if callee_name: @@ -2040,6 +2448,7 @@ def walk_calls(node, caller_nid: str) -> None: "source": caller_nid, "target": tgt_nid, "relation": "calls", + "context": "call", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", @@ -2049,6 +2458,7 @@ def walk_calls(node, caller_nid: str) -> None: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2087,7 +2497,7 @@ def extract_rust(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -2106,8 +2516,9 @@ def add_node(nid: str, label: str, line: int) -> None: }) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({ + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = { "source": src, "target": tgt, "relation": relation, @@ -2115,7 +2526,10 @@ def add_edge(src: str, tgt: str, relation: str, line: int, "source_file": str_path, "source_location": f"L{line}", "weight": weight, - }) + } + if context: + edge["context"] = context + edges.append(edge) file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -2172,7 +2586,7 @@ def walk(node, parent_impl_nid: str | None = None) -> None: module_name = clean.split("::")[-1].strip() if module_name: tgt_nid = _make_id(module_name) - add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1) + add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1, context="import") return for child in node.children: @@ -2195,10 +2609,12 @@ def walk_calls(node, caller_nid: str) -> None: if node.type == "call_expression": func_node = node.child_by_field_name("function") callee_name: str | None = None + is_member_call: bool = False if func_node: if func_node.type == "identifier": callee_name = _read_text(func_node, source) elif func_node.type == "field_expression": + is_member_call = True field = func_node.child_by_field_name("field") if field: callee_name = _read_text(field, source) @@ -2217,6 +2633,7 @@ def walk_calls(node, caller_nid: str) -> None: "source": caller_nid, "target": tgt_nid, "relation": "calls", + "context": "call", "confidence": "EXTRACTED", "source_file": str_path, "source_location": f"L{line}", @@ -2226,6 +2643,7 @@ def walk_calls(node, caller_nid: str) -> None: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2264,7 +2682,7 @@ def extract_zig(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -2278,10 +2696,14 @@ def add_node(nid: str, label: str, line: int) -> None: "source_file": str_path, "source_location": f"L{line}"}) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "source_file": str_path, - "source_location": f"L{line}", "weight": weight}) + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": weight} + if context: + edge["context"] = context + edges.append(edge) file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -2380,7 +2802,9 @@ def walk_calls(node, caller_nid: str) -> None: if node.type == "call_expression": fn = node.child_by_field_name("function") if fn: - callee = _read_text(fn, source).split(".")[-1] + fn_text = _read_text(fn, source) + callee = fn_text.split(".")[-1] + is_member_call = "." in fn_text tgt_nid = next((n["id"] for n in nodes if n["label"] in (f"{callee}()", f".{callee}()")), None) if tgt_nid and tgt_nid != caller_nid: @@ -2394,6 +2818,7 @@ def walk_calls(node, caller_nid: str) -> None: raw_calls.append({ "caller_nid": caller_nid, "callee": callee, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2427,7 +2852,7 @@ def extract_powershell(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -2441,10 +2866,14 @@ def add_node(nid: str, label: str, line: int) -> None: "source_file": str_path, "source_location": f"L{line}"}) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "source_file": str_path, - "source_location": f"L{line}", "weight": weight}) + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": weight} + if context: + edge["context"] = context + edges.append(edge) file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -2558,6 +2987,7 @@ def walk_calls(node, caller_nid: str) -> None: raw_calls.append({ "caller_nid": caller_nid, "callee": cmd_text, + "is_member_call": False, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -2612,8 +3042,17 @@ def _resolve_cross_file_imports( stem = Path(src).stem label = node.get("label", "") nid = node.get("id", "") - # Only index real classes/functions (not file nodes, not method stubs) - if label and not label.endswith((")", ".py")) and "_" not in label[:1]: + # Index class-level entities only. Function/method labels end in "()" + # so are excluded by the `endswith(")")` filter; file nodes end in ".py"; + # private/internal labels start with "_"; rationale nodes carry + # file_type=="rationale" and must never participate in cross-file + # import resolution (#563). + if ( + label + and not label.endswith((")", ".py")) + and "_" not in label[:1] + and node.get("file_type") != "rationale" + ): stem_to_entities.setdefault(stem, {})[label] = nid # Pass 2: for each file, find `from .X import A, B, C` and resolve @@ -2621,15 +3060,18 @@ def _resolve_cross_file_imports( stem_to_path: dict[str, Path] = {p.stem: p for p in paths} for file_result, path in zip(per_file, paths): - stem = path.stem + stem = _file_stem(path) str_path = str(path) - # Find all classes defined in this file (the importers) + # Find all classes defined in this file (the importers). + # Excludes rationale nodes whose labels happen not to end in ")" or ".py" + # but which must never be treated as importing entities (#563). local_classes = [ n["id"] for n in file_result.get("nodes", []) if n.get("source_file") == str_path and not n["label"].endswith((")", ".py")) and n["id"] != _make_id(stem) # exclude file-level node + and n.get("file_type") != "rationale" ] if not local_classes: continue @@ -2745,7 +3187,7 @@ def _resolve_cross_file_java_imports( new_edges: list[dict] = [] seen_pairs: set[tuple[str, str]] = set() for path in paths: - file_nid = _make_id(path.stem) + file_nid = _make_id(str(path)) try: source = path.read_bytes() tree = parser.parse(source) @@ -2809,7 +3251,7 @@ def extract_objc(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -2823,10 +3265,14 @@ def add_node(nid: str, label: str, line: int) -> None: "source_file": str_path, "source_location": f"L{line}"}) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "source_file": str_path, - "source_location": f"L{line}", "weight": weight}) + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": weight} + if context: + edge["context"] = context + edges.append(edge) file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -2850,7 +3296,7 @@ def walk(node, parent_nid: str | None = None) -> None: module = raw.split("/")[-1].replace(".h", "") if module: tgt_nid = _make_id(module) - add_edge(file_nid, tgt_nid, "imports", line) + add_edge(file_nid, tgt_nid, "imports", line, context="import") elif child.type == "string_literal": # recurse into string_literal to find string_content for sub in child.children: @@ -2859,7 +3305,7 @@ def walk(node, parent_nid: str | None = None) -> None: module = raw.split("/")[-1].replace(".h", "") if module: tgt_nid = _make_id(module) - add_edge(file_nid, tgt_nid, "imports", line) + add_edge(file_nid, tgt_nid, "imports", line, context="import") return if t == "class_interface": @@ -2890,7 +3336,7 @@ def walk(node, parent_nid: str | None = None) -> None: for s in sub.children: if s.type == "type_identifier": proto_nid = _make_id(_read(s)) - add_edge(cls_nid, proto_nid, "imports", line) + add_edge(cls_nid, proto_nid, "imports", line, context="import") elif child.type == "method_declaration": walk(child, cls_nid) return @@ -2982,7 +3428,7 @@ def walk_calls(n) -> None: if pair not in seen_calls and caller_nid != candidate: seen_calls.add(pair) add_edge(caller_nid, candidate, "calls", body_node.start_point[0] + 1, - confidence="EXTRACTED", weight=1.0) + confidence="EXTRACTED", weight=1.0, context="call") for child in n.children: walk_calls(child) walk_calls(body_node) @@ -3007,7 +3453,7 @@ def extract_elixir(path: Path) -> dict: except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} - stem = path.stem + stem = _file_stem(path) str_path = str(path) nodes: list[dict] = [] edges: list[dict] = [] @@ -3021,10 +3467,14 @@ def add_node(nid: str, label: str, line: int) -> None: "source_file": str_path, "source_location": f"L{line}"}) def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "source_file": str_path, - "source_location": f"L{line}", "weight": weight}) + confidence: str = "EXTRACTED", weight: float = 1.0, + context: str | None = None) -> None: + edge = {"source": src, "target": tgt, "relation": relation, + "confidence": confidence, "source_file": str_path, + "source_location": f"L{line}", "weight": weight} + if context: + edge["context"] = context + edges.append(edge) file_nid = _make_id(str(path)) add_node(file_nid, path.name, 1) @@ -3103,7 +3553,7 @@ def walk(node, parent_module_nid: str | None = None) -> None: module_name = _get_alias_text(arguments_node) if module_name: tgt_nid = _make_id(module_name) - add_edge(file_nid, tgt_nid, "imports", line) + add_edge(file_nid, tgt_nid, "imports", line, context="import") return for child in node.children: @@ -3139,8 +3589,10 @@ def walk_calls(node, caller_nid: str) -> None: return break callee_name: str | None = None + is_member_call: bool = False for child in node.children: if child.type == "dot": + is_member_call = True dot_text = source[child.start_byte:child.end_byte].decode("utf-8", errors="replace") parts = dot_text.rstrip(".").split(".") if parts: @@ -3156,11 +3608,13 @@ def walk_calls(node, caller_nid: str) -> None: if pair not in seen_call_pairs: seen_call_pairs.add(pair) add_edge(caller_nid, tgt_nid, "calls", - node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0) + node.start_point[0] + 1, confidence="EXTRACTED", weight=1.0, + context="call") else: raw_calls.append({ "caller_nid": caller_nid, "callee": callee_name, + "is_member_call": is_member_call, "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", }) @@ -3196,7 +3650,167 @@ def _check_tree_sitter_version() -> None: ) -def extract(paths: list[Path], cache_root: Path | None = None) -> dict: +_DISPATCH: dict[str, Any] = { + ".py": extract_python, + ".js": extract_js, + ".jsx": extract_js, + ".mjs": extract_js, + ".ts": extract_js, + ".tsx": extract_js, + ".go": extract_go, + ".rs": extract_rust, + ".java": extract_java, + ".c": extract_c, + ".h": extract_c, + ".cpp": extract_cpp, + ".cc": extract_cpp, + ".cxx": extract_cpp, + ".hpp": extract_cpp, + ".rb": extract_ruby, + ".cs": extract_csharp, + ".kt": extract_kotlin, + ".kts": extract_kotlin, + ".scala": extract_scala, + ".php": extract_php, + ".swift": extract_swift, + ".lua": extract_lua, + ".toc": extract_lua, + ".zig": extract_zig, + ".ps1": extract_powershell, + ".ex": extract_elixir, + ".exs": extract_elixir, + ".m": extract_objc, + ".mm": extract_objc, + ".jl": extract_julia, + ".vue": extract_js, + ".svelte": extract_js, + ".dart": extract_dart, + ".v": extract_verilog, + ".sv": extract_verilog, + ".sql": extract_sql, +} + + +def _get_extractor(path: Path) -> Any | None: + """Return the correct extractor function for a file, or None if unsupported.""" + if path.name.endswith(".blade.php"): + return extract_blade + return _DISPATCH.get(path.suffix) + + +def _extract_single_file(args: tuple) -> tuple[int, dict]: + """Worker function for parallel extraction. Runs in a subprocess. + + Must be at module level (not a closure) so it can be pickled by + ProcessPoolExecutor. + + Args: + args: (index, path_str, cache_root_str) tuple + + Returns: + (index, result_dict) so results can be placed back in order. + """ + idx, path_str, cache_root_str = args + path = Path(path_str) + cache_root = Path(cache_root_str) + + # Check cache first (avoid re-extraction) + cached = load_cached(path, cache_root) + if cached is not None: + return idx, cached + + extractor = _get_extractor(path) + if extractor is None: + return idx, {"nodes": [], "edges": []} + + result = extractor(path) + if "error" not in result: + save_cached(path, result, cache_root) + return idx, result + + +def _extract_parallel( + uncached_work: list[tuple[int, Path]], + per_file: list[dict | None], + effective_root: Path, + max_workers: int | None, + total_files: int, +) -> None: + """Extract uncached files in parallel using ProcessPoolExecutor.""" + import concurrent.futures + + if max_workers is None: + max_workers = min(os.cpu_count() or 4, len(uncached_work), 8) + + root_str = str(effective_root) + work_items = [(idx, str(path), root_str) for idx, path in uncached_work] + + done_count = 0 + _PROGRESS_INTERVAL = 100 + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as pool: + futures = { + pool.submit(_extract_single_file, item): item[0] for item in work_items + } + for future in concurrent.futures.as_completed(futures): + idx, result = future.result() + per_file[idx] = result + done_count += 1 + if ( + total_files >= _PROGRESS_INTERVAL + and done_count % _PROGRESS_INTERVAL == 0 + ): + print( + f" AST extraction: {done_count}/{len(uncached_work)} uncached files " + f"({done_count * 100 // len(uncached_work)}%) [{max_workers} workers]", + flush=True, + ) + if total_files >= _PROGRESS_INTERVAL: + print( + f" AST extraction: {total_files}/{total_files} files (100%) [{max_workers} workers]", + flush=True, + ) + + +def _extract_sequential( + uncached_work: list[tuple[int, Path]], + per_file: list[dict | None], + effective_root: Path, + total_files: int, +) -> None: + """Extract uncached files sequentially (fallback for small batches).""" + _PROGRESS_INTERVAL = 100 + for work_idx, (idx, path) in enumerate(uncached_work): + if ( + total_files >= _PROGRESS_INTERVAL + and work_idx % _PROGRESS_INTERVAL == 0 + and work_idx > 0 + ): + print( + f" AST extraction: {work_idx}/{len(uncached_work)} uncached files ({work_idx * 100 // len(uncached_work)}%)", + flush=True, + ) + extractor = _get_extractor(path) + if extractor is None: + per_file[idx] = {"nodes": [], "edges": []} + continue + result = extractor(path) + if "error" not in result: + save_cached(path, result, effective_root) + per_file[idx] = result + if total_files >= _PROGRESS_INTERVAL: + print(f" AST extraction: {total_files}/{total_files} files (100%)", flush=True) + + +_PARALLEL_THRESHOLD = 20 + + +def extract( + paths: list[Path], + cache_root: Path | None = None, + *, + parallel: bool = True, + max_workers: int | None = None, +) -> dict: """Extract AST nodes and edges from a list of code files. Two-pass process: @@ -3209,9 +3823,11 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: cache_root: explicit root for graphify-out/cache/ (overrides the inferred common path prefix). Pass Path('.') when running on a subdirectory so the cache stays at ./graphify-out/cache/. + parallel: if True and there are >= _PARALLEL_THRESHOLD uncached files, + use ProcessPoolExecutor for multi-core extraction. + max_workers: max subprocess count. Defaults to min(cpu_count, 8). """ _check_tree_sitter_version() - per_file: list[dict] = [] # Infer a common root for cache keys (use first diverging segment, not sum of all matches) try: @@ -3232,67 +3848,36 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: root = Path(".") root = root.resolve() - _DISPATCH: dict[str, Any] = { - ".py": extract_python, - ".js": extract_js, - ".jsx": extract_js, - ".mjs": extract_js, - ".ts": extract_js, - ".tsx": extract_js, - ".go": extract_go, - ".rs": extract_rust, - ".java": extract_java, - ".c": extract_c, - ".h": extract_c, - ".cpp": extract_cpp, - ".cc": extract_cpp, - ".cxx": extract_cpp, - ".hpp": extract_cpp, - ".rb": extract_ruby, - ".cs": extract_csharp, - ".kt": extract_kotlin, - ".kts": extract_kotlin, - ".scala": extract_scala, - ".php": extract_php, - ".swift": extract_swift, - ".lua": extract_lua, - ".toc": extract_lua, - ".zig": extract_zig, - ".ps1": extract_powershell, - ".ex": extract_elixir, - ".exs": extract_elixir, - ".m": extract_objc, - ".mm": extract_objc, - ".jl": extract_julia, - ".vue": extract_js, - ".svelte": extract_js, - ".dart": extract_dart, - ".v": extract_verilog, - ".sv": extract_verilog, - } - + effective_root = cache_root or root total = len(paths) - _PROGRESS_INTERVAL = 100 + + # Phase 1: separate cached hits from uncached work + per_file: list[dict | None] = [None] * total + uncached_work: list[tuple[int, Path]] = [] + for i, path in enumerate(paths): - if total >= _PROGRESS_INTERVAL and i % _PROGRESS_INTERVAL == 0 and i > 0: - print(f" AST extraction: {i}/{total} files ({i * 100 // total}%)", flush=True) - # .blade.php must be checked before suffix lookup since Path.suffix returns .php - if path.name.endswith(".blade.php"): - extractor = extract_blade - else: - extractor = _DISPATCH.get(path.suffix) - if extractor is None: + if _get_extractor(path) is None: + per_file[i] = {"nodes": [], "edges": []} continue - cached = load_cached(path, cache_root or root) + cached = load_cached(path, effective_root) if cached is not None: - per_file.append(cached) + per_file[i] = cached continue - result = extractor(path) - if "error" not in result: - save_cached(path, result, cache_root or root) - per_file.append(result) - if total >= _PROGRESS_INTERVAL: - print(f" AST extraction: {total}/{total} files (100%)", flush=True) + uncached_work.append((i, path)) + + # Phase 2: extract uncached files (parallel or sequential) + if uncached_work: + if parallel and len(uncached_work) >= _PARALLEL_THRESHOLD: + _extract_parallel( + uncached_work, per_file, effective_root, max_workers, total + ) + else: + _extract_sequential(uncached_work, per_file, effective_root, total) + + # Fill any remaining None slots (shouldn't happen, but defensive) + for i in range(total): + if per_file[i] is None: + per_file[i] = {"nodes": [], "edges": []} all_nodes: list[dict] = [] all_edges: list[dict] = [] @@ -3345,12 +3930,21 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: # Cross-file call resolution for all languages # Each extractor saved unresolved calls in raw_calls. Now that we have all # nodes from all files, resolve any callee that exists in another file. - global_label_to_nid: dict[str, str] = {} + # Build name → ALL matching node IDs so we can skip ambiguous common names + # (e.g. "log", "execute", "find") that appear in multiple files — resolving + # those inflates god_nodes ranking with spurious cross-file edges. + # Build label -> node_id index for cross-file call resolution. + # Skip rationale nodes (their labels are docstring text, not callable + # identifiers, and they were polluting matches for short names — #563). + global_label_to_nids: dict[str, list[str]] = {} for n in all_nodes: + if n.get("file_type") == "rationale": + continue raw = n.get("label", "") normalised = raw.strip("()").lstrip(".") if normalised: - global_label_to_nid[normalised.lower()] = n["id"] + key = normalised.lower() + global_label_to_nids.setdefault(key, []).append(n["id"]) existing_pairs = {(e["source"], e["target"]) for e in all_edges} for result in per_file: @@ -3358,14 +3952,25 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: callee = rc.get("callee", "") if not callee: continue - tgt = global_label_to_nid.get(callee.lower()) + # Skip member-call callees: obj.log() → "log" has no import evidence + # and collides with any top-level function named "log" in the corpus. + if rc.get("is_member_call"): + continue + candidates = global_label_to_nids.get(callee.lower(), []) + # Skip ambiguous names that resolve to multiple nodes — these are + # common short names (log, execute, find) with no import evidence + # to pick the right target; emitting all edges inflates god_nodes. + if len(candidates) != 1: + continue + tgt = candidates[0] caller = rc["caller_nid"] - if tgt and tgt != caller and (caller, tgt) not in existing_pairs: + if tgt != caller and (caller, tgt) not in existing_pairs: existing_pairs.add((caller, tgt)) all_edges.append({ "source": caller, "target": tgt, "relation": "calls", + "context": "call", "confidence": "INFERRED", "confidence_score": 0.8, "source_file": rc.get("source_file", ""), @@ -3373,6 +3978,19 @@ def extract(paths: list[Path], cache_root: Path | None = None) -> dict: "weight": 1.0, }) + # Relativize source_file fields so paths are portable across machines (#555) + for item in all_nodes + all_edges: + sf = item.get("source_file") + if not sf: + continue + sf_path = Path(sf) + if not sf_path.is_absolute(): + continue + try: + item["source_file"] = str(sf_path.relative_to(root)) + except ValueError: + pass + return { "nodes": all_nodes, "edges": all_edges, diff --git a/graphify/hooks.py b/graphify/hooks.py index 3fa7d2e53..eebd92ae0 100644 --- a/graphify/hooks.py +++ b/graphify/hooks.py @@ -61,7 +61,13 @@ """ + _PYTHON_DETECT + """ export GRAPHIFY_CHANGED="$CHANGED" -$GRAPHIFY_PYTHON -c " + +# Run rebuild detached so git commit returns immediately. +# Full repo rebuilds can take hours; blocking the post-commit hook stalls the shell. +_GRAPHIFY_LOG="${HOME}/.cache/graphify-rebuild.log" +mkdir -p "$(dirname "$_GRAPHIFY_LOG")" +echo "[graphify hook] launching background rebuild (log: $_GRAPHIFY_LOG)" +nohup $GRAPHIFY_PYTHON -c " import os, sys from pathlib import Path @@ -74,12 +80,15 @@ print(f'[graphify hook] {len(changed)} file(s) changed - rebuilding graph...') try: + import os as _os from graphify.watch import _rebuild_code - _rebuild_code(Path('.')) + _force = _os.environ.get('GRAPHIFY_FORCE', '').lower() in ('1', 'true', 'yes') + _rebuild_code(Path('.'), force=_force) except Exception as exc: print(f'[graphify hook] Rebuild failed: {exc}') sys.exit(1) -" +" > "$_GRAPHIFY_LOG" 2>&1 < /dev/null & +disown 2>/dev/null || true # graphify-hook-end """ @@ -111,17 +120,21 @@ [ -f "$GIT_DIR/CHERRY_PICK_HEAD" ] && exit 0 """ + _PYTHON_DETECT + """ -echo "[graphify] Branch switched - rebuilding knowledge graph (code files)..." -$GRAPHIFY_PYTHON -c " +_GRAPHIFY_LOG="${HOME}/.cache/graphify-rebuild.log" +mkdir -p "$(dirname "$_GRAPHIFY_LOG")" +echo "[graphify] Branch switched - launching background rebuild (log: $_GRAPHIFY_LOG)" +nohup $GRAPHIFY_PYTHON -c " from graphify.watch import _rebuild_code from pathlib import Path -import sys +import os, sys try: - _rebuild_code(Path('.')) + _force = os.environ.get('GRAPHIFY_FORCE', '').lower() in ('1', 'true', 'yes') + _rebuild_code(Path('.'), force=_force) except Exception as exc: print(f'[graphify] Rebuild failed: {exc}') sys.exit(1) -" +" > "$_GRAPHIFY_LOG" 2>&1 < /dev/null & +disown 2>/dev/null || true # graphify-checkout-hook-end """ @@ -145,7 +158,7 @@ def _hooks_dir(root: Path) -> Path: if result.returncode == 0: custom = result.stdout.strip() if custom: - p = Path(custom) + p = Path(custom).expanduser() if not p.is_absolute(): p = root / p p.mkdir(parents=True, exist_ok=True) diff --git a/graphify/ingest.py b/graphify/ingest.py index 62d8386b7..d811bfcda 100644 --- a/graphify/ingest.py +++ b/graphify/ingest.py @@ -49,19 +49,16 @@ def _fetch_html(url: str) -> str: def _html_to_markdown(html: str, url: str) -> str: - """Convert HTML to clean markdown. Uses html2text if available, else basic strip.""" + """Convert HTML to clean markdown. Uses markdownify if available, else basic strip.""" + # Always pre-strip script/style so their text content never leaks into output + html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) + html = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) try: - import html2text - h = html2text.HTML2Text() - h.ignore_links = False - h.ignore_images = True - h.body_width = 0 - return h.handle(html) + from markdownify import markdownify + return markdownify(html, heading_style="ATX", bullets="-", strip=["img"]) except ImportError: - # Fallback: strip tags - text = re.sub(r"]*>.*?", "", html, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE) - text = re.sub(r"<[^>]+>", " ", text) + # Fallback: basic tag strip + text = re.sub(r"<[^>]+>", " ", html) text = re.sub(r"\s+", " ", text).strip() return text[:8000] diff --git a/graphify/llm.py b/graphify/llm.py new file mode 100644 index 000000000..9cba0ff81 --- /dev/null +++ b/graphify/llm.py @@ -0,0 +1,478 @@ +# Direct LLM backend for semantic extraction — supports Claude and Kimi K2.6. +# Used by `graphify . --backend kimi` and the benchmark scripts. +# The default graphify pipeline uses Claude Code subagents via skill.md; +# this module provides a direct API path for non-Claude-Code environments. +from __future__ import annotations + +import json +import os +import sys +import time +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +# `_read_files` truncates each file at this many characters before joining into +# the user message. Token estimates use the same cap so packing matches reality. +_FILE_CHAR_CAP = 20_000 +# `_read_files` also wraps each file in a `=== {rel} ===\n...\n\n` separator; +# this is roughly the per-file overhead in characters that the prompt adds. +_PER_FILE_OVERHEAD_CHARS = 80 +# Coarse fallback used only when `tiktoken` is not installed. 1 token ≈ 4 chars +# is the standard heuristic for English/code on BPE tokenizers. +_CHARS_PER_TOKEN = 4 + + +def _get_tokenizer(): + """Return a tiktoken encoder for accurate token counts, or None if tiktoken + is not installed. We use `cl100k_base` (GPT-4 / GPT-3.5-turbo) as a proxy: + Kimi-K2 ships a tiktoken-based tokenizer with very similar BPE behaviour, + and Claude's tokenizer has a comparable token-to-char ratio for prose/code. + Estimates only need to be within ~5%, not exact. + """ + try: + import tiktoken + except ImportError: + return None + try: + return tiktoken.get_encoding("cl100k_base") + except Exception: # network failure on first-use download, etc. + return None + + +# Cached at import time. None if tiktoken is unavailable; consumers must handle. +_TOKENIZER = _get_tokenizer() + +BACKENDS: dict[str, dict] = { + "claude": { + "base_url": "https://api.anthropic.com", + "default_model": "claude-sonnet-4-6", + "env_key": "ANTHROPIC_API_KEY", + "pricing": {"input": 3.0, "output": 15.0}, # USD per 1M tokens + "temperature": 0, + }, + "kimi": { + "base_url": "https://api.moonshot.ai/v1", + "default_model": "kimi-k2.6", + "env_key": "MOONSHOT_API_KEY", + "pricing": {"input": 0.74, "output": 4.66}, # USD per 1M tokens + "temperature": None, # kimi-k2.6 enforces its own fixed temperature; sending any value raises 400 + }, +} + +_EXTRACTION_SYSTEM = """\ +You are a graphify semantic extraction agent. Extract a knowledge graph fragment from the files provided. +Output ONLY valid JSON — no explanation, no markdown fences, no preamble. + +Rules: +- EXTRACTED: relationship explicit in source (import, call, citation, reference) +- INFERRED: reasonable inference (shared data structure, implied dependency) +- AMBIGUOUS: uncertain — flag for review, do not omit + +Node ID format: lowercase, only [a-z0-9_], no dots or slashes. +Format: {stem}_{entity} where stem = filename without extension, entity = symbol name (both normalised). + +Output exactly this schema: +{"nodes":[{"id":"stem_entity","label":"Human Readable Name","file_type":"code|document|paper|image|concept","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[],"input_tokens":0,"output_tokens":0} +""" + + +def _read_files(paths: list[Path], root: Path) -> str: + """Return file contents formatted for the extraction prompt.""" + parts: list[str] = [] + for p in paths: + try: + rel = p.relative_to(root) + except ValueError: + rel = p + try: + content = p.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + parts.append(f"=== {rel} ===\n{content[:20000]}") + return "\n\n".join(parts) + + +def _parse_llm_json(raw: str) -> dict: + """Strip optional markdown fences and parse JSON. Returns empty fragment on failure.""" + if raw.startswith("```"): + raw = raw.split("```", 2)[1] + if raw.startswith("json"): + raw = raw[4:] + raw = raw.rsplit("```", 1)[0] + try: + return json.loads(raw.strip()) + except json.JSONDecodeError as exc: + print(f"[graphify] LLM returned invalid JSON, skipping chunk: {exc}", file=sys.stderr) + return {"nodes": [], "edges": [], "hyperedges": []} + + +def _call_openai_compat( + base_url: str, + api_key: str, + model: str, + user_message: str, + temperature: float | None = 0, +) -> dict: + """Call any OpenAI-compatible API (Kimi, OpenAI, etc.) and return parsed JSON.""" + try: + from openai import OpenAI + except ImportError as exc: + raise ImportError( + "Kimi/OpenAI-compatible extraction requires the openai package. " + "Run: pip install openai" + ) from exc + + client = OpenAI(api_key=api_key, base_url=base_url) + kwargs: dict = { + "model": model, + "messages": [ + {"role": "system", "content": _EXTRACTION_SYSTEM}, + {"role": "user", "content": user_message}, + ], + "max_completion_tokens": 8192, + } + if temperature is not None: + kwargs["temperature"] = temperature + # Kimi-k2.6 is a reasoning model — disable thinking so content isn't empty + if "moonshot" in base_url: + kwargs["extra_body"] = {"thinking": {"type": "disabled"}} + resp = client.chat.completions.create(**kwargs) + result = _parse_llm_json(resp.choices[0].message.content or "{}") + result["input_tokens"] = resp.usage.prompt_tokens if resp.usage else 0 + result["output_tokens"] = resp.usage.completion_tokens if resp.usage else 0 + result["model"] = model + # `finish_reason == "length"` means the model hit max_completion_tokens + # mid-generation. The JSON we got back is truncated; callers should + # treat this as a signal to retry with smaller input. + result["finish_reason"] = resp.choices[0].finish_reason + return result + + +def _call_claude(api_key: str, model: str, user_message: str) -> dict: + """Call Anthropic Claude directly (not via OpenAI compat layer).""" + try: + import anthropic + except ImportError as exc: + raise ImportError( + "Claude direct extraction requires the anthropic package. " + "Run: pip install anthropic" + ) from exc + + client = anthropic.Anthropic(api_key=api_key) + resp = client.messages.create( + model=model, + max_tokens=8192, + system=_EXTRACTION_SYSTEM, + messages=[{"role": "user", "content": user_message}], + ) + result = _parse_llm_json(resp.content[0].text if resp.content else "{}") + result["input_tokens"] = resp.usage.input_tokens if resp.usage else 0 + result["output_tokens"] = resp.usage.output_tokens if resp.usage else 0 + result["model"] = model + # Normalise Anthropic's `stop_reason` to the OpenAI-compat `finish_reason` + # vocabulary so the adaptive-retry layer doesn't have to know which + # backend produced the result. + result["finish_reason"] = "length" if resp.stop_reason == "max_tokens" else "stop" + return result + + +def extract_files_direct( + files: list[Path], + backend: str = "kimi", + api_key: str | None = None, + model: str | None = None, + root: Path = Path("."), +) -> dict: + """Extract semantic nodes/edges from a list of files using the given backend. + + Returns dict with nodes, edges, hyperedges, input_tokens, output_tokens. + Raises ValueError for unknown backends. Raises ImportError if SDK missing. + """ + if backend not in BACKENDS: + raise ValueError(f"Unknown backend {backend!r}. Available: {sorted(BACKENDS)}") + + cfg = BACKENDS[backend] + key = api_key or os.environ.get(cfg["env_key"], "") + if not key: + raise ValueError( + f"No API key for backend '{backend}'. " + f"Set {cfg['env_key']} or pass api_key=." + ) + mdl = model or cfg["default_model"] + user_msg = _read_files(files, root) + + if backend == "claude": + return _call_claude(key, mdl, user_msg) + else: + return _call_openai_compat(cfg["base_url"], key, mdl, user_msg, temperature=cfg.get("temperature", 0)) + + +def _estimate_file_tokens(path: Path) -> int: + """Estimate the prompt-token cost of a single file under `_read_files` rules. + + Uses tiktoken (`cl100k_base`) when available for accurate counts. Falls back + to the chars/4 heuristic if tiktoken is not installed. Both paths cap at + `_FILE_CHAR_CAP` to match `_read_files`'s truncation, plus a constant for + the `=== rel ===` separator. Returns 0 for unreadable paths so they don't + blow up packing. + """ + if _TOKENIZER is None: + try: + size = path.stat().st_size + except OSError: + return 0 + chars = min(size, _FILE_CHAR_CAP) + _PER_FILE_OVERHEAD_CHARS + return chars // _CHARS_PER_TOKEN + + try: + content = path.read_text(encoding="utf-8", errors="replace")[:_FILE_CHAR_CAP] + except OSError: + return 0 + return len(_TOKENIZER.encode(content)) + (_PER_FILE_OVERHEAD_CHARS // _CHARS_PER_TOKEN) + + +def _pack_chunks_by_tokens( + files: list[Path], + token_budget: int, +) -> list[list[Path]]: + """Greedily pack files into chunks that fit a token budget. + + Files are first grouped by parent directory so related artifacts share a + chunk (cross-file edges are more likely to be extracted within a chunk + than across chunks). Within each directory, files are added one at a + time; a chunk is closed when adding the next file would exceed the + budget. A single file larger than the budget gets its own chunk and the + caller is expected to handle the API error if it actually overflows the + model's context window — packing can't shrink one big file. + """ + if token_budget <= 0: + raise ValueError(f"token_budget must be positive, got {token_budget}") + + by_dir: dict[Path, list[Path]] = {} + for f in files: + by_dir.setdefault(f.parent, []).append(f) + + chunks: list[list[Path]] = [] + current: list[Path] = [] + current_tokens = 0 + + for directory in sorted(by_dir): + for path in by_dir[directory]: + cost = _estimate_file_tokens(path) + if current and current_tokens + cost > token_budget: + chunks.append(current) + current = [] + current_tokens = 0 + current.append(path) + current_tokens += cost + + if current: + chunks.append(current) + return chunks + + +def _extract_with_adaptive_retry( + chunk: list[Path], + backend: str, + api_key: str | None, + model: str | None, + root: Path, + max_depth: int, + _depth: int = 0, +) -> dict: + """Extract a chunk; if the response is truncated (`finish_reason="length"`), + split the chunk in half and recurse. + + The signal driving the retry is the API's own `finish_reason` — `"length"` + means the model hit `max_completion_tokens` mid-output. The truncated JSON + has nothing useful in it (parse fails partway through a string or array), + so we discard it and re-extract on smaller inputs that produce shorter + outputs. + + Recursion is capped at `max_depth` to bound worst-case cost. A chunk of N + files can split into up to 2**max_depth pieces — at depth=3 that's 8x. If + still truncated at the cap, we surface the (likely empty) result with a + warning rather than infinite-loop. + + A single-file chunk that truncates is unrecoverable here — we can't make + one file smaller than itself, so we return what we got and warn. + """ + result = extract_files_direct( + chunk, backend=backend, api_key=api_key, model=model, root=root + ) + + if result.get("finish_reason") != "length": + return result + + if len(chunk) <= 1: + print( + f"[graphify] single-file chunk {chunk[0]} truncated at " + f"max_completion_tokens — partial result kept", + file=sys.stderr, + ) + return result + + if _depth >= max_depth: + print( + f"[graphify] chunk of {len(chunk)} still truncated at recursion " + f"depth {_depth} (max {max_depth}) — partial result kept", + file=sys.stderr, + ) + return result + + print( + f"[graphify] chunk of {len(chunk)} truncated at depth {_depth}, " + f"splitting into halves of {len(chunk) // 2} and " + f"{len(chunk) - len(chunk) // 2}", + file=sys.stderr, + ) + mid = len(chunk) // 2 + left = _extract_with_adaptive_retry( + chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1 + ) + right = _extract_with_adaptive_retry( + chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1 + ) + + return { + "nodes": left.get("nodes", []) + right.get("nodes", []), + "edges": left.get("edges", []) + right.get("edges", []), + "hyperedges": left.get("hyperedges", []) + right.get("hyperedges", []), + "input_tokens": left.get("input_tokens", 0) + right.get("input_tokens", 0), + "output_tokens": left.get("output_tokens", 0) + right.get("output_tokens", 0), + "model": result.get("model"), + # Both halves either succeeded or have already surfaced their own + # truncation warning; the merged result is no longer truncated as a + # logical unit. + "finish_reason": "stop", + } + + +def extract_corpus_parallel( + files: list[Path], + backend: str = "kimi", + api_key: str | None = None, + model: str | None = None, + root: Path = Path("."), + chunk_size: int = 20, + on_chunk_done: Callable | None = None, + token_budget: int | None = 60_000, + max_concurrency: int = 4, + max_retry_depth: int = 3, +) -> dict: + """Extract a corpus in chunks, merging results. + + Chunking strategy: + - If `token_budget` is set (default 60_000), files are packed to fit + the budget and grouped by parent directory. This avoids the worst + case where 20 randomly-grouped files exceed a model's context + window in a single request. + - If `token_budget=None`, falls back to the legacy fixed-count + `chunk_size` packing for backwards compatibility. + + Concurrency: + - Chunks run in parallel via a thread pool capped at `max_concurrency` + (default 4 — conservative to stay under provider rate limits). + - Set `max_concurrency=1` to force sequential execution. + + Adaptive retry on truncation: + - When the LLM returns `finish_reason="length"` (output truncated at + `max_completion_tokens`), the chunk is split in half and each half + re-extracted recursively, up to `max_retry_depth` levels deep + (default 3 → max 8x expansion of one chunk). + - This is signal-driven: chunks too dense to fit in one response + self-heal by splitting until they do, while well-sized chunks pay + no extra cost. Set `max_retry_depth=0` to disable retries. + + `on_chunk_done(idx, total, chunk_result)` fires once per chunk as it + completes (in completion order, not submission order). `idx` is the + chunk's submission index so callers can correlate progress. The + callback fires once per top-level chunk; recursive splits are merged + transparently before the callback is invoked. + + Returns merged dict with nodes, edges, hyperedges, input_tokens, + output_tokens. Failed chunks are logged to stderr and skipped — one bad + chunk does not abort the run. + """ + if token_budget is not None: + chunks = _pack_chunks_by_tokens(files, token_budget=token_budget) + else: + chunks = [files[i:i + chunk_size] for i in range(0, len(files), chunk_size)] + + merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + total = len(chunks) + + def _run_one(idx: int, chunk: list[Path]) -> tuple[int, dict | None, Exception | None]: + t0 = time.time() + try: + result = _extract_with_adaptive_retry( + chunk, + backend=backend, + api_key=api_key, + model=model, + root=root, + max_depth=max_retry_depth, + ) + result["elapsed_seconds"] = round(time.time() - t0, 2) + return idx, result, None + except Exception as exc: # noqa: BLE001 — caller-facing surface, log + continue + return idx, None, exc + + workers = max(1, min(max_concurrency, total)) + if workers == 1: + # Avoid thread pool overhead for single-worker runs (and keep + # callback ordering identical to the pre-refactor sequential path). + for idx, chunk in enumerate(chunks): + _, result, exc = _run_one(idx, chunk) + if exc is not None: + print(f"[graphify] chunk {idx + 1}/{total} failed: {exc}", file=sys.stderr) + continue + assert result is not None + _merge_into(merged, result) + if callable(on_chunk_done): + on_chunk_done(idx, total, result) + return merged + + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = [pool.submit(_run_one, idx, chunk) for idx, chunk in enumerate(chunks)] + for future in as_completed(futures): + idx, result, exc = future.result() + if exc is not None: + print(f"[graphify] chunk {idx + 1}/{total} failed: {exc}", file=sys.stderr) + continue + assert result is not None + _merge_into(merged, result) + if callable(on_chunk_done): + on_chunk_done(idx, total, result) + return merged + + +def _merge_into(merged: dict, result: dict) -> None: + """Append a chunk result into the running merged accumulator.""" + merged["nodes"].extend(result.get("nodes", [])) + merged["edges"].extend(result.get("edges", [])) + merged["hyperedges"].extend(result.get("hyperedges", [])) + merged["input_tokens"] += result.get("input_tokens", 0) + merged["output_tokens"] += result.get("output_tokens", 0) + + +def estimate_cost(backend: str, input_tokens: int, output_tokens: int) -> float: + """Estimate USD cost for a given token count using published pricing.""" + if backend not in BACKENDS: + return 0.0 + p = BACKENDS[backend]["pricing"] + return (input_tokens * p["input"] + output_tokens * p["output"]) / 1_000_000 + + +def detect_backend() -> str | None: + """Return the name of whichever backend has an API key set, or None. + + Kimi is checked first (opt-in). Falls back to Claude if ANTHROPIC_API_KEY is set. + Claude is the default for the skill.md subagent pipeline and is never forced here. + """ + if os.environ.get("MOONSHOT_API_KEY"): + return "kimi" + if os.environ.get("ANTHROPIC_API_KEY"): + return "claude" + return None diff --git a/graphify/report.py b/graphify/report.py index 103765dcf..7183b18f9 100644 --- a/graphify/report.py +++ b/graphify/report.py @@ -23,6 +23,7 @@ def generate( token_cost: dict, root: str, suggested_questions: list[dict] | None = None, + min_community_size: int = 3, ) -> str: today = date.today().isoformat() @@ -108,7 +109,11 @@ def generate( conf_tag = f"{conf} {cscore:.2f}" if cscore is not None else conf lines.append(f"- **{h.get('label', h.get('id', ''))}** — {node_labels} [{conf_tag}]") - lines += ["", "## Communities"] + thin_count = sum( + 1 for nodes in communities.values() + if 0 < sum(1 for n in nodes if not _ifn(G, n)) < min_community_size + ) + lines += ["", f"## Communities ({len(communities)} total, {thin_count} thin omitted)"] for cid, nodes in communities.items(): label = community_labels.get(cid, f"Community {cid}") score = cohesion_scores.get(cid, 0.0) @@ -116,6 +121,8 @@ def generate( real_nodes = [n for n in nodes if not _ifn(G, n)] if not real_nodes: continue + if len(real_nodes) < min_community_size: + continue display = [G.nodes[n].get("label", n) for n in real_nodes[:8]] suffix = f" (+{len(real_nodes)-8} more)" if len(real_nodes) > 8 else "" lines += [ @@ -157,11 +164,7 @@ def generate( lines.append(f"- **{len(isolated)} isolated node(s):** {', '.join(f'`{l}`' for l in isolated_labels)}{suffix}") lines.append(" These have ≤1 connection - possible missing edges or undocumented components.") if thin_communities: - for cid, nodes in thin_communities.items(): - label = community_labels.get(cid, f"Community {cid}") - node_labels = [G.nodes[n].get("label", n) for n in nodes] - lines.append(f"- **Thin community `{label}`** ({len(nodes)} nodes): {', '.join(f'`{l}`' for l in node_labels)}") - lines.append(" Too small to be a meaningful cluster - may be noise or needs more connections extracted.") + lines.append(f"- **{len(thin_communities)} thin communities (<{min_community_size} nodes) omitted from report** — run `graphify query` to explore isolated nodes.") if amb_pct > 20: lines.append(f"- **High ambiguity: {amb_pct}% of edges are AMBIGUOUS.** Review the Ambiguous Edges section above.") diff --git a/graphify/security.py b/graphify/security.py index 0d9060130..42c08fd14 100644 --- a/graphify/security.py +++ b/graphify/security.py @@ -1,6 +1,7 @@ # Security helpers - URL validation, safe fetch, path guards, label sanitisation from __future__ import annotations +import contextlib import html import re import urllib.error @@ -58,12 +59,45 @@ def validate_url(url: str) -> str: f"Blocked private/internal IP {addr} (resolved from '{hostname}'). " f"Got: {url!r}" ) - except socket.gaierror: - pass # DNS failure will surface later during fetch + except socket.gaierror as exc: + raise ValueError( + f"DNS resolution failed for '{hostname}': {exc}. Got: {url!r}" + ) from exc return url +@contextlib.contextmanager +def _ssrf_guarded_socket(): + """Patch socket.getaddrinfo for the duration of a fetch to catch DNS rebinding. + + Validates every IP that urllib resolves so a DNS server cannot return a public IP + for validate_url and swap to a private IP for the actual connection (TOCTOU fix). + Not thread-safe, but graphify is a single-threaded CLI tool. + """ + original = socket.getaddrinfo + + def _guarded(host, port, *args, **kwargs): + results = original(host, port, *args, **kwargs) + for info in results: + addr = info[4][0] + try: + ip = ipaddress.ip_address(addr) + except ValueError: + continue + if ip.is_private or ip.is_reserved or ip.is_loopback or ip.is_link_local: + raise OSError( + f"SSRF blocked: IP {addr} resolved from '{host}' is private/reserved" + ) + return results + + socket.getaddrinfo = _guarded + try: + yield + finally: + socket.getaddrinfo = original + + class _NoFileRedirectHandler(urllib.request.HTTPRedirectHandler): """Redirect handler that re-validates every redirect target. @@ -104,7 +138,7 @@ def safe_fetch(url: str, max_bytes: int = _MAX_FETCH_BYTES, timeout: int = 30) - opener = _build_opener() req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 graphify/1.0"}) - with opener.open(req, timeout=timeout) as resp: + with _ssrf_guarded_socket(), opener.open(req, timeout=timeout) as resp: # urllib raises HTTPError for non-2xx when using urlopen directly; # with a custom opener we check manually to be safe. status = getattr(resp, "status", None) or getattr(resp, "code", None) diff --git a/graphify/serve.py b/graphify/serve.py index 361dec3c0..708f3f3c8 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -45,6 +45,9 @@ def _strip_diacritics(text: str) -> str: return "".join(c for c in nfkd if not unicodedata.combining(c)) +_EXACT_MATCH_BONUS = 100.0 + + def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: scored = [] norm_terms = [_strip_diacritics(t).lower() for t in terms] @@ -52,11 +55,76 @@ def _score_nodes(G: nx.Graph, terms: list[str]) -> list[tuple[float, str]]: norm_label = data.get("norm_label") or _strip_diacritics(data.get("label") or "").lower() source = (data.get("source_file") or "").lower() score = sum(1 for t in norm_terms if t in norm_label) + sum(0.5 for t in norm_terms if t in source) + # Exact match: single term equals the full label (strip trailing () for functions) + if any(t == norm_label or t == norm_label.rstrip("()") for t in norm_terms): + score += _EXACT_MATCH_BONUS if score > 0: scored.append((score, nid)) return sorted(scored, reverse=True) +_CONTEXT_HINTS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("call", ("call", "calls", "called", "invoke", "invokes", "invoked")), + ("import", ("import", "imports", "imported", "module", "modules")), + ("field", ("field", "fields", "member", "members", "property", "properties")), + ("parameter_type", ("parameter", "parameters", "param", "params", "argument", "arguments")), + ("return_type", ("return", "returns", "returned")), + ("generic_arg", ("generic", "generics", "template", "templates")), +) + + +def _normalize_context_filters(filters: list[str] | None) -> list[str]: + if not filters: + return [] + normalized: list[str] = [] + seen: set[str] = set() + for value in filters: + key = _strip_diacritics(str(value)).strip().lower() + if key and key not in seen: + seen.add(key) + normalized.append(key) + return normalized + + +def _infer_context_filters(question: str) -> list[str]: + lowered = { + _strip_diacritics(token).lower() + for token in question.replace("?", " ").replace(",", " ").split() + } + inferred: list[str] = [] + for context, hints in _CONTEXT_HINTS: + if any(hint in lowered for hint in hints): + inferred.append(context) + return inferred + + +def _resolve_context_filters(question: str, explicit_filters: list[str] | None = None) -> tuple[list[str], str | None]: + normalized = _normalize_context_filters(explicit_filters) + if normalized: + return normalized, "explicit" + inferred = _infer_context_filters(question) + if inferred: + return inferred, "heuristic" + return [], None + + +def _filter_graph_by_context(G: nx.Graph, context_filters: list[str] | None) -> nx.Graph: + filters = set(_normalize_context_filters(context_filters)) + if not filters: + return G + H = G.__class__() + H.add_nodes_from(G.nodes(data=True)) + if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): + for u, v, key, data in G.edges(keys=True, data=True): + if data.get("context") in filters: + H.add_edge(u, v, key=key, **data) + else: + for u, v, data in G.edges(data=True): + if data.get("context") in filters: + H.add_edge(u, v, **data) + return H + + def _bfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], list[tuple]]: visited: set[str] = set(start_nodes) frontier = set(start_nodes) @@ -89,11 +157,18 @@ def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], lis return visited, edges_seen -def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000) -> str: - """Render subgraph as text, cutting at token_budget (approx 3 chars/token).""" +def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000, *, seeds: list[str] | None = None) -> str: + """Render subgraph as text, cutting at token_budget (approx 3 chars/token). + + seeds: exact-match nodes rendered first before the degree-sorted expansion, + so the queried symbol always appears at the top of the output. + """ char_budget = token_budget * 3 lines = [] - for nid in sorted(nodes, key=lambda n: G.degree(n), reverse=True): + seed_set = set(seeds or []) + ordered = [n for n in (seeds or []) if n in nodes] + \ + sorted(nodes - seed_set, key=lambda n: G.degree(n), reverse=True) + for nid in ordered: d = G.nodes[nid] line = f"NODE {sanitize_label(d.get('label', nid))} [src={d.get('source_file', '')} loc={d.get('source_location', '')} community={d.get('community', '')}]" lines.append(line) @@ -101,7 +176,13 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu if u in nodes and v in nodes: raw = G[u][v] d = next(iter(raw.values()), {}) if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)) else raw - line = f"EDGE {sanitize_label(G.nodes[u].get('label', u))} --{d.get('relation', '')} [{d.get('confidence', '')}]--> {sanitize_label(G.nodes[v].get('label', v))}" + context = d.get("context") + context_suffix = f" context={context}" if context else "" + line = ( + f"EDGE {sanitize_label(G.nodes[u].get('label', u))} " + f"--{d.get('relation', '')} [{d.get('confidence', '')}{context_suffix}]--> " + f"{sanitize_label(G.nodes[v].get('label', v))}" + ) lines.append(line) output = "\n".join(lines) if len(output) > char_budget: @@ -109,6 +190,34 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu return output +def _query_graph_text( + G: nx.Graph, + question: str, + *, + mode: str = "bfs", + depth: int = 3, + token_budget: int = 2000, + context_filters: list[str] | None = None, +) -> str: + terms = [t.lower() for t in question.split() if len(t) > 2] + scored = _score_nodes(G, terms) + start_nodes = [nid for _, nid in scored[:3]] + if not start_nodes: + return "No matching nodes found." + resolved_filters, filter_source = _resolve_context_filters(question, context_filters) + traversal_graph = _filter_graph_by_context(G, resolved_filters) + nodes, edges = _dfs(traversal_graph, start_nodes, depth) if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth) + header_parts = [ + f"Traversal: {mode.upper()} depth={depth}", + f"Start: {[G.nodes[n].get('label', n) for n in start_nodes]}", + ] + if resolved_filters: + header_parts.append(f"Context: {', '.join(resolved_filters)} ({filter_source})") + header_parts.append(f"{len(nodes)} nodes found") + header = " | ".join(header_parts) + "\n\n" + return header + _subgraph_to_text(traversal_graph, nodes, edges, token_budget) + + def _find_node(G: nx.Graph, label: str) -> list[str]: """Return node IDs whose label or ID matches the search term (diacritic-insensitive).""" term = _strip_diacritics(label).lower() @@ -175,6 +284,11 @@ async def list_tools() -> list[types.Tool]: "description": "bfs=broad context, dfs=trace a specific path"}, "depth": {"type": "integer", "default": 3, "description": "Traversal depth (1-6)"}, "token_budget": {"type": "integer", "default": 2000, "description": "Max output tokens"}, + "context_filter": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional explicit edge-context filter, e.g. ['call', 'field']", + }, }, "required": ["question"], }, @@ -239,14 +353,15 @@ def _tool_query_graph(arguments: dict) -> str: mode = arguments.get("mode", "bfs") depth = min(int(arguments.get("depth", 3)), 6) budget = int(arguments.get("token_budget", 2000)) - terms = [t.lower() for t in question.split() if len(t) > 2] - scored = _score_nodes(G, terms) - start_nodes = [nid for _, nid in scored[:3]] - if not start_nodes: - return "No matching nodes found." - nodes, edges = _dfs(G, start_nodes, depth) if mode == "dfs" else _bfs(G, start_nodes, depth) - header = f"Traversal: {mode.upper()} depth={depth} | Start: {[G.nodes[n].get('label', n) for n in start_nodes]} | {len(nodes)} nodes found\n\n" - return header + _subgraph_to_text(G, nodes, edges, budget) + context_filter = arguments.get("context_filter") + return _query_graph_text( + G, + question, + mode=mode, + depth=depth, + token_budget=budget, + context_filters=context_filter, + ) def _tool_get_node(arguments: dict) -> str: label = arguments["label"].lower() diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 7c745f3cb..de5704459 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -235,7 +235,7 @@ Process each file one at a time. For each file: - INFERRED: reasonable inference (shared structure, implied dependency) - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations, and rationale nodes (WHY decisions were made → `rationale_for` edges) + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. @@ -244,7 +244,7 @@ Process each file one at a time. For each file: 3. Accumulate results across all files Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} After processing all files, write the accumulated result to `.graphify_semantic_new.json`. @@ -254,6 +254,30 @@ For the accumulated result: If more than half the chunks failed, stop and tell the user. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat .graphify_python) -c " diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 2f3b3a7f2..865eae265 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -235,7 +235,7 @@ Process each file one at a time. For each file: - INFERRED: reasonable inference (shared structure, implied dependency) - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations, and rationale nodes (WHY decisions were made → `rationale_for` edges) + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. @@ -244,7 +244,7 @@ Process each file one at a time. For each file: 3. Accumulate results across all files Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} After processing all files, write the accumulated result to `.graphify_semantic_new.json`. @@ -254,6 +254,30 @@ For the accumulated result: If more than half the chunks failed, stop and tell the user. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat .graphify_python) -c " diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index f3c440812..febdb6ee0 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -263,7 +263,8 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -299,7 +300,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -312,6 +313,30 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat .graphify_python) -c " diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index cfccee543..826798c3c 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -259,7 +259,8 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -295,7 +296,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -308,6 +309,30 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat graphify-out/.graphify_python) -c " diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 1f2573576..8c649da1b 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -260,7 +260,8 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -296,7 +297,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -309,6 +310,30 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat .graphify_python) -c " diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index f04bc0f7c..8109adbe1 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -234,7 +234,7 @@ Process each file one at a time. For each file: - INFERRED: reasonable inference (shared structure, implied dependency) - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. - - Doc/paper files: named concepts, entities, citations, and rationale nodes (WHY decisions were made → `rationale_for` edges) + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. @@ -243,7 +243,7 @@ Process each file one at a time. For each file: 3. Accumulate results across all files Schema for each file's output: -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} After processing all files, write the accumulated result to `.graphify_semantic_new.json`. @@ -253,6 +253,30 @@ For the accumulated result: If more than half the chunks failed, stop and tell the user. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat .graphify_python) -c " diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 479b677d2..3f1c0053d 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -261,7 +261,8 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -297,7 +298,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -310,6 +311,30 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat graphify-out/.graphify_python) -c " diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md new file mode 100644 index 000000000..8109adbe1 --- /dev/null +++ b/graphify/skill-pi.md @@ -0,0 +1,1207 @@ +--- +name: graphify +description: Turn any folder of files (code, docs, papers, images, video) into a queryable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. Use when asked to analyze a codebase, understand architecture, map dependencies, or build a knowledge graph. +--- + +# /graphify + +Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md. + +## Usage + +``` +/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on specific path +/graphify --mode deep # thorough extraction, richer INFERRED edges +/graphify --update # incremental - re-extract only new/changed files +/graphify --cluster-only # rerun clustering on existing graph +/graphify --no-viz # skip visualization, just report + JSON +/graphify --html # (HTML is generated by default - this flag is a no-op) +/graphify --svg # also export graph.svg (embeds in Notion, GitHub) +/graphify --graphml # export graph.graphml (Gephi, yEd) +/graphify --neo4j # generate graphify-out/cypher.txt for Neo4j +/graphify --neo4j-push bolt://localhost:7687 # push directly to Neo4j +/graphify --mcp # start MCP stdio server for agent access +/graphify --watch # watch folder, auto-rebuild on code changes (no LLM needed) +/graphify add # fetch URL, save to ./raw, update graph +/graphify add --author "Name" # tag who wrote it +/graphify add --contributor "Name" # tag who added it to the corpus +/graphify query "" # BFS traversal - broad context +/graphify query "" --dfs # DFS - trace a specific path +/graphify query "" --budget 1500 # cap answer at N tokens +/graphify path "AuthModule" "Database" # shortest path between two concepts +/graphify explain "SwinTransformer" # plain-language explanation of a node +``` + +## What graphify is for + +graphify is built around Andrej Karpathy's /raw folder workflow: drop anything into a folder - papers, tweets, screenshots, code, notes - and get a structured knowledge graph that shows you what you didn't know was connected. + +Three things it does that your AI assistant alone cannot: +1. **Persistent graph** - relationships are stored in `graphify-out/graph.json` and survive across sessions. Ask questions weeks later without re-reading everything. +2. **Honest audit trail** - every edge is tagged EXTRACTED, INFERRED, or AMBIGUOUS. You know what was found vs invented. +3. **Cross-document surprise** - community detection finds connections between concepts in different files that you would never think to ask about directly. + +Use it for: +- A codebase you're new to (understand architecture before touching anything) +- A reading list (papers + tweets + notes → one navigable graph) +- A research corpus (citation graph + concept graph in one) +- Your personal /raw folder (drop everything in, let it grow, query it) + +## What You Must Do When Invoked + +If no path was given, use `.` (current directory). Do not ask the user for a path. + +Follow these steps in order. Do not skip steps. + +### Step 1 - Ensure graphify is installed + +```bash +# Detect the correct Python interpreter (handles pipx, venv, system installs) +GRAPHIFY_BIN=$(which graphify 2>/dev/null) +if [ -n "$GRAPHIFY_BIN" ]; then + PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!') + case "$PYTHON" in + *[!a-zA-Z0-9/_.-]*) PYTHON="python3" ;; + esac +else + PYTHON="python3" +fi +"$PYTHON" -c "import graphify" 2>/dev/null || "$PYTHON" -m pip install graphifyy -q 2>/dev/null || "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3 +mkdir -p graphify-out +# Write interpreter path for all subsequent steps +"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +``` + +If the import succeeds, print nothing and move straight to Step 2. + +**In every subsequent bash block, replace `python3` with `$(cat .graphify_python)` to use the correct interpreter.** + +### Step 2 - Detect files + +```bash +$(cat .graphify_python) -c " +import json +from graphify.detect import detect +from pathlib import Path +result = detect(Path('INPUT_PATH')) +print(json.dumps(result)) +" > .graphify_detect.json +``` + +Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead: + +``` +Corpus: X files · ~Y words + code: N files (.py .ts .go ...) + docs: N files (.md .txt ...) + papers: N files (.pdf ...) + images: N files + video: N files (.mp4 .mp3 ...) +``` + +Omit any category with 0 files from the summary. + +Then act on it: +- If `total_files` is 0: stop with "No supported files found in [path]." +- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. +- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. + +### Step 2.5 - Transcribe video / audio files (only if video files detected) + +Skip this step entirely if `detect` returned zero `video` files. + +Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3. + +**Strategy:** Read the god nodes from the detect output or analysis file. You are already a language model - write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed. + +**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."` + +**Step 1 - Write the Whisper prompt yourself.** + +Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example: + +- Labels: `transformer, attention, encoder, decoder` -> `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."` +- Labels: `kubernetes, deployment, pod, helm` -> `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."` + +Set it as `GRAPHIFY_WHISPER_PROMPT` in the environment before running the transcription command. + +**Step 2 - Transcribe:** + +```bash +$(cat graphify-out/.graphify_python) -c " +import json, os +from pathlib import Path +from graphify.transcribe import transcribe_all + +detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) +video_files = detect.get('files', {}).get('video', []) +prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.') + +transcript_paths = transcribe_all(video_files, initial_prompt=prompt) +print(json.dumps(transcript_paths)) +" > graphify-out/.graphify_transcripts.json +``` + +After transcription: +- Read the transcript paths from `graphify-out/.graphify_transcripts.json` +- Add them to the docs list before dispatching semantic subagents in Step 3B +- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs` +- If transcription fails for a file, print a warning and continue with the rest + +**Whisper model:** Default is `base`. If the user passed `--whisper-model `, set `GRAPHIFY_WHISPER_MODEL=` in the environment before running the command above. + +### Step 3 - Extract entities and relationships + +**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. + +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (your AI model, costs tokens). + +**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** + +Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. + +#### Part A - Structural extraction for code files + +For any code files detected, run AST extraction in parallel with Part B subagents: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.extract import collect_files, extract +from pathlib import Path +import json + +code_files = [] +detect = json.loads(Path('.graphify_detect.json').read_text()) +for f in detect.get('files', {}).get('code', []): + code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) + +if code_files: + result = extract(code_files) + Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) + print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') +else: + Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) + print('No code files - skipping AST extraction') +" +``` + +#### Part B - Semantic extraction (parallel subagents) + +**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. + +> **OpenClaw platform:** Multi-agent support is still early on OpenClaw. Extraction runs sequentially — you read and extract each file yourself. This is slower than parallel platforms but fully reliable. + +Print: `"Semantic extraction: N files (sequential — OpenClaw)"` + +**Step B0 - Check extraction cache first** + +Before dispatching any subagents, check which files already have cached extraction results: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import check_semantic_cache +from pathlib import Path + +detect = json.loads(Path('.graphify_detect.json').read_text()) +all_files = [f for files in detect['files'].values() for f in files] + +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files) + +if cached_nodes or cached_edges or cached_hyperedges: + Path('.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges})) +Path('.graphify_uncached.txt').write_text('\n'.join(uncached)) +print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction') +" +``` + +Only dispatch subagents for files listed in `.graphify_uncached.txt`. If all files are cached, skip to Part C directly. + +**Step B1 - Split into chunks** + +Load files from `.graphify_uncached.txt`. Split into chunks of 20-25 files each. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. + +**Step B2 - Sequential extraction (OpenClaw)** + +Process each file one at a time. For each file: + +1. Read the file contents +2. Extract nodes, edges, and hyperedges applying the same rules: + - EXTRACTED: relationship explicit in source (import, call, citation) + - INFERRED: reasonable inference (shared structure, implied dependency) + - AMBIGUOUS: uncertain — flag it, do not omit + - Code files: semantic edges AST cannot find. Do not re-extract imports. + - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms). Do NOT invent file_types like `concept`. When adding `calls` edges: source is caller, target is callee. + - Image files: use vision — understand what the image IS, not just OCR + - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges + - Semantic similarity: if two concepts solve the same problem without a structural link, add `semantically_similar_to` INFERRED edge (confidence 0.6-0.95). Non-obvious cross-file links only. + - Hyperedges: if 3+ nodes share a concept/flow not captured by pairwise edges, add a hyperedge. Max 3 per file. + - confidence_score REQUIRED on every edge: EXTRACTED=1.0, INFERRED=0.6-0.9 (reason individually), AMBIGUOUS=0.1-0.3 +3. Accumulate results across all files + +Schema for each file's output: +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} + +After processing all files, write the accumulated result to `.graphify_semantic_new.json`. + +**Step B3 - Cache and merge** + +For the accumulated result: + +If more than half the chunks failed, stop and tell the user. + +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + +Save new results to cache: +```bash +$(cat .graphify_python) -c " +import json +from graphify.cache import save_semantic_cache +from pathlib import Path + +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +print(f'Cached {saved} files') +" +``` + +Merge cached + new results into `.graphify_semantic.json`: +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +cached = json.loads(Path('.graphify_cached.json').read_text()) if Path('.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} +new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} + +all_nodes = cached['nodes'] + new.get('nodes', []) +all_edges = cached['edges'] + new.get('edges', []) +all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', []) +seen = set() +deduped = [] +for n in all_nodes: + if n['id'] not in seen: + seen.add(n['id']) + deduped.append(n) + +merged = { + 'nodes': deduped, + 'edges': all_edges, + 'hyperedges': all_hyperedges, + 'input_tokens': new.get('input_tokens', 0), + 'output_tokens': new.get('output_tokens', 0), +} +Path('.graphify_semantic.json').write_text(json.dumps(merged, indent=2)) +print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') +" +``` +Clean up temp files: `rm -f .graphify_cached.json .graphify_uncached.txt .graphify_semantic_new.json` + +#### Part C - Merge AST + semantic into final extraction + +```bash +$(cat .graphify_python) -c " +import sys, json +from pathlib import Path + +ast = json.loads(Path('.graphify_ast.json').read_text()) +sem = json.loads(Path('.graphify_semantic.json').read_text()) + +# Merge: AST nodes first, semantic nodes deduplicated by id +seen = {n['id'] for n in ast['nodes']} +merged_nodes = list(ast['nodes']) +for n in sem['nodes']: + if n['id'] not in seen: + merged_nodes.append(n) + seen.add(n['id']) + +merged_edges = ast['edges'] + sem['edges'] +merged_hyperedges = sem.get('hyperedges', []) +merged = { + 'nodes': merged_nodes, + 'edges': merged_edges, + 'hyperedges': merged_hyperedges, + 'input_tokens': sem.get('input_tokens', 0), + 'output_tokens': sem.get('output_tokens', 0), +} +Path('.graphify_extract.json').write_text(json.dumps(merged, indent=2)) +total = len(merged_nodes) +edges = len(merged_edges) +print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)') +" +``` + +### Step 4 - Build graph, cluster, analyze, generate outputs + +```bash +mkdir -p graphify-out +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from graphify.export import to_json +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) + +G = build_from_json(extraction) +communities = cluster(G) +cohesion = score_all(G, communities) +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} +# Placeholder questions - regenerated with real labels in Step 5 +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, + 'questions': questions, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +if G.number_of_nodes() == 0: + print('ERROR: Graph is empty - extraction produced no nodes.') + print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.') + raise SystemExit(1) +print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities') +" +``` + +If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization. + +Replace INPUT_PATH with the actual path. + +### Step 5 - Label communities + +Read `.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading"). + +Then regenerate the report and save the labels for the visualizer: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import score_all +from graphify.analyze import god_nodes, surprising_connections, suggest_questions +from graphify.report import generate +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +detection = json.loads(Path('.graphify_detect.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)} + +# LABELS - replace these with the names you chose above +labels = LABELS_DICT + +# Regenerate questions with real community labels (labels affect question phrasing) +questions = suggest_questions(G, communities, labels) + +report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions) +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +Path('.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()})) +print('Report updated with community labels') +" +``` + +Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`). +Replace INPUT_PATH with the actual path. + +### Step 6 - Generate Obsidian vault (opt-in) + HTML + +**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node. + +If `--obsidian` was given: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_obsidian, to_canvas +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +cohesion = {int(k): v for k, v in analysis['cohesion'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +n = to_obsidian(G, communities, 'graphify-out/obsidian', community_labels=labels or None, cohesion=cohesion) +print(f'Obsidian vault: {n} notes in graphify-out/obsidian/') + +to_canvas(G, communities, 'graphify-out/obsidian/graph.canvas', community_labels=labels or None) +print('Canvas: graphify-out/obsidian/graph.canvas - open in Obsidian for structured community layout') +print() +print('Open graphify-out/obsidian/ as a vault in Obsidian.') +print(' Graph view - nodes colored by community (set automatically)') +print(' graph.canvas - structured layout with communities as groups') +print(' _COMMUNITY_* - overview notes with cohesion scores and dataview queries') +" +``` + +Generate the HTML graph (always, unless `--no-viz`): + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_html +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +if G.number_of_nodes() > 5000: + print(f'Graph has {G.number_of_nodes()} nodes - too large for HTML viz. Use Obsidian vault instead.') +else: + to_html(G, communities, 'graphify-out/graph.html', community_labels=labels or None) + print('graph.html written - open in any browser, no server needed') +" +``` + +### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag) + +**If `--neo4j`** - generate a Cypher file for manual import: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_cypher +from pathlib import Path + +G = build_from_json(json.loads(Path('.graphify_extract.json').read_text())) +to_cypher(G, 'graphify-out/cypher.txt') +print('cypher.txt written - import with: cypher-shell < graphify-out/cypher.txt') +" +``` + +**If `--neo4j-push `** - push directly to a running Neo4j instance. Ask the user for credentials if not provided: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.cluster import cluster +from graphify.export import push_to_neo4j +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +result = push_to_neo4j(G, uri='NEO4J_URI', user='NEO4J_USER', password='NEO4J_PASSWORD', communities=communities) +print(f'Pushed to Neo4j: {result[\"nodes\"]} nodes, {result[\"edges\"]} edges') +" +``` + +Replace `NEO4J_URI`, `NEO4J_USER`, `NEO4J_PASSWORD` with actual values. Default URI is `bolt://localhost:7687`, default user is `neo4j`. Uses MERGE - safe to re-run without creating duplicates. + +### Step 7b - SVG export (only if --svg flag) + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_svg +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) +labels_raw = json.loads(Path('.graphify_labels.json').read_text()) if Path('.graphify_labels.json').exists() else {} + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} +labels = {int(k): v for k, v in labels_raw.items()} + +to_svg(G, communities, 'graphify-out/graph.svg', community_labels=labels or None) +print('graph.svg written - embeds in Obsidian, Notion, GitHub READMEs') +" +``` + +### Step 7c - GraphML export (only if --graphml flag) + +```bash +$(cat .graphify_python) -c " +import json +from graphify.build import build_from_json +from graphify.export import to_graphml +from pathlib import Path + +extraction = json.loads(Path('.graphify_extract.json').read_text()) +analysis = json.loads(Path('.graphify_analysis.json').read_text()) + +G = build_from_json(extraction) +communities = {int(k): v for k, v in analysis['communities'].items()} + +to_graphml(G, communities, 'graphify-out/graph.graphml') +print('graph.graphml written - open in Gephi, yEd, or any GraphML tool') +" +``` + +### Step 7d - MCP server (only if --mcp flag) + +```bash +python3 -m graphify.serve graphify-out/graph.json +``` + +This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live. + +To configure in Claude Desktop, add to `claude_desktop_config.json`: +```json +{ + "mcpServers": { + "graphify": { + "command": "python3", + "args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"] + } + } +} +``` + +### Step 8 - Token reduction benchmark (only if total_words > 5000) + +If `total_words` from `.graphify_detect.json` is greater than 5,000, run: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.benchmark import run_benchmark, print_benchmark +from pathlib import Path + +detection = json.loads(Path('.graphify_detect.json').read_text()) +result = run_benchmark('graphify-out/graph.json', corpus_words=detection['total_words']) +print_benchmark(result) +" +``` + +Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora. + +--- + +### Step 9 - Save manifest, update cost tracker, clean up, and report + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path +from datetime import datetime, timezone +from graphify.detect import save_manifest + +# Save manifest for --update +detect = json.loads(Path('.graphify_detect.json').read_text()) +save_manifest(detect['files']) + +# Update cumulative cost tracker +extract = json.loads(Path('.graphify_extract.json').read_text()) +input_tok = extract.get('input_tokens', 0) +output_tok = extract.get('output_tokens', 0) + +cost_path = Path('graphify-out/cost.json') +if cost_path.exists(): + cost = json.loads(cost_path.read_text()) +else: + cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0} + +cost['runs'].append({ + 'date': datetime.now(timezone.utc).isoformat(), + 'input_tokens': input_tok, + 'output_tokens': output_tok, + 'files': detect.get('total_files', 0), +}) +cost['total_input_tokens'] += input_tok +cost['total_output_tokens'] += output_tok +cost_path.write_text(json.dumps(cost, indent=2)) + +print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') +print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') +" +rm -f .graphify_detect.json .graphify_extract.json .graphify_ast.json .graphify_semantic.json .graphify_analysis.json .graphify_labels.json .graphify_chunk_*.json +rm -f graphify-out/.needs_update 2>/dev/null || true +``` + +Tell the user (omit the obsidian line unless --obsidian was given): +``` +Graph complete. Outputs in PATH_TO_DIR/graphify-out/ + + graph.html - interactive graph, open in browser + GRAPH_REPORT.md - audit report + graph.json - raw graph data + obsidian/ - Obsidian vault (only if --obsidian was given) +``` + +If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi + +Replace PATH_TO_DIR with the actual absolute path of the directory that was processed. + +Then paste these sections from GRAPH_REPORT.md directly into the chat: +- God Nodes +- Surprising Connections +- Suggested Questions + +Do NOT paste the full report - just those three sections. Keep it concise. + +Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask: + +> "The most interesting question this graph can answer: **[question]**. Want me to trace it?" + +If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report. + +The graph is the map. Your job after the pipeline is to be the guide. + +--- + +## For --update (incremental re-extraction) + +Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time. + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.detect import detect_incremental, save_manifest +from pathlib import Path + +result = detect_incremental(Path('INPUT_PATH')) +new_total = result.get('new_total', 0) +print(json.dumps(result, indent=2)) +Path('.graphify_incremental.json').write_text(json.dumps(result)) +if new_total == 0: + print('No files changed since last run. Nothing to update.') + raise SystemExit(0) +print(f'{new_total} new/changed file(s) to re-extract.') +" +``` + +If new files exist, first check whether all changed files are code files: + +```bash +$(cat .graphify_python) -c " +import json +from pathlib import Path + +result = json.loads(open('.graphify_incremental.json').read()) if Path('.graphify_incremental.json').exists() else {} +code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts'} +new_files = result.get('new_files', {}) +all_changed = [f for files in new_files.values() for f in files] +code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed) +print('code_only:', code_only) +" +``` + +If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run only Step 3A (AST) on the changed files, skip Step 3B entirely (no subagents), then go straight to merge and Steps 4–8. + +If `code_only` is False (any changed file is a doc/paper/image): run the full Steps 3A–3C pipeline as normal. + +Then: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.build import build_from_json +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load existing graph +existing_data = json.loads(Path('graphify-out/graph.json').read_text()) +G_existing = json_graph.node_link_graph(existing_data, edges='links') + +# Load new extraction +new_extraction = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extraction) + +# Merge: new nodes/edges into existing graph +G_existing.update(G_new) +print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') +" +``` + +Then run Steps 4–8 on the merged graph as normal. + +After Step 4, show the graph diff: + +```bash +$(cat .graphify_python) -c " +import json +from graphify.analyze import graph_diff +from graphify.build import build_from_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +# Load old graph (before update) from backup written before merge +old_data = json.loads(Path('.graphify_old.json').read_text()) if Path('.graphify_old.json').exists() else None +new_extract = json.loads(Path('.graphify_extract.json').read_text()) +G_new = build_from_json(new_extract) + +if old_data: + G_old = json_graph.node_link_graph(old_data, edges='links') + diff = graph_diff(G_old, G_new) + print(diff['summary']) + if diff['new_nodes']: + print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5])) + if diff['new_edges']: + print('New edges:', len(diff['new_edges'])) +" +``` + +Before the merge step, save the old graph: `cp graphify-out/graph.json .graphify_old.json` +Clean up after: `rm -f .graphify_old.json` + +--- + +## For --cluster-only + +Skip Steps 1–3. Load the existing graph from `graphify-out/graph.json` and re-run clustering: + +```bash +$(cat .graphify_python) -c " +import sys, json +from graphify.cluster import cluster, score_all +from graphify.analyze import god_nodes, surprising_connections +from graphify.report import generate +from graphify.export import to_json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, + 'files': {'code': [], 'document': [], 'paper': []}} +tokens = {'input': 0, 'output': 0} + +communities = cluster(G) +cohesion = score_all(G, communities) +gods = god_nodes(G) +surprises = surprising_connections(G, communities) +labels = {cid: 'Community ' + str(cid) for cid in communities} + +report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, '.') +Path('graphify-out/GRAPH_REPORT.md').write_text(report) +to_json(G, communities, 'graphify-out/graph.json') + +analysis = { + 'communities': {str(k): v for k, v in communities.items()}, + 'cohesion': {str(k): v for k, v in cohesion.items()}, + 'gods': gods, + 'surprises': surprises, +} +Path('.graphify_analysis.json').write_text(json.dumps(analysis, indent=2)) +print(f'Re-clustered: {len(communities)} communities') +" +``` + +Then run Steps 5–9 as normal (label communities, generate viz, benchmark, clean up, report). + +--- + +## For /graphify query + +Two traversal modes - choose based on the question: + +| Mode | Flag | Best for | +|------|------|----------| +| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first | +| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path | + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +Load `graphify-out/graph.json`, then: + +1. Find the 1-3 nodes whose label best matches key terms in the question. +2. Run the appropriate traversal from each starting node. +3. Read the subgraph - node labels, edge relations, confidence tags, source locations. +4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact. +5. If the graph lacks enough information, say so - do not hallucinate edges. + +```bash +$(cat .graphify_python) -c " +import sys, json +from networkx.readwrite import json_graph +import networkx as nx +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +question = 'QUESTION' +mode = 'MODE' # 'bfs' or 'dfs' +terms = [t.lower() for t in question.split() if len(t) > 3] + +# Find best-matching start nodes +scored = [] +for nid, ndata in G.nodes(data=True): + label = ndata.get('label', '').lower() + score = sum(1 for t in terms if t in label) + if score > 0: + scored.append((score, nid)) +scored.sort(reverse=True) +start_nodes = [nid for _, nid in scored[:3]] + +if not start_nodes: + print('No matching nodes found for query terms:', terms) + sys.exit(0) + +subgraph_nodes = set() +subgraph_edges = [] + +if mode == 'dfs': + # DFS: follow one path as deep as possible before backtracking. + # Depth-limited to 6 to avoid traversing the whole graph. + visited = set() + stack = [(n, 0) for n in reversed(start_nodes)] + while stack: + node, depth = stack.pop() + if node in visited or depth > 6: + continue + visited.add(node) + subgraph_nodes.add(node) + for neighbor in G.neighbors(node): + if neighbor not in visited: + stack.append((neighbor, depth + 1)) + subgraph_edges.append((node, neighbor)) +else: + # BFS: explore all neighbors layer by layer up to depth 3. + frontier = set(start_nodes) + subgraph_nodes = set(start_nodes) + for _ in range(3): + next_frontier = set() + for n in frontier: + for neighbor in G.neighbors(n): + if neighbor not in subgraph_nodes: + next_frontier.add(neighbor) + subgraph_edges.append((n, neighbor)) + subgraph_nodes.update(next_frontier) + frontier = next_frontier + +# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token) +token_budget = BUDGET # default 2000 +char_budget = token_budget * 4 + +# Score each node by term overlap for ranked output +def relevance(nid): + label = G.nodes[nid].get('label', '').lower() + return sum(1 for t in terms if t in label) + +ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True) + +lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes'] +for nid in ranked_nodes: + d = G.nodes[nid] + lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]') +for u, v in subgraph_edges: + if u in subgraph_nodes and v in subgraph_nodes: + d = G.edges[u, v] + lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{d.get(\"relation\",\"\")} [{d.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}') + +output = '\n'.join(lines) +if len(output) > char_budget: + output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)' +print(output) +" +``` + +Replace `QUESTION` with the user's actual question, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above. + +After writing the answer, save it back into the graph so it improves future queries: + +```bash +$(cat .graphify_python) -m graphify save-result --question "QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2 +``` + +Replace `QUESTION` with the question, `ANSWER` with your full answer text, `SOURCE_NODES` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph. + +--- + +## For /graphify path + +Find the shortest path between two named concepts in the graph. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +a_term = 'NODE_A' +b_term = 'NODE_B' + +def find_node(term): + term = term.lower() + scored = sorted( + [(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True + ) + return scored[0][1] if scored and scored[0][0] > 0 else None + +src = find_node(a_term) +tgt = find_node(b_term) + +if not src or not tgt: + print(f'Could not find nodes matching: {a_term!r} or {b_term!r}') + sys.exit(0) + +try: + path = nx.shortest_path(G, src, tgt) + print(f'Shortest path ({len(path)-1} hops):') + for i, nid in enumerate(path): + label = G.nodes[nid].get('label', nid) + if i < len(path) - 1: + edge = G.edges[nid, path[i+1]] + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + print(f' {label} --{rel}--> [{conf}]') + else: + print(f' {label}') +except nx.NetworkXNoPath: + print(f'No path found between {a_term!r} and {b_term!r}') +except nx.NodeNotFound as e: + print(f'Node not found: {e}') +" +``` + +Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B +``` + +--- + +## For /graphify explain + +Give a plain-language explanation of a single node - everything connected to it. + +First check the graph exists: +```bash +$(cat .graphify_python) -c " +from pathlib import Path +if not Path('graphify-out/graph.json').exists(): + print('ERROR: No graph found. Run /graphify first to build the graph.') + raise SystemExit(1) +" +``` +If it fails, stop and tell the user to run `/graphify ` first. + +```bash +$(cat .graphify_python) -c " +import json, sys +import networkx as nx +from networkx.readwrite import json_graph +from pathlib import Path + +data = json.loads(Path('graphify-out/graph.json').read_text()) +G = json_graph.node_link_graph(data, edges='links') + +term = 'NODE_NAME' +term_lower = term.lower() + +# Find best matching node +scored = sorted( + [(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n) + for n in G.nodes()], + reverse=True +) +if not scored or scored[0][0] == 0: + print(f'No node matching {term!r}') + sys.exit(0) + +nid = scored[0][1] +data_n = G.nodes[nid] +print(f'NODE: {data_n.get(\"label\", nid)}') +print(f' source: {data_n.get(\"source_file\",\"unknown\")}') +print(f' type: {data_n.get(\"file_type\",\"unknown\")}') +print(f' degree: {G.degree(nid)}') +print() +print('CONNECTIONS:') +for neighbor in G.neighbors(nid): + edge = G.edges[nid, neighbor] + nlabel = G.nodes[neighbor].get('label', neighbor) + rel = edge.get('relation', '') + conf = edge.get('confidence', '') + src_file = G.nodes[neighbor].get('source_file', '') + print(f' --{rel}--> {nlabel} [{conf}] ({src_file})') +" +``` + +Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations. + +After writing the explanation, save it back: + +```bash +$(cat .graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME +``` + +--- + +## For /graphify add + +Fetch a URL and add it to the corpus, then update the graph. + +```bash +$(cat .graphify_python) -c " +import sys +from graphify.ingest import ingest +from pathlib import Path + +try: + out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR') + print(f'Saved to {out}') +except ValueError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +except RuntimeError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +" +``` + +Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph. + +Supported URL types (auto-detected): +- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author +- arXiv → abstract + metadata saved as `.md` +- PDF → downloaded as `.pdf` +- Images (.png/.jpg/.webp) → downloaded, vision extraction runs on next build +- Any webpage → converted to markdown via html2text + +--- + +## For --watch + +Start a background watcher that monitors a folder and auto-updates the graph when files change. + +```bash +python3 -m graphify.watch INPUT_PATH --debounce 3 +``` + +Replace INPUT_PATH with the folder to watch. Behavior depends on what changed: + +- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically. +- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required). + +Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file. + +Press Ctrl+C to stop. + +For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves. + +--- + +## For git commit hook + +Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor. + +```bash +graphify hook install # install +graphify hook uninstall # remove +graphify hook status # check +``` + +After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those. + +If a post-commit hook already exists, graphify appends to it rather than replacing it. + +--- + +## For native CLAUDE.md integration + +Run once per project to make graphify always-on in Claude Code sessions: + +```bash +graphify claude install +``` + +This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions. + +```bash +graphify claude uninstall # remove the section +``` + +--- + +## Honesty Rules + +- Never invent an edge. If unsure, use AMBIGUOUS. +- Never skip the corpus check warning. +- Always show token cost in the report. +- Never hide cohesion scores behind symbols - show the raw number. +- Never run HTML viz on a graph with more than 5,000 nodes without warning the user. diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 5dfdc2d15..58ed168a8 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -250,7 +250,8 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -286,7 +287,7 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` After all subagents complete, collect their results. For each result: @@ -305,6 +306,30 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat .graphify_python) -c " @@ -825,7 +850,7 @@ G = json_graph.node_link_graph(data, edges='links') detection = {'total_files': 0, 'total_words': 99999, 'needs_graph': True, 'warning': None, 'files': {'code': [], 'document': [], 'paper': []}} -tokens = {'input': 0, 'output':': 0} +tokens = {'input': 0, 'output': 0} communities = cluster(G) cohesion = score_all(G, communities) diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 7d427ac97..9e5e3fbe0 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -151,13 +151,16 @@ if cached_path.exists(): all_hyperedges.extend(cached.get('hyperedges', [])) # PASTE each subagent response here as chunk_1, chunk_2, etc. +total_in, total_out = 0, 0 for chunk_json in []: # replace [] with your chunk results chunk = json.loads(chunk_json) if isinstance(chunk_json, str) else chunk_json all_nodes.extend(chunk.get('nodes', [])) all_edges.extend(chunk.get('edges', [])) all_hyperedges.extend(chunk.get('hyperedges', [])) + total_in += chunk.get('input_tokens', 0) + total_out += chunk.get('output_tokens', 0) -merged = {'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': 0, 'output_tokens': 0} +merged = {'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, 'input_tokens': total_in, 'output_tokens': total_out} Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2)) print(f'Merged: {len(all_nodes)} nodes, {len(all_edges)} edges') " diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 8aa048238..f668c2f16 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -249,7 +249,8 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -278,14 +279,20 @@ If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, auth confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: - EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 Output exactly this JSON (no other text): -{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"filestem_entityname","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -298,6 +305,30 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```powershell python -c " @@ -784,9 +815,28 @@ G_existing = json_graph.node_link_graph(existing_data, edges='links') new_extraction = json.loads(Path('.graphify_extract.json').read_text()) G_new = build_from_json(new_extraction) +# Prune nodes from deleted files +incremental = json.loads(Path('.graphify_incremental.json').read_text()) +deleted = set(incremental.get('deleted_files', [])) +if deleted: + to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] + G_existing.remove_nodes_from(to_remove) + if to_remove: + print(f'Pruned {len(to_remove)} ghost node(s) from {len(deleted)} deleted file(s) — drift detected and corrected.') + else: + print(f'{len(deleted)} file(s) deleted since last run, but no ghost nodes were present in the graph — no drift.') + # Merge: new nodes/edges into existing graph G_existing.update(G_new) print(f'Merged: {G_existing.number_of_nodes()} nodes, {G_existing.number_of_edges()} edges') + +# Save manifest with the CURRENT full file list so the next --update +# diffs against today's filesystem state, not the prior --update's +# baseline. Without this, deleted files get reported as ghosts again +# on every subsequent --update until a full rebuild runs. +from graphify.detect import save_manifest +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') " ``` diff --git a/graphify/skill.md b/graphify/skill.md index 0674b0a79..5e4d90570 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -1,6 +1,6 @@ --- name: graphify -description: "any input (code, docs, papers, images) - knowledge graph - clustered communities - HTML + JSON + audit report" +description: "any input (code, docs, papers, images, videos) to knowledge graph. Use when user asks any question about a codebase, documents, or project content - especially if graphify-out/ exists, treat the question as a /graphify query." trigger: /graphify --- @@ -13,6 +13,9 @@ Turn any folder of files into a navigable knowledge graph with community detecti ``` /graphify # full pipeline on current directory → Obsidian vault /graphify # full pipeline on specific path +/graphify https://github.com// # clone repo then run full pipeline on it +/graphify https://github.com// --branch # clone a specific branch +/graphify ... # clone multiple repos, build each, merge into one cross-repo graph /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files /graphify --directed # build directed graph (preserves edge direction: source→target) @@ -57,8 +60,33 @@ Use it for: If no path was given, use `.` (current directory). Do not ask the user for a path. +If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL — run Step 0 before anything else, then continue with the resolved local path. + Follow these steps in order. Do not skip steps. +### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given) + +**Single repo:** +```bash +LOCAL_PATH=$(graphify clone [--branch ]) +# Use LOCAL_PATH as the target for all subsequent steps +``` + +**Multiple repos (cross-repo graph):** +```bash +# Clone each repo, run the full pipeline on each, then merge +graphify clone # → ~/.graphify/repos// +graphify clone # → ~/.graphify/repos// +# Run /graphify on each local path to produce their graph.json files +# Then merge: +graphify merge-graphs \ + ~/.graphify/repos///graphify-out/graph.json \ + ~/.graphify/repos///graphify-out/graph.json \ + --out graphify-out/cross-repo-graph.json +``` + +Graphify clones into `~/.graphify/repos//` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin. + ### Step 1 - Ensure graphify is installed ```bash @@ -84,6 +112,8 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi # Write interpreter path for all subsequent steps (persists across invocations) mkdir -p graphify-out "$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w').write(sys.executable)" +# Save scan root so `graphify update` (no args) knows where to look next time +echo "$(cd INPUT_PATH && pwd)" > graphify-out/.graphify_root ``` If the import succeeds, print nothing and move straight to Step 2. @@ -170,7 +200,12 @@ After transcription: **Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it. -This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (Claude, costs tokens). +This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens). + +**Before dispatching subagents:** check whether `MOONSHOT_API_KEY` is set. If it is NOT set, print this one-liner to the user: +> Tip: set `MOONSHOT_API_KEY` to use Kimi K2.6 for semantic extraction — 3x cheaper, richer graphs (`pip install 'graphifyy[kimi]'`). + +Print it once, then continue. If `MOONSHOT_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="kimi")` for semantic extraction instead of dispatching Claude subagents. **Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.** @@ -272,7 +307,8 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. -Doc/paper files: extract named concepts, entities, citations. Also extract rationale — sections that explain WHY a decision was made, trade-offs chosen, or design intent. These become nodes with `rationale_for` edges pointing to the concept they explain. +Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). Do NOT invent file_types like `concept` — valid values are only `code|document|paper|image|rationale`. +Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. Chart: metric, trend/insight, data source. @@ -301,16 +337,22 @@ If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, auth confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default: - EXTRACTED edges: confidence_score = 1.0 always -- INFERRED edges: reason about each edge individually. - Direct structural evidence (shared data structure, clear dependency): 0.8-0.9. - Reasonable inference with some uncertainty: 0.6-0.7. - Weak or speculative: 0.4-0.5. Most edges should be 0.6-0.9, not 0.5. +- INFERRED edges: pick exactly ONE value from this set — never 0.5: + 0.95 direct structural evidence (shared data structure, named cross-file reference). + 0.85 strong inference (clear functional alignment, no direct symbol link). + 0.75 reasonable inference (shared problem domain + similar shape, requires interpretation). + 0.65 weak inference (thematically related, no shape evidence). + 0.55 speculative but plausible (surface-level co-occurrence only). + Models follow discrete rubrics better than continuous ranges; the bimodal + distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the + range guidance is being collapsed to a binary. If no value above fits, mark + the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the filename without extension and entity is the symbol name, both normalized (lowercase, non-alphanumeric chars replaced with `_`). Example: `src/auth/session.py` + `ValidateToken` → `session_validatetoken`. This must match the ID the AST extractor generates so cross-references between code and semantic nodes connect correctly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Output exactly this JSON (no other text): -{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} +{"nodes":[{"id":"session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale","source_file":"relative/path","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"relative/path","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"relative/path"}],"input_tokens":0,"output_tokens":0} ``` **Step B3 - Collect, cache, and merge** @@ -323,6 +365,30 @@ Wait for all subagents. For each result: If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used. +Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run: +```bash +$(cat graphify-out/.graphify_python) -c " +import json, glob +from pathlib import Path + +chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json')) +all_nodes, all_edges, all_hyperedges = [], [], [] +total_in, total_out = 0, 0 +for c in chunks: + d = json.loads(Path(c).read_text()) + all_nodes += d.get('nodes', []) + all_edges += d.get('edges', []) + all_hyperedges += d.get('hyperedges', []) + total_in += d.get('input_tokens', 0) + total_out += d.get('output_tokens', 0) +Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({ + 'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges, + 'input_tokens': total_in, 'output_tokens': total_out, +}, indent=2)) +print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens') +" +``` + Save new results to cache: ```bash $(cat graphify-out/.graphify_python) -c " @@ -768,7 +834,7 @@ cost_path.write_text(json.dumps(cost, indent=2)) print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens') print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)') " -rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_labels.json graphify-out/.graphify_chunk_*.json +rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_chunk_*.json rm -f graphify-out/.needs_update 2>/dev/null || true ``` @@ -887,7 +953,10 @@ deleted = set(incremental.get('deleted_files', [])) if deleted: to_remove = [n for n, d in G_existing.nodes(data=True) if d.get('source_file') in deleted] G_existing.remove_nodes_from(to_remove) - print(f'Pruned {len(to_remove)} ghost nodes from {len(deleted)} deleted file(s)') + if to_remove: + print(f'Pruned {len(to_remove)} ghost node(s) from {len(deleted)} deleted file(s) — drift detected and corrected.') + else: + print(f'{len(deleted)} file(s) deleted since last run, but no ghost nodes were present in the graph — no drift.') # Merge: new nodes/edges into existing graph G_existing.update(G_new) @@ -903,6 +972,14 @@ merged_out = { } Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out)) print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)') + +# Save manifest with the CURRENT full file list so the next --update +# diffs against today's filesystem state, not the prior --update's +# baseline. Without this, deleted files get reported as ghosts again +# on every subsequent --update until a full rebuild runs. +from graphify.detect import save_manifest +save_manifest(incremental['files']) +print('[graphify update] Manifest saved.') " ``` diff --git a/graphify/transcribe.py b/graphify/transcribe.py index 70000757a..6d21038f3 100644 --- a/graphify/transcribe.py +++ b/graphify/transcribe.py @@ -51,12 +51,14 @@ def download_audio(url: str, output_dir: Path) -> Path: Returns the path to the downloaded audio file (.m4a or .opus). Uses cached file if already downloaded. """ + from graphify.security import validate_url + validate_url(url) # blocks private IPs, bad schemes before yt-dlp runs yt_dlp = _get_yt_dlp() output_dir.mkdir(parents=True, exist_ok=True) # yt-dlp uses %(title)s which can be long/weird — use a stable name based on URL hash import hashlib - url_hash = hashlib.sha1(url.encode()).hexdigest()[:12] + url_hash = hashlib.sha1(url.encode(), usedforsecurity=False).hexdigest()[:12] out_template = str(output_dir / f"yt_{url_hash}.%(ext)s") # Check for already-downloaded file diff --git a/graphify/tree_html.py b/graphify/tree_html.py new file mode 100644 index 000000000..8ef177aeb --- /dev/null +++ b/graphify/tree_html.py @@ -0,0 +1,580 @@ +"""tree_html — emit a D3 v7 collapsible-tree HTML view of a graph. + +A self-contained printable / browseable tree-of-modules view +intended to complement the existing force-directed ``graph.html``. +Key visual elements: + + * Expand-all / collapse-all / reset-view buttons. + * Multi-line label wrapping (``wrapText``) with separately-coloured + name and descendant-count. + * Depth-based colour palette (top-level directories get distinct + accent colours; deeper levels follow a level-specific palette). + * Click-to-toggle subtree. + +Tree-data shape: + + { + "name": "", + "total_count": , + "children": [ { "name", "total_count", "children": [...] }, ... ] + } + +CLI: ``graphify tree [--graph PATH] [--output HTML] [--root PATH] +[--max-children N] [--label NAME]``. + +Implementation notes: + - ``total_count`` is the descendant leaf count, so collapsed nodes + can show ``(Total Count: 95)`` without needing the children loaded. + - ``--max-children`` (default 200) caps how many children render + under any one node; a synthetic ``(+N more)`` leaf appears when the + cap fires so very wide directories stay usable. + - The first-level palette is auto-populated from the live top-level + directories so each gets a stable accent colour. +""" + +from __future__ import annotations + +import html as _html +import json +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List, Optional + +DEFAULT_MAX_CHILDREN = 200 + + +# ── Tree builder (filesystem hierarchy → JSON) ────────────────── + + +def _common_root(paths: List[str]) -> str: + if not paths: + return "" + parts = [Path(p).parts for p in paths if p] + if not parts: + return "" + common = parts[0] + for p in parts[1:]: + i = 0 + while i < len(common) and i < len(p) and common[i] == p[i]: + i += 1 + common = common[:i] + return str(Path(*common)) if common else "" + + +def _make_truncation_leaf(extra: int) -> Dict[str, Any]: + return {"name": f"(+{extra} more)", "total_count": extra, "children": []} + + +def build_tree( + graph: Dict[str, Any], + *, + root: Optional[str] = None, + max_children: int = DEFAULT_MAX_CHILDREN, + project_label: Optional[str] = None, +) -> Dict[str, Any]: + """Build a ``{name, total_count, children}`` hierarchy. + + Each leaf is either a code symbol (class / top-level function) or + a synthetic "(+N more)" placeholder for truncated wide directories. + Each interior node carries ``total_count = sum of leaf counts``. + """ + nodes: List[Dict[str, Any]] = list(graph.get("nodes", [])) + file_nodes = [n for n in nodes if n.get("source_file")] + if not file_nodes: + return {"name": "(empty graph)", "total_count": 0, "children": []} + + if root is None: + root = _common_root([n["source_file"] for n in file_nodes]) + root_path = Path(root) + + by_file: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for n in file_nodes: + by_file[n["source_file"]].append(n) + + # Build dir tree. + dir_index: Dict[str, Dict[str, Any]] = {} + label_root = project_label or root_path.name or root or "/" + root_node: Dict[str, Any] = { + "name": label_root, "total_count": 0, "children": [], + } + dir_index[str(root_path)] = root_node + + def _ensure_dir(abs_path: Path) -> Dict[str, Any]: + key = str(abs_path) + if key in dir_index: + return dir_index[key] + if abs_path == abs_path.parent: + return root_node + parent = (_ensure_dir(abs_path.parent) + if abs_path.parent != abs_path else root_node) + node = {"name": abs_path.name, "total_count": 0, "children": []} + dir_index[key] = node + parent["children"].append(node) + return node + + for src_file, syms in sorted(by_file.items()): + src_path = Path(src_file) + try: + rel = src_path.relative_to(root_path) + parent_path = (root_path / rel).parent + except ValueError: + parent_path = root_path + parent_dir = _ensure_dir(parent_path) + + # File node — children are the symbols. + sym_children: List[Dict[str, Any]] = [] + for n in syms: + label = n.get("label", n.get("id", "?")) + # Skip the redundant file-name node graphify emits. + if label == src_path.name and n.get("file_type") == "code": + continue + sym_children.append({ + "name": label, + "total_count": 1, + "children": [], + }) + # Sort: code symbols first by name, then anything else. + sym_children.sort(key=lambda c: ( + c["name"].startswith("_"), + c["name"].lower(), + )) + if len(sym_children) > max_children: + extra = len(sym_children) - max_children + sym_children = sym_children[:max_children] + [ + _make_truncation_leaf(extra), + ] + file_node = { + "name": src_path.name, + "total_count": len(sym_children) or 1, + "children": sym_children, + } + parent_dir["children"].append(file_node) + + # Sort each dir's children + propagate total_count up. + def _finalise(d: Dict[str, Any]) -> int: + kids = d.get("children") or [] + kids.sort(key=lambda c: ( + 0 if (c.get("children") and len(c["children"]) > 0) else 1, + c["name"].lower(), + )) + if not kids: + return d.get("total_count") or 1 + n = 0 + for c in kids: + n += _finalise(c) + d["total_count"] = n or 1 + return d["total_count"] + + _finalise(root_node) + return root_node + + +# ── HTML emitter (single-data-blob substitution) ────────────────── + + +# We emit a Python f-string with literal CSS/JS braces escaped as {{ }}. +_HTML_TEMPLATE = r""" + + + + {title} + + + +

{header}

+
+ + + +
+
+ +
+ + + + + +""" + + +def emit_html( + tree: Dict[str, Any], + *, + title: str, + header: str, + svg_width: int = 6000, + svg_height: int = 8000, +) -> str: + # Escape sequences so embedded JSON cannot break out of the + #