From 695c281816240e836bfe493fc2e31d8848ec0c13 Mon Sep 17 00:00:00 2001 From: johan Date: Thu, 9 Jul 2026 21:40:07 +0200 Subject: [PATCH] feat: add `ctx sql` raw read-only SQL query surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new top-level `ctx sql` command that runs read-only SQL against the code-intelligence index through DuckDB, exposing a versioned, stable `v1` view layer (v1.symbols, v1.edges, v1.files, v1.meta) over the physical tables. - Engine (src/analytics/duckdb_impl.rs): `open_sql_sandbox` attaches the SQLite index READ_ONLY, builds the `v1` schema, then hardens the connection (enable_external_access=false, lock_configuration=true, memory_limit) so untrusted SQL cannot touch the filesystem, load extensions, attach files, or re-enable any of it. Safety is enforced by engine configuration only — no SQL text inspection or keyword filtering. Streaming row reader caps output without materializing the full result; interrupt-handle watchdog enforces --timeout. - Handler (src/commands/sql.rs): input from arg/--file/stdin, table/csv/json output with a JSON envelope, --max-rows cap, --fail-on-rows gate mode, and the three-way exit convention (0 ok / 1 findings / 2 error). One-result-set rule allows `CREATE TEMP TABLE ...; SELECT ...` while rejecting multiple queries. - `--schema` prints the public schema reference from a single source (src/commands/sql_schema.md) that also feeds the docs site; a drift-guard test keeps them byte-identical. - Docs: docs/website/docs/sql-schema.md + commands/sql.md, sidebar entries, and a README "ctx sql" section. - Tests: tests/sql.rs (assert_cmd) covers the security, schema, execution, and gate acceptance criteria; inline unit tests cover statement splitting + drift. The command appears in --help in all builds; without the `duckdb` feature it exits 2 with a clear message. Note: DuckDB 1.4 has no engine toggle to reject a bare in-memory `ATTACH ':memory:'`; it is permitted but inert (no filesystem access, index stays read-only, and the one-result-set rule blocks chaining a write). File-based ATTACH, COPY, read_csv, INSTALL, and config changes are all blocked. --- Cargo.lock | 84 +++++ Cargo.toml | 5 + README.md | 32 ++ docs/website/docs/commands/sql.md | 157 +++++++++ docs/website/docs/sql-schema.md | 100 ++++++ docs/website/sidebars.ts | 2 + src/analytics/duckdb_impl.rs | 301 +++++++++++++++- src/analytics/mod.rs | 3 + src/cli.rs | 44 +++ src/commands/mod.rs | 2 + src/commands/sql.rs | 569 ++++++++++++++++++++++++++++++ src/commands/sql_schema.md | 94 +++++ src/main.rs | 23 ++ tests/sql.rs | 375 ++++++++++++++++++++ 14 files changed, 1790 insertions(+), 1 deletion(-) create mode 100644 docs/website/docs/commands/sql.md create mode 100644 docs/website/docs/sql-schema.md create mode 100644 src/commands/sql.rs create mode 100644 src/commands/sql_schema.md create mode 100644 tests/sql.rs diff --git a/Cargo.lock b/Cargo.lock index a2eb839..ac05561 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,9 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" name = "agentis-ctx" version = "0.2.1" dependencies = [ + "assert_cmd", "async-trait", + "chrono", "clap", "dirs", "duckdb", @@ -22,6 +24,7 @@ dependencies = [ "ignore", "notify", "notify-debouncer-mini", + "predicates", "rayon", "reqwest", "rmcp", @@ -369,6 +372,21 @@ dependencies = [ "term", ] +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -1041,6 +1059,12 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.10.7" @@ -1339,6 +1363,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2528,6 +2561,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "notify" version = "6.1.1" @@ -2948,6 +2987,36 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -4062,6 +4131,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "thiserror" version = "1.0.69" @@ -4688,6 +4763,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index be5f8f8..ebc4d52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,9 @@ notify = "6.1" notify-debouncer-mini = "0.4" dirs = "6.0.0" +# Timestamps for `ctx sql` JSON envelope (generated_at, ISO-8601 UTC) +chrono = { version = "0.4", features = ["clock"] } + [package.metadata.docs.rs] all-features = true @@ -103,6 +106,8 @@ mcp = ["rmcp", "schemars", "async-trait"] [dev-dependencies] tempfile = "3" tokio-test = "0.4" +assert_cmd = "2" +predicates = "3" [profile.release] lto = true diff --git a/README.md b/README.md index 1e83b41..a690230 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,37 @@ ctx query stats ctx query files ``` +### `ctx sql` +Run read-only SQL against the code intelligence index through DuckDB, over a +stable `v1` view layer (`v1.symbols`, `v1.edges`, `v1.files`, `v1.meta`). Reach +for `ctx sql` instead of the canned `ctx query` subcommands whenever you need an +aggregation, a join, or a custom `WHERE` condition. Query `v1.*` only — anything +else (e.g. `code.*`) is internal and unstable. For agents and scripts, use +`--json` with `--max-rows`. Run `ctx sql --schema` for the full column reference. + +```bash +# Ten most complex symbols +ctx sql "SELECT name, file, complexity FROM v1.symbols ORDER BY complexity DESC LIMIT 10;" + +# Symbol counts by kind +ctx sql "SELECT kind, COUNT(*) AS n FROM v1.symbols GROUP BY kind ORDER BY n DESC;" + +# Public functions that nothing calls (dead-code candidates) +ctx sql "SELECT name, file FROM v1.symbols WHERE kind IN ('function', 'method') AND is_public AND fan_in = 0 ORDER BY file, name;" +``` + +Use `--fail-on-rows` to turn a query into a gate: any returned row is a +violation, and the command exits `1`. + +```bash +# Fails (exit 1) if the query returns any row +ctx sql --fail-on-rows --file .ctx/gates/no-utils-imports.sql +``` + +Access is read-only and engine-hardened (filesystem access, extension loading, +and file-based `ATTACH` are disabled; the index cannot be modified), so +`Bash(ctx sql *)` is safe to add to a Claude Code allow-list. + ### `ctx explain ` Get detailed information about a symbol including its relationships. @@ -597,6 +628,7 @@ USAGE: COMMANDS: index Build or update the code intelligence index query Query the code intelligence database + sql Run read-only SQL against the index (v1 schema) search Search for symbols using keyword matching semantic Search using embeddings (natural language) embed Generate embeddings for semantic search diff --git a/docs/website/docs/commands/sql.md b/docs/website/docs/commands/sql.md new file mode 100644 index 0000000..4c51f51 --- /dev/null +++ b/docs/website/docs/commands/sql.md @@ -0,0 +1,157 @@ +--- +id: sql +title: ctx sql +sidebar_position: 6 +--- + +# ctx sql + +Run read-only SQL against the code-intelligence index through DuckDB, over a stable `v1` view layer. + +## Synopsis + +```bash +ctx sql [QUERY] [OPTIONS] +``` + +## Description + +`ctx sql` gives you raw SQL access to the code-intelligence index. Where the +canned `ctx query` subcommands answer fixed questions (callers, deps, impact), +`ctx sql` lets you ask arbitrary ones: aggregations, joins across symbols and +edges, and custom `WHERE` conditions. **Prefer `ctx sql` whenever you need a +grouping, a join, or a filter the canned commands do not expose.** + +The query surface is the versioned **`v1`** schema — a set of stable views +(`v1.symbols`, `v1.edges`, `v1.files`, `v1.meta`). See the full +[SQL Schema (v1)](../sql-schema.md) reference for every column. + +- **Query `v1.*` only.** It is the compatibility contract: columns and views may + be added within `schema_version` 1, but nothing is renamed or removed without + bumping `v1.meta.schema_version`. +- **Anything outside `v1.*` is internal and unstable.** The raw index is + reachable as `code.*`, but its shape can change at any time. Do not depend on it. + +The query is read from the first of: the `[QUERY]` argument, `--file`, or stdin +(when `QUERY` is `-` or omitted and stdin is piped). + +### Programmatic use + +For agents and scripts, pass `--json` for machine-readable rows and +`--max-rows` to bound the result set: + +```bash +ctx sql --json --max-rows 50 "SELECT name, complexity FROM v1.symbols ORDER BY complexity DESC" +``` + +An index must exist first (`ctx index`); querying without one exits with code 2. + +## Options + +| Option | Description | Default | +|--------|-------------|---------| +| `[QUERY]` | SQL text. If `-` or omitted while stdin is piped, read from stdin. | — | +| `--file ` | Read the query from a file (mutually exclusive with `QUERY`). | — | +| `--output ` | Output format: `table`, `csv`, or `json`. | `table` | +| `--json` | Alias for `--output json`. | — | +| `--max-rows ` | Cap returned rows (`0` = unlimited). | 1000 | +| `--timeout ` | Abort the query after N seconds. | 10 | +| `--fail-on-rows` | Exit `1` if the query returns `>= 1` row (for gate usage). | false | +| `--schema` | Print the `v1` schema reference and exit `0`. | — | + +> **Note:** the flag is `--output`, not `--format` — `ctx` already has a global +> `-f/--format` for context output. `--json` is the convenient alias for +> `--output json`. + +### Exit codes + +| Code | Meaning | +|------|---------| +| `0` | Query ran successfully (with `--fail-on-rows`: zero rows returned). | +| `1` | `--fail-on-rows` was set and the query returned `>= 1` row. | +| `2` | Any error: SQL error, timeout, missing index, invalid flags, or a build without the `duckdb` feature. | + +## Examples + +### Ten most complex symbols + +```bash +ctx sql "SELECT name, file, complexity +FROM v1.symbols +ORDER BY complexity DESC +LIMIT 10;" +``` + +### Symbol counts by kind + +```bash +ctx sql "SELECT kind, COUNT(*) AS n +FROM v1.symbols +GROUP BY kind +ORDER BY n DESC;" +``` + +### Public functions that nothing calls (dead-code candidates) + +```bash +ctx sql "SELECT name, file +FROM v1.symbols +WHERE kind IN ('function', 'method') AND is_public AND fan_in = 0 +ORDER BY file, name;" +``` + +### Print the schema + +```bash +ctx sql --schema +``` + +## Gates + +`--fail-on-rows` turns a query into a CI or pre-commit gate: write SQL that +selects the *violations*, and any returned row fails the check. Keep gate +queries in files under `.ctx/gates/` and run them by path: + +```bash +ctx sql --fail-on-rows --file .ctx/gates/no-utils-imports.sql +``` + +If the query returns any row, `ctx sql` exits `1` and the gate fails; zero rows +exits `0`. For example, a `.ctx/gates/no-utils-imports.sql` that flags forbidden +imports: + +```sql +-- Fail if anything imports the legacy utils module +SELECT source_file, line +FROM v1.edges +WHERE kind = 'imports' AND target_name = 'utils'; +``` + +Wire it into a pre-commit hook: + +```bash +#!/usr/bin/env bash +# .git/hooks/pre-commit +ctx index >/dev/null +for gate in .ctx/gates/*.sql; do + ctx sql --fail-on-rows --file "$gate" || { + echo "Gate failed: $gate" >&2 + exit 1 + } +done +``` + +## Security + +Access is **read-only and engine-hardened**. Filesystem access, extension +loading, and file-based `ATTACH` are disabled, and the index cannot be +modified — a query can only read the `v1` (and internal `code`) views. Because +`ctx sql` cannot write to disk, mutate the index, or reach the network, it is +safe to add `Bash(ctx sql *)` to a Claude Code (or other harness) plugin +allow-list. + +## See Also + +- [SQL Schema (v1)](../sql-schema.md) - Full column reference for the `v1` views +- [Code Intelligence](../code-intelligence.md) - Indexing and the canned `ctx query` commands +- [ctx audit](./audit.md) - Automated quality gates diff --git a/docs/website/docs/sql-schema.md b/docs/website/docs/sql-schema.md new file mode 100644 index 0000000..a601809 --- /dev/null +++ b/docs/website/docs/sql-schema.md @@ -0,0 +1,100 @@ +--- +id: sql-schema +title: SQL Schema (v1) +sidebar_position: 8 +--- + +# ctx sql — Public Schema (`v1`) + +`ctx sql` runs read-only SQL against the code-intelligence index through DuckDB. +The query surface is the versioned **`v1`** schema: a set of stable views over +the physical index. Query `v1.*` — not the underlying tables. + +## Stability contract + +- **`v1.*` is the contract.** Columns and views may be *added* within + `schema_version` 1. Renaming or removing anything increments + `v1.meta.schema_version` and is noted in the changelog. +- **Everything outside `v1.*` is internal and unstable.** The raw index is + reachable as `code.*`, but its shape can change at any time and it is excluded + from all compatibility guarantees. Do not depend on it. +- **Access is read-only and engine-hardened.** Filesystem access, extension + loading, and attaching other databases are disabled; the index cannot be + modified. + +## Views + +### `v1.symbols` — one row per symbol + +| Column | Type | Description | +|------------------|---------|----------------------------------------------------| +| `id` | VARCHAR | Stable symbol identifier | +| `name` | VARCHAR | Symbol name | +| `qualified_name` | VARCHAR | Fully-qualified name, when known | +| `kind` | VARCHAR | `function`, `method`, `struct`, `enum`, `trait`, … | +| `file` | VARCHAR | Path of the file that defines the symbol | +| `line_start` | BIGINT | First line of the symbol | +| `line_end` | BIGINT | Last line of the symbol | +| `is_public` | BOOLEAN | Whether the symbol is publicly visible | +| `complexity` | BIGINT | `fan_out * 2 + fan_in` heuristic complexity score | +| `fan_in` | BIGINT | Number of resolved incoming `calls` edges | +| `fan_out` | BIGINT | Number of outgoing `calls` edges | +| `doc` | VARCHAR | Docstring or brief, when present | + +### `v1.edges` — one row per relationship + +| Column | Type | Description | +|---------------|---------|---------------------------------------------------------| +| `source_id` | VARCHAR | `id` of the source symbol | +| `source_name` | VARCHAR | Name of the source symbol | +| `source_file` | VARCHAR | File of the source symbol | +| `target_id` | VARCHAR | `id` of the target symbol; NULL when unresolved | +| `target_name` | VARCHAR | Name of the target; retained even when unresolved | +| `target_file` | VARCHAR | File of the target symbol; NULL when unresolved | +| `kind` | VARCHAR | `calls`, `extends`, `implements`, or `imports` | +| `line` | BIGINT | Line of the reference in the source file | + +### `v1.files` — one row per indexed file + +| Column | Type | Description | +|--------------------|---------|----------------------------------------------| +| `path` | VARCHAR | File path | +| `language` | VARCHAR | Detected language | +| `symbol_count` | BIGINT | Number of symbols defined in the file | +| `total_complexity` | BIGINT | Sum of `v1.symbols.complexity` for the file | +| `indexed_at` | BIGINT | Unix time the file was last indexed | + +### `v1.meta` — single row of index metadata + +| Column | Type | Description | +|--------------------|---------|----------------------------------------------| +| `schema_version` | INTEGER | Public schema version (starts at 1) | +| `ctx_version` | VARCHAR | Version of ctx that produced this output | +| `index_created_at` | BIGINT | Earliest file index time (Unix seconds) | +| `index_root` | VARCHAR | Absolute root path of the indexed project | + +## Examples + +```sql +-- Ten most complex symbols +SELECT name, file, complexity +FROM v1.symbols +ORDER BY complexity DESC +LIMIT 10; +``` + +```sql +-- Symbol counts by kind +SELECT kind, COUNT(*) AS n +FROM v1.symbols +GROUP BY kind +ORDER BY n DESC; +``` + +```sql +-- Public functions that nothing calls (dead-code candidates) +SELECT name, file +FROM v1.symbols +WHERE kind IN ('function', 'method') AND is_public AND fan_in = 0 +ORDER BY file, name; +``` diff --git a/docs/website/sidebars.ts b/docs/website/sidebars.ts index 0233ecf..1ada033 100644 --- a/docs/website/sidebars.ts +++ b/docs/website/sidebars.ts @@ -9,6 +9,7 @@ const sidebars: SidebarsConfig = { 'configuration', 'language-support', 'architecture', + 'sql-schema', { type: 'category', label: 'Commands', @@ -16,6 +17,7 @@ const sidebars: SidebarsConfig = { 'commands/audit', 'commands/diff', 'commands/smart', + 'commands/sql', 'commands/shell', 'commands/serve', ], diff --git a/src/analytics/duckdb_impl.rs b/src/analytics/duckdb_impl.rs index 76a0c4f..e7062e9 100644 --- a/src/analytics/duckdb_impl.rs +++ b/src/analytics/duckdb_impl.rs @@ -7,8 +7,10 @@ //! - Codebase statistics aggregations use std::path::Path; +use std::sync::Arc; -use duckdb::{params, Connection, Result}; +use duckdb::types::ValueRef; +use duckdb::{params, Connection, InterruptHandle, Result}; use super::{CallGraphNode, ComplexityResult, FileStats, ImpactNode}; use crate::index::{CTX_DIR, DB_FILE}; @@ -521,6 +523,303 @@ impl Analytics { rows.collect() } + + // ======================================================================== + // Raw SQL surface (`ctx sql`) — hardened, read-only query sandbox. + // ======================================================================== + + /// Open DuckDB for the `ctx sql` command: attach the SQLite index read-only, + /// build the public `v1` view layer, then lock the engine down so untrusted + /// user SQL cannot touch the filesystem, load extensions, or re-enable any + /// of that. Safety is enforced entirely by engine configuration — never by + /// inspecting the SQL text. + pub fn open_sql_sandbox(root: &Path) -> Result { + let ctx_dir = root.join(CTX_DIR); + let sqlite_path = ctx_dir.join(DB_FILE); + + let conn = Connection::open_in_memory()?; + + // Attach the SQLite index read-only (single-quote-escape the path). + let path_str = sqlite_path.display().to_string(); + let escaped_path = path_str.replace('\'', "''"); + conn.execute( + &format!("ATTACH '{}' AS code (TYPE sqlite, READ_ONLY)", escaped_path), + [], + )?; + + let analytics = Self { conn }; + + // Build the public `v1` contract views BEFORE hardening — creating views + // and reading the attached DB must happen while access is still allowed. + let index_root = root.display().to_string(); + analytics.create_public_schema_v1(env!("CARGO_PKG_VERSION"), &index_root)?; + + // Engine-level hardening (order matters: memory + external-access first, + // then lock configuration, which blocks any further `SET`). + // + // `enable_external_access = false` disables the filesystem (COPY, + // read_csv, file-based ATTACH), extension installation, and config + // changes; `lock_configuration = true` prevents user SQL from re-enabling + // any of it. The SQLite index is attached READ_ONLY, so it cannot be + // mutated. Note: DuckDB 1.4 has no engine toggle to reject a bare + // in-memory `ATTACH ':memory:'`; that is permitted but inert (no + // filesystem access, the index stays read-only, and the one-result-set + // rule blocks chaining a write onto it). We do not add SQL-text filtering + // for it — safety is enforced by engine configuration, not by parsing. + let mem_limit = + std::env::var("CTX_SQL_MEMORY_LIMIT").unwrap_or_else(|_| "512MB".to_string()); + let mem_escaped = mem_limit.replace('\'', "''"); + analytics.conn.execute_batch(&format!( + "SET memory_limit = '{}';\n\ + SET enable_external_access = false;\n\ + SET lock_configuration = true;", + mem_escaped + ))?; + + Ok(analytics) + } + + /// Create the versioned public schema `v1` — the stable query surface. + /// + /// These views (not the physical `code.*` tables) are the contract. Column + /// lists here are the documented `v1` schema; derived columns (fan-in/out, + /// complexity) are computed in-view. + fn create_public_schema_v1(&self, ctx_version: &str, index_root: &str) -> Result<()> { + // Literals injected into `v1.meta`; single-quote-escape defensively. + let ctx_version_lit = ctx_version.replace('\'', "''"); + let index_root_lit = index_root.replace('\'', "''"); + + self.conn.execute_batch(&format!( + r#" + CREATE SCHEMA v1; + + CREATE VIEW v1.symbols AS + WITH fan_out_counts AS ( + SELECT source_id AS id, COUNT(*) AS fan_out + FROM code.edges + WHERE kind = 'calls' + GROUP BY source_id + ), + fan_in_counts AS ( + SELECT target_id AS id, COUNT(*) AS fan_in + FROM code.edges + WHERE kind = 'calls' AND target_id IS NOT NULL + GROUP BY target_id + ) + SELECT + s.id, + s.name, + s.qualified_name, + s.kind, + s.file_path AS file, + s.line_start, + s.line_end, + (s.visibility = 'public') AS is_public, + (COALESCE(fo.fan_out, 0) * 2 + COALESCE(fi.fan_in, 0)) AS complexity, + COALESCE(fi.fan_in, 0) AS fan_in, + COALESCE(fo.fan_out, 0) AS fan_out, + COALESCE(s.docstring, s.brief) AS doc + FROM code.symbols s + LEFT JOIN fan_out_counts fo ON fo.id = s.id + LEFT JOIN fan_in_counts fi ON fi.id = s.id; + + CREATE VIEW v1.edges AS + SELECT + e.source_id, + s.name AS source_name, + s.file_path AS source_file, + e.target_id, + e.target_name, + t.file_path AS target_file, + e.kind, + e.line + FROM code.edges e + JOIN code.symbols s ON e.source_id = s.id + LEFT JOIN code.symbols t ON e.target_id = t.id; + + CREATE VIEW v1.files AS + SELECT + f.path, + f.language, + COALESCE(agg.symbol_count, 0) AS symbol_count, + COALESCE(agg.total_complexity, 0) AS total_complexity, + f.last_indexed AS indexed_at + FROM code.files f + LEFT JOIN ( + SELECT file, COUNT(*) AS symbol_count, SUM(complexity) AS total_complexity + FROM v1.symbols + GROUP BY file + ) agg ON agg.file = f.path; + + CREATE VIEW v1.meta AS + SELECT + 1 AS schema_version, + '{ctx_version}' AS ctx_version, + (SELECT MIN(last_indexed) FROM code.files) AS index_created_at, + '{index_root}' AS index_root; + "#, + ctx_version = ctx_version_lit, + index_root = index_root_lit, + ))?; + + Ok(()) + } + + /// A handle that can interrupt an in-flight query from another thread + /// (used to enforce `--timeout`). `InterruptHandle` is `Send + Sync`. + pub fn interrupt_handle(&self) -> Arc { + self.conn.interrupt_handle() + } + + /// Execute a non-final statement in a multi-statement submission. + /// + /// Returns `Ok(true)` if the statement produces a result set — the caller + /// rejects that (only the final statement may return rows, per the one + /// result-set rule). Statements that produce no result set (e.g. + /// `CREATE TEMP TABLE …`, `SET …`) are executed for their side effects and + /// return `Ok(false)`. + /// + /// (DuckDB only knows a statement's column count *after* execution, so the + /// statement is always run — harmless here, since the index is read-only + /// and the in-memory DuckDB is throwaway.) + pub fn exec_non_final_statement(&self, sql: &str) -> Result { + let mut stmt = self.conn.prepare(sql)?; + let rows = stmt.query([])?; + let produced = rows + .as_ref() + .map(|s| { + let n = s.column_count(); + if n == 0 { + return false; + } + // DuckDB reports DDL/DML — CREATE/INSERT/UPDATE/DELETE, including + // `CREATE TABLE … AS SELECT` — as a single BIGINT column named + // "Count" (rows affected). That is not a user-facing result set, + // so such statements are allowed to precede the final one. + !(n == 1 && s.column_name(0).map(|c| c == "Count").unwrap_or(false)) + }) + .unwrap_or(false); + Ok(produced) + } + + /// Run the final (result-producing) statement, streaming rows and stopping + /// once `max_rows` is reached (`0` = unlimited). One extra row is fetched to + /// detect truncation, so the full result is never materialized in Rust. + pub fn run_final_statement(&self, sql: &str, max_rows: usize) -> Result { + let mut stmt = self.conn.prepare(sql)?; + let mut rows = stmt.query([])?; + + // Column metadata is only available after the statement has executed. + let columns: Vec = { + let executed = rows + .as_ref() + .expect("statement is available after query()"); + let col_count = executed.column_count(); + let names = executed.column_names(); + (0..col_count) + .map(|i| SqlColumn { + name: names.get(i).cloned().unwrap_or_default(), + type_name: duckdb_type_name(&executed.column_type(i)), + }) + .collect() + }; + let col_count = columns.len(); + + let cap = if max_rows == 0 { usize::MAX } else { max_rows }; + let mut rows_out: Vec> = Vec::new(); + let mut truncated = false; + + while let Some(row) = rows.next()? { + if rows_out.len() >= cap { + truncated = true; + break; + } + let mut record = Vec::with_capacity(col_count); + for i in 0..col_count { + record.push(value_ref_to_json(row.get_ref(i)?)); + } + rows_out.push(record); + } + + Ok(SqlResult { + columns, + rows: rows_out, + truncated, + }) + } +} + +/// A column in a `ctx sql` result: its name and DuckDB type name. +#[derive(Debug, Clone)] +pub struct SqlColumn { + pub name: String, + pub type_name: String, +} + +/// The result of a `ctx sql` query: columns plus row-capped data. +#[derive(Debug, Clone)] +pub struct SqlResult { + pub columns: Vec, + pub rows: Vec>, + pub truncated: bool, +} + +/// Map an Arrow `DataType` (what duckdb-rs reports for a column via +/// `column_type`) to a DuckDB type name for the JSON envelope (e.g. `VARCHAR`, +/// `BIGINT`). +fn duckdb_type_name(dt: &duckdb::arrow::datatypes::DataType) -> String { + use duckdb::arrow::datatypes::DataType as D; + match dt { + D::Null => "NULL", + D::Boolean => "BOOLEAN", + D::Int8 => "TINYINT", + D::Int16 => "SMALLINT", + D::Int32 => "INTEGER", + D::Int64 => "BIGINT", + D::UInt8 => "UTINYINT", + D::UInt16 => "USMALLINT", + D::UInt32 => "UINTEGER", + D::UInt64 => "UBIGINT", + D::Float16 | D::Float32 => "FLOAT", + D::Float64 => "DOUBLE", + D::Utf8 | D::LargeUtf8 | D::Utf8View => "VARCHAR", + D::Binary | D::LargeBinary | D::BinaryView => "BLOB", + D::Date32 | D::Date64 => "DATE", + D::Timestamp(_, _) => "TIMESTAMP", + D::Time32(_) | D::Time64(_) => "TIME", + D::Decimal128(_, _) | D::Decimal256(_, _) => "DECIMAL", + other => return format!("{:?}", other), + } + .to_string() +} + +/// Convert a DuckDB `ValueRef` into a `serde_json::Value` for output. Exotic +/// types (temporal, decimal, nested) fall back to a debug string so rendering +/// never panics. +fn value_ref_to_json(v: ValueRef<'_>) -> serde_json::Value { + use serde_json::Value as J; + match v { + ValueRef::Null => J::Null, + ValueRef::Boolean(b) => J::Bool(b), + ValueRef::TinyInt(n) => J::from(n), + ValueRef::SmallInt(n) => J::from(n), + ValueRef::Int(n) => J::from(n), + ValueRef::BigInt(n) => J::from(n), + ValueRef::UTinyInt(n) => J::from(n), + ValueRef::USmallInt(n) => J::from(n), + ValueRef::UInt(n) => J::from(n), + ValueRef::UBigInt(n) => J::from(n), + ValueRef::HugeInt(n) => J::String(n.to_string()), + ValueRef::Float(f) => serde_json::Number::from_f64(f as f64) + .map(J::Number) + .unwrap_or(J::Null), + ValueRef::Double(f) => serde_json::Number::from_f64(f) + .map(J::Number) + .unwrap_or(J::Null), + ValueRef::Text(bytes) => J::String(String::from_utf8_lossy(bytes).into_owned()), + ValueRef::Blob(bytes) => J::String(format!("", bytes.len())), + other => J::String(format!("{:?}", other)), + } } #[cfg(test)] diff --git a/src/analytics/mod.rs b/src/analytics/mod.rs index ea97031..9fe7234 100644 --- a/src/analytics/mod.rs +++ b/src/analytics/mod.rs @@ -74,5 +74,8 @@ mod stub; #[cfg(feature = "duckdb")] pub use duckdb_impl::Analytics; +#[cfg(feature = "duckdb")] +pub use duckdb_impl::{SqlColumn, SqlResult}; + #[cfg(not(feature = "duckdb"))] pub use stub::Analytics; diff --git a/src/cli.rs b/src/cli.rs index 9e51035..2774bf8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -146,6 +146,50 @@ pub enum Command { query: QueryCommand, }, + /// Run read-only SQL against the code index (via the stable `v1.*` views) + #[command(after_help = r#"EXAMPLES: + ctx sql "SELECT name, file, complexity FROM v1.symbols ORDER BY complexity DESC LIMIT 10" + ctx sql --json "SELECT kind, COUNT(*) n FROM v1.symbols GROUP BY kind" + ctx sql --fail-on-rows --file .ctx/gates/no-utils-imports.sql + ctx sql --schema # print the v1 schema reference and exit + +The query surface is the versioned `v1` schema; anything outside `v1.*` is +internal and unstable. Access is read-only and engine-hardened. +"#)] + Sql { + /// SQL text. If "-" or omitted while stdin is piped, read from stdin. + query: Option, + + /// Read the query from a file (mutually exclusive with the QUERY arg) + #[arg(long)] + file: Option, + + /// Output format: table, csv, or json (the global -f/--format does not + /// apply here; use this or --json) + #[arg(long = "output", default_value = "table")] + output: String, + + /// Alias for --output json + #[arg(long)] + json: bool, + + /// Cap returned rows (0 = unlimited) + #[arg(long, default_value = "1000")] + max_rows: usize, + + /// Abort the query after N seconds + #[arg(long, default_value = "10")] + timeout: u64, + + /// Exit 1 if the query returns >= 1 row (for gate usage) + #[arg(long)] + fail_on_rows: bool, + + /// Print the public schema reference and exit + #[arg(long)] + schema: bool, + }, + /// Search for symbols using semantic or text search Search { /// Search query (symbol name or natural language) diff --git a/src/commands/mod.rs b/src/commands/mod.rs index c0cb66a..a970d4c 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -18,6 +18,7 @@ pub mod query; pub mod search; pub mod similar; pub mod smart_cmd; +pub mod sql; pub mod symbol; pub use analysis::{run_audit, run_complexity}; @@ -37,6 +38,7 @@ pub use query::run_query; pub use search::run_search; pub use similar::run_similar; pub use smart_cmd::run_smart; +pub use sql::{run_sql, SqlArgs}; pub use symbol::{run_explain, run_source}; /// Format a token count as a human-readable string (e.g. 241_502 → "241.5k"). diff --git a/src/commands/sql.rs b/src/commands/sql.rs new file mode 100644 index 0000000..5982a3c --- /dev/null +++ b/src/commands/sql.rs @@ -0,0 +1,569 @@ +//! `ctx sql` — raw, read-only SQL access to the code index. +//! +//! Queries run through DuckDB against the stable `v1.*` view layer. Safety is +//! enforced entirely by engine configuration (see +//! [`ctx::analytics::Analytics::open_sql_sandbox`]) — this command never +//! inspects, filters, or blocklists SQL text. + +use std::io::{IsTerminal, Read}; +use std::path::PathBuf; + +use ctx::error::{CtxError, Result}; +use ctx::exit::Outcome; + +/// The reference for the public `v1` schema, printed by `ctx sql --schema`. +/// +/// This constant is the single source of truth; the docs site page +/// `docs/website/docs/sql-schema.md` mirrors it verbatim (a drift-guard test +/// keeps them byte-identical). +pub const SQL_SCHEMA_REFERENCE: &str = include_str!("sql_schema.md"); + +/// Arguments for `ctx sql`, mirrored from the CLI variant. +// Several fields are only read on the `duckdb` execution path. +#[cfg_attr(not(feature = "duckdb"), allow(dead_code))] +pub struct SqlArgs { + pub query: Option, + pub file: Option, + pub format: String, + pub json: bool, + pub max_rows: usize, + pub timeout: u64, + pub fail_on_rows: bool, + pub schema: bool, +} + +/// Run `ctx sql`. +/// +/// Exit convention (via [`Outcome`] / `Err`): +/// - `Ok(Outcome::Clean)` → 0 (ran; with `--fail-on-rows`: zero rows) +/// - `Ok(Outcome::Findings)` → 1 (only with `--fail-on-rows`: ≥ 1 row) +/// - `Err(_)` → 2 (SQL error, timeout, missing index, invalid flags, stub build) +pub fn run_sql(args: SqlArgs) -> Result { + // `--schema` needs neither an index nor the engine; works in every build. + if args.schema { + print!("{}", SQL_SCHEMA_REFERENCE); + return Ok(Outcome::Clean); + } + + // Resolve the effective output format (`--json` is an alias for json). + let format = if args.json { + "json" + } else { + args.format.as_str() + }; + if !matches!(format, "table" | "csv" | "json") { + return Err(CtxError::Other(format!( + "unknown --format '{}'; expected table, csv, or json", + format + ))); + } + + let sql_text = resolve_query_text(&args)?; + if sql_text.trim().is_empty() { + return Err(CtxError::Other( + "no query provided; pass SQL as an argument, via --file, or on stdin".into(), + )); + } + + #[cfg(not(feature = "duckdb"))] + { + let _ = format; + Err(CtxError::Other( + "ctx sql requires the duckdb feature; reinstall with default features".into(), + )) + } + + #[cfg(feature = "duckdb")] + { + run_sql_duckdb(&args, format, &sql_text) + } +} + +/// Resolve the query text from the positional arg, `--file`, or stdin. +fn resolve_query_text(args: &SqlArgs) -> Result { + match (&args.query, &args.file) { + (Some(q), Some(_)) if q != "-" => Err(CtxError::Other( + "provide the query as an argument OR via --file, not both".into(), + )), + (_, Some(path)) => Ok(std::fs::read_to_string(path)?), + (Some(q), None) if q != "-" => Ok(q.clone()), + _ => { + // Omitted or "-": read stdin, but don't hang an interactive terminal. + if std::io::stdin().is_terminal() { + return Err(CtxError::Other( + "no query provided; pass SQL as an argument, via --file, or on stdin".into(), + )); + } + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + Ok(buf) + } + } +} + +/// Split a SQL submission into individual statements on top-level `;`, +/// respecting single-quoted strings, double-quoted identifiers, line comments +/// (`--`), and block comments (`/* */`). This governs execution semantics +/// (the one-result-set rule) only — it is never a safety filter. +#[cfg_attr(not(feature = "duckdb"), allow(dead_code))] +fn split_sql_statements(sql: &str) -> Vec { + #[derive(PartialEq)] + enum State { + Normal, + Single, + Double, + Line, + Block, + } + + let mut statements = Vec::new(); + let mut current = String::new(); + let mut state = State::Normal; + let mut chars = sql.chars().peekable(); + + while let Some(c) = chars.next() { + match state { + State::Normal => match c { + '\'' => { + current.push(c); + state = State::Single; + } + '"' => { + current.push(c); + state = State::Double; + } + '-' if chars.peek() == Some(&'-') => { + current.push(c); + current.push(chars.next().unwrap()); + state = State::Line; + } + '/' if chars.peek() == Some(&'*') => { + current.push(c); + current.push(chars.next().unwrap()); + state = State::Block; + } + ';' => { + let trimmed = current.trim(); + if !trimmed.is_empty() { + statements.push(trimmed.to_string()); + } + current.clear(); + } + _ => current.push(c), + }, + State::Single => { + current.push(c); + if c == '\'' { + if chars.peek() == Some(&'\'') { + current.push(chars.next().unwrap()); + } else { + state = State::Normal; + } + } + } + State::Double => { + current.push(c); + if c == '"' { + if chars.peek() == Some(&'"') { + current.push(chars.next().unwrap()); + } else { + state = State::Normal; + } + } + } + State::Line => { + current.push(c); + if c == '\n' { + state = State::Normal; + } + } + State::Block => { + current.push(c); + if c == '*' && chars.peek() == Some(&'/') { + current.push(chars.next().unwrap()); + state = State::Normal; + } + } + } + } + + let trimmed = current.trim(); + if !trimmed.is_empty() { + statements.push(trimmed.to_string()); + } + statements +} + +#[cfg(feature = "duckdb")] +mod engine { + use super::*; + use ctx::analytics::{Analytics, SqlResult}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + /// Internal execution error, kept separate from `CtxError` so the caller can + /// distinguish a timeout and the one-result-set rule from a plain SQL error. + enum EngineError { + MultipleResultSets, + Duck(duckdb::Error), + } + + pub fn run_sql_duckdb(args: &SqlArgs, format: &str, sql_text: &str) -> Result { + use ctx::index::{CTX_DIR, DB_FILE}; + + let root = std::env::current_dir()?; + let db_path = root.join(CTX_DIR).join(DB_FILE); + if !db_path.exists() { + return Err(CtxError::Other(format!( + "no index found at {}; run `ctx index` first", + db_path.display() + ))); + } + + let statements = split_sql_statements(sql_text); + if statements.is_empty() { + return Err(CtxError::Other("no SQL statement found".into())); + } + + let analytics = Analytics::open_sql_sandbox(&root)?; + + // Timeout watchdog: interrupt the in-flight query if it overruns. + let timed_out = Arc::new(AtomicBool::new(false)); + let done = Arc::new(AtomicBool::new(false)); + let watchdog = if args.timeout > 0 { + let handle = analytics.interrupt_handle(); + let timed_out = timed_out.clone(); + let done = done.clone(); + let timeout = args.timeout; + Some(std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(timeout); + while Instant::now() < deadline { + if done.load(Ordering::SeqCst) { + return; + } + std::thread::sleep(Duration::from_millis(50)); + } + if !done.load(Ordering::SeqCst) { + timed_out.store(true, Ordering::SeqCst); + handle.interrupt(); + } + })) + } else { + None + }; + + let start = Instant::now(); + let result = execute_statements(&analytics, &statements, args.max_rows); + let elapsed_ms = start.elapsed().as_millis(); + + // Retire the watchdog before we touch the result. + done.store(true, Ordering::SeqCst); + if let Some(w) = watchdog { + let _ = w.join(); + } + + let result = match result { + Ok(r) => r, + Err(EngineError::MultipleResultSets) => { + return Err(CtxError::Other( + "only the final statement may return rows; earlier statements must not produce a result set" + .into(), + )); + } + Err(EngineError::Duck(e)) => { + if timed_out.load(Ordering::SeqCst) { + return Err(CtxError::Other(format!( + "query exceeded the {}s timeout; raise it with --timeout", + args.timeout + ))); + } + return Err(CtxError::Other(format!("sql error: {}", e))); + } + }; + + render(&result, format, elapsed_ms); + + if args.fail_on_rows && !result.rows.is_empty() { + Ok(Outcome::Findings) + } else { + Ok(Outcome::Clean) + } + } + + /// Execute leading statements (which must not produce result sets), then run + /// the final statement and return its rows. + fn execute_statements( + analytics: &Analytics, + statements: &[String], + max_rows: usize, + ) -> std::result::Result { + let (last, leading) = statements + .split_last() + .expect("statements is non-empty (checked by caller)"); + for stmt in leading { + let produced_result = analytics + .exec_non_final_statement(stmt) + .map_err(EngineError::Duck)?; + if produced_result { + return Err(EngineError::MultipleResultSets); + } + } + analytics + .run_final_statement(last, max_rows) + .map_err(EngineError::Duck) + } + + fn render(result: &SqlResult, format: &str, elapsed_ms: u128) { + match format { + "json" => render_json(result, elapsed_ms), + "csv" => render_csv(result), + _ => render_table(result), + } + } + + fn render_json(result: &SqlResult, elapsed_ms: u128) { + use serde::Serialize; + + #[derive(Serialize)] + struct Column<'a> { + name: &'a str, + #[serde(rename = "type")] + type_name: &'a str, + } + #[derive(Serialize)] + struct Data<'a> { + columns: Vec>, + rows: &'a [Vec], + row_count: usize, + truncated: bool, + elapsed_ms: u128, + } + #[derive(Serialize)] + struct Envelope<'a> { + ctx_version: &'a str, + command: &'a str, + generated_at: String, + data: Data<'a>, + } + + let envelope = Envelope { + ctx_version: env!("CARGO_PKG_VERSION"), + command: "sql", + generated_at: chrono::Utc::now().to_rfc3339(), + data: Data { + columns: result + .columns + .iter() + .map(|c| Column { + name: &c.name, + type_name: &c.type_name, + }) + .collect(), + rows: &result.rows, + row_count: result.rows.len(), + truncated: result.truncated, + elapsed_ms, + }, + }; + + match serde_json::to_string_pretty(&envelope) { + Ok(s) => println!("{}", s), + Err(e) => eprintln!("failed to serialize result: {}", e), + } + } + + const MAX_CELL: usize = 200; + + /// Render a JSON value to a display string, returning whether it was + /// truncated at [`MAX_CELL`] characters. + fn cell_string(value: &serde_json::Value) -> (String, bool) { + let raw = match value { + serde_json::Value::Null => return ("∅".to_string(), false), + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Bool(b) => b.to_string(), + serde_json::Value::Number(n) => n.to_string(), + other => other.to_string(), + }; + if raw.chars().count() > MAX_CELL { + let truncated: String = raw.chars().take(MAX_CELL).collect(); + (format!("{}…", truncated), true) + } else { + (raw, false) + } + } + + fn render_table(result: &SqlResult) { + let headers: Vec<&str> = result.columns.iter().map(|c| c.name.as_str()).collect(); + let mut any_cell_truncated = false; + + // Pre-render every cell so widths and truncation are computed once. + let rendered: Vec> = result + .rows + .iter() + .map(|row| { + row.iter() + .map(|v| { + let (s, truncated) = cell_string(v); + any_cell_truncated |= truncated; + s + }) + .collect() + }) + .collect(); + + let mut widths: Vec = headers.iter().map(|h| h.chars().count()).collect(); + for row in &rendered { + for (i, cell) in row.iter().enumerate() { + if i < widths.len() { + widths[i] = widths[i].max(cell.chars().count()); + } + } + } + + let mut out = String::new(); + push_row(&mut out, &headers.iter().map(|h| h.to_string()).collect::>(), &widths); + push_separator(&mut out, &widths); + for row in &rendered { + push_row(&mut out, row, &widths); + } + print!("{}", out); + + if result.truncated { + eprintln!( + "note: output capped at {} rows; raise the limit with --max-rows", + result.rows.len() + ); + } + if any_cell_truncated { + eprintln!( + "note: some cells were truncated at {} characters; use --format json or csv for full values", + MAX_CELL + ); + } + } + + fn push_row(out: &mut String, cells: &[String], widths: &[usize]) { + let mut parts = Vec::with_capacity(cells.len()); + for (i, cell) in cells.iter().enumerate() { + let width = widths.get(i).copied().unwrap_or(0); + let pad = width.saturating_sub(cell.chars().count()); + parts.push(format!("{}{}", cell, " ".repeat(pad))); + } + out.push_str(parts.join(" ").trim_end()); + out.push('\n'); + } + + fn push_separator(out: &mut String, widths: &[usize]) { + let parts: Vec = widths.iter().map(|w| "-".repeat(*w)).collect(); + out.push_str(parts.join(" ").trim_end()); + out.push('\n'); + } + + fn render_csv(result: &SqlResult) { + let mut out = String::new(); + let headers: Vec = result + .columns + .iter() + .map(|c| csv_field(&c.name)) + .collect(); + out.push_str(&headers.join(",")); + out.push_str("\r\n"); + + for row in &result.rows { + let fields: Vec = row + .iter() + .map(|v| match v { + serde_json::Value::Null => String::new(), + serde_json::Value::String(s) => csv_field(s), + serde_json::Value::Bool(b) => b.to_string(), + serde_json::Value::Number(n) => n.to_string(), + other => csv_field(&other.to_string()), + }) + .collect(); + out.push_str(&fields.join(",")); + out.push_str("\r\n"); + } + print!("{}", out); + + if result.truncated { + eprintln!( + "note: output capped at {} rows; raise the limit with --max-rows", + result.rows.len() + ); + } + } + + /// Quote a CSV field per RFC-4180 when it contains a comma, quote, CR, or LF. + fn csv_field(s: &str) -> String { + if s.contains([',', '"', '\n', '\r']) { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_string() + } + } +} + +#[cfg(feature = "duckdb")] +use engine::run_sql_duckdb; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_simple_statements() { + let stmts = split_sql_statements("SELECT 1; SELECT 2"); + assert_eq!(stmts, vec!["SELECT 1", "SELECT 2"]); + } + + #[test] + fn ignores_semicolons_in_strings_and_comments() { + let stmts = split_sql_statements( + "SELECT ';' AS a; -- comment; not a split\nSELECT /* also; not */ 2", + ); + assert_eq!(stmts.len(), 2); + assert!(stmts[0].contains("';'")); + assert!(stmts[1].contains("2")); + } + + #[test] + fn single_statement_yields_one() { + assert_eq!(split_sql_statements("SELECT * FROM v1.symbols").len(), 1); + } + + #[test] + fn trailing_semicolon_is_not_an_empty_statement() { + assert_eq!(split_sql_statements("SELECT 1;").len(), 1); + } + + /// The `--schema` output and the published docs page must be byte-identical + /// in their content sections (generated from one source). + #[test] + fn schema_reference_matches_docs_page() { + let manifest = env!("CARGO_MANIFEST_DIR"); + let doc_path = + std::path::Path::new(manifest).join("docs/website/docs/sql-schema.md"); + let doc = std::fs::read_to_string(&doc_path) + .unwrap_or_else(|e| panic!("read {}: {}", doc_path.display(), e)); + + // Strip a leading YAML front-matter block (--- … ---) if present. + let body = strip_front_matter(&doc); + assert_eq!( + body.trim(), + SQL_SCHEMA_REFERENCE.trim(), + "docs/website/docs/sql-schema.md content must match SQL_SCHEMA_REFERENCE" + ); + } + + fn strip_front_matter(doc: &str) -> &str { + if let Some(rest) = doc.strip_prefix("---") { + if let Some(end) = rest.find("\n---") { + // Skip past the closing delimiter line. + let after = &rest[end + 4..]; + return after.trim_start_matches(['\n', '\r']); + } + } + doc + } +} diff --git a/src/commands/sql_schema.md b/src/commands/sql_schema.md new file mode 100644 index 0000000..0483518 --- /dev/null +++ b/src/commands/sql_schema.md @@ -0,0 +1,94 @@ +# ctx sql — Public Schema (`v1`) + +`ctx sql` runs read-only SQL against the code-intelligence index through DuckDB. +The query surface is the versioned **`v1`** schema: a set of stable views over +the physical index. Query `v1.*` — not the underlying tables. + +## Stability contract + +- **`v1.*` is the contract.** Columns and views may be *added* within + `schema_version` 1. Renaming or removing anything increments + `v1.meta.schema_version` and is noted in the changelog. +- **Everything outside `v1.*` is internal and unstable.** The raw index is + reachable as `code.*`, but its shape can change at any time and it is excluded + from all compatibility guarantees. Do not depend on it. +- **Access is read-only and engine-hardened.** Filesystem access, extension + loading, and attaching other databases are disabled; the index cannot be + modified. + +## Views + +### `v1.symbols` — one row per symbol + +| Column | Type | Description | +|------------------|---------|----------------------------------------------------| +| `id` | VARCHAR | Stable symbol identifier | +| `name` | VARCHAR | Symbol name | +| `qualified_name` | VARCHAR | Fully-qualified name, when known | +| `kind` | VARCHAR | `function`, `method`, `struct`, `enum`, `trait`, … | +| `file` | VARCHAR | Path of the file that defines the symbol | +| `line_start` | BIGINT | First line of the symbol | +| `line_end` | BIGINT | Last line of the symbol | +| `is_public` | BOOLEAN | Whether the symbol is publicly visible | +| `complexity` | BIGINT | `fan_out * 2 + fan_in` heuristic complexity score | +| `fan_in` | BIGINT | Number of resolved incoming `calls` edges | +| `fan_out` | BIGINT | Number of outgoing `calls` edges | +| `doc` | VARCHAR | Docstring or brief, when present | + +### `v1.edges` — one row per relationship + +| Column | Type | Description | +|---------------|---------|---------------------------------------------------------| +| `source_id` | VARCHAR | `id` of the source symbol | +| `source_name` | VARCHAR | Name of the source symbol | +| `source_file` | VARCHAR | File of the source symbol | +| `target_id` | VARCHAR | `id` of the target symbol; NULL when unresolved | +| `target_name` | VARCHAR | Name of the target; retained even when unresolved | +| `target_file` | VARCHAR | File of the target symbol; NULL when unresolved | +| `kind` | VARCHAR | `calls`, `extends`, `implements`, or `imports` | +| `line` | BIGINT | Line of the reference in the source file | + +### `v1.files` — one row per indexed file + +| Column | Type | Description | +|--------------------|---------|----------------------------------------------| +| `path` | VARCHAR | File path | +| `language` | VARCHAR | Detected language | +| `symbol_count` | BIGINT | Number of symbols defined in the file | +| `total_complexity` | BIGINT | Sum of `v1.symbols.complexity` for the file | +| `indexed_at` | BIGINT | Unix time the file was last indexed | + +### `v1.meta` — single row of index metadata + +| Column | Type | Description | +|--------------------|---------|----------------------------------------------| +| `schema_version` | INTEGER | Public schema version (starts at 1) | +| `ctx_version` | VARCHAR | Version of ctx that produced this output | +| `index_created_at` | BIGINT | Earliest file index time (Unix seconds) | +| `index_root` | VARCHAR | Absolute root path of the indexed project | + +## Examples + +```sql +-- Ten most complex symbols +SELECT name, file, complexity +FROM v1.symbols +ORDER BY complexity DESC +LIMIT 10; +``` + +```sql +-- Symbol counts by kind +SELECT kind, COUNT(*) AS n +FROM v1.symbols +GROUP BY kind +ORDER BY n DESC; +``` + +```sql +-- Public functions that nothing calls (dead-code candidates) +SELECT name, file +FROM v1.symbols +WHERE kind IN ('function', 'method') AND is_public AND fan_in = 0 +ORDER BY file, name; +``` diff --git a/src/main.rs b/src/main.rs index d4087a3..a1022bf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,6 +73,29 @@ fn run(args: Args) -> Result { commands::run_index(config) } Some(Command::Query { query }) => commands::run_query(query, json), + Some(Command::Sql { + query, + file, + output, + json: json_flag, + max_rows, + timeout, + fail_on_rows, + schema, + }) => { + // `ctx sql` owns its exit code (0 clean / 1 with --fail-on-rows / 2 error), + // so it returns an Outcome directly like the other quality commands. + return commands::run_sql(commands::SqlArgs { + query, + file, + format: output, + json: json || json_flag, + max_rows, + timeout, + fail_on_rows, + schema, + }); + } Some(Command::Search { query, limit, diff --git a/tests/sql.rs b/tests/sql.rs new file mode 100644 index 0000000..b9699f6 --- /dev/null +++ b/tests/sql.rs @@ -0,0 +1,375 @@ +//! Integration tests for the `ctx sql` subcommand. +//! +//! These run the real `ctx` binary against a real DuckDB-backed index built by +//! `ctx index`. Safety is enforced entirely by engine configuration, so the +//! security tests below assert that dangerous operations fail (exit code 2) and, +//! where relevant, that they leave the filesystem and index untouched. +//! +//! Assertions deliberately use substring matching (`predicates::str::contains`) +//! and exit-code checks rather than parsing JSON, to avoid depending on +//! `serde_json` from an integration-test target. + +use std::path::Path; + +use assert_cmd::Command; +use predicates::prelude::*; +use tempfile::TempDir; + +/// Build a minimal, real Rust project in a fresh `TempDir` and run `ctx index` +/// so the `v1.*` views are populated with symbols and `calls` edges. +/// +/// The returned `TempDir` must be kept alive for the duration of a test; its +/// path is the working directory for every `ctx sql` invocation. +fn indexed_fixture() -> TempDir { + let temp = TempDir::new().expect("create temp dir"); + let src = temp.path().join("src"); + std::fs::create_dir_all(&src).expect("create src dir"); + + std::fs::write( + src.join("main.rs"), + r#" +struct Point { + x: i32, + y: i32, +} + +enum Color { + Red, + Green, + Blue, +} + +fn helper(n: i32) -> i32 { + n * 2 +} + +fn main() { + let p = Point { x: 1, y: 2 }; + let _c = Color::Red; + let _sum = helper(p.x) + helper(p.y); +} +"#, + ) + .expect("write main.rs"); + + std::fs::write( + src.join("lib.rs"), + r#" +pub fn public_api() -> u32 { + private_impl() + 1 +} + +fn private_impl() -> u32 { + 7 +} +"#, + ) + .expect("write lib.rs"); + + Command::cargo_bin("ctx") + .unwrap() + .current_dir(temp.path()) + .arg("index") + .assert() + .success(); + + temp +} + +/// Convenience: a `ctx sql` command rooted in `dir`. +fn sql(dir: &Path) -> Command { + let mut cmd = Command::cargo_bin("ctx").unwrap(); + cmd.current_dir(dir).arg("sql"); + cmd +} + +/// Path to the on-disk index for a fixture project. +fn index_db(dir: &Path) -> std::path::PathBuf { + dir.join(".ctx").join("codebase.sqlite") +} + +// --------------------------------------------------------------------------- +// Security: the engine must reject filesystem access, extension installs, +// configuration changes, and file-based ATTACH. +// --------------------------------------------------------------------------- + +#[test] +fn copy_to_file_is_blocked_and_writes_nothing() { + let temp = indexed_fixture(); + let leak = temp.path().join("leak.csv"); + let query = format!("COPY (SELECT 1) TO '{}'", leak.display()); + + sql(temp.path()).arg(&query).assert().code(2); + + assert!( + !leak.exists(), + "COPY must not create a file on disk: {}", + leak.display() + ); +} + +#[test] +fn read_csv_from_filesystem_is_blocked() { + let temp = indexed_fixture(); + sql(temp.path()) + .arg("SELECT * FROM read_csv('/etc/passwd')") + .assert() + .code(2); +} + +#[test] +fn install_extension_is_blocked() { + let temp = indexed_fixture(); + sql(temp.path()).arg("INSTALL httpfs").assert().code(2); +} + +#[test] +fn changing_locked_configuration_is_blocked() { + let temp = indexed_fixture(); + sql(temp.path()) + .arg("SET enable_external_access = true") + .assert() + .code(2); +} + +#[test] +fn attach_file_database_is_blocked() { + let temp = indexed_fixture(); + // A file-based ATTACH must be rejected. (`:memory:` is intentionally + // permitted-but-inert and is deliberately NOT tested here.) + let evil = temp.path().join("ctx_evil_attach.db"); + let query = format!("ATTACH '{}' AS x", evil.display()); + sql(temp.path()).arg(&query).assert().code(2); + assert!( + !evil.exists(), + "file-based ATTACH must not create a database file" + ); +} + +#[test] +fn updates_are_rejected_and_index_bytes_are_unchanged() { + let temp = indexed_fixture(); + let db = index_db(temp.path()); + + let before = std::fs::read(&db).expect("read index before"); + + // Both the public view layer and the underlying read-only `code` database + // must reject writes. + for stmt in [ + "UPDATE v1.symbols SET name='x'", + "UPDATE code.symbols SET name='x'", + ] { + sql(temp.path()).arg(stmt).assert().code(2); + } + + let after = std::fs::read(&db).expect("read index after"); + assert!( + before == after, + "the on-disk index must be byte-for-byte unchanged after rejected UPDATEs" + ); +} + +#[test] +fn runaway_query_is_interrupted_by_timeout() { + let temp = indexed_fixture(); + let start = std::time::Instant::now(); + sql(temp.path()) + .arg("--timeout") + .arg("2") + .arg("WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM r) SELECT count(*) FROM r") + .assert() + .code(2); + // Generous upper bound so the test is not flaky under load, but still proves + // the query did not run unbounded. + assert!( + start.elapsed().as_secs() < 30, + "timeout should abort the query promptly" + ); +} + +// --------------------------------------------------------------------------- +// Schema: the versioned `v1` views are queryable and `v1.meta` reports the +// contract version and crate version. +// --------------------------------------------------------------------------- + +#[test] +fn meta_reports_schema_and_crate_version() { + let temp = indexed_fixture(); + sql(temp.path()) + .arg("--json") + .arg("SELECT * FROM v1.meta") + .assert() + .success() + .stdout(predicate::str::contains("\"command\": \"sql\"")) + .stdout(predicate::str::contains("schema_version")) + .stdout(predicate::str::contains(env!("CARGO_PKG_VERSION"))) + // schema_version is 1 (single-row meta). + .stdout(predicate::str::contains("1")); +} + +#[test] +fn symbol_analytics_columns_are_selectable() { + let temp = indexed_fixture(); + sql(temp.path()) + .arg("SELECT name, complexity, fan_in, fan_out FROM v1.symbols LIMIT 1") + .assert() + .success(); +} + +#[test] +fn every_documented_view_is_queryable() { + let temp = indexed_fixture(); + for view in ["v1.symbols", "v1.edges", "v1.files", "v1.meta"] { + sql(temp.path()) + .arg(format!("SELECT * FROM {} LIMIT 0", view)) + .assert() + .success(); + } +} + +#[test] +fn schema_flag_prints_reference_without_index() { + // `--schema` needs neither an index nor the engine. + let temp = TempDir::new().unwrap(); + sql(temp.path()) + .arg("--schema") + .assert() + .success() + .stdout(predicate::str::contains("v1.symbols")) + .stdout(predicate::str::contains("Public Schema")); +} + +// --------------------------------------------------------------------------- +// Execution: exit-code convention, row caps, JSON envelope, multi-statement. +// --------------------------------------------------------------------------- + +#[test] +fn fail_on_rows_exit_codes() { + let temp = indexed_fixture(); + + // >= 1 row -> exit 1. + sql(temp.path()) + .arg("--fail-on-rows") + .arg("SELECT 1") + .assert() + .code(1); + + // zero rows -> exit 0. + sql(temp.path()) + .arg("--fail-on-rows") + .arg("SELECT 1 WHERE false") + .assert() + .code(0); + + // error -> exit 2. + sql(temp.path()) + .arg("--fail-on-rows") + .arg("SELCT bad syntax") + .assert() + .code(2); +} + +#[test] +fn row_cap_truncates_in_json_and_warns_in_table() { + let temp = indexed_fixture(); + + // JSON: default cap is 1000 rows; the envelope reports the cap and truncation. + sql(temp.path()) + .arg("--json") + .arg("SELECT * FROM range(5000) t(n)") + .assert() + .success() + .stdout(predicate::str::contains("\"row_count\": 1000")) + .stdout(predicate::str::contains("\"truncated\": true")); + + // Table (default): a cap notice mentioning --max-rows goes to stderr. + sql(temp.path()) + .arg("SELECT * FROM range(5000) t(n)") + .assert() + .success() + .stderr(predicate::str::contains("--max-rows")); +} + +#[test] +fn json_output_is_a_well_formed_envelope() { + let temp = indexed_fixture(); + sql(temp.path()) + .arg("--json") + .arg("SELECT 1 AS one") + .assert() + .success() + .stdout(predicate::str::starts_with("{")) + .stdout(predicate::str::contains("\"command\": \"sql\"")) + .stdout(predicate::str::contains("\"data\"")); +} + +#[test] +fn multi_statement_semantics() { + let temp = indexed_fixture(); + + // Leading non-result statement, then a final SELECT -> ok, value shown. + sql(temp.path()) + .arg("CREATE TEMP TABLE t AS SELECT 42 AS x; SELECT * FROM t") + .assert() + .success() + .stdout(predicate::str::contains("42")); + + // Two result-producing statements -> error (only the final may return rows). + sql(temp.path()) + .arg("SELECT 1; SELECT 2") + .assert() + .code(2); +} + +#[test] +fn missing_index_errors_and_creates_nothing() { + // Fresh temp dir with NO `ctx index` run. + let temp = TempDir::new().unwrap(); + sql(temp.path()) + .arg("SELECT 1") + .assert() + .code(2) + .stderr(predicate::str::contains("ctx index")); + + assert!( + !temp.path().join(".ctx").exists(), + "a failed `ctx sql` must not create a .ctx directory" + ); +} + +// --------------------------------------------------------------------------- +// Integration: the gate pattern (`--fail-on-rows --file .sql`). +// --------------------------------------------------------------------------- + +#[test] +fn gate_files_drive_pass_and_fail_exit_codes() { + let temp = indexed_fixture(); + let gates = temp.path().join(".ctx").join("gates"); + std::fs::create_dir_all(&gates).expect("create gates dir"); + + // A passing gate returns no rows. + std::fs::write( + gates.join("pass.sql"), + "SELECT name FROM v1.symbols WHERE 1=0", + ) + .unwrap(); + sql(temp.path()) + .arg("--fail-on-rows") + .arg("--file") + .arg(".ctx/gates/pass.sql") + .assert() + .code(0); + + // A failing gate returns at least one row. + std::fs::write( + gates.join("fail.sql"), + "SELECT name FROM v1.symbols LIMIT 1", + ) + .unwrap(); + sql(temp.path()) + .arg("--fail-on-rows") + .arg("--file") + .arg(".ctx/gates/fail.sql") + .assert() + .code(1); +}