From a4c292f20b2e06692d15823321dbdb88210332e4 Mon Sep 17 00:00:00 2001 From: Jerry Yuan Date: Thu, 6 Aug 2026 14:36:49 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(db):=20the=20db=20module=20=E2=80=94?= =?UTF-8?q?=20SQLite=20behind=20a=20five-op=20spec,=20sim=20host,=20pocket?= =?UTF-8?q?-db=20reference=20core?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fourth module-shaped vertical slice (after ui, strike and audio), built spec-first per the module discipline: contracts/spec/db.ts pins five synchronous ops (open/close/exec/query/lastError), the JSON value encoding (blobs as {"$b": base64}, integers past 2^53-1 fail loudly), logical database names the host maps under the app's own data root, and the resource ceilings (4 databases, 4096 result rows per query). Statement caching is host-side, keyed by the sql string — the guest holds no statement handles, so there is nothing to finalize and nothing to leak. The module owns no clock and emits no events: every op completes inside the guest's single per-tick turn, and golden-tested apps must not depend on random()/'now'-relative SQL (the Date.now rule applied to the dialect). ATTACH — the one SQL statement that names a file — is refused so the app's data root stays the sandbox boundary; load_extension stays off. - gen-rust emits pub mod db into engine/core/src/spec.rs (drift-guarded) - data.sqlite capability registered ahead of any stock TARGET advertising it, the audio.pcm precedent: the sim host and the reference core implement and test the whole contract - @pocketjs/framework/db SDK: the bun:sqlite shape (Database, cached Statement .get/.all/.values/.run, transaction with savepoint nesting), throwing where the namespace is unmounted — data code that silently drops writes is a corruption bug, not a missing enhancement - hosts/sim/db.ts: bun:sqlite behind the op namespace, injected via bootWorld extraGlobals; tests/db.test.ts runs the op contract, the SDK, and an oracle comparison against bun:sqlite directly - engine/crates/pocket-db: the reference core over rusqlite (bundled), a real SQLite authorizer for the ATTACH refusal, Storage::Memory/Dir, mountable as globalThis.db on any pocket-mod guest — the adoption path a device host copies, with SQLite's own VFS as the port point - docs/DB.md maps the boundary and the three-move adoption path Verified: bun run test 11/11 stages green (tests/db.test.ts 17 pass; suite 313+ pass), cargo test -p pocket-db 9/9 including a live QuickJS guest round-trip, cargo check --workspace clean. Co-Authored-By: Claude Fable 5 --- contracts/spec/db.ts | 139 ++++++++ contracts/spec/gen-rust.ts | 34 +- contracts/spec/platforms.ts | 9 + docs/DB.md | 112 +++++++ engine/Cargo.lock | 104 +++++- engine/Cargo.toml | 6 + engine/core/src/spec.rs | 20 ++ engine/crates/pocket-db/Cargo.toml | 15 + engine/crates/pocket-db/src/lib.rs | 505 +++++++++++++++++++++++++++++ framework/compiler/subpaths.ts | 1 + framework/src/db-api.ts | 291 +++++++++++++++++ hosts/sim/db.ts | 178 ++++++++++ hosts/sim/sim.ts | 1 + package.json | 3 + tests/db.test.ts | 270 +++++++++++++++ tools/test.ts | 1 + 16 files changed, 1687 insertions(+), 2 deletions(-) create mode 100644 contracts/spec/db.ts create mode 100644 docs/DB.md create mode 100644 engine/crates/pocket-db/Cargo.toml create mode 100644 engine/crates/pocket-db/src/lib.rs create mode 100644 framework/src/db-api.ts create mode 100644 hosts/sim/db.ts create mode 100644 tests/db.test.ts diff --git a/contracts/spec/db.ts b/contracts/spec/db.ts new file mode 100644 index 00000000..805dc91e --- /dev/null +++ b/contracts/spec/db.ts @@ -0,0 +1,139 @@ +// PocketJS db spec — the boundary of the DB module (`globalThis.db`). +// +// This is a MODULE spec in the docs/RUNTIMES.md §5 sense: a vertical slice +// with its own vocabulary, mounted as its own namespace, pinned here as data. +// It is deliberately NOT part of the `ui` op table — db evolves append-only +// in its own op space, and a host adopts it independently of the UI surface +// (capability id `data.sqlite` in contracts/spec/platforms.ts). +// +// The module is SQLite behind a five-op namespace. The engine, the SQL +// dialect, and the file format are SQLite's — the spec pins only what +// crosses the boundary: op codes, the JSON row encoding, the resource +// ceilings, and the storage/clock rules a host must follow. +// +// The four parts of the boundary: +// +// ops guest -> core intent (numeric codes below, append-only) +// events none — every op is synchronous; the module owns no clock +// data contract the JSON value encoding + database name rules (this file) +// frame contract every op completes inside the guest's single per-tick +// turn (law 3 holds unchanged). SQL time and randomness +// resolve host-side: deterministic hosts pin them, and a +// golden-tested app must not depend on `random()` or +// 'now'-relative SQL (same rule as Date.now in guest code). +// +// Storage rule: `open(name)` is the ONLY path to a database. Names are +// logical (below); the HOST maps them to real files under the app's own +// data root (or memory). The guest never sees a path, and a database file +// is never shared between apps. Hosts MUST refuse `ATTACH` — it is the one +// SQL statement that names a file — so the app's data root stays the +// sandbox boundary. `sqlite3_load_extension` stays disabled (SQLite's +// default). +// +// If you change ANY value here: run `bun contracts/spec/gen-rust.ts`, commit +// the regenerated engine/core/src/spec.rs (tests/contract.ts byte-compares). + +// --------------------------------------------------------------------------- +// Db ops (the `db.*` native contract) +// --------------------------------------------------------------------------- +// Numeric codes are the FFI ABI identity of each op. 0 is reserved +// (invalid/nop). Codes are append-only: never renumber, never reuse. +// +// Signatures (authoritative; hosts marshal them however they like): +// open(name:string) -> handle | -1 +// [name is DB_MEMORY or matches DB_NAME_PATTERN; -1 = +// refused (bad name or DB_MAX_DATABASES already open). +// Opening the same persistent name twice returns the +// SAME handle; DB_MEMORY always opens a fresh private +// database] +// close(handle) [idempotent; a closed handle is +// dead and every later op on it +// fails with "database is closed"] +// exec(handle, sql:string) -> 0 | 1 [run one or more statements, no +// result rows — the schema and +// migration path. 1 = failed; the +// detail is lastError()] +// query(handle, sql:string, args:string) -> string +// [run ONE statement with bound parameters and return +// the complete result as one JSON line (shape below). +// `args` is a JSON array (positional) or object (named +// $x / :x / @x) of encoded values. Statement caching is +// HOST-side, keyed by the sql string — the guest holds +// no statement handles] +// lastError(handle) -> string [detail for the last failed +// exec/query on this handle; "" +// when none] +// +// query() result, one JSON object per call (fields append-only): +// +// { "cols": ["id","name"], "rows": [[1,"a"],[2,"b"]], +// "changes": 0, "lastInsertRowid": 0 } +// `cols` are the statement's column names ([] for non-readers), +// `rows` are arrays in column order using the value encoding below. +// `changes` / `lastInsertRowid` are SQLite's counters after the call. +// { "error": "no such table: t" } +// The statement failed (parse, bind, step, or a ceiling below). +// lastError() returns the same string. + +export const DB_OP = { + open: 1, + close: 2, + exec: 3, + query: 4, + lastError: 5, +} as const; + +// --------------------------------------------------------------------------- +// Data contract — value encoding (rows out, parameters in) +// --------------------------------------------------------------------------- +// SQLite value -> JSON value, both directions: +// +// NULL <-> null +// INTEGER <-> number [|v| must be <= 2^53 - 1. A larger stored +// integer makes the op FAIL — a loud error +// beats silent precision loss. Store money +// in cents and ids under 2^53.] +// REAL <-> number [non-finite REAL fails the same way] +// TEXT <-> string +// BLOB <-> { "$b": "" } +// +// Binding accepts additionally: true/false bind as INTEGER 1/0, and a JSON +// number binds as INTEGER when integer-valued, else REAL (the bun:sqlite / +// better-sqlite3 convention the SDK mirrors). + +/** Marker key for a BLOB value inside a row or a parameter list. */ +export const DB_BLOB_KEY = "$b"; + +/** Largest integer magnitude that crosses the boundary losslessly. */ +export const DB_MAX_SAFE_INTEGER = 9007199254740991; + +// --------------------------------------------------------------------------- +// Data contract — database names +// --------------------------------------------------------------------------- + +/** The in-memory database name (private to the handle, gone on close). */ +export const DB_MEMORY = ":memory:"; + +/** + * Logical persistent-database names: a filename-safe token, no paths, no + * extensions games. The host maps a name to a real file under the app's own + * data root; the mapping is host policy and never guest-visible. + */ +export const DB_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; + +// --------------------------------------------------------------------------- +// Data contract — resource ceilings +// --------------------------------------------------------------------------- + +/** Open databases per guest. An app has its own db plus room for a scratch + * or migration companion. Deliberately tiny — more simultaneous databases + * is a schema smell, not a bigger constant. */ +export const DB_MAX_DATABASES = 4; + +/** + * Result-row ceiling per query() call. A query producing more rows FAILS + * ("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate") — the row + * budget of a 480x272..720x1280 UI is far below this, and an unbounded + * SELECT on a device heap is a bug surfaced early, not a workload. + */ +export const DB_MAX_RESULT_ROWS = 4096; diff --git a/contracts/spec/gen-rust.ts b/contracts/spec/gen-rust.ts index b19bd9f6..6fc96750 100644 --- a/contracts/spec/gen-rust.ts +++ b/contracts/spec/gen-rust.ts @@ -1,4 +1,4 @@ -// Deterministic codegen: contracts/spec/{spec,audio}.ts -> engine/core/src/spec.rs. +// Deterministic codegen: contracts/spec/{spec,audio,db}.ts -> engine/core/src/spec.rs. // // Run from PocketJS/: bun contracts/spec/gen-rust.ts // @@ -16,6 +16,14 @@ import { AUDIO_RING_FRAMES, audioFramesForTick, } from "./audio.ts"; +import { + DB_BLOB_KEY, + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MAX_SAFE_INTEGER, + DB_MEMORY, + DB_OP, +} from "./db.ts"; import { ANALOG_CENTER, ANIMATABLE, @@ -462,6 +470,30 @@ export function generateRust(): string { put(` pub const EVENT_${screaming(name)}: &str = ${JSON.stringify(v)};`); } put("}"); + put(""); + + // --- db module ----------------------------------------------------------- + // The db MODULE's boundary (contracts/spec/db.ts): SQLite behind five + // synchronous ops, mounted as `globalThis.db`, independent of the ui + // surface. A native host implementing it reads these constants; the + // reference implementation is engine/crates/pocket-db. + put("/// DB module boundary (contracts/spec/db.ts — `globalThis.db`)."); + put("/// SQLite behind five synchronous ops; rows cross as one JSON line per"); + put("/// query() call. The module owns no clock and emits no events."); + put("pub mod db {"); + for (const [name, v] of Object.entries(DB_OP)) { + put(` pub const OP_${screaming(name)}: u8 = ${v};`); + } + put(` /// The in-memory database name (private to the handle).`); + put(` pub const MEMORY: &str = ${JSON.stringify(DB_MEMORY)};`); + put(` /// Marker key for a BLOB value inside a row or a parameter list.`); + put(` pub const BLOB_KEY: &str = ${JSON.stringify(DB_BLOB_KEY)};`); + put(` /// Largest integer magnitude that crosses the boundary losslessly.`); + put(` pub const MAX_SAFE_INTEGER: i64 = ${DB_MAX_SAFE_INTEGER};`); + put(` pub const MAX_DATABASES: usize = ${DB_MAX_DATABASES};`); + put(` /// Result-row ceiling per query() call (exceeding it fails the op).`); + put(` pub const MAX_RESULT_ROWS: usize = ${DB_MAX_RESULT_ROWS};`); + put("}"); return L.join("\n") + "\n"; } diff --git a/contracts/spec/platforms.ts b/contracts/spec/platforms.ts index 88ece438..9d44a319 100644 --- a/contracts/spec/platforms.ts +++ b/contracts/spec/platforms.ts @@ -144,6 +144,15 @@ export const POCKET_CAPABILITIES = defineCapabilityRegistry([ // appends the id to its profile only when its native host ships the module // (the ring/thread discipline to copy is hosts/psp/src/audio.rs). "audio.pcm", + // SQLite behind the db module's own namespace (`globalThis.db`, + // contracts/spec/db.ts): five synchronous ops, rows as one JSON line per + // query() call, per-app storage the host confines. Registered ahead of any + // stock TARGET advertising it: the sim host and the engine/crates/pocket-db + // reference core implement and test the whole contract, so apps can already + // declare the requirement and fail admission where the module is absent. A + // device target appends the id to its profile only when its native host + // ships the module. + "data.sqlite", // Copy/cut/paste round-trips with the OS clipboard. "host.clipboard", // The logical viewport is runtime-mutable: the app is told about live diff --git a/docs/DB.md b/docs/DB.md new file mode 100644 index 00000000..e817d12e --- /dev/null +++ b/docs/DB.md @@ -0,0 +1,112 @@ +# The DB Module + +DB is PocketJS's fourth module (after `ui`, `strike` and `audio`): SQLite +mounted as `globalThis.db` behind five synchronous ops. Like audio it was +written spec-first — the boundary existed before any host code, every host +implements the same pinned protocol, and a developer adding a data feature +extends the spec instead of forking a host. `contracts/spec/db.ts` is +normative; this page is the map. + +``` +platform storage (POSIX file · LittleFS · memory) Host / substrate + ↑ SQLite's own VFS is the port point +db core: SQLite + handle table + statement cache the module +db spec: ops (open, close, exec, query, lastError) + events (none — every op is synchronous) + data contract (JSON value encoding · name rules · ceilings) + frame contract (no module clock; ops complete in the guest turn) +SDK: @pocketjs/framework/db (Database, Statement — the bun:sqlite shape) + ↓ +app: schema in exec(), rows out of query(), state in tables Guest +``` + +## The boundary in one page + +**Mount.** The module is its own namespace: `globalThis.db`, one method per +op (`DB_OP` codes are the ABI identity, append-only). Capability id +`data.sqlite`. Unlike audio, absence does **not** degrade to a no-op — data +code that silently drops writes is a corruption bug, so the SDK throws where +the namespace is unmounted, and an app declares `data.sqlite` in +`pocket.json` `requires` so admission catches the gap before eval does. + +**Ops** (guest → core, all synchronous): `open(name)` → handle, +`close(handle)`, `exec(handle, sql)` → 0/1 for schema and migrations, +`query(handle, sql, argsJson)` → one JSON line +(`{cols, rows, changes, lastInsertRowid}` or `{error}`), and +`lastError(handle)`. Statement caching is **host-side**, keyed by the sql +string — the guest holds no statement handles, so there is nothing to +finalize and nothing to leak. + +**Values** cross as JSON: NULL ↔ `null`, INTEGER ↔ number, REAL ↔ number, +TEXT ↔ string, BLOB ↔ `{"$b": ""}`. Integers beyond 2^53 − 1 and +non-finite REALs **fail the op** instead of losing precision silently — +store money in cents. Booleans bind as 1/0; integer-valued numbers bind as +INTEGER, fractional as REAL (the bun:sqlite convention). + +**Storage rule.** `open(name)` is the only path to a database: names are +filename-safe tokens (`DB_NAME_PATTERN`) or `:memory:`, and the host maps a +name to a real file under the app's own data root. The guest never sees a +path. `ATTACH` — the one SQL statement that names a file — is refused +(engine/crates/pocket-db uses a real SQLite authorizer), and +`load_extension` stays disabled, so the data root stays the sandbox +boundary. + +**Ceilings.** `DB_MAX_DATABASES` (4) open handles; `DB_MAX_RESULT_ROWS` +(4096) rows per `query()` call, above which the op fails with "add LIMIT or +aggregate" — an unbounded SELECT on a device heap is a bug surfaced early, +not a workload. + +**Frame contract.** The module owns no clock and emits no events: every op +completes inside the guest's single per-tick turn (law 3 unchanged). SQL +time and randomness resolve host-side — deterministic hosts pin them, and a +golden-tested app must not depend on `random()` or 'now'-relative SQL, the +same rule as `Date.now` in guest code. + +## The SDK + +`@pocketjs/framework/db` is the bun:sqlite shape, so code written against +Bun's built-in SQLite runs against the mounted module unchanged: + +```ts +import { Database } from "@pocketjs/framework/db"; + +const db = new Database("portfolio"); +db.exec(`CREATE TABLE IF NOT EXISTS history ( + bucket TEXT PRIMARY KEY, total_cents INTEGER NOT NULL +)`); + +const insert = db.query("INSERT OR REPLACE INTO history VALUES ($bucket, $cents)"); +const record = db.transaction((bucket: string, cents: number) => { + insert.run({ $bucket: bucket, $cents: cents }); +}); +record("2026-08-06T14:35", 1_532_042); + +db.query("SELECT * FROM history ORDER BY bucket DESC LIMIT 288").all(); +``` + +`query()` returns a cached `Statement` (`.get/.all/.values/.run`); +`prepare()` skips the cache; `transaction(fn)` wraps BEGIN/COMMIT with +ROLLBACK on throw and nests as savepoints — batching writes into one +transaction is also the flash-wear discipline on device hosts. + +## Host status + +| Host | Implementation | Status | +|---|---|---| +| sim (`hosts/sim/db.ts`) | bun:sqlite behind the op namespace, injected via `bootWorld` `extraGlobals` | ships with the test host; `tests/db.test.ts` runs the contract and an oracle comparison against bun:sqlite directly | +| reference core (`engine/crates/pocket-db`) | rusqlite (bundled SQLite) + the ATTACH authorizer, mountable on any `pocket-mod` guest as `globalThis.db` | tested including a live QuickJS guest round-trip | +| consoles / devices | — | a target appends `data.sqlite` to its profile when its native host ships the module; the port point is SQLite's VFS (POSIX on desktop, LittleFS-backed on MCU hosts) | + +## Adoption path + +A device host that wants the module makes three moves, none of which touch +the spec, the SDK, or any app: + +1. compile SQLite with the platform toolchain and register a VFS for the + platform filesystem (the desktop default VFS already serves + `Storage::Dir`); +2. mount the namespace beside `ui` — `pocket_db::mount(&guest, module)` on + `pocket-mod` hosts, or the raw-QuickJS spelling of the same five + functions elsewhere; +3. append `data.sqlite` to the target's profile in + `contracts/spec/platforms.ts`. diff --git a/engine/Cargo.lock b/engine/Cargo.lock index 4138dfe9..65a35f92 100644 --- a/engine/Cargo.lock +++ b/engine/Cargo.lock @@ -187,6 +187,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-set" version = "0.8.0" @@ -535,6 +541,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fdeflate" version = "0.3.7" @@ -689,7 +707,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3ce1918195723ce6ac74e80542c5a96a40c2b26162c1957a5cd70799b8cacf7" dependencies = [ - "base64", + "base64 0.13.1", "byteorder", "gltf-json", "image", @@ -809,6 +827,15 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -820,6 +847,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -1042,6 +1078,17 @@ dependencies = [ "redox_syscall 0.9.0", ] +[[package]] +name = "libsqlite3-sys" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1587,6 +1634,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "pocket-db" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.23.1", + "pocket-mod", + "pocketjs-core", + "rusqlite", + "serde_json", +] + [[package]] name = "pocket-mod" version = "0.1.0" @@ -1915,6 +1974,31 @@ dependencies = [ "cc", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + +[[package]] +name = "rusqlite" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11438310b19e3109b6446c33d1ed5e889428cf2e278407bc7896bc4aaea43323" +dependencies = [ + "bitflags 2.13.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2137,6 +2221,18 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2367,6 +2463,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/engine/Cargo.toml b/engine/Cargo.toml index d4a627f8..66be04c5 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -11,6 +11,7 @@ [workspace] resolver = "2" members = [ + "crates/pocket-db", "crates/pocket-mod", "crates/pocket-ui-surface", "crates/pocket-ui-wgpu", @@ -42,6 +43,7 @@ repository = "https://github.com/pocket-stack/pocketjs" [workspace.dependencies] pocket3d = { path = "pocket3d/crates/pocket3d" } pocket3d-bsp = { path = "pocket3d/crates/pocket3d-bsp" } +pocket-db = { path = "crates/pocket-db" } pocket-mod = { path = "crates/pocket-mod" } pocket-ui-surface = { path = "crates/pocket-ui-surface" } pocket-ui-wgpu = { path = "crates/pocket-ui-wgpu" } @@ -65,6 +67,10 @@ ab_glyph = "0.2" memmap2 = "0.9" serde = { version = "1", features = ["derive"] } serde_json = "1" +# bundled: compile the vendored sqlite3.c — the same source a device host +# compiles with its own toolchain; hooks: the authorizer that refuses ATTACH. +rusqlite = { version = "0.40", features = ["bundled", "hooks"] } +base64 = "0.23" env_logger = "0.11" font8x8 = "0.3" diff --git a/engine/core/src/spec.rs b/engine/core/src/spec.rs index f5d6ac2e..a9035ec3 100644 --- a/engine/core/src/spec.rs +++ b/engine/core/src/spec.rs @@ -498,3 +498,23 @@ pub mod audio { pub const EVENT_UNDERRUN: &str = "underrun"; pub const EVENT_ENDED: &str = "ended"; } + +/// DB module boundary (contracts/spec/db.ts — `globalThis.db`). +/// SQLite behind five synchronous ops; rows cross as one JSON line per +/// query() call. The module owns no clock and emits no events. +pub mod db { + pub const OP_OPEN: u8 = 1; + pub const OP_CLOSE: u8 = 2; + pub const OP_EXEC: u8 = 3; + pub const OP_QUERY: u8 = 4; + pub const OP_LAST_ERROR: u8 = 5; + /// The in-memory database name (private to the handle). + pub const MEMORY: &str = ":memory:"; + /// Marker key for a BLOB value inside a row or a parameter list. + pub const BLOB_KEY: &str = "$b"; + /// Largest integer magnitude that crosses the boundary losslessly. + pub const MAX_SAFE_INTEGER: i64 = 9007199254740991; + pub const MAX_DATABASES: usize = 4; + /// Result-row ceiling per query() call (exceeding it fails the op). + pub const MAX_RESULT_ROWS: usize = 4096; +} diff --git a/engine/crates/pocket-db/Cargo.toml b/engine/crates/pocket-db/Cargo.toml new file mode 100644 index 00000000..ee36f770 --- /dev/null +++ b/engine/crates/pocket-db/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "pocket-db" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "The db module's reference core: SQLite behind the five-op contracts/spec/db.ts boundary, mountable as globalThis.db via pocket-mod" + +[dependencies] +pocketjs-core = { workspace = true } +pocket-mod = { workspace = true } +anyhow = { workspace = true } +rusqlite = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } diff --git a/engine/crates/pocket-db/src/lib.rs b/engine/crates/pocket-db/src/lib.rs new file mode 100644 index 00000000..0d0fc463 --- /dev/null +++ b/engine/crates/pocket-db/src/lib.rs @@ -0,0 +1,505 @@ +//! pocket-db — the db module's reference core. +//! +//! SQLite behind the five-op boundary pinned in contracts/spec/db.ts +//! (`pocketjs_core::spec::db` is the generated mirror): open / close / exec / +//! query / lastError, mounted as `globalThis.db` through [`mount`]. Rows +//! cross as one JSON line per `query()` call using the spec's value +//! encoding; statement caching is host-side (rusqlite's prepared-statement +//! cache, keyed by the sql string), so the guest holds no statement handles. +//! +//! Storage policy is the host's: [`Storage::Memory`] for tests and +//! throwaway guests, [`Storage::Dir`] to map each logical database name to +//! `/.sqlite` — the app's own data root. `ATTACH` is refused by +//! a real SQLite authorizer (not string inspection), which keeps that root +//! the sandbox boundary; `load_extension` stays off (rusqlite's default). +//! +//! Porting note (the ESP32/LittleFS path): everything in this file above +//! rusqlite is portable. A device host swaps the storage layer — SQLite's +//! own VFS is the port point, exactly as a DrawList backend is the ui +//! module's — and keeps the op semantics bit-identical. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::PathBuf; +use std::rc::Rc; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use pocketjs_core::spec::db as spec; +use rusqlite::hooks::Authorization; +use rusqlite::types::{Value, ValueRef}; +use rusqlite::Connection; +use serde_json::{json, Map as JsonMap, Value as Json}; + +/// Where logical database names live. +pub enum Storage { + /// Every database, named or `:memory:`, is in-memory (tests, previews). + /// Named databases still share a handle for the module's lifetime. + Memory, + /// A named database maps to `/.sqlite`. The host passes the + /// app's own data root; the guest never sees the path. + Dir(PathBuf), +} + +struct Db { + conn: Connection, + name: String, + last_error: String, +} + +/// The db module: every op as a method, [`mount`] to install the namespace. +pub struct DbModule { + storage: Storage, + dbs: HashMap, + by_name: HashMap, + next_handle: i32, +} + +impl DbModule { + pub fn new(storage: Storage) -> DbModule { + DbModule { + storage, + dbs: HashMap::new(), + by_name: HashMap::new(), + next_handle: 1, + } + } + + /// `open(name) -> handle | -1` (spec OP_OPEN). + pub fn open(&mut self, name: &str) -> i32 { + let memory = name == spec::MEMORY; + if !memory { + if !valid_name(name) { + return -1; + } + if let Some(handle) = self.by_name.get(name) { + return *handle; + } + } + if self.dbs.len() >= spec::MAX_DATABASES { + return -1; + } + let conn = match &self.storage { + Storage::Memory => Connection::open_in_memory(), + Storage::Dir(dir) if !memory => Connection::open(dir.join(format!("{name}.sqlite"))), + Storage::Dir(_) => Connection::open_in_memory(), + }; + let conn = match conn { + Ok(conn) => conn, + Err(_) => return -1, + }; + // The storage rule's teeth: ATTACH names a file, so it is denied at + // the engine level. A database the authorizer cannot guard is a + // database this module refuses to open. + if conn + .authorizer(Some(|ctx: rusqlite::hooks::AuthContext<'_>| { + match ctx.action { + rusqlite::hooks::AuthAction::Attach { .. } => Authorization::Deny, + _ => Authorization::Allow, + } + })) + .is_err() + { + return -1; + } + let handle = self.next_handle; + self.next_handle += 1; + self.dbs.insert( + handle, + Db { + conn, + name: name.to_owned(), + last_error: String::new(), + }, + ); + if !memory { + self.by_name.insert(name.to_owned(), handle); + } + handle + } + + /// `close(handle)` (spec OP_CLOSE) — idempotent. + pub fn close(&mut self, handle: i32) { + if let Some(db) = self.dbs.remove(&handle) { + if db.name != spec::MEMORY { + self.by_name.remove(&db.name); + } + } + } + + /// `exec(handle, sql) -> 0 | 1` (spec OP_EXEC). + pub fn exec(&mut self, handle: i32, sql: &str) -> i32 { + let Some(db) = self.dbs.get_mut(&handle) else { + return 1; + }; + match db.conn.execute_batch(sql) { + Ok(()) => { + db.last_error.clear(); + 0 + } + Err(error) => { + db.last_error = error.to_string(); + 1 + } + } + } + + /// `query(handle, sql, args) -> json line` (spec OP_QUERY). + pub fn query(&mut self, handle: i32, sql: &str, args: &str) -> String { + let Some(db) = self.dbs.get_mut(&handle) else { + return json!({ "error": "database is closed" }).to_string(); + }; + match run_query(&db.conn, sql, args) { + Ok(line) => { + db.last_error.clear(); + line + } + Err(message) => { + db.last_error = message.clone(); + json!({ "error": message }).to_string() + } + } + } + + /// `lastError(handle) -> string` (spec OP_LAST_ERROR). + pub fn last_error(&self, handle: i32) -> String { + match self.dbs.get(&handle) { + Some(db) => db.last_error.clone(), + None => "database is closed".to_owned(), + } + } +} + +/// Logical persistent-database names (spec DB_NAME_PATTERN): +/// `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`, spelled out to keep regex out of +/// the dependency tree. +fn valid_name(name: &str) -> bool { + let bytes = name.as_bytes(); + if bytes.is_empty() || bytes.len() > 64 { + return false; + } + if !bytes[0].is_ascii_alphanumeric() { + return false; + } + bytes[1..] + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +fn run_query(conn: &Connection, sql: &str, args: &str) -> Result { + let params: Json = serde_json::from_str(args).map_err(|e| format!("malformed args: {e}"))?; + let mut statement = conn + .prepare_cached(sql) + .map_err(|e| e.to_string())?; + + match ¶ms { + Json::Array(list) => { + if list.len() != statement.parameter_count() { + return Err(format!( + "expected {} parameters, got {}", + statement.parameter_count(), + list.len() + )); + } + for (i, value) in list.iter().enumerate() { + statement + .raw_bind_parameter(i + 1, decode_param(value)?) + .map_err(|e| e.to_string())?; + } + } + Json::Object(named) => { + for (key, value) in named { + let index = statement + .parameter_index(key) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("unknown parameter: {key}"))?; + statement + .raw_bind_parameter(index, decode_param(value)?) + .map_err(|e| e.to_string())?; + } + } + _ => return Err("args must be a JSON array or object".to_owned()), + } + + let cols: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let column_count = cols.len(); + + let mut rows_out: Vec = Vec::new(); + let mut rows = statement.raw_query(); + loop { + let row = rows.next().map_err(|e| e.to_string())?; + let Some(row) = row else { break }; + if rows_out.len() >= spec::MAX_RESULT_ROWS { + return Err("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate".to_owned()); + } + let mut cells: Vec = Vec::with_capacity(column_count); + for i in 0..column_count { + cells.push(encode_cell(row.get_ref(i).map_err(|e| e.to_string())?)?); + } + rows_out.push(Json::Array(cells)); + } + drop(rows); + + Ok(json!({ + "cols": cols, + "rows": rows_out, + "changes": conn.changes(), + "lastInsertRowid": conn.last_insert_rowid(), + }) + .to_string()) +} + +/// JSON parameter -> SQLite value (the spec's binding rules). +fn decode_param(value: &Json) -> Result { + Ok(match value { + Json::Null => Value::Null, + Json::Bool(b) => Value::Integer(i64::from(*b)), + Json::Number(n) => { + if let Some(i) = n.as_i64() { + if i.unsigned_abs() > spec::MAX_SAFE_INTEGER as u64 { + return Err("integer exceeds DB_MAX_SAFE_INTEGER".to_owned()); + } + Value::Integer(i) + } else { + let f = n.as_f64().ok_or("unrepresentable number")?; + if !f.is_finite() { + return Err("cannot bind a non-finite number".to_owned()); + } + Value::Real(f) + } + } + Json::String(s) => Value::Text(s.clone()), + Json::Object(map) => Value::Blob(decode_blob(map)?), + Json::Array(_) => return Err("cannot bind an array value".to_owned()), + }) +} + +fn decode_blob(map: &JsonMap) -> Result, String> { + let Some(Json::String(b64)) = map.get(spec::BLOB_KEY) else { + return Err("malformed blob parameter".to_owned()); + }; + BASE64 + .decode(b64) + .map_err(|e| format!("malformed blob parameter: {e}")) +} + +/// SQLite cell -> JSON value (the spec's row encoding; loud on lossy). +fn encode_cell(cell: ValueRef<'_>) -> Result { + Ok(match cell { + ValueRef::Null => Json::Null, + ValueRef::Integer(i) => { + if i.unsigned_abs() > spec::MAX_SAFE_INTEGER as u64 { + return Err("integer result exceeds DB_MAX_SAFE_INTEGER".to_owned()); + } + json!(i) + } + ValueRef::Real(f) => { + if !f.is_finite() { + return Err("non-finite REAL result".to_owned()); + } + json!(f) + } + ValueRef::Text(t) => { + Json::String(String::from_utf8(t.to_vec()).map_err(|_| "non-UTF-8 TEXT result")?) + } + ValueRef::Blob(b) => json!({ spec::BLOB_KEY: BASE64.encode(b) }), + }) +} + +/// Mount the module as `globalThis.db` on a pocket-mod [`Guest`] — one JS +/// function per spec op, marshaled as (i32, String) -> i32/String. +pub fn mount(guest: &pocket_mod::Guest, module: Rc>) -> anyhow::Result<()> { + use pocket_mod::qjs::Function; + guest.mount("db", |ctx, ns| { + let m = module.clone(); + ns.set( + "open", + Function::new(ctx.clone(), move |name: String| -> i32 { + m.borrow_mut().open(&name) + })?, + )?; + let m = module.clone(); + ns.set( + "close", + Function::new(ctx.clone(), move |handle: i32| { + m.borrow_mut().close(handle); + })?, + )?; + let m = module.clone(); + ns.set( + "exec", + Function::new(ctx.clone(), move |handle: i32, sql: String| -> i32 { + m.borrow_mut().exec(handle, &sql) + })?, + )?; + let m = module.clone(); + ns.set( + "query", + Function::new( + ctx.clone(), + move |handle: i32, sql: String, args: String| -> String { + m.borrow_mut().query(handle, &sql, &args) + }, + )?, + )?; + let m = module.clone(); + ns.set( + "lastError", + Function::new(ctx.clone(), move |handle: i32| -> String { + m.borrow().last_error(handle) + })?, + )?; + Ok(()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn module() -> DbModule { + DbModule::new(Storage::Memory) + } + + fn rows(line: &str) -> Json { + serde_json::from_str(line).unwrap() + } + + #[test] + fn crud_round_trip_with_positional_and_named_parameters() { + let mut m = module(); + let h = m.open(spec::MEMORY); + assert!(h > 0); + assert_eq!(m.exec(h, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"), 0); + let ins = rows(&m.query(h, "INSERT INTO t (v) VALUES ($v)", r#"{"$v":"hello"}"#)); + assert_eq!(ins["changes"], 1); + assert_eq!(ins["lastInsertRowid"], 1); + let sel = rows(&m.query(h, "SELECT id, v FROM t WHERE id = ?", "[1]")); + assert_eq!(sel["cols"], json!(["id", "v"])); + assert_eq!(sel["rows"], json!([[1, "hello"]])); + } + + #[test] + fn open_refuses_bad_names_and_over_limit() { + let mut m = module(); + assert_eq!(m.open("../escape"), -1); + assert_eq!(m.open(".hidden"), -1); + assert_eq!(m.open(""), -1); + assert_eq!(m.open(&"a".repeat(65)), -1); + for i in 0..spec::MAX_DATABASES { + assert!(m.open(&format!("app-{i}")) > 0); + } + assert_eq!(m.open("one-too-many"), -1); + } + + #[test] + fn same_persistent_name_shares_a_handle_and_memory_never_does() { + let mut m = module(); + let a = m.open("app"); + assert_eq!(m.open("app"), a); + assert_ne!(m.open(spec::MEMORY), m.open(spec::MEMORY)); + } + + #[test] + fn attach_is_denied_by_the_authorizer() { + let mut m = module(); + let h = m.open(spec::MEMORY); + assert_eq!(m.exec(h, "ATTACH DATABASE ':memory:' AS other"), 1); + assert!(m.last_error(h).contains("not authorized"), "{}", m.last_error(h)); + let line = m.query(h, "ATTACH DATABASE ':memory:' AS other", "[]"); + assert!(rows(&line)["error"].as_str().unwrap().contains("not authorized")); + } + + #[test] + fn closed_handles_fail_loudly_and_close_is_idempotent() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.close(h); + m.close(h); + assert_eq!(m.exec(h, "SELECT 1"), 1); + assert_eq!(rows(&m.query(h, "SELECT 1", "[]"))["error"], "database is closed"); + assert_eq!(m.last_error(h), "database is closed"); + } + + #[test] + fn blobs_round_trip_and_big_integers_fail_loudly() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.exec(h, "CREATE TABLE b (data BLOB)"); + let ins = m.query(h, "INSERT INTO b VALUES (?)", r#"[{"$b":"AAEC+vv8/f7/"}]"#); + assert_eq!(rows(&ins)["changes"], 1); + let sel = rows(&m.query(h, "SELECT data FROM b", "[]")); + assert_eq!(sel["rows"][0][0][spec::BLOB_KEY], "AAEC+vv8/f7/"); + + let over = rows(&m.query(h, "SELECT 9007199254740993", "[]")); + assert!(over["error"].as_str().unwrap().contains("DB_MAX_SAFE_INTEGER")); + let ok = rows(&m.query(h, "SELECT 9007199254740991", "[]")); + assert_eq!(ok["rows"], json!([[9007199254740991i64]])); + } + + #[test] + fn result_rows_beyond_the_ceiling_fail() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.exec(h, "CREATE TABLE n (v INTEGER)"); + let fill = format!( + "WITH RECURSIVE seq(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM seq LIMIT {}) + INSERT INTO n SELECT x FROM seq", + spec::MAX_RESULT_ROWS + 1 + ); + assert_eq!(m.exec(h, &fill), 0); + let over = rows(&m.query(h, "SELECT v FROM n", "[]")); + assert!(over["error"].as_str().unwrap().contains("DB_MAX_RESULT_ROWS")); + let capped = rows(&m.query( + h, + &format!("SELECT v FROM n LIMIT {}", spec::MAX_RESULT_ROWS), + "[]", + )); + assert_eq!(capped["rows"].as_array().unwrap().len(), spec::MAX_RESULT_ROWS); + } + + #[test] + fn persistent_dir_storage_survives_reopen() { + let dir = std::env::temp_dir().join(format!("pocket-db-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + { + let mut m = DbModule::new(Storage::Dir(dir.clone())); + let h = m.open("ledger"); + m.exec(h, "CREATE TABLE snap (v INTEGER); INSERT INTO snap VALUES (42)"); + } + { + let mut m = DbModule::new(Storage::Dir(dir.clone())); + let h = m.open("ledger"); + let sel = rows(&m.query(h, "SELECT v FROM snap", "[]")); + assert_eq!(sel["rows"], json!([[42]])); + } + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn mounted_namespace_serves_a_quickjs_guest() { + let guest = pocket_mod::Guest::new().unwrap(); + let module = Rc::new(RefCell::new(module())); + mount(&guest, module).unwrap(); + guest + .eval( + "boot", + r#" + const h = db.open(":memory:"); + if (h < 0) throw new Error("open failed"); + if (db.exec(h, "CREATE TABLE t (v TEXT)") !== 0) throw new Error(db.lastError(h)); + const ins = JSON.parse(db.query(h, "INSERT INTO t VALUES (?)", '["from-guest"]')); + if (ins.changes !== 1) throw new Error("insert failed"); + const sel = JSON.parse(db.query(h, "SELECT v FROM t", "[]")); + globalThis.result = sel.rows[0][0]; + "#, + ) + .unwrap(); + let result: String = guest.with(|ctx| ctx.globals().get("result").unwrap()); + assert_eq!(result, "from-guest"); + } +} diff --git a/framework/compiler/subpaths.ts b/framework/compiler/subpaths.ts index 451dad23..3eca570b 100644 --- a/framework/compiler/subpaths.ts +++ b/framework/compiler/subpaths.ts @@ -55,6 +55,7 @@ export const SUBPATHS: Record = { audio: { file: "framework/src/audio-api.ts", aliases: TWINS }, clock: { file: "framework/src/clock.ts", aliases: TWINS }, config: { file: "framework/src/config.ts" }, + db: { file: "framework/src/db-api.ts", aliases: TWINS }, components: { file: { solid: "framework/src/components.ts", diff --git a/framework/src/db-api.ts b/framework/src/db-api.ts new file mode 100644 index 00000000..8e64d0f1 --- /dev/null +++ b/framework/src/db-api.ts @@ -0,0 +1,291 @@ +// Db module SDK — the thin guest-side algebra over the `db` spec +// (contracts/spec/db.ts). Framework-agnostic (no solid-js, no JSX): the +// same file serves ./db, ./vue-vapor/db and ./octane/db. +// +// The API is the bun:sqlite shape — `new Database(name)`, `db.query(sql)` +// returning a cached Statement with `.get/.all/.values/.run`, `db.exec`, +// `db.transaction(fn)` — so code written against Bun's built-in SQLite runs +// against the mounted module unchanged, and the model of "SQL in, plain +// objects out" carries across hosts. Two deliberate deviations from +// bun:sqlite, both from the spec: +// +// - a Statement is a guest-side (db, sql) pair — statement caching is +// HOST-side, keyed by the sql string, so there is nothing to finalize +// and no handle to leak. `columnNames` is populated by execution +// (empty before the first run). +// - integers beyond 2^53 - 1 and non-finite REALs fail loudly instead of +// losing precision (DB_MAX_SAFE_INTEGER; store money in cents). +// +// Unlike the audio SDK, absence does NOT degrade to a no-op: data code that +// silently drops writes is a corruption bug, not a missing enhancement. +// `new Database(...)` throws where `globalThis.db` is unmounted — declare +// `data.sqlite` in pocket.json `requires` so admission catches it first. + +import { DB_BLOB_KEY, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; + +export { DB_MAX_RESULT_ROWS, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; + +/** The mounted db namespace — one method per spec op (DB_OP codes). */ +export interface DbOps { + open(name: string): number; + close(handle: number): void; + exec(handle: number, sql: string): number; + query(handle: number, sql: string, args: string): string; + lastError(handle: number): string; +} + +/** The db module namespace, or null where the host doesn't mount one. + * A live lookup (not cached): hosts install `globalThis.db` before eval + * and reset it per app load, exactly like `globalThis.ui`. */ +export function dbHost(): DbOps | null { + const ns = (globalThis as { db?: unknown }).db; + if (!ns || typeof ns !== "object") return null; + return typeof (ns as DbOps).open === "function" ? (ns as DbOps) : null; +} + +// --------------------------------------------------------------------------- +// Value encoding (the contracts/spec/db.ts data contract, both directions) +// --------------------------------------------------------------------------- + +/** A value crossing the boundary: what a row cell or a bound parameter is. */ +export type SqlValue = null | number | string | boolean | Uint8Array; +export type SqlParams = readonly SqlValue[] | Readonly>; + +const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/** QuickJS has no btoa/Buffer; the codec is spelled out (cold path). */ +function bytesToBase64(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += 3) { + const a = bytes[i]; + const b = i + 1 < bytes.length ? bytes[i + 1] : 0; + const c = i + 2 < bytes.length ? bytes[i + 2] : 0; + out += B64[a >> 2] + B64[((a & 3) << 4) | (b >> 4)]; + out += i + 1 < bytes.length ? B64[((b & 15) << 2) | (c >> 6)] : "="; + out += i + 2 < bytes.length ? B64[c & 63] : "="; + } + return out; +} + +const B64_INV: Record = {}; +for (let i = 0; i < B64.length; i++) B64_INV[B64[i]] = i; + +function base64ToBytes(s: string): Uint8Array { + let pad = 0; + while (s.endsWith("=")) { + pad++; + s = s.slice(0, -1); + } + const out = new Uint8Array(Math.floor((s.length * 3) / 4)); + let o = 0; + for (let i = 0; i < s.length; i += 4) { + const n = + (B64_INV[s[i]] << 18) | + ((B64_INV[s[i + 1]] ?? 0) << 12) | + ((B64_INV[s[i + 2]] ?? 0) << 6) | + (B64_INV[s[i + 3]] ?? 0); + out[o++] = n >> 16; + if (o < out.length) out[o++] = (n >> 8) & 0xff; + if (o < out.length) out[o++] = n & 0xff; + } + return out; +} + +function encodeValue(v: SqlValue): unknown { + if (v instanceof Uint8Array) return { [DB_BLOB_KEY]: bytesToBase64(v) }; + if (typeof v === "number" && !Number.isFinite(v)) { + throw new Error("db: cannot bind a non-finite number"); + } + if (typeof v === "number" && Number.isInteger(v) && Math.abs(v) > DB_MAX_SAFE_INTEGER) { + throw new Error("db: integer exceeds DB_MAX_SAFE_INTEGER"); + } + return v; +} + +function decodeValue(v: unknown): SqlValue { + if (v !== null && typeof v === "object") { + return base64ToBytes((v as Record)[DB_BLOB_KEY]); + } + return v as SqlValue; +} + +function encodeParams(params: SqlParams): string { + if (Array.isArray(params)) return JSON.stringify(params.map(encodeValue)); + const out: Record = {}; + for (const [k, v] of Object.entries(params)) out[k] = encodeValue(v as SqlValue); + return JSON.stringify(out); +} + +// --------------------------------------------------------------------------- +// Database / Statement (the bun:sqlite shape) +// --------------------------------------------------------------------------- + +interface QueryResult { + cols?: string[]; + rows?: unknown[][]; + changes?: number; + lastInsertRowid?: number; + error?: string; +} + +export interface RunResult { + changes: number; + lastInsertRowid: number; +} + +export class Statement { + private cols: string[] = []; + + constructor( + private readonly ops: DbOps, + private readonly handle: number, + private readonly sql: string, + ) {} + + /** Column names of the last execution ([] before the first run). */ + get columnNames(): readonly string[] { + return this.cols; + } + + private execute(params: SqlParams): QueryResult { + const line = this.ops.query(this.handle, this.sql, encodeParams(params)); + const result = JSON.parse(line) as QueryResult; + if (result.error !== undefined) throw new Error(`db: ${result.error}`); + this.cols = result.cols ?? []; + return result; + } + + /** First row as a column-name keyed object, or null. */ + get(...params: SqlValue[]): Record | null; + get(params: SqlParams): Record | null; + get(...args: unknown[]): Record | null { + const rows = this.values(...(args as SqlValue[])); + if (rows.length === 0) return null; + const out: Record = {}; + this.cols.forEach((c, i) => (out[c] = rows[0][i])); + return out; + } + + /** Every row as a column-name keyed object. */ + all(...params: SqlValue[]): Record[]; + all(params: SqlParams): Record[]; + all(...args: unknown[]): Record[] { + const rows = this.values(...(args as SqlValue[])); + return rows.map((r) => { + const out: Record = {}; + this.cols.forEach((c, i) => (out[c] = r[i])); + return out; + }); + } + + /** Every row as an array in column order. */ + values(...params: SqlValue[]): SqlValue[][]; + values(params: SqlParams): SqlValue[][]; + values(...args: unknown[]): SqlValue[][] { + const params = normalizeArgs(args); + const result = this.execute(params); + return (result.rows ?? []).map((r) => r.map(decodeValue)); + } + + /** Execute for effect; rows (if any) are discarded. */ + run(...params: SqlValue[]): RunResult; + run(params: SqlParams): RunResult; + run(...args: unknown[]): RunResult { + const result = this.execute(normalizeArgs(args)); + return { changes: result.changes ?? 0, lastInsertRowid: result.lastInsertRowid ?? 0 }; + } +} + +/** Spread positional values, one array, or one named-parameter object. */ +function normalizeArgs(args: unknown[]): SqlParams { + if (args.length === 1 && Array.isArray(args[0])) return args[0] as SqlValue[]; + if ( + args.length === 1 && + args[0] !== null && + typeof args[0] === "object" && + !(args[0] instanceof Uint8Array) + ) { + return args[0] as Record; + } + return args as SqlValue[]; +} + +export class Database { + private readonly ops: DbOps; + private readonly handle: number; + private readonly statements = new Map(); + private txDepth = 0; + + constructor(name: string = DB_MEMORY) { + const ops = dbHost(); + if (!ops) { + throw new Error( + "db: globalThis.db is not mounted — declare `data.sqlite` in pocket.json requires", + ); + } + const handle = ops.open(name); + if (handle < 0) throw new Error(`db: open(${JSON.stringify(name)}) refused`); + this.ops = ops; + this.handle = handle; + } + + /** Cached statement for `sql` (host-side prepare cache backs it). */ + query(sql: string): Statement { + let statement = this.statements.get(sql); + if (!statement) { + statement = new Statement(this.ops, this.handle, sql); + this.statements.set(sql, statement); + } + return statement; + } + + /** Uncached statement (the bun:sqlite `prepare` spelling). */ + prepare(sql: string): Statement { + return new Statement(this.ops, this.handle, sql); + } + + /** Run one statement with parameters, for effect. */ + run(sql: string, params: SqlParams = []): RunResult { + return this.query(sql).run(params); + } + + /** Run one or more statements with no parameters and no result rows — + * the schema/migration path. */ + exec(sql: string): void { + if (this.ops.exec(this.handle, sql) !== 0) { + throw new Error(`db: ${this.ops.lastError(this.handle)}`); + } + } + + /** + * Wrap `fn` in BEGIN/COMMIT with ROLLBACK on throw; nested calls become + * savepoints (the bun:sqlite convention). Batching writes into one + * transaction is also the flash-wear discipline on device hosts. + */ + transaction(fn: (...args: A) => R): (...args: A) => R { + return (...args: A): R => { + const name = `pocket_tx_${this.txDepth}`; + const [begin, commit, rollback] = + this.txDepth === 0 + ? ["BEGIN", "COMMIT", "ROLLBACK"] + : [`SAVEPOINT ${name}`, `RELEASE ${name}`, `ROLLBACK TO ${name}; RELEASE ${name}`]; + this.exec(begin); + this.txDepth++; + try { + const result = fn(...args); + this.txDepth--; + this.exec(commit); + return result; + } catch (error) { + this.txDepth--; + this.exec(rollback); + throw error; + } + }; + } + + close(): void { + this.statements.clear(); + this.ops.close(this.handle); + } +} diff --git a/hosts/sim/db.ts b/hosts/sim/db.ts new file mode 100644 index 00000000..539ed1df --- /dev/null +++ b/hosts/sim/db.ts @@ -0,0 +1,178 @@ +// hosts/sim/db.ts — the bun:sqlite-backed implementation of the db module +// (contracts/spec/db.ts) for the headless sim host. +// +// Bun's built-in SQLite is the same engine a device host links, so the sim +// runs the real dialect — only the storage policy is sim-shaped: every +// database, named or DB_MEMORY, lives in memory (no disk, no cleanup), and +// named databases persist for the life of the host object so an app reload +// inside one scenario keeps its data, the way a device keeps its files. +// +// Two dev-host caveats, both spec-permitted: +// - ATTACH is refused by a whole-string match, not a real authorizer +// (engine/crates/pocket-db carries the authoritative refusal); a string +// literal containing "attach database" is a false positive a test can +// spell around. +// - SQL time and randomness are NOT pinned — bun:sqlite exposes no VFS +// hook. The spec already forbids golden-tested apps from depending on +// them; tests/db.test.ts stays on deterministic SQL. +// +// Inject via bootWorld's extraGlobals: { db: host.ns }, the way a device +// host mounts the namespace beside `ui`. + +import { Database as BunDatabase } from "bun:sqlite"; +import { + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MAX_SAFE_INTEGER, + DB_MEMORY, + DB_NAME_PATTERN, + DB_BLOB_KEY, +} from "../../contracts/spec/db.ts"; + +interface SimDb { + bun: BunDatabase; + name: string; + lastError: string; +} + +export interface SimDbHost { + /** The `globalThis.db` namespace (one method per DB_OP). */ + ns: Record; + /** Every op call in order (for trace assertions). */ + log: string[]; + /** Close every underlying database (end of scenario). */ + dispose(): void; +} + +function encodeCell(v: unknown): unknown { + if (v === null || typeof v === "string") return v; + if (typeof v === "bigint") { + if (v > BigInt(DB_MAX_SAFE_INTEGER) || v < -BigInt(DB_MAX_SAFE_INTEGER)) { + throw new Error("integer result exceeds DB_MAX_SAFE_INTEGER"); + } + return Number(v); + } + if (typeof v === "number") { + if (!Number.isFinite(v)) throw new Error("non-finite REAL result"); + return v; + } + if (v instanceof Uint8Array) return { [DB_BLOB_KEY]: Buffer.from(v).toString("base64") }; + throw new Error(`unencodable result value: ${typeof v}`); +} + +function decodeParam(v: unknown): unknown { + if (v !== null && typeof v === "object" && !Array.isArray(v)) { + const b64 = (v as Record)[DB_BLOB_KEY]; + if (typeof b64 !== "string") throw new Error("malformed blob parameter"); + return new Uint8Array(Buffer.from(b64, "base64")); + } + if (typeof v === "number" && !Number.isFinite(v)) { + throw new Error("cannot bind a non-finite number"); + } + return v; +} + +const ATTACH = /\battach\s+(database\b|')/i; + +export function createSimDbHost(): SimDbHost { + const dbs = new Map(); + const byName = new Map(); + const log: string[] = []; + let nextHandle = 1; + + function live(handle: number): SimDb | null { + return dbs.get(handle) ?? null; + } + + const ns = { + open(name: string): number { + log.push(`op open ${name}`); + if (name !== DB_MEMORY) { + if (!DB_NAME_PATTERN.test(name)) return -1; + const existing = byName.get(name); + if (existing !== undefined) return existing; + } + if (dbs.size >= DB_MAX_DATABASES) return -1; + const bun = new BunDatabase(":memory:", { safeIntegers: true }); + const handle = nextHandle++; + dbs.set(handle, { bun, name, lastError: "" }); + if (name !== DB_MEMORY) byName.set(name, handle); + return handle; + }, + close(handle: number): void { + log.push(`op close ${handle}`); + const db = live(handle); + if (!db) return; + db.bun.close(); + dbs.delete(handle); + if (db.name !== DB_MEMORY) byName.delete(db.name); + }, + exec(handle: number, sql: string): number { + log.push(`op exec ${handle}`); + const db = live(handle); + if (!db) return 1; + if (ATTACH.test(sql)) { + db.lastError = "ATTACH is refused (contracts/spec/db.ts storage rule)"; + return 1; + } + try { + db.bun.exec(sql); + db.lastError = ""; + return 0; + } catch (error) { + db.lastError = error instanceof Error ? error.message : String(error); + return 1; + } + }, + query(handle: number, sql: string, args: string): string { + log.push(`op query ${handle} ${sql}`); + const db = live(handle); + if (!db) return JSON.stringify({ error: "database is closed" }); + if (ATTACH.test(sql)) { + db.lastError = "ATTACH is refused (contracts/spec/db.ts storage rule)"; + return JSON.stringify({ error: db.lastError }); + } + try { + const parsed = JSON.parse(args) as unknown[] | Record; + const params = Array.isArray(parsed) + ? parsed.map(decodeParam) + : Object.fromEntries(Object.entries(parsed).map(([k, v]) => [k, decodeParam(v)])); + const statement = db.bun.query(sql); + // bun returns null (not []) for statements that produce no rows. + const rows = ((Array.isArray(params) + ? statement.values(...(params as never[])) + : statement.values(params as never)) ?? []) as unknown[][]; + if (rows.length > DB_MAX_RESULT_ROWS) { + throw new Error("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate"); + } + const counters = db.bun + .query("SELECT changes() AS c, last_insert_rowid() AS r") + .get() as { c: bigint; r: bigint }; + db.lastError = ""; + return JSON.stringify({ + cols: statement.columnNames, + rows: rows.map((r) => r.map(encodeCell)), + changes: Number(counters.c), + lastInsertRowid: Number(counters.r), + }); + } catch (error) { + db.lastError = error instanceof Error ? error.message : String(error); + return JSON.stringify({ error: db.lastError }); + } + }, + lastError(handle: number): string { + const db = live(handle); + return db ? db.lastError : "database is closed"; + }, + }; + + return { + ns, + log, + dispose(): void { + for (const db of dbs.values()) db.bun.close(); + dbs.clear(); + byName.clear(); + }, + }; +} diff --git a/hosts/sim/sim.ts b/hosts/sim/sim.ts index 30c7bc6a..b986b8e5 100644 --- a/hosts/sim/sim.ts +++ b/hosts/sim/sim.ts @@ -241,6 +241,7 @@ export async function bootWorld( : undefined; g.frame = undefined; g.audio = undefined; // audio module namespace: absent unless extraGlobals mounts one + g.db = undefined; // db module namespace: absent unless extraGlobals mounts one g.__pocketApp = app; g.__simHz = hz; g.__pocketEffectTrace = (e: EffectEvent) => effects.push(e); diff --git a/package.json b/package.json index 6a15b57e..6a21826d 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "./audio": "./framework/src/audio-api.ts", "./clock": "./framework/src/clock.ts", "./config": "./framework/src/config.ts", + "./db": "./framework/src/db-api.ts", "./components": "./framework/src/components.ts", "./devtools": "./framework/src/devtools.ts", "./effects": "./framework/src/effects.ts", @@ -100,6 +101,7 @@ "./vue-vapor/animation": "./framework/src/animation.ts", "./vue-vapor/audio": "./framework/src/audio-api.ts", "./vue-vapor/clock": "./framework/src/clock.ts", + "./vue-vapor/db": "./framework/src/db-api.ts", "./vue-vapor/components": "./framework/src/components-vue-vapor.ts", "./vue-vapor/effects": "./framework/src/effects.ts", "./vue-vapor/lifecycle": "./framework/src/lifecycle-vue-vapor.ts", @@ -109,6 +111,7 @@ "./octane/animation": "./framework/src/animation.ts", "./octane/audio": "./framework/src/audio-api.ts", "./octane/clock": "./framework/src/clock.ts", + "./octane/db": "./framework/src/db-api.ts", "./octane/components": "./framework/src/components-octane.tsx", "./octane/effects": "./framework/src/effects.ts", "./octane/lifecycle": "./framework/src/lifecycle-octane.ts", diff --git a/tests/db.test.ts b/tests/db.test.ts new file mode 100644 index 00000000..a7049c9b --- /dev/null +++ b/tests/db.test.ts @@ -0,0 +1,270 @@ +// Db module unit tests: the sim host against the pinned op contract, the +// bun:sqlite-shaped SDK over it, and Bun's own SQLite as the oracle for row +// results. Runs entirely in-process; no built bundle, no disk. + +import { afterEach, describe, expect, test } from "bun:test"; +import { Database as BunDatabase } from "bun:sqlite"; +import { + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MEMORY, + DB_NAME_PATTERN, +} from "../contracts/spec/db.ts"; +import { Database, dbHost } from "../framework/src/db-api.ts"; +import { createSimDbHost, type SimDbHost } from "../hosts/sim/db.ts"; + +const g = globalThis as { db?: unknown }; +let host: SimDbHost | null = null; + +/** Mount a fresh sim host as globalThis.db, the way bootWorld's + * extraGlobals does for a scenario. */ +function mount(): SimDbHost { + host = createSimDbHost(); + g.db = host.ns; + return host; +} + +afterEach(() => { + host?.dispose(); + host = null; + g.db = undefined; +}); + +// --- the namespace contract (ops, straight through) ------------------------- + +describe("sim host ops", () => { + test("open refuses bad names, path traversal, and over-limit opens", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + expect(open("../escape")).toBe(-1); + expect(open("a/b")).toBe(-1); + expect(open("")).toBe(-1); + expect(open(".hidden")).toBe(-1); + const handles = []; + for (let i = 0; i < DB_MAX_DATABASES; i++) handles.push(open(`app-${i}`)); + for (const h of handles) expect(h).toBeGreaterThan(0); + expect(open("one-too-many")).toBe(-1); + }); + + test("the same persistent name returns the same handle; :memory: never does", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const a = open("app"); + expect(open("app")).toBe(a); + expect(open(DB_MEMORY)).not.toBe(open(DB_MEMORY)); + }); + + test("ATTACH is refused on both exec and query", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const lastError = ns.lastError as (h: number) => string; + const h = open(DB_MEMORY); + expect(exec(h, "ATTACH DATABASE '/tmp/x' AS other")).toBe(1); + expect(lastError(h)).toContain("ATTACH"); + const result = JSON.parse(query(h, "attach database ':memory:' as other", "[]")); + expect(result.error).toContain("ATTACH"); + }); + + test("ops on a closed handle fail with 'database is closed'", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const close = ns.close as (h: number) => void; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + close(h); + expect(JSON.parse(query(h, "SELECT 1", "[]")).error).toBe("database is closed"); + }); + + test("query result carries cols, rows, changes and lastInsertRowid", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + expect(exec(h, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")).toBe(0); + const ins = JSON.parse(query(h, "INSERT INTO t (v) VALUES (?)", '["hello"]')); + expect(ins.changes).toBe(1); + expect(ins.lastInsertRowid).toBe(1); + const sel = JSON.parse(query(h, "SELECT id, v FROM t", "[]")); + expect(sel.cols).toEqual(["id", "v"]); + expect(sel.rows).toEqual([[1, "hello"]]); + }); + + test("a result beyond DB_MAX_RESULT_ROWS fails loudly", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + exec(h, "CREATE TABLE n (v INTEGER)"); + exec( + h, + `WITH RECURSIVE seq(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM seq LIMIT ${DB_MAX_RESULT_ROWS + 1}) + INSERT INTO n SELECT x FROM seq`, + ); + const over = JSON.parse(query(h, "SELECT v FROM n", "[]")); + expect(over.error).toContain("DB_MAX_RESULT_ROWS"); + const capped = JSON.parse(query(h, `SELECT v FROM n LIMIT ${DB_MAX_RESULT_ROWS}`, "[]")); + expect(capped.rows.length).toBe(DB_MAX_RESULT_ROWS); + }); + + test("an integer beyond 2^53-1 fails instead of losing precision", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + const over = JSON.parse(query(h, "SELECT 9007199254740993", "[]")); + expect(over.error).toContain("DB_MAX_SAFE_INTEGER"); + const ok = JSON.parse(query(h, "SELECT 9007199254740991", "[]")); + expect(ok.rows).toEqual([[9007199254740991]]); + }); +}); + +// --- the SDK (the bun:sqlite shape over the mounted namespace) --------------- + +describe("Database SDK", () => { + test("throws where the module is unmounted", () => { + expect(dbHost()).toBeNull(); + expect(() => new Database()).toThrow("data.sqlite"); + }); + + test("CRUD round-trip with objects, values and named parameters", () => { + mount(); + const db = new Database(); + db.exec(`CREATE TABLE positions ( + symbol TEXT PRIMARY KEY, qty REAL NOT NULL, cost_cents INTEGER NOT NULL + )`); + const insert = db.query("INSERT INTO positions VALUES ($s, $q, $c)"); + insert.run({ $s: "AAPL", $q: 10, $c: 190_00 }); + insert.run({ $s: "NVDA", $q: 2.5, $c: 121_50 }); + expect(db.query("SELECT count(*) AS n FROM positions").get()).toEqual({ n: 2 }); + expect(db.query("SELECT symbol FROM positions ORDER BY symbol").values()).toEqual([ + ["AAPL"], + ["NVDA"], + ]); + const row = db.query("SELECT * FROM positions WHERE symbol = ?").get("NVDA"); + expect(row).toEqual({ symbol: "NVDA", qty: 2.5, cost_cents: 12150 }); + expect(db.query("SELECT * FROM positions WHERE symbol = ?").get("MSFT")).toBeNull(); + const del = db.run("DELETE FROM positions WHERE qty < ?", [5]); + expect(del.changes).toBe(1); + }); + + test("statement caching: query() reuses, prepare() does not", () => { + mount(); + const db = new Database(); + expect(db.query("SELECT 1")).toBe(db.query("SELECT 1")); + expect(db.prepare("SELECT 1")).not.toBe(db.prepare("SELECT 1")); + }); + + test("columnNames populate on execution", () => { + mount(); + const db = new Database(); + const q = db.query("SELECT 1 AS one, 2 AS two"); + expect(q.columnNames).toEqual([]); + q.get(); + expect(q.columnNames).toEqual(["one", "two"]); + }); + + test("blobs round-trip as Uint8Array", () => { + mount(); + const db = new Database(); + db.exec("CREATE TABLE b (data BLOB)"); + const bytes = new Uint8Array([0, 1, 2, 250, 251, 252, 253, 254, 255]); + db.run("INSERT INTO b VALUES (?)", [bytes]); + const out = db.query("SELECT data FROM b").get(); + expect(out?.data).toBeInstanceOf(Uint8Array); + expect(Array.from(out?.data as Uint8Array)).toEqual(Array.from(bytes)); + }); + + test("transaction commits, rolls back on throw, and nests as savepoints", () => { + mount(); + const db = new Database(); + db.exec("CREATE TABLE t (v INTEGER)"); + const insertBoth = db.transaction((a: number, b: number) => { + db.run("INSERT INTO t VALUES (?)", [a]); + db.run("INSERT INTO t VALUES (?)", [b]); + return a + b; + }); + expect(insertBoth(1, 2)).toBe(3); + expect(db.query("SELECT count(*) AS n FROM t").get()).toEqual({ n: 2 }); + + const failing = db.transaction(() => { + db.run("INSERT INTO t VALUES (99)"); + throw new Error("boom"); + }); + expect(() => failing()).toThrow("boom"); + expect(db.query("SELECT count(*) AS n FROM t").get()).toEqual({ n: 2 }); + + const outer = db.transaction(() => { + db.run("INSERT INTO t VALUES (10)"); + const inner = db.transaction(() => { + db.run("INSERT INTO t VALUES (11)"); + throw new Error("inner"); + }); + expect(() => inner()).toThrow("inner"); + }); + outer(); + const values = db.query("SELECT v FROM t ORDER BY v").values(); + expect(values).toEqual([[1], [2], [10]]); + }); + + test("a persistent name survives close/reopen inside one host", () => { + mount(); + const first = new Database("ledger"); + first.exec("CREATE TABLE snap (v INTEGER)"); + first.run("INSERT INTO snap VALUES (42)"); + const second = new Database("ledger"); + expect(second.query("SELECT v FROM snap").get()).toEqual({ v: 42 }); + }); + + test("SQL errors surface as thrown Errors with SQLite's message", () => { + mount(); + const db = new Database(); + expect(() => db.query("SELECT * FROM missing").all()).toThrow("missing"); + expect(() => db.exec("NOT SQL AT ALL")).toThrow(); + }); +}); + +// --- oracle: the SDK over the sim host agrees with bun:sqlite directly ------- + +describe("bun:sqlite oracle", () => { + test("identical statements produce identical rows", () => { + mount(); + const ours = new Database(); + const oracle = new BunDatabase(":memory:"); + const ddl = `CREATE TABLE history ( + bucket TEXT PRIMARY KEY, total_cents INTEGER, day_pnl_cents INTEGER + )`; + const rows: [string, number, number][] = [ + ["2026-08-06T14:30", 1_532_042, 1824], + ["2026-08-06T14:35", 1_531_010, 792], + ["2026-08-06T14:40", 1_540_500, 10_282], + ]; + ours.exec(ddl); + oracle.exec(ddl); + for (const r of rows) { + ours.run("INSERT INTO history VALUES (?, ?, ?)", r); + oracle.query("INSERT INTO history VALUES (?, ?, ?)").run(...r); + } + const sql = `SELECT bucket, total_cents FROM history + WHERE day_pnl_cents > ? ORDER BY bucket DESC`; + expect(ours.query(sql).values(1000)).toEqual( + oracle.query(sql).values(1000) as never, + ); + }); +}); + +// --- spec sanity -------------------------------------------------------------- + +describe("spec constants", () => { + test("DB_NAME_PATTERN accepts tokens and refuses paths", () => { + for (const good of ["app", "portfolio-history", "a.b_c-1", "A"]) { + expect(DB_NAME_PATTERN.test(good)).toBe(true); + } + for (const bad of ["", ".hidden", "-lead", "a/b", "a\\b", "..", "a".repeat(65)]) { + expect(DB_NAME_PATTERN.test(bad)).toBe(false); + } + }); +}); diff --git a/tools/test.ts b/tools/test.ts index 12b251d9..e2799740 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -55,6 +55,7 @@ const SUITE: readonly Stage[] = [ "tests/kinetics.test.ts", "tests/osk-controller.test.ts", "tests/audio.test.ts", + "tests/db.test.ts", "tests/vita-package.test.ts", "tests/psp-toolchain.test.ts", "tests/symbian-data.test.ts", From 8a1407ab452e38857bf91ff7ea3ce840dcb7100b Mon Sep 17 00:00:00 2001 From: Jerry Yuan Date: Thu, 6 Aug 2026 21:19:11 +0800 Subject: [PATCH 2/3] feat(db): ESP-IDF support in-crate; databases are ordinary files in the app root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes from the ESP32-P4 bring-up, both invisible on desktop: - cfg(target_os = "espidf") support ships in the crate: the newlib shims SQLite's syscall table references (geteuid/fchmod/fchown/utimes/readlink no-ops — honest on a filesystem with no users or symlinks — and nanosleep routed through usleep for the busy handler), plus the unix-none VFS on open (LittleFS has no fcntl locks; a module instance is its files' only writer) with the flash-friendly pragmas (journal_mode=TRUNCATE, synchronous=NORMAL, cache_size=-32). `mount` is now a default feature — default-features = false drops the pocket-mod/rquickjs dependency for firmware with its own QuickJS wiring, so an MCU build compiles only the module core plus SQLite. - Storage::Dir creates the data root on first open and maps a name to /.sqlite — an ORDINARY file in the app's own home, the same root the fs module is typically bound to. The database is the app's own asset, deliberately visible and touchable like any of its files (backup = a file copy); overwriting it corrupts the app's own data, the same trust class as deleting its own files, and SQLite fails loudly on a corrupt image. The build-environment half a firmware must supply (LIBSQLITE3_FLAGS, the empty sys/ioctl.h shim, arch CFLAGS) is documented in docs/DB.md — values validated on a Waveshare ESP32-P4 over a LittleFS workspace: open 15 ms, 288-row transaction ~0.4 s, ~70–80 KB heap, data intact across reopen and power cycling. Verified: cargo test -p pocket-db 9/9, clippy clean, cargo check --no-default-features clean, cargo check --target riscv32imafc-esp-espidf (no default features) clean incl. the bundled libsqlite3.a. Co-Authored-By: Claude Fable 5 --- docs/DB.md | 59 ++++++++++-- engine/crates/pocket-db/Cargo.toml | 12 ++- engine/crates/pocket-db/src/lib.rs | 145 ++++++++++++++++++++++++++--- 3 files changed, 193 insertions(+), 23 deletions(-) diff --git a/docs/DB.md b/docs/DB.md index e817d12e..eac03264 100644 --- a/docs/DB.md +++ b/docs/DB.md @@ -45,11 +45,16 @@ INTEGER, fractional as REAL (the bun:sqlite convention). **Storage rule.** `open(name)` is the only path to a database: names are filename-safe tokens (`DB_NAME_PATTERN`) or `:memory:`, and the host maps a -name to a real file under the app's own data root. The guest never sees a -path. `ATTACH` — the one SQL statement that names a file — is refused -(engine/crates/pocket-db uses a real SQLite authorizer), and -`load_extension` stays disabled, so the data root stays the sandbox -boundary. +name to a real file under the app's own data root — the reference core +spells that mapping `/.sqlite`. The database is an +ORDINARY file in the app's home, deliberately visible to a co-mounted fs +module: it is the app's own asset (backup = a file copy), and an app that +overwrites its own database corrupts its own data — the same trust class +as deleting its own files (SQLite fails loudly on a corrupt image). The +guest never sees a path. `ATTACH` — the one SQL statement that names a +file — is refused (engine/crates/pocket-db uses a real SQLite +authorizer), and `load_extension` stays disabled, so the data root stays +the sandbox boundary. **Ceilings.** `DB_MAX_DATABASES` (4) open handles; `DB_MAX_RESULT_ROWS` (4096) rows per `query()` call, above which the op fails with "add LIMIT or @@ -95,7 +100,7 @@ transaction is also the flash-wear discipline on device hosts. |---|---|---| | sim (`hosts/sim/db.ts`) | bun:sqlite behind the op namespace, injected via `bootWorld` `extraGlobals` | ships with the test host; `tests/db.test.ts` runs the contract and an oracle comparison against bun:sqlite directly | | reference core (`engine/crates/pocket-db`) | rusqlite (bundled SQLite) + the ATTACH authorizer, mountable on any `pocket-mod` guest as `globalThis.db` | tested including a live QuickJS guest round-trip | -| consoles / devices | — | a target appends `data.sqlite` to its profile when its native host ships the module; the port point is SQLite's VFS (POSIX on desktop, LittleFS-backed on MCU hosts) | +| consoles / devices | — | a target appends `data.sqlite` to its profile when its native host ships the module; the port point is SQLite's VFS (POSIX on desktop, LittleFS-backed on MCU hosts). ESP-IDF support ships in the crate — see below | ## Adoption path @@ -110,3 +115,45 @@ the spec, the SDK, or any app: functions elsewhere; 3. append `data.sqlite` to the target's profile in `contracts/spec/platforms.ts`. + +## ESP32 / ESP-IDF + +`pocket-db` carries its own ESP-IDF support behind +`cfg(target_os = "espidf")` — desktop builds never see it: + +- **newlib shims** for the POSIX symbols SQLite's syscall table references + but newlib lacks (`geteuid`/`fchmod`/`fchown`/`utimes`/`readlink` no-ops + that are honest on a filesystem with no users or symlinks, and + `nanosleep` routed through `usleep` for the busy handler); +- **the `unix-none` VFS** on open — LittleFS has no fcntl file locks, and + a module instance is its files' only writer — plus flash-friendly + pragmas (`journal_mode=TRUNCATE`, `synchronous=NORMAL`, + `cache_size=-32`). + +What the crate cannot carry is the build environment; a firmware adds, in +its `.cargo/config.toml` (values validated on an ESP32-P4, ESP-IDF v5.5.x, +LittleFS workspace — where a 288-row transaction landed in ~0.4 s on +~70–80 KB of heap and survived power cycling): + +```toml +[env] +# The vendored sqlite3.c, tuned for the device: temp tables in memory, no +# mmap, no WAL (shared memory), no dynamic extension loading; lstat does +# not exist on newlib and the VFS never follows symlinks anyway. +LIBSQLITE3_FLAGS = "-DSQLITE_TEMP_STORE=3 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_MAX_MMAP_SIZE=0 -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -Dlstat=stat" +# CC_/AR_/CFLAGS_riscv32imafc_esp_espidf point at the ESP-IDF gcc as usual. +# newlib has no sys/ioctl.h: put an EMPTY sys/ioctl.h in a shim directory +# and add `-isystem ` to CFLAGS. Setting CFLAGS in [env] replaces +# cargo's derived flags, so repeat the arch flags (-mabi/-march) alongside. +``` + +`sqlite3_os_init` for the `unix-none` VFS, journal-file creation, and +power-loss recovery all run against ESP-IDF's VFS layer over LittleFS — +no SQLite source patches, no custom VFS to write. + +A firmware that brings its own QuickJS embedding depends with +`default-features = false`: that drops the `mount` helper and its +pocket-mod/rquickjs dependency, so the MCU build compiles only the module +core plus SQLite. Verified: `cargo check` for `riscv32imafc-esp-espidf` +compiles the crate and the bundled `libsqlite3.a` clean under this +recipe. diff --git a/engine/crates/pocket-db/Cargo.toml b/engine/crates/pocket-db/Cargo.toml index ee36f770..85386356 100644 --- a/engine/crates/pocket-db/Cargo.toml +++ b/engine/crates/pocket-db/Cargo.toml @@ -6,10 +6,18 @@ license.workspace = true repository.workspace = true description = "The db module's reference core: SQLite behind the five-op contracts/spec/db.ts boundary, mountable as globalThis.db via pocket-mod" +[features] +# `mount` brings pocket-mod (and its QuickJS embedding) for the one-line +# globalThis.db install. A device host with its own QuickJS wiring turns +# it off (`default-features = false`) and drives DbModule directly — the +# MCU build then never compiles an engine it doesn't use. +default = ["mount"] +mount = ["dep:pocket-mod", "dep:anyhow"] + [dependencies] pocketjs-core = { workspace = true } -pocket-mod = { workspace = true } -anyhow = { workspace = true } +pocket-mod = { workspace = true, optional = true } +anyhow = { workspace = true, optional = true } rusqlite = { workspace = true } serde_json = { workspace = true } base64 = { workspace = true } diff --git a/engine/crates/pocket-db/src/lib.rs b/engine/crates/pocket-db/src/lib.rs index 0d0fc463..72e9bf4e 100644 --- a/engine/crates/pocket-db/src/lib.rs +++ b/engine/crates/pocket-db/src/lib.rs @@ -9,18 +9,29 @@ //! //! Storage policy is the host's: [`Storage::Memory`] for tests and //! throwaway guests, [`Storage::Dir`] to map each logical database name to -//! `/.sqlite` — the app's own data root. `ATTACH` is refused by -//! a real SQLite authorizer (not string inspection), which keeps that root -//! the sandbox boundary; `load_extension` stays off (rusqlite's default). +//! `/.sqlite` — an ORDINARY file in the app's own data root, +//! the same root the fs module is typically bound to. That is deliberate: +//! the database is the app's own asset, visible and touchable like any of +//! its files (backup = a file copy). An app that overwrites its own +//! database corrupts its own data — the same trust class as deleting its +//! own files, and SQLite fails loudly (SQLITE_CORRUPT), not unsafely. +//! `ATTACH` is refused by a real SQLite authorizer (not string +//! inspection), which keeps that root the sandbox boundary; +//! `load_extension` stays off (rusqlite's default). //! -//! Porting note (the ESP32/LittleFS path): everything in this file above -//! rusqlite is portable. A device host swaps the storage layer — SQLite's -//! own VFS is the port point, exactly as a DrawList backend is the ui -//! module's — and keeps the op semantics bit-identical. - +//! ESP32/LittleFS: this crate carries its own ESP-IDF support (the +//! `espidf` module below — newlib symbol shims, the `unix-none` VFS, the +//! flash-friendly pragmas), all behind `cfg(target_os = "espidf")`; +//! desktop builds never see it. The build-environment recipe a firmware +//! needs (C flags, header shim) is documented in docs/DB.md — validated +//! on an ESP32-P4 with a LittleFS workspace, where data survived reopen +//! and power cycling. + +#[cfg(feature = "mount")] use std::cell::RefCell; use std::collections::HashMap; use std::path::PathBuf; +#[cfg(feature = "mount")] use std::rc::Rc; use base64::Engine as _; @@ -36,8 +47,9 @@ pub enum Storage { /// Every database, named or `:memory:`, is in-memory (tests, previews). /// Named databases still share a handle for the module's lifetime. Memory, - /// A named database maps to `/.sqlite`. The host passes the - /// app's own data root; the guest never sees the path. + /// A named database maps to `/.sqlite` — an ordinary file + /// in the app's own data root (created on first open), the same root + /// the fs module is typically bound to. The guest never sees the path. Dir(PathBuf), } @@ -81,7 +93,12 @@ impl DbModule { } let conn = match &self.storage { Storage::Memory => Connection::open_in_memory(), - Storage::Dir(dir) if !memory => Connection::open(dir.join(format!("{name}.sqlite"))), + Storage::Dir(dir) if !memory => { + if std::fs::create_dir_all(dir).is_err() { + return -1; + } + open_file(&dir.join(format!("{name}.sqlite"))) + } Storage::Dir(_) => Connection::open_in_memory(), }; let conn = match conn { @@ -120,10 +137,10 @@ impl DbModule { /// `close(handle)` (spec OP_CLOSE) — idempotent. pub fn close(&mut self, handle: i32) { - if let Some(db) = self.dbs.remove(&handle) { - if db.name != spec::MEMORY { - self.by_name.remove(&db.name); - } + if let Some(db) = self.dbs.remove(&handle) + && db.name != spec::MEMORY + { + self.by_name.remove(&db.name); } } @@ -170,6 +187,97 @@ impl DbModule { } } +/// Open a persistent database file the platform way. Desktop: the default +/// VFS. ESP-IDF: the `unix-none` VFS — LittleFS has no fcntl file locks, +/// and a Pocket guest's module instance is the file's only writer — plus +/// the flash-friendly pragmas the ESP32-P4 probe validated (TRUNCATE +/// journal: WAL is compiled out on MCU builds; NORMAL sync; a 32 KiB page +/// cache sized for a device heap). +fn open_file(path: &std::path::Path) -> rusqlite::Result { + #[cfg(not(target_os = "espidf"))] + { + Connection::open(path) + } + #[cfg(target_os = "espidf")] + { + let conn = Connection::open_with_flags_and_vfs( + path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + "unix-none", + )?; + conn.query_row("PRAGMA journal_mode=TRUNCATE", [], |_| Ok(()))?; + conn.execute_batch("PRAGMA synchronous=NORMAL; PRAGMA cache_size=-32;")?; + Ok(conn) + } +} + +// --------------------------------------------------------------------------- +// ESP-IDF (ESP32) support +// --------------------------------------------------------------------------- +// SQLite's unix VFS keeps a syscall table referencing a handful of POSIX +// symbols newlib does not provide. Under this crate's configuration +// (unix-none VFS, `-Dlstat=stat`, no symlinks, no dotlock files) the first +// five are never actually called — no-op successes are the honest +// implementations for a filesystem with no users, permissions or symlinks. +// nanosleep IS called (the busy handler sleeps); it routes through +// ESP-IDF's usleep. Compiled only for espidf; a desktop build never sees +// these. The build-environment recipe these link against is in docs/DB.md. +#[cfg(target_os = "espidf")] +mod espidf { + #[repr(C)] + pub struct Timespec { + tv_sec: i64, // espidf_time64: 64-bit time_t + tv_nsec: i32, + } + + unsafe extern "C" { + fn usleep(microseconds: u32) -> i32; + } + + #[unsafe(no_mangle)] + extern "C" fn geteuid() -> u32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn fchmod(_fd: i32, _mode: u32) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn fchown(_fd: i32, _owner: u32, _group: u32) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn utimes( + _path: *const core::ffi::c_char, + _times: *const core::ffi::c_void, + ) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn readlink( + _path: *const core::ffi::c_char, + _buf: *mut core::ffi::c_char, + _len: usize, + ) -> isize { + -1 // never a symlink on LittleFS + } + + #[unsafe(no_mangle)] + extern "C" fn nanosleep(request: *const Timespec, _remain: *mut Timespec) -> i32 { + let request = unsafe { &*request }; + let micros = (request.tv_sec as u64) + .saturating_mul(1_000_000) + .saturating_add((request.tv_nsec as u64) / 1_000); + unsafe { usleep(micros.min(u32::MAX as u64) as u32) } + } +} + /// Logical persistent-database names (spec DB_NAME_PATTERN): /// `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`, spelled out to keep regex out of /// the dependency tree. @@ -312,6 +420,9 @@ fn encode_cell(cell: ValueRef<'_>) -> Result { /// Mount the module as `globalThis.db` on a pocket-mod [`Guest`] — one JS /// function per spec op, marshaled as (i32, String) -> i32/String. +/// Feature `mount` (default); a host with its own QuickJS wiring turns it +/// off and spells these five functions itself. +#[cfg(feature = "mount")] pub fn mount(guest: &pocket_mod::Guest, module: Rc>) -> anyhow::Result<()> { use pocket_mod::qjs::Function; guest.mount("db", |ctx, ns| { @@ -477,9 +588,13 @@ mod tests { let sel = rows(&m.query(h, "SELECT v FROM snap", "[]")); assert_eq!(sel["rows"], json!([[42]])); } + // The database is an ordinary file in the data root — the app's own + // asset, visible to a co-mounted fs module like any of its files. + assert!(dir.join("ledger.sqlite").is_file()); std::fs::remove_dir_all(&dir).unwrap(); } + #[cfg(feature = "mount")] #[test] fn mounted_namespace_serves_a_quickjs_guest() { let guest = pocket_mod::Guest::new().unwrap(); From 6dbb0f472b0ed73df4790abd51586a725ee01d06 Mon Sep 17 00:00:00 2001 From: "Yifeng \"Evan\" Wang" <7312949+doodlewind@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:50:38 +0800 Subject: [PATCH 3/3] fix(db): close the expression-ATTACH escape and align the sim host with the reference core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on #231, verified by probing both hosts with the same binding/ATTACH matrix: - ATTACH AS x bypassed BOTH refusals: rusqlite maps a NULL filename (any non-literal ATTACH argument) to AuthAction::Unknown, so the authorizer's catch-all allowed it, and the sim regex only matched the DATABASE-keyword and string-literal spellings — the probe left a real file on disk from each host. The core now also sets SQLITE_LIMIT_ATTACHED=0 (rusqlite "limits" feature) so every spelling is refused at the engine level; the sim matches the word "attach" anywhere (the documented false-positive trade widens accordingly). - The sim silently accepted named parameters without the $/:/@ prefix (bun binds bare keys) where the reference core fails with "unknown parameter" — an app developed on the sim would break on device. The sim now refuses them with the core's message; the remaining leniency (a PREFIXED key the statement never names is ignored by bun, loud on the core) is documented, since bun exposes no parameter-name introspection. - The sim host's header claimed named databases persist "the way a device keeps its files", but close() dropped the data that a Storage::Dir host keeps. Closed named databases are now stashed with serialize() and restored on reopen, and the close/reopen path is pinned in the SDK test. - DB_NAME_PATTERN tightened from 64 to 57 chars so the reference mapping .sqlite (+7 bytes) stays within the fs module's 64-byte segment ceiling (#238) — without this, a max-length database file is invisible to a co-mounted fs module, contradicting the "visible like any of its files" contract both PRs document. cargo test -p pocket-db 10/10 (new expression-ATTACH test), clippy clean, tests/db.test.ts 18/18, bunx tsc --noEmit clean. Co-Authored-By: Claude Fable 5 --- contracts/spec/db.ts | 7 ++-- docs/DB.md | 13 +++++--- engine/Cargo.toml | 6 ++-- engine/crates/pocket-db/src/lib.rs | 51 ++++++++++++++++++++++++++---- hosts/sim/db.ts | 48 ++++++++++++++++++++++------ tests/db.test.ts | 28 ++++++++++++++-- 6 files changed, 125 insertions(+), 28 deletions(-) diff --git a/contracts/spec/db.ts b/contracts/spec/db.ts index 805dc91e..0c613ef5 100644 --- a/contracts/spec/db.ts +++ b/contracts/spec/db.ts @@ -117,9 +117,12 @@ export const DB_MEMORY = ":memory:"; /** * Logical persistent-database names: a filename-safe token, no paths, no * extensions games. The host maps a name to a real file under the app's own - * data root; the mapping is host policy and never guest-visible. + * data root; the mapping is host policy and never guest-visible. The 57-char + * ceiling keeps the reference mapping `.sqlite` (+7 bytes) within the + * fs module's 64-byte segment ceiling, so a co-mounted fs module can always + * address the database file the docs call "visible like any of its files". */ -export const DB_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +export const DB_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,56}$/; // --------------------------------------------------------------------------- // Data contract — resource ceilings diff --git a/docs/DB.md b/docs/DB.md index eac03264..1076975e 100644 --- a/docs/DB.md +++ b/docs/DB.md @@ -44,17 +44,20 @@ store money in cents. Booleans bind as 1/0; integer-valued numbers bind as INTEGER, fractional as REAL (the bun:sqlite convention). **Storage rule.** `open(name)` is the only path to a database: names are -filename-safe tokens (`DB_NAME_PATTERN`) or `:memory:`, and the host maps a -name to a real file under the app's own data root — the reference core +filename-safe tokens (`DB_NAME_PATTERN`, ≤ 57 chars so `.sqlite` +stays within the fs module's 64-byte segment ceiling) or `:memory:`, and +the host maps a name to a real file under the app's own data root — the reference core spells that mapping `/.sqlite`. The database is an ORDINARY file in the app's home, deliberately visible to a co-mounted fs module: it is the app's own asset (backup = a file copy), and an app that overwrites its own database corrupts its own data — the same trust class as deleting its own files (SQLite fails loudly on a corrupt image). The guest never sees a path. `ATTACH` — the one SQL statement that names a -file — is refused (engine/crates/pocket-db uses a real SQLite -authorizer), and `load_extension` stays disabled, so the data root stays -the sandbox boundary. +file — is refused twice over (engine/crates/pocket-db uses a real SQLite +authorizer for the literal spelling and **`SQLITE_LIMIT_ATTACHED=0`** for +the expression spelling that reaches an authorizer with a NULL filename), +and `load_extension` stays disabled, so the data root stays the sandbox +boundary. **Ceilings.** `DB_MAX_DATABASES` (4) open handles; `DB_MAX_RESULT_ROWS` (4096) rows per `query()` call, above which the op fails with "add LIMIT or diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 66be04c5..9c70faae 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -68,8 +68,10 @@ memmap2 = "0.9" serde = { version = "1", features = ["derive"] } serde_json = "1" # bundled: compile the vendored sqlite3.c — the same source a device host -# compiles with its own toolchain; hooks: the authorizer that refuses ATTACH. -rusqlite = { version = "0.40", features = ["bundled", "hooks"] } +# compiles with its own toolchain; hooks: the authorizer that refuses ATTACH; +# limits: SQLITE_LIMIT_ATTACHED=0, the backstop that refuses ATTACH spellings +# the authorizer cannot see (an expression filename reaches it as NULL). +rusqlite = { version = "0.40", features = ["bundled", "hooks", "limits"] } base64 = "0.23" env_logger = "0.11" font8x8 = "0.3" diff --git a/engine/crates/pocket-db/src/lib.rs b/engine/crates/pocket-db/src/lib.rs index 72e9bf4e..bcf6d1a0 100644 --- a/engine/crates/pocket-db/src/lib.rs +++ b/engine/crates/pocket-db/src/lib.rs @@ -15,9 +15,10 @@ //! its files (backup = a file copy). An app that overwrites its own //! database corrupts its own data — the same trust class as deleting its //! own files, and SQLite fails loudly (SQLITE_CORRUPT), not unsafely. -//! `ATTACH` is refused by a real SQLite authorizer (not string -//! inspection), which keeps that root the sandbox boundary; -//! `load_extension` stays off (rusqlite's default). +//! `ATTACH` is refused twice over — a real SQLite authorizer for the +//! literal spelling, and `SQLITE_LIMIT_ATTACHED=0` for the expression +//! spelling the authorizer cannot see — which keeps that root the sandbox +//! boundary; `load_extension` stays off (rusqlite's default). //! //! ESP32/LittleFS: this crate carries its own ESP-IDF support (the //! `espidf` module below — newlib symbol shims, the `unix-none` VFS, the @@ -119,6 +120,17 @@ impl DbModule { { return -1; } + // The authorizer only sees a FILENAME literal: `ATTACH AS x` + // reaches it with a NULL filename, which rusqlite maps to + // AuthAction::Unknown — allowed by the catch-all above. The engine + // attach limit closes every spelling; the authorizer stays for the + // clearer "not authorized" on the literal form. + if conn + .set_limit(rusqlite::limits::Limit::SQLITE_LIMIT_ATTACHED, 0) + .is_err() + { + return -1; + } let handle = self.next_handle; self.next_handle += 1; self.dbs.insert( @@ -279,11 +291,12 @@ mod espidf { } /// Logical persistent-database names (spec DB_NAME_PATTERN): -/// `^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`, spelled out to keep regex out of -/// the dependency tree. +/// `^[A-Za-z0-9][A-Za-z0-9._-]{0,56}$`, spelled out to keep regex out of +/// the dependency tree. 57 chars keeps `.sqlite` within the fs +/// module's 64-byte segment ceiling. fn valid_name(name: &str) -> bool { let bytes = name.as_bytes(); - if bytes.is_empty() || bytes.len() > 64 { + if bytes.is_empty() || bytes.len() > 57 { return false; } if !bytes[0].is_ascii_alphanumeric() { @@ -500,7 +513,11 @@ mod tests { assert_eq!(m.open("../escape"), -1); assert_eq!(m.open(".hidden"), -1); assert_eq!(m.open(""), -1); - assert_eq!(m.open(&"a".repeat(65)), -1); + // 57 is the ceiling: `.sqlite` stays a valid fs segment. + assert_eq!(m.open(&"a".repeat(58)), -1); + let longest = m.open(&"a".repeat(57)); + assert!(longest > 0); + m.close(longest); for i in 0..spec::MAX_DATABASES { assert!(m.open(&format!("app-{i}")) > 0); } @@ -525,6 +542,26 @@ mod tests { assert!(rows(&line)["error"].as_str().unwrap().contains("not authorized")); } + #[test] + fn attach_with_an_expression_filename_is_refused_by_the_attach_limit() { + // `ATTACH AS x` reaches the authorizer with a NULL filename + // (AuthAction::Unknown), so only SQLITE_LIMIT_ATTACHED=0 refuses it. + let mut m = module(); + let h = m.open(spec::MEMORY); + assert_eq!(m.exec(h, "ATTACH hex('2f746d702f78') AS other"), 1); + assert!( + m.last_error(h).contains("attached databases"), + "{}", + m.last_error(h) + ); + let line = m.query(h, "ATTACH ':memory:' AS other", "[]"); + let error = rows(&line)["error"].as_str().unwrap().to_owned(); + assert!( + error.contains("not authorized") || error.contains("attached databases"), + "{error}" + ); + } + #[test] fn closed_handles_fail_loudly_and_close_is_idempotent() { let mut m = module(); diff --git a/hosts/sim/db.ts b/hosts/sim/db.ts index 539ed1df..11fb8898 100644 --- a/hosts/sim/db.ts +++ b/hosts/sim/db.ts @@ -4,14 +4,22 @@ // Bun's built-in SQLite is the same engine a device host links, so the sim // runs the real dialect — only the storage policy is sim-shaped: every // database, named or DB_MEMORY, lives in memory (no disk, no cleanup), and -// named databases persist for the life of the host object so an app reload -// inside one scenario keeps its data, the way a device keeps its files. +// named databases persist for the life of the host object — across close() +// and reopen (the image is stashed with serialize()) and across an app +// reload inside one scenario — the way a device keeps its files. // -// Two dev-host caveats, both spec-permitted: -// - ATTACH is refused by a whole-string match, not a real authorizer -// (engine/crates/pocket-db carries the authoritative refusal); a string -// literal containing "attach database" is a false positive a test can +// Three dev-host caveats, all spec-permitted: +// - ATTACH is refused by matching the word anywhere in the sql — the only +// net that catches every spelling SQLite accepts (`ATTACH DATABASE f`, +// `ATTACH 'f'`, `ATTACH hex(...)`) without an authorizer, which +// bun:sqlite does not expose (engine/crates/pocket-db carries the +// authoritative refusal: authorizer + SQLITE_LIMIT_ATTACHED=0). A string +// literal containing the word "attach" is a false positive a test can // spell around. +// - a named parameter whose key lacks the $ / : / @ prefix is refused here +// (the reference core fails it with "unknown parameter"), but a PREFIXED +// key the statement never mentions is silently ignored — bun:sqlite +// offers no parameter-name introspection to close that last gap. // - SQL time and randomness are NOT pinned — bun:sqlite exposes no VFS // hook. The spec already forbids golden-tested apps from depending on // them; tests/db.test.ts stays on deterministic SQL. @@ -72,11 +80,14 @@ function decodeParam(v: unknown): unknown { return v; } -const ATTACH = /\battach\s+(database\b|')/i; +const ATTACH = /\battach\b/i; export function createSimDbHost(): SimDbHost { const dbs = new Map(); const byName = new Map(); + // Serialized images of closed named databases — reopening restores them, + // the way Storage::Dir keeps the file after close on a device host. + const closedImages = new Map(); const log: string[] = []; let nextHandle = 1; @@ -93,19 +104,28 @@ export function createSimDbHost(): SimDbHost { if (existing !== undefined) return existing; } if (dbs.size >= DB_MAX_DATABASES) return -1; - const bun = new BunDatabase(":memory:", { safeIntegers: true }); + const image = name !== DB_MEMORY ? closedImages.get(name) : undefined; + const bun = image + ? BunDatabase.deserialize(image, { safeIntegers: true }) + : new BunDatabase(":memory:", { safeIntegers: true }); const handle = nextHandle++; dbs.set(handle, { bun, name, lastError: "" }); - if (name !== DB_MEMORY) byName.set(name, handle); + if (name !== DB_MEMORY) { + closedImages.delete(name); + byName.set(name, handle); + } return handle; }, close(handle: number): void { log.push(`op close ${handle}`); const db = live(handle); if (!db) return; + if (db.name !== DB_MEMORY) { + closedImages.set(db.name, db.bun.serialize()); + byName.delete(db.name); + } db.bun.close(); dbs.delete(handle); - if (db.name !== DB_MEMORY) byName.delete(db.name); }, exec(handle: number, sql: string): number { log.push(`op exec ${handle}`); @@ -134,6 +154,13 @@ export function createSimDbHost(): SimDbHost { } try { const parsed = JSON.parse(args) as unknown[] | Record; + if (!Array.isArray(parsed)) { + // The spec spells named parameters $x / :x / @x; bun would bind a + // bare key anyway, the reference core fails it — fail like the core. + for (const key of Object.keys(parsed)) { + if (!/^[$:@]/.test(key)) throw new Error(`unknown parameter: ${key}`); + } + } const params = Array.isArray(parsed) ? parsed.map(decodeParam) : Object.fromEntries(Object.entries(parsed).map(([k, v]) => [k, decodeParam(v)])); @@ -173,6 +200,7 @@ export function createSimDbHost(): SimDbHost { for (const db of dbs.values()) db.bun.close(); dbs.clear(); byName.clear(); + closedImages.clear(); }, }; } diff --git a/tests/db.test.ts b/tests/db.test.ts index a7049c9b..9db44922 100644 --- a/tests/db.test.ts +++ b/tests/db.test.ts @@ -65,6 +65,22 @@ describe("sim host ops", () => { expect(lastError(h)).toContain("ATTACH"); const result = JSON.parse(query(h, "attach database ':memory:' as other", "[]")); expect(result.error).toContain("ATTACH"); + // Every spelling SQLite accepts, not just the DATABASE-keyword form: + // bare string, and an expression filename (which on the reference core + // reaches the authorizer as NULL and is caught by SQLITE_LIMIT_ATTACHED). + expect(exec(h, "ATTACH ':memory:' AS o1")).toBe(1); + expect(exec(h, "ATTACH hex('2f746d702f78') AS o2")).toBe(1); + }); + + test("a named parameter without the $/:/@ prefix fails like the reference core", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + exec(h, "CREATE TABLE t (a, b)"); + const result = JSON.parse(query(h, "INSERT INTO t VALUES ($a, $b)", '{"a":1,"$b":2}')); + expect(result.error).toBe("unknown parameter: a"); }); test("ops on a closed handle fail with 'database is closed'", () => { @@ -217,6 +233,11 @@ describe("Database SDK", () => { first.run("INSERT INTO snap VALUES (42)"); const second = new Database("ledger"); expect(second.query("SELECT v FROM snap").get()).toEqual({ v: 42 }); + // Through an actual close(), too — the way Storage::Dir keeps the file + // on a device host. + second.close(); + const third = new Database("ledger"); + expect(third.query("SELECT v FROM snap").get()).toEqual({ v: 42 }); }); test("SQL errors surface as thrown Errors with SQLite's message", () => { @@ -260,10 +281,13 @@ describe("bun:sqlite oracle", () => { describe("spec constants", () => { test("DB_NAME_PATTERN accepts tokens and refuses paths", () => { - for (const good of ["app", "portfolio-history", "a.b_c-1", "A"]) { + // 57 chars is the ceiling: `.sqlite` (+7 bytes) stays within the + // fs module's 64-byte segment ceiling, keeping the database file + // addressable by a co-mounted fs module. + for (const good of ["app", "portfolio-history", "a.b_c-1", "A", "a".repeat(57)]) { expect(DB_NAME_PATTERN.test(good)).toBe(true); } - for (const bad of ["", ".hidden", "-lead", "a/b", "a\\b", "..", "a".repeat(65)]) { + for (const bad of ["", ".hidden", "-lead", "a/b", "a\\b", "..", "a".repeat(58)]) { expect(DB_NAME_PATTERN.test(bad)).toBe(false); } });