diff --git a/contracts/spec/db.ts b/contracts/spec/db.ts new file mode 100644 index 00000000..0c613ef5 --- /dev/null +++ b/contracts/spec/db.ts @@ -0,0 +1,142 @@ +// PocketJS db spec — the boundary of the DB module (`globalThis.db`). +// +// This is a MODULE spec in the docs/RUNTIMES.md §5 sense: a vertical slice +// with its own vocabulary, mounted as its own namespace, pinned here as data. +// It is deliberately NOT part of the `ui` op table — db evolves append-only +// in its own op space, and a host adopts it independently of the UI surface +// (capability id `data.sqlite` in contracts/spec/platforms.ts). +// +// The module is SQLite behind a five-op namespace. The engine, the SQL +// dialect, and the file format are SQLite's — the spec pins only what +// crosses the boundary: op codes, the JSON row encoding, the resource +// ceilings, and the storage/clock rules a host must follow. +// +// The four parts of the boundary: +// +// ops guest -> core intent (numeric codes below, append-only) +// events none — every op is synchronous; the module owns no clock +// data contract the JSON value encoding + database name rules (this file) +// frame contract every op completes inside the guest's single per-tick +// turn (law 3 holds unchanged). SQL time and randomness +// resolve host-side: deterministic hosts pin them, and a +// golden-tested app must not depend on `random()` or +// 'now'-relative SQL (same rule as Date.now in guest code). +// +// Storage rule: `open(name)` is the ONLY path to a database. Names are +// logical (below); the HOST maps them to real files under the app's own +// data root (or memory). The guest never sees a path, and a database file +// is never shared between apps. Hosts MUST refuse `ATTACH` — it is the one +// SQL statement that names a file — so the app's data root stays the +// sandbox boundary. `sqlite3_load_extension` stays disabled (SQLite's +// default). +// +// If you change ANY value here: run `bun contracts/spec/gen-rust.ts`, commit +// the regenerated engine/core/src/spec.rs (tests/contract.ts byte-compares). + +// --------------------------------------------------------------------------- +// Db ops (the `db.*` native contract) +// --------------------------------------------------------------------------- +// Numeric codes are the FFI ABI identity of each op. 0 is reserved +// (invalid/nop). Codes are append-only: never renumber, never reuse. +// +// Signatures (authoritative; hosts marshal them however they like): +// open(name:string) -> handle | -1 +// [name is DB_MEMORY or matches DB_NAME_PATTERN; -1 = +// refused (bad name or DB_MAX_DATABASES already open). +// Opening the same persistent name twice returns the +// SAME handle; DB_MEMORY always opens a fresh private +// database] +// close(handle) [idempotent; a closed handle is +// dead and every later op on it +// fails with "database is closed"] +// exec(handle, sql:string) -> 0 | 1 [run one or more statements, no +// result rows — the schema and +// migration path. 1 = failed; the +// detail is lastError()] +// query(handle, sql:string, args:string) -> string +// [run ONE statement with bound parameters and return +// the complete result as one JSON line (shape below). +// `args` is a JSON array (positional) or object (named +// $x / :x / @x) of encoded values. Statement caching is +// HOST-side, keyed by the sql string — the guest holds +// no statement handles] +// lastError(handle) -> string [detail for the last failed +// exec/query on this handle; "" +// when none] +// +// query() result, one JSON object per call (fields append-only): +// +// { "cols": ["id","name"], "rows": [[1,"a"],[2,"b"]], +// "changes": 0, "lastInsertRowid": 0 } +// `cols` are the statement's column names ([] for non-readers), +// `rows` are arrays in column order using the value encoding below. +// `changes` / `lastInsertRowid` are SQLite's counters after the call. +// { "error": "no such table: t" } +// The statement failed (parse, bind, step, or a ceiling below). +// lastError() returns the same string. + +export const DB_OP = { + open: 1, + close: 2, + exec: 3, + query: 4, + lastError: 5, +} as const; + +// --------------------------------------------------------------------------- +// Data contract — value encoding (rows out, parameters in) +// --------------------------------------------------------------------------- +// SQLite value -> JSON value, both directions: +// +// NULL <-> null +// INTEGER <-> number [|v| must be <= 2^53 - 1. A larger stored +// integer makes the op FAIL — a loud error +// beats silent precision loss. Store money +// in cents and ids under 2^53.] +// REAL <-> number [non-finite REAL fails the same way] +// TEXT <-> string +// BLOB <-> { "$b": "" } +// +// Binding accepts additionally: true/false bind as INTEGER 1/0, and a JSON +// number binds as INTEGER when integer-valued, else REAL (the bun:sqlite / +// better-sqlite3 convention the SDK mirrors). + +/** Marker key for a BLOB value inside a row or a parameter list. */ +export const DB_BLOB_KEY = "$b"; + +/** Largest integer magnitude that crosses the boundary losslessly. */ +export const DB_MAX_SAFE_INTEGER = 9007199254740991; + +// --------------------------------------------------------------------------- +// Data contract — database names +// --------------------------------------------------------------------------- + +/** The in-memory database name (private to the handle, gone on close). */ +export const DB_MEMORY = ":memory:"; + +/** + * Logical persistent-database names: a filename-safe token, no paths, no + * extensions games. The host maps a name to a real file under the app's own + * data root; the mapping is host policy and never guest-visible. The 57-char + * ceiling keeps the reference mapping `.sqlite` (+7 bytes) within the + * fs module's 64-byte segment ceiling, so a co-mounted fs module can always + * address the database file the docs call "visible like any of its files". + */ +export const DB_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,56}$/; + +// --------------------------------------------------------------------------- +// Data contract — resource ceilings +// --------------------------------------------------------------------------- + +/** Open databases per guest. An app has its own db plus room for a scratch + * or migration companion. Deliberately tiny — more simultaneous databases + * is a schema smell, not a bigger constant. */ +export const DB_MAX_DATABASES = 4; + +/** + * Result-row ceiling per query() call. A query producing more rows FAILS + * ("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate") — the row + * budget of a 480x272..720x1280 UI is far below this, and an unbounded + * SELECT on a device heap is a bug surfaced early, not a workload. + */ +export const DB_MAX_RESULT_ROWS = 4096; diff --git a/contracts/spec/gen-rust.ts b/contracts/spec/gen-rust.ts index b19bd9f6..6fc96750 100644 --- a/contracts/spec/gen-rust.ts +++ b/contracts/spec/gen-rust.ts @@ -1,4 +1,4 @@ -// Deterministic codegen: contracts/spec/{spec,audio}.ts -> engine/core/src/spec.rs. +// Deterministic codegen: contracts/spec/{spec,audio,db}.ts -> engine/core/src/spec.rs. // // Run from PocketJS/: bun contracts/spec/gen-rust.ts // @@ -16,6 +16,14 @@ import { AUDIO_RING_FRAMES, audioFramesForTick, } from "./audio.ts"; +import { + DB_BLOB_KEY, + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MAX_SAFE_INTEGER, + DB_MEMORY, + DB_OP, +} from "./db.ts"; import { ANALOG_CENTER, ANIMATABLE, @@ -462,6 +470,30 @@ export function generateRust(): string { put(` pub const EVENT_${screaming(name)}: &str = ${JSON.stringify(v)};`); } put("}"); + put(""); + + // --- db module ----------------------------------------------------------- + // The db MODULE's boundary (contracts/spec/db.ts): SQLite behind five + // synchronous ops, mounted as `globalThis.db`, independent of the ui + // surface. A native host implementing it reads these constants; the + // reference implementation is engine/crates/pocket-db. + put("/// DB module boundary (contracts/spec/db.ts — `globalThis.db`)."); + put("/// SQLite behind five synchronous ops; rows cross as one JSON line per"); + put("/// query() call. The module owns no clock and emits no events."); + put("pub mod db {"); + for (const [name, v] of Object.entries(DB_OP)) { + put(` pub const OP_${screaming(name)}: u8 = ${v};`); + } + put(` /// The in-memory database name (private to the handle).`); + put(` pub const MEMORY: &str = ${JSON.stringify(DB_MEMORY)};`); + put(` /// Marker key for a BLOB value inside a row or a parameter list.`); + put(` pub const BLOB_KEY: &str = ${JSON.stringify(DB_BLOB_KEY)};`); + put(` /// Largest integer magnitude that crosses the boundary losslessly.`); + put(` pub const MAX_SAFE_INTEGER: i64 = ${DB_MAX_SAFE_INTEGER};`); + put(` pub const MAX_DATABASES: usize = ${DB_MAX_DATABASES};`); + put(` /// Result-row ceiling per query() call (exceeding it fails the op).`); + put(` pub const MAX_RESULT_ROWS: usize = ${DB_MAX_RESULT_ROWS};`); + put("}"); return L.join("\n") + "\n"; } diff --git a/contracts/spec/platforms.ts b/contracts/spec/platforms.ts index 88ece438..9d44a319 100644 --- a/contracts/spec/platforms.ts +++ b/contracts/spec/platforms.ts @@ -144,6 +144,15 @@ export const POCKET_CAPABILITIES = defineCapabilityRegistry([ // appends the id to its profile only when its native host ships the module // (the ring/thread discipline to copy is hosts/psp/src/audio.rs). "audio.pcm", + // SQLite behind the db module's own namespace (`globalThis.db`, + // contracts/spec/db.ts): five synchronous ops, rows as one JSON line per + // query() call, per-app storage the host confines. Registered ahead of any + // stock TARGET advertising it: the sim host and the engine/crates/pocket-db + // reference core implement and test the whole contract, so apps can already + // declare the requirement and fail admission where the module is absent. A + // device target appends the id to its profile only when its native host + // ships the module. + "data.sqlite", // Copy/cut/paste round-trips with the OS clipboard. "host.clipboard", // The logical viewport is runtime-mutable: the app is told about live diff --git a/docs/DB.md b/docs/DB.md new file mode 100644 index 00000000..1076975e --- /dev/null +++ b/docs/DB.md @@ -0,0 +1,162 @@ +# The DB Module + +DB is PocketJS's fourth module (after `ui`, `strike` and `audio`): SQLite +mounted as `globalThis.db` behind five synchronous ops. Like audio it was +written spec-first — the boundary existed before any host code, every host +implements the same pinned protocol, and a developer adding a data feature +extends the spec instead of forking a host. `contracts/spec/db.ts` is +normative; this page is the map. + +``` +platform storage (POSIX file · LittleFS · memory) Host / substrate + ↑ SQLite's own VFS is the port point +db core: SQLite + handle table + statement cache the module +db spec: ops (open, close, exec, query, lastError) + events (none — every op is synchronous) + data contract (JSON value encoding · name rules · ceilings) + frame contract (no module clock; ops complete in the guest turn) +SDK: @pocketjs/framework/db (Database, Statement — the bun:sqlite shape) + ↓ +app: schema in exec(), rows out of query(), state in tables Guest +``` + +## The boundary in one page + +**Mount.** The module is its own namespace: `globalThis.db`, one method per +op (`DB_OP` codes are the ABI identity, append-only). Capability id +`data.sqlite`. Unlike audio, absence does **not** degrade to a no-op — data +code that silently drops writes is a corruption bug, so the SDK throws where +the namespace is unmounted, and an app declares `data.sqlite` in +`pocket.json` `requires` so admission catches the gap before eval does. + +**Ops** (guest → core, all synchronous): `open(name)` → handle, +`close(handle)`, `exec(handle, sql)` → 0/1 for schema and migrations, +`query(handle, sql, argsJson)` → one JSON line +(`{cols, rows, changes, lastInsertRowid}` or `{error}`), and +`lastError(handle)`. Statement caching is **host-side**, keyed by the sql +string — the guest holds no statement handles, so there is nothing to +finalize and nothing to leak. + +**Values** cross as JSON: NULL ↔ `null`, INTEGER ↔ number, REAL ↔ number, +TEXT ↔ string, BLOB ↔ `{"$b": ""}`. Integers beyond 2^53 − 1 and +non-finite REALs **fail the op** instead of losing precision silently — +store money in cents. Booleans bind as 1/0; integer-valued numbers bind as +INTEGER, fractional as REAL (the bun:sqlite convention). + +**Storage rule.** `open(name)` is the only path to a database: names are +filename-safe tokens (`DB_NAME_PATTERN`, ≤ 57 chars so `.sqlite` +stays within the fs module's 64-byte segment ceiling) or `:memory:`, and +the host maps a name to a real file under the app's own data root — the reference core +spells that mapping `/.sqlite`. The database is an +ORDINARY file in the app's home, deliberately visible to a co-mounted fs +module: it is the app's own asset (backup = a file copy), and an app that +overwrites its own database corrupts its own data — the same trust class +as deleting its own files (SQLite fails loudly on a corrupt image). The +guest never sees a path. `ATTACH` — the one SQL statement that names a +file — is refused twice over (engine/crates/pocket-db uses a real SQLite +authorizer for the literal spelling and **`SQLITE_LIMIT_ATTACHED=0`** for +the expression spelling that reaches an authorizer with a NULL filename), +and `load_extension` stays disabled, so the data root stays the sandbox +boundary. + +**Ceilings.** `DB_MAX_DATABASES` (4) open handles; `DB_MAX_RESULT_ROWS` +(4096) rows per `query()` call, above which the op fails with "add LIMIT or +aggregate" — an unbounded SELECT on a device heap is a bug surfaced early, +not a workload. + +**Frame contract.** The module owns no clock and emits no events: every op +completes inside the guest's single per-tick turn (law 3 unchanged). SQL +time and randomness resolve host-side — deterministic hosts pin them, and a +golden-tested app must not depend on `random()` or 'now'-relative SQL, the +same rule as `Date.now` in guest code. + +## The SDK + +`@pocketjs/framework/db` is the bun:sqlite shape, so code written against +Bun's built-in SQLite runs against the mounted module unchanged: + +```ts +import { Database } from "@pocketjs/framework/db"; + +const db = new Database("portfolio"); +db.exec(`CREATE TABLE IF NOT EXISTS history ( + bucket TEXT PRIMARY KEY, total_cents INTEGER NOT NULL +)`); + +const insert = db.query("INSERT OR REPLACE INTO history VALUES ($bucket, $cents)"); +const record = db.transaction((bucket: string, cents: number) => { + insert.run({ $bucket: bucket, $cents: cents }); +}); +record("2026-08-06T14:35", 1_532_042); + +db.query("SELECT * FROM history ORDER BY bucket DESC LIMIT 288").all(); +``` + +`query()` returns a cached `Statement` (`.get/.all/.values/.run`); +`prepare()` skips the cache; `transaction(fn)` wraps BEGIN/COMMIT with +ROLLBACK on throw and nests as savepoints — batching writes into one +transaction is also the flash-wear discipline on device hosts. + +## Host status + +| Host | Implementation | Status | +|---|---|---| +| sim (`hosts/sim/db.ts`) | bun:sqlite behind the op namespace, injected via `bootWorld` `extraGlobals` | ships with the test host; `tests/db.test.ts` runs the contract and an oracle comparison against bun:sqlite directly | +| reference core (`engine/crates/pocket-db`) | rusqlite (bundled SQLite) + the ATTACH authorizer, mountable on any `pocket-mod` guest as `globalThis.db` | tested including a live QuickJS guest round-trip | +| consoles / devices | — | a target appends `data.sqlite` to its profile when its native host ships the module; the port point is SQLite's VFS (POSIX on desktop, LittleFS-backed on MCU hosts). ESP-IDF support ships in the crate — see below | + +## Adoption path + +A device host that wants the module makes three moves, none of which touch +the spec, the SDK, or any app: + +1. compile SQLite with the platform toolchain and register a VFS for the + platform filesystem (the desktop default VFS already serves + `Storage::Dir`); +2. mount the namespace beside `ui` — `pocket_db::mount(&guest, module)` on + `pocket-mod` hosts, or the raw-QuickJS spelling of the same five + functions elsewhere; +3. append `data.sqlite` to the target's profile in + `contracts/spec/platforms.ts`. + +## ESP32 / ESP-IDF + +`pocket-db` carries its own ESP-IDF support behind +`cfg(target_os = "espidf")` — desktop builds never see it: + +- **newlib shims** for the POSIX symbols SQLite's syscall table references + but newlib lacks (`geteuid`/`fchmod`/`fchown`/`utimes`/`readlink` no-ops + that are honest on a filesystem with no users or symlinks, and + `nanosleep` routed through `usleep` for the busy handler); +- **the `unix-none` VFS** on open — LittleFS has no fcntl file locks, and + a module instance is its files' only writer — plus flash-friendly + pragmas (`journal_mode=TRUNCATE`, `synchronous=NORMAL`, + `cache_size=-32`). + +What the crate cannot carry is the build environment; a firmware adds, in +its `.cargo/config.toml` (values validated on an ESP32-P4, ESP-IDF v5.5.x, +LittleFS workspace — where a 288-row transaction landed in ~0.4 s on +~70–80 KB of heap and survived power cycling): + +```toml +[env] +# The vendored sqlite3.c, tuned for the device: temp tables in memory, no +# mmap, no WAL (shared memory), no dynamic extension loading; lstat does +# not exist on newlib and the VFS never follows symlinks anyway. +LIBSQLITE3_FLAGS = "-DSQLITE_TEMP_STORE=3 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_MAX_MMAP_SIZE=0 -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -Dlstat=stat" +# CC_/AR_/CFLAGS_riscv32imafc_esp_espidf point at the ESP-IDF gcc as usual. +# newlib has no sys/ioctl.h: put an EMPTY sys/ioctl.h in a shim directory +# and add `-isystem ` to CFLAGS. Setting CFLAGS in [env] replaces +# cargo's derived flags, so repeat the arch flags (-mabi/-march) alongside. +``` + +`sqlite3_os_init` for the `unix-none` VFS, journal-file creation, and +power-loss recovery all run against ESP-IDF's VFS layer over LittleFS — +no SQLite source patches, no custom VFS to write. + +A firmware that brings its own QuickJS embedding depends with +`default-features = false`: that drops the `mount` helper and its +pocket-mod/rquickjs dependency, so the MCU build compiles only the module +core plus SQLite. Verified: `cargo check` for `riscv32imafc-esp-espidf` +compiles the crate and the bundled `libsqlite3.a` clean under this +recipe. diff --git a/engine/Cargo.lock b/engine/Cargo.lock index 4138dfe9..65a35f92 100644 --- a/engine/Cargo.lock +++ b/engine/Cargo.lock @@ -187,6 +187,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-set" version = "0.8.0" @@ -535,6 +541,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fdeflate" version = "0.3.7" @@ -689,7 +707,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3ce1918195723ce6ac74e80542c5a96a40c2b26162c1957a5cd70799b8cacf7" dependencies = [ - "base64", + "base64 0.13.1", "byteorder", "gltf-json", "image", @@ -809,6 +827,15 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -820,6 +847,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -1042,6 +1078,17 @@ dependencies = [ "redox_syscall 0.9.0", ] +[[package]] +name = "libsqlite3-sys" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1587,6 +1634,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "pocket-db" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.23.1", + "pocket-mod", + "pocketjs-core", + "rusqlite", + "serde_json", +] + [[package]] name = "pocket-mod" version = "0.1.0" @@ -1915,6 +1974,31 @@ dependencies = [ "cc", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + +[[package]] +name = "rusqlite" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11438310b19e3109b6446c33d1ed5e889428cf2e278407bc7896bc4aaea43323" +dependencies = [ + "bitflags 2.13.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2137,6 +2221,18 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2367,6 +2463,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/engine/Cargo.toml b/engine/Cargo.toml index d4a627f8..9c70faae 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -11,6 +11,7 @@ [workspace] resolver = "2" members = [ + "crates/pocket-db", "crates/pocket-mod", "crates/pocket-ui-surface", "crates/pocket-ui-wgpu", @@ -42,6 +43,7 @@ repository = "https://github.com/pocket-stack/pocketjs" [workspace.dependencies] pocket3d = { path = "pocket3d/crates/pocket3d" } pocket3d-bsp = { path = "pocket3d/crates/pocket3d-bsp" } +pocket-db = { path = "crates/pocket-db" } pocket-mod = { path = "crates/pocket-mod" } pocket-ui-surface = { path = "crates/pocket-ui-surface" } pocket-ui-wgpu = { path = "crates/pocket-ui-wgpu" } @@ -65,6 +67,12 @@ ab_glyph = "0.2" memmap2 = "0.9" serde = { version = "1", features = ["derive"] } serde_json = "1" +# bundled: compile the vendored sqlite3.c — the same source a device host +# compiles with its own toolchain; hooks: the authorizer that refuses ATTACH; +# limits: SQLITE_LIMIT_ATTACHED=0, the backstop that refuses ATTACH spellings +# the authorizer cannot see (an expression filename reaches it as NULL). +rusqlite = { version = "0.40", features = ["bundled", "hooks", "limits"] } +base64 = "0.23" env_logger = "0.11" font8x8 = "0.3" diff --git a/engine/core/src/spec.rs b/engine/core/src/spec.rs index f5d6ac2e..a9035ec3 100644 --- a/engine/core/src/spec.rs +++ b/engine/core/src/spec.rs @@ -498,3 +498,23 @@ pub mod audio { pub const EVENT_UNDERRUN: &str = "underrun"; pub const EVENT_ENDED: &str = "ended"; } + +/// DB module boundary (contracts/spec/db.ts — `globalThis.db`). +/// SQLite behind five synchronous ops; rows cross as one JSON line per +/// query() call. The module owns no clock and emits no events. +pub mod db { + pub const OP_OPEN: u8 = 1; + pub const OP_CLOSE: u8 = 2; + pub const OP_EXEC: u8 = 3; + pub const OP_QUERY: u8 = 4; + pub const OP_LAST_ERROR: u8 = 5; + /// The in-memory database name (private to the handle). + pub const MEMORY: &str = ":memory:"; + /// Marker key for a BLOB value inside a row or a parameter list. + pub const BLOB_KEY: &str = "$b"; + /// Largest integer magnitude that crosses the boundary losslessly. + pub const MAX_SAFE_INTEGER: i64 = 9007199254740991; + pub const MAX_DATABASES: usize = 4; + /// Result-row ceiling per query() call (exceeding it fails the op). + pub const MAX_RESULT_ROWS: usize = 4096; +} diff --git a/engine/crates/pocket-db/Cargo.toml b/engine/crates/pocket-db/Cargo.toml new file mode 100644 index 00000000..85386356 --- /dev/null +++ b/engine/crates/pocket-db/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "pocket-db" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "The db module's reference core: SQLite behind the five-op contracts/spec/db.ts boundary, mountable as globalThis.db via pocket-mod" + +[features] +# `mount` brings pocket-mod (and its QuickJS embedding) for the one-line +# globalThis.db install. A device host with its own QuickJS wiring turns +# it off (`default-features = false`) and drives DbModule directly — the +# MCU build then never compiles an engine it doesn't use. +default = ["mount"] +mount = ["dep:pocket-mod", "dep:anyhow"] + +[dependencies] +pocketjs-core = { workspace = true } +pocket-mod = { workspace = true, optional = true } +anyhow = { workspace = true, optional = true } +rusqlite = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } diff --git a/engine/crates/pocket-db/src/lib.rs b/engine/crates/pocket-db/src/lib.rs new file mode 100644 index 00000000..bcf6d1a0 --- /dev/null +++ b/engine/crates/pocket-db/src/lib.rs @@ -0,0 +1,657 @@ +//! pocket-db — the db module's reference core. +//! +//! SQLite behind the five-op boundary pinned in contracts/spec/db.ts +//! (`pocketjs_core::spec::db` is the generated mirror): open / close / exec / +//! query / lastError, mounted as `globalThis.db` through [`mount`]. Rows +//! cross as one JSON line per `query()` call using the spec's value +//! encoding; statement caching is host-side (rusqlite's prepared-statement +//! cache, keyed by the sql string), so the guest holds no statement handles. +//! +//! Storage policy is the host's: [`Storage::Memory`] for tests and +//! throwaway guests, [`Storage::Dir`] to map each logical database name to +//! `/.sqlite` — an ORDINARY file in the app's own data root, +//! the same root the fs module is typically bound to. That is deliberate: +//! the database is the app's own asset, visible and touchable like any of +//! its files (backup = a file copy). An app that overwrites its own +//! database corrupts its own data — the same trust class as deleting its +//! own files, and SQLite fails loudly (SQLITE_CORRUPT), not unsafely. +//! `ATTACH` is refused twice over — a real SQLite authorizer for the +//! literal spelling, and `SQLITE_LIMIT_ATTACHED=0` for the expression +//! spelling the authorizer cannot see — which keeps that root the sandbox +//! boundary; `load_extension` stays off (rusqlite's default). +//! +//! ESP32/LittleFS: this crate carries its own ESP-IDF support (the +//! `espidf` module below — newlib symbol shims, the `unix-none` VFS, the +//! flash-friendly pragmas), all behind `cfg(target_os = "espidf")`; +//! desktop builds never see it. The build-environment recipe a firmware +//! needs (C flags, header shim) is documented in docs/DB.md — validated +//! on an ESP32-P4 with a LittleFS workspace, where data survived reopen +//! and power cycling. + +#[cfg(feature = "mount")] +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::PathBuf; +#[cfg(feature = "mount")] +use std::rc::Rc; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use pocketjs_core::spec::db as spec; +use rusqlite::hooks::Authorization; +use rusqlite::types::{Value, ValueRef}; +use rusqlite::Connection; +use serde_json::{json, Map as JsonMap, Value as Json}; + +/// Where logical database names live. +pub enum Storage { + /// Every database, named or `:memory:`, is in-memory (tests, previews). + /// Named databases still share a handle for the module's lifetime. + Memory, + /// A named database maps to `/.sqlite` — an ordinary file + /// in the app's own data root (created on first open), the same root + /// the fs module is typically bound to. The guest never sees the path. + Dir(PathBuf), +} + +struct Db { + conn: Connection, + name: String, + last_error: String, +} + +/// The db module: every op as a method, [`mount`] to install the namespace. +pub struct DbModule { + storage: Storage, + dbs: HashMap, + by_name: HashMap, + next_handle: i32, +} + +impl DbModule { + pub fn new(storage: Storage) -> DbModule { + DbModule { + storage, + dbs: HashMap::new(), + by_name: HashMap::new(), + next_handle: 1, + } + } + + /// `open(name) -> handle | -1` (spec OP_OPEN). + pub fn open(&mut self, name: &str) -> i32 { + let memory = name == spec::MEMORY; + if !memory { + if !valid_name(name) { + return -1; + } + if let Some(handle) = self.by_name.get(name) { + return *handle; + } + } + if self.dbs.len() >= spec::MAX_DATABASES { + return -1; + } + let conn = match &self.storage { + Storage::Memory => Connection::open_in_memory(), + Storage::Dir(dir) if !memory => { + if std::fs::create_dir_all(dir).is_err() { + return -1; + } + open_file(&dir.join(format!("{name}.sqlite"))) + } + Storage::Dir(_) => Connection::open_in_memory(), + }; + let conn = match conn { + Ok(conn) => conn, + Err(_) => return -1, + }; + // The storage rule's teeth: ATTACH names a file, so it is denied at + // the engine level. A database the authorizer cannot guard is a + // database this module refuses to open. + if conn + .authorizer(Some(|ctx: rusqlite::hooks::AuthContext<'_>| { + match ctx.action { + rusqlite::hooks::AuthAction::Attach { .. } => Authorization::Deny, + _ => Authorization::Allow, + } + })) + .is_err() + { + return -1; + } + // The authorizer only sees a FILENAME literal: `ATTACH AS x` + // reaches it with a NULL filename, which rusqlite maps to + // AuthAction::Unknown — allowed by the catch-all above. The engine + // attach limit closes every spelling; the authorizer stays for the + // clearer "not authorized" on the literal form. + if conn + .set_limit(rusqlite::limits::Limit::SQLITE_LIMIT_ATTACHED, 0) + .is_err() + { + return -1; + } + let handle = self.next_handle; + self.next_handle += 1; + self.dbs.insert( + handle, + Db { + conn, + name: name.to_owned(), + last_error: String::new(), + }, + ); + if !memory { + self.by_name.insert(name.to_owned(), handle); + } + handle + } + + /// `close(handle)` (spec OP_CLOSE) — idempotent. + pub fn close(&mut self, handle: i32) { + if let Some(db) = self.dbs.remove(&handle) + && db.name != spec::MEMORY + { + self.by_name.remove(&db.name); + } + } + + /// `exec(handle, sql) -> 0 | 1` (spec OP_EXEC). + pub fn exec(&mut self, handle: i32, sql: &str) -> i32 { + let Some(db) = self.dbs.get_mut(&handle) else { + return 1; + }; + match db.conn.execute_batch(sql) { + Ok(()) => { + db.last_error.clear(); + 0 + } + Err(error) => { + db.last_error = error.to_string(); + 1 + } + } + } + + /// `query(handle, sql, args) -> json line` (spec OP_QUERY). + pub fn query(&mut self, handle: i32, sql: &str, args: &str) -> String { + let Some(db) = self.dbs.get_mut(&handle) else { + return json!({ "error": "database is closed" }).to_string(); + }; + match run_query(&db.conn, sql, args) { + Ok(line) => { + db.last_error.clear(); + line + } + Err(message) => { + db.last_error = message.clone(); + json!({ "error": message }).to_string() + } + } + } + + /// `lastError(handle) -> string` (spec OP_LAST_ERROR). + pub fn last_error(&self, handle: i32) -> String { + match self.dbs.get(&handle) { + Some(db) => db.last_error.clone(), + None => "database is closed".to_owned(), + } + } +} + +/// Open a persistent database file the platform way. Desktop: the default +/// VFS. ESP-IDF: the `unix-none` VFS — LittleFS has no fcntl file locks, +/// and a Pocket guest's module instance is the file's only writer — plus +/// the flash-friendly pragmas the ESP32-P4 probe validated (TRUNCATE +/// journal: WAL is compiled out on MCU builds; NORMAL sync; a 32 KiB page +/// cache sized for a device heap). +fn open_file(path: &std::path::Path) -> rusqlite::Result { + #[cfg(not(target_os = "espidf"))] + { + Connection::open(path) + } + #[cfg(target_os = "espidf")] + { + let conn = Connection::open_with_flags_and_vfs( + path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + "unix-none", + )?; + conn.query_row("PRAGMA journal_mode=TRUNCATE", [], |_| Ok(()))?; + conn.execute_batch("PRAGMA synchronous=NORMAL; PRAGMA cache_size=-32;")?; + Ok(conn) + } +} + +// --------------------------------------------------------------------------- +// ESP-IDF (ESP32) support +// --------------------------------------------------------------------------- +// SQLite's unix VFS keeps a syscall table referencing a handful of POSIX +// symbols newlib does not provide. Under this crate's configuration +// (unix-none VFS, `-Dlstat=stat`, no symlinks, no dotlock files) the first +// five are never actually called — no-op successes are the honest +// implementations for a filesystem with no users, permissions or symlinks. +// nanosleep IS called (the busy handler sleeps); it routes through +// ESP-IDF's usleep. Compiled only for espidf; a desktop build never sees +// these. The build-environment recipe these link against is in docs/DB.md. +#[cfg(target_os = "espidf")] +mod espidf { + #[repr(C)] + pub struct Timespec { + tv_sec: i64, // espidf_time64: 64-bit time_t + tv_nsec: i32, + } + + unsafe extern "C" { + fn usleep(microseconds: u32) -> i32; + } + + #[unsafe(no_mangle)] + extern "C" fn geteuid() -> u32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn fchmod(_fd: i32, _mode: u32) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn fchown(_fd: i32, _owner: u32, _group: u32) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn utimes( + _path: *const core::ffi::c_char, + _times: *const core::ffi::c_void, + ) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn readlink( + _path: *const core::ffi::c_char, + _buf: *mut core::ffi::c_char, + _len: usize, + ) -> isize { + -1 // never a symlink on LittleFS + } + + #[unsafe(no_mangle)] + extern "C" fn nanosleep(request: *const Timespec, _remain: *mut Timespec) -> i32 { + let request = unsafe { &*request }; + let micros = (request.tv_sec as u64) + .saturating_mul(1_000_000) + .saturating_add((request.tv_nsec as u64) / 1_000); + unsafe { usleep(micros.min(u32::MAX as u64) as u32) } + } +} + +/// Logical persistent-database names (spec DB_NAME_PATTERN): +/// `^[A-Za-z0-9][A-Za-z0-9._-]{0,56}$`, spelled out to keep regex out of +/// the dependency tree. 57 chars keeps `.sqlite` within the fs +/// module's 64-byte segment ceiling. +fn valid_name(name: &str) -> bool { + let bytes = name.as_bytes(); + if bytes.is_empty() || bytes.len() > 57 { + return false; + } + if !bytes[0].is_ascii_alphanumeric() { + return false; + } + bytes[1..] + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +fn run_query(conn: &Connection, sql: &str, args: &str) -> Result { + let params: Json = serde_json::from_str(args).map_err(|e| format!("malformed args: {e}"))?; + let mut statement = conn + .prepare_cached(sql) + .map_err(|e| e.to_string())?; + + match ¶ms { + Json::Array(list) => { + if list.len() != statement.parameter_count() { + return Err(format!( + "expected {} parameters, got {}", + statement.parameter_count(), + list.len() + )); + } + for (i, value) in list.iter().enumerate() { + statement + .raw_bind_parameter(i + 1, decode_param(value)?) + .map_err(|e| e.to_string())?; + } + } + Json::Object(named) => { + for (key, value) in named { + let index = statement + .parameter_index(key) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("unknown parameter: {key}"))?; + statement + .raw_bind_parameter(index, decode_param(value)?) + .map_err(|e| e.to_string())?; + } + } + _ => return Err("args must be a JSON array or object".to_owned()), + } + + let cols: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let column_count = cols.len(); + + let mut rows_out: Vec = Vec::new(); + let mut rows = statement.raw_query(); + loop { + let row = rows.next().map_err(|e| e.to_string())?; + let Some(row) = row else { break }; + if rows_out.len() >= spec::MAX_RESULT_ROWS { + return Err("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate".to_owned()); + } + let mut cells: Vec = Vec::with_capacity(column_count); + for i in 0..column_count { + cells.push(encode_cell(row.get_ref(i).map_err(|e| e.to_string())?)?); + } + rows_out.push(Json::Array(cells)); + } + drop(rows); + + Ok(json!({ + "cols": cols, + "rows": rows_out, + "changes": conn.changes(), + "lastInsertRowid": conn.last_insert_rowid(), + }) + .to_string()) +} + +/// JSON parameter -> SQLite value (the spec's binding rules). +fn decode_param(value: &Json) -> Result { + Ok(match value { + Json::Null => Value::Null, + Json::Bool(b) => Value::Integer(i64::from(*b)), + Json::Number(n) => { + if let Some(i) = n.as_i64() { + if i.unsigned_abs() > spec::MAX_SAFE_INTEGER as u64 { + return Err("integer exceeds DB_MAX_SAFE_INTEGER".to_owned()); + } + Value::Integer(i) + } else { + let f = n.as_f64().ok_or("unrepresentable number")?; + if !f.is_finite() { + return Err("cannot bind a non-finite number".to_owned()); + } + Value::Real(f) + } + } + Json::String(s) => Value::Text(s.clone()), + Json::Object(map) => Value::Blob(decode_blob(map)?), + Json::Array(_) => return Err("cannot bind an array value".to_owned()), + }) +} + +fn decode_blob(map: &JsonMap) -> Result, String> { + let Some(Json::String(b64)) = map.get(spec::BLOB_KEY) else { + return Err("malformed blob parameter".to_owned()); + }; + BASE64 + .decode(b64) + .map_err(|e| format!("malformed blob parameter: {e}")) +} + +/// SQLite cell -> JSON value (the spec's row encoding; loud on lossy). +fn encode_cell(cell: ValueRef<'_>) -> Result { + Ok(match cell { + ValueRef::Null => Json::Null, + ValueRef::Integer(i) => { + if i.unsigned_abs() > spec::MAX_SAFE_INTEGER as u64 { + return Err("integer result exceeds DB_MAX_SAFE_INTEGER".to_owned()); + } + json!(i) + } + ValueRef::Real(f) => { + if !f.is_finite() { + return Err("non-finite REAL result".to_owned()); + } + json!(f) + } + ValueRef::Text(t) => { + Json::String(String::from_utf8(t.to_vec()).map_err(|_| "non-UTF-8 TEXT result")?) + } + ValueRef::Blob(b) => json!({ spec::BLOB_KEY: BASE64.encode(b) }), + }) +} + +/// Mount the module as `globalThis.db` on a pocket-mod [`Guest`] — one JS +/// function per spec op, marshaled as (i32, String) -> i32/String. +/// Feature `mount` (default); a host with its own QuickJS wiring turns it +/// off and spells these five functions itself. +#[cfg(feature = "mount")] +pub fn mount(guest: &pocket_mod::Guest, module: Rc>) -> anyhow::Result<()> { + use pocket_mod::qjs::Function; + guest.mount("db", |ctx, ns| { + let m = module.clone(); + ns.set( + "open", + Function::new(ctx.clone(), move |name: String| -> i32 { + m.borrow_mut().open(&name) + })?, + )?; + let m = module.clone(); + ns.set( + "close", + Function::new(ctx.clone(), move |handle: i32| { + m.borrow_mut().close(handle); + })?, + )?; + let m = module.clone(); + ns.set( + "exec", + Function::new(ctx.clone(), move |handle: i32, sql: String| -> i32 { + m.borrow_mut().exec(handle, &sql) + })?, + )?; + let m = module.clone(); + ns.set( + "query", + Function::new( + ctx.clone(), + move |handle: i32, sql: String, args: String| -> String { + m.borrow_mut().query(handle, &sql, &args) + }, + )?, + )?; + let m = module.clone(); + ns.set( + "lastError", + Function::new(ctx.clone(), move |handle: i32| -> String { + m.borrow().last_error(handle) + })?, + )?; + Ok(()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn module() -> DbModule { + DbModule::new(Storage::Memory) + } + + fn rows(line: &str) -> Json { + serde_json::from_str(line).unwrap() + } + + #[test] + fn crud_round_trip_with_positional_and_named_parameters() { + let mut m = module(); + let h = m.open(spec::MEMORY); + assert!(h > 0); + assert_eq!(m.exec(h, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"), 0); + let ins = rows(&m.query(h, "INSERT INTO t (v) VALUES ($v)", r#"{"$v":"hello"}"#)); + assert_eq!(ins["changes"], 1); + assert_eq!(ins["lastInsertRowid"], 1); + let sel = rows(&m.query(h, "SELECT id, v FROM t WHERE id = ?", "[1]")); + assert_eq!(sel["cols"], json!(["id", "v"])); + assert_eq!(sel["rows"], json!([[1, "hello"]])); + } + + #[test] + fn open_refuses_bad_names_and_over_limit() { + let mut m = module(); + assert_eq!(m.open("../escape"), -1); + assert_eq!(m.open(".hidden"), -1); + assert_eq!(m.open(""), -1); + // 57 is the ceiling: `.sqlite` stays a valid fs segment. + assert_eq!(m.open(&"a".repeat(58)), -1); + let longest = m.open(&"a".repeat(57)); + assert!(longest > 0); + m.close(longest); + for i in 0..spec::MAX_DATABASES { + assert!(m.open(&format!("app-{i}")) > 0); + } + assert_eq!(m.open("one-too-many"), -1); + } + + #[test] + fn same_persistent_name_shares_a_handle_and_memory_never_does() { + let mut m = module(); + let a = m.open("app"); + assert_eq!(m.open("app"), a); + assert_ne!(m.open(spec::MEMORY), m.open(spec::MEMORY)); + } + + #[test] + fn attach_is_denied_by_the_authorizer() { + let mut m = module(); + let h = m.open(spec::MEMORY); + assert_eq!(m.exec(h, "ATTACH DATABASE ':memory:' AS other"), 1); + assert!(m.last_error(h).contains("not authorized"), "{}", m.last_error(h)); + let line = m.query(h, "ATTACH DATABASE ':memory:' AS other", "[]"); + assert!(rows(&line)["error"].as_str().unwrap().contains("not authorized")); + } + + #[test] + fn attach_with_an_expression_filename_is_refused_by_the_attach_limit() { + // `ATTACH AS x` reaches the authorizer with a NULL filename + // (AuthAction::Unknown), so only SQLITE_LIMIT_ATTACHED=0 refuses it. + let mut m = module(); + let h = m.open(spec::MEMORY); + assert_eq!(m.exec(h, "ATTACH hex('2f746d702f78') AS other"), 1); + assert!( + m.last_error(h).contains("attached databases"), + "{}", + m.last_error(h) + ); + let line = m.query(h, "ATTACH ':memory:' AS other", "[]"); + let error = rows(&line)["error"].as_str().unwrap().to_owned(); + assert!( + error.contains("not authorized") || error.contains("attached databases"), + "{error}" + ); + } + + #[test] + fn closed_handles_fail_loudly_and_close_is_idempotent() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.close(h); + m.close(h); + assert_eq!(m.exec(h, "SELECT 1"), 1); + assert_eq!(rows(&m.query(h, "SELECT 1", "[]"))["error"], "database is closed"); + assert_eq!(m.last_error(h), "database is closed"); + } + + #[test] + fn blobs_round_trip_and_big_integers_fail_loudly() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.exec(h, "CREATE TABLE b (data BLOB)"); + let ins = m.query(h, "INSERT INTO b VALUES (?)", r#"[{"$b":"AAEC+vv8/f7/"}]"#); + assert_eq!(rows(&ins)["changes"], 1); + let sel = rows(&m.query(h, "SELECT data FROM b", "[]")); + assert_eq!(sel["rows"][0][0][spec::BLOB_KEY], "AAEC+vv8/f7/"); + + let over = rows(&m.query(h, "SELECT 9007199254740993", "[]")); + assert!(over["error"].as_str().unwrap().contains("DB_MAX_SAFE_INTEGER")); + let ok = rows(&m.query(h, "SELECT 9007199254740991", "[]")); + assert_eq!(ok["rows"], json!([[9007199254740991i64]])); + } + + #[test] + fn result_rows_beyond_the_ceiling_fail() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.exec(h, "CREATE TABLE n (v INTEGER)"); + let fill = format!( + "WITH RECURSIVE seq(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM seq LIMIT {}) + INSERT INTO n SELECT x FROM seq", + spec::MAX_RESULT_ROWS + 1 + ); + assert_eq!(m.exec(h, &fill), 0); + let over = rows(&m.query(h, "SELECT v FROM n", "[]")); + assert!(over["error"].as_str().unwrap().contains("DB_MAX_RESULT_ROWS")); + let capped = rows(&m.query( + h, + &format!("SELECT v FROM n LIMIT {}", spec::MAX_RESULT_ROWS), + "[]", + )); + assert_eq!(capped["rows"].as_array().unwrap().len(), spec::MAX_RESULT_ROWS); + } + + #[test] + fn persistent_dir_storage_survives_reopen() { + let dir = std::env::temp_dir().join(format!("pocket-db-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + { + let mut m = DbModule::new(Storage::Dir(dir.clone())); + let h = m.open("ledger"); + m.exec(h, "CREATE TABLE snap (v INTEGER); INSERT INTO snap VALUES (42)"); + } + { + let mut m = DbModule::new(Storage::Dir(dir.clone())); + let h = m.open("ledger"); + let sel = rows(&m.query(h, "SELECT v FROM snap", "[]")); + assert_eq!(sel["rows"], json!([[42]])); + } + // The database is an ordinary file in the data root — the app's own + // asset, visible to a co-mounted fs module like any of its files. + assert!(dir.join("ledger.sqlite").is_file()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[cfg(feature = "mount")] + #[test] + fn mounted_namespace_serves_a_quickjs_guest() { + let guest = pocket_mod::Guest::new().unwrap(); + let module = Rc::new(RefCell::new(module())); + mount(&guest, module).unwrap(); + guest + .eval( + "boot", + r#" + const h = db.open(":memory:"); + if (h < 0) throw new Error("open failed"); + if (db.exec(h, "CREATE TABLE t (v TEXT)") !== 0) throw new Error(db.lastError(h)); + const ins = JSON.parse(db.query(h, "INSERT INTO t VALUES (?)", '["from-guest"]')); + if (ins.changes !== 1) throw new Error("insert failed"); + const sel = JSON.parse(db.query(h, "SELECT v FROM t", "[]")); + globalThis.result = sel.rows[0][0]; + "#, + ) + .unwrap(); + let result: String = guest.with(|ctx| ctx.globals().get("result").unwrap()); + assert_eq!(result, "from-guest"); + } +} diff --git a/framework/compiler/subpaths.ts b/framework/compiler/subpaths.ts index 451dad23..3eca570b 100644 --- a/framework/compiler/subpaths.ts +++ b/framework/compiler/subpaths.ts @@ -55,6 +55,7 @@ export const SUBPATHS: Record = { audio: { file: "framework/src/audio-api.ts", aliases: TWINS }, clock: { file: "framework/src/clock.ts", aliases: TWINS }, config: { file: "framework/src/config.ts" }, + db: { file: "framework/src/db-api.ts", aliases: TWINS }, components: { file: { solid: "framework/src/components.ts", diff --git a/framework/src/db-api.ts b/framework/src/db-api.ts new file mode 100644 index 00000000..8e64d0f1 --- /dev/null +++ b/framework/src/db-api.ts @@ -0,0 +1,291 @@ +// Db module SDK — the thin guest-side algebra over the `db` spec +// (contracts/spec/db.ts). Framework-agnostic (no solid-js, no JSX): the +// same file serves ./db, ./vue-vapor/db and ./octane/db. +// +// The API is the bun:sqlite shape — `new Database(name)`, `db.query(sql)` +// returning a cached Statement with `.get/.all/.values/.run`, `db.exec`, +// `db.transaction(fn)` — so code written against Bun's built-in SQLite runs +// against the mounted module unchanged, and the model of "SQL in, plain +// objects out" carries across hosts. Two deliberate deviations from +// bun:sqlite, both from the spec: +// +// - a Statement is a guest-side (db, sql) pair — statement caching is +// HOST-side, keyed by the sql string, so there is nothing to finalize +// and no handle to leak. `columnNames` is populated by execution +// (empty before the first run). +// - integers beyond 2^53 - 1 and non-finite REALs fail loudly instead of +// losing precision (DB_MAX_SAFE_INTEGER; store money in cents). +// +// Unlike the audio SDK, absence does NOT degrade to a no-op: data code that +// silently drops writes is a corruption bug, not a missing enhancement. +// `new Database(...)` throws where `globalThis.db` is unmounted — declare +// `data.sqlite` in pocket.json `requires` so admission catches it first. + +import { DB_BLOB_KEY, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; + +export { DB_MAX_RESULT_ROWS, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; + +/** The mounted db namespace — one method per spec op (DB_OP codes). */ +export interface DbOps { + open(name: string): number; + close(handle: number): void; + exec(handle: number, sql: string): number; + query(handle: number, sql: string, args: string): string; + lastError(handle: number): string; +} + +/** The db module namespace, or null where the host doesn't mount one. + * A live lookup (not cached): hosts install `globalThis.db` before eval + * and reset it per app load, exactly like `globalThis.ui`. */ +export function dbHost(): DbOps | null { + const ns = (globalThis as { db?: unknown }).db; + if (!ns || typeof ns !== "object") return null; + return typeof (ns as DbOps).open === "function" ? (ns as DbOps) : null; +} + +// --------------------------------------------------------------------------- +// Value encoding (the contracts/spec/db.ts data contract, both directions) +// --------------------------------------------------------------------------- + +/** A value crossing the boundary: what a row cell or a bound parameter is. */ +export type SqlValue = null | number | string | boolean | Uint8Array; +export type SqlParams = readonly SqlValue[] | Readonly>; + +const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/** QuickJS has no btoa/Buffer; the codec is spelled out (cold path). */ +function bytesToBase64(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += 3) { + const a = bytes[i]; + const b = i + 1 < bytes.length ? bytes[i + 1] : 0; + const c = i + 2 < bytes.length ? bytes[i + 2] : 0; + out += B64[a >> 2] + B64[((a & 3) << 4) | (b >> 4)]; + out += i + 1 < bytes.length ? B64[((b & 15) << 2) | (c >> 6)] : "="; + out += i + 2 < bytes.length ? B64[c & 63] : "="; + } + return out; +} + +const B64_INV: Record = {}; +for (let i = 0; i < B64.length; i++) B64_INV[B64[i]] = i; + +function base64ToBytes(s: string): Uint8Array { + let pad = 0; + while (s.endsWith("=")) { + pad++; + s = s.slice(0, -1); + } + const out = new Uint8Array(Math.floor((s.length * 3) / 4)); + let o = 0; + for (let i = 0; i < s.length; i += 4) { + const n = + (B64_INV[s[i]] << 18) | + ((B64_INV[s[i + 1]] ?? 0) << 12) | + ((B64_INV[s[i + 2]] ?? 0) << 6) | + (B64_INV[s[i + 3]] ?? 0); + out[o++] = n >> 16; + if (o < out.length) out[o++] = (n >> 8) & 0xff; + if (o < out.length) out[o++] = n & 0xff; + } + return out; +} + +function encodeValue(v: SqlValue): unknown { + if (v instanceof Uint8Array) return { [DB_BLOB_KEY]: bytesToBase64(v) }; + if (typeof v === "number" && !Number.isFinite(v)) { + throw new Error("db: cannot bind a non-finite number"); + } + if (typeof v === "number" && Number.isInteger(v) && Math.abs(v) > DB_MAX_SAFE_INTEGER) { + throw new Error("db: integer exceeds DB_MAX_SAFE_INTEGER"); + } + return v; +} + +function decodeValue(v: unknown): SqlValue { + if (v !== null && typeof v === "object") { + return base64ToBytes((v as Record)[DB_BLOB_KEY]); + } + return v as SqlValue; +} + +function encodeParams(params: SqlParams): string { + if (Array.isArray(params)) return JSON.stringify(params.map(encodeValue)); + const out: Record = {}; + for (const [k, v] of Object.entries(params)) out[k] = encodeValue(v as SqlValue); + return JSON.stringify(out); +} + +// --------------------------------------------------------------------------- +// Database / Statement (the bun:sqlite shape) +// --------------------------------------------------------------------------- + +interface QueryResult { + cols?: string[]; + rows?: unknown[][]; + changes?: number; + lastInsertRowid?: number; + error?: string; +} + +export interface RunResult { + changes: number; + lastInsertRowid: number; +} + +export class Statement { + private cols: string[] = []; + + constructor( + private readonly ops: DbOps, + private readonly handle: number, + private readonly sql: string, + ) {} + + /** Column names of the last execution ([] before the first run). */ + get columnNames(): readonly string[] { + return this.cols; + } + + private execute(params: SqlParams): QueryResult { + const line = this.ops.query(this.handle, this.sql, encodeParams(params)); + const result = JSON.parse(line) as QueryResult; + if (result.error !== undefined) throw new Error(`db: ${result.error}`); + this.cols = result.cols ?? []; + return result; + } + + /** First row as a column-name keyed object, or null. */ + get(...params: SqlValue[]): Record | null; + get(params: SqlParams): Record | null; + get(...args: unknown[]): Record | null { + const rows = this.values(...(args as SqlValue[])); + if (rows.length === 0) return null; + const out: Record = {}; + this.cols.forEach((c, i) => (out[c] = rows[0][i])); + return out; + } + + /** Every row as a column-name keyed object. */ + all(...params: SqlValue[]): Record[]; + all(params: SqlParams): Record[]; + all(...args: unknown[]): Record[] { + const rows = this.values(...(args as SqlValue[])); + return rows.map((r) => { + const out: Record = {}; + this.cols.forEach((c, i) => (out[c] = r[i])); + return out; + }); + } + + /** Every row as an array in column order. */ + values(...params: SqlValue[]): SqlValue[][]; + values(params: SqlParams): SqlValue[][]; + values(...args: unknown[]): SqlValue[][] { + const params = normalizeArgs(args); + const result = this.execute(params); + return (result.rows ?? []).map((r) => r.map(decodeValue)); + } + + /** Execute for effect; rows (if any) are discarded. */ + run(...params: SqlValue[]): RunResult; + run(params: SqlParams): RunResult; + run(...args: unknown[]): RunResult { + const result = this.execute(normalizeArgs(args)); + return { changes: result.changes ?? 0, lastInsertRowid: result.lastInsertRowid ?? 0 }; + } +} + +/** Spread positional values, one array, or one named-parameter object. */ +function normalizeArgs(args: unknown[]): SqlParams { + if (args.length === 1 && Array.isArray(args[0])) return args[0] as SqlValue[]; + if ( + args.length === 1 && + args[0] !== null && + typeof args[0] === "object" && + !(args[0] instanceof Uint8Array) + ) { + return args[0] as Record; + } + return args as SqlValue[]; +} + +export class Database { + private readonly ops: DbOps; + private readonly handle: number; + private readonly statements = new Map(); + private txDepth = 0; + + constructor(name: string = DB_MEMORY) { + const ops = dbHost(); + if (!ops) { + throw new Error( + "db: globalThis.db is not mounted — declare `data.sqlite` in pocket.json requires", + ); + } + const handle = ops.open(name); + if (handle < 0) throw new Error(`db: open(${JSON.stringify(name)}) refused`); + this.ops = ops; + this.handle = handle; + } + + /** Cached statement for `sql` (host-side prepare cache backs it). */ + query(sql: string): Statement { + let statement = this.statements.get(sql); + if (!statement) { + statement = new Statement(this.ops, this.handle, sql); + this.statements.set(sql, statement); + } + return statement; + } + + /** Uncached statement (the bun:sqlite `prepare` spelling). */ + prepare(sql: string): Statement { + return new Statement(this.ops, this.handle, sql); + } + + /** Run one statement with parameters, for effect. */ + run(sql: string, params: SqlParams = []): RunResult { + return this.query(sql).run(params); + } + + /** Run one or more statements with no parameters and no result rows — + * the schema/migration path. */ + exec(sql: string): void { + if (this.ops.exec(this.handle, sql) !== 0) { + throw new Error(`db: ${this.ops.lastError(this.handle)}`); + } + } + + /** + * Wrap `fn` in BEGIN/COMMIT with ROLLBACK on throw; nested calls become + * savepoints (the bun:sqlite convention). Batching writes into one + * transaction is also the flash-wear discipline on device hosts. + */ + transaction(fn: (...args: A) => R): (...args: A) => R { + return (...args: A): R => { + const name = `pocket_tx_${this.txDepth}`; + const [begin, commit, rollback] = + this.txDepth === 0 + ? ["BEGIN", "COMMIT", "ROLLBACK"] + : [`SAVEPOINT ${name}`, `RELEASE ${name}`, `ROLLBACK TO ${name}; RELEASE ${name}`]; + this.exec(begin); + this.txDepth++; + try { + const result = fn(...args); + this.txDepth--; + this.exec(commit); + return result; + } catch (error) { + this.txDepth--; + this.exec(rollback); + throw error; + } + }; + } + + close(): void { + this.statements.clear(); + this.ops.close(this.handle); + } +} diff --git a/hosts/sim/db.ts b/hosts/sim/db.ts new file mode 100644 index 00000000..11fb8898 --- /dev/null +++ b/hosts/sim/db.ts @@ -0,0 +1,206 @@ +// hosts/sim/db.ts — the bun:sqlite-backed implementation of the db module +// (contracts/spec/db.ts) for the headless sim host. +// +// Bun's built-in SQLite is the same engine a device host links, so the sim +// runs the real dialect — only the storage policy is sim-shaped: every +// database, named or DB_MEMORY, lives in memory (no disk, no cleanup), and +// named databases persist for the life of the host object — across close() +// and reopen (the image is stashed with serialize()) and across an app +// reload inside one scenario — the way a device keeps its files. +// +// Three dev-host caveats, all spec-permitted: +// - ATTACH is refused by matching the word anywhere in the sql — the only +// net that catches every spelling SQLite accepts (`ATTACH DATABASE f`, +// `ATTACH 'f'`, `ATTACH hex(...)`) without an authorizer, which +// bun:sqlite does not expose (engine/crates/pocket-db carries the +// authoritative refusal: authorizer + SQLITE_LIMIT_ATTACHED=0). A string +// literal containing the word "attach" is a false positive a test can +// spell around. +// - a named parameter whose key lacks the $ / : / @ prefix is refused here +// (the reference core fails it with "unknown parameter"), but a PREFIXED +// key the statement never mentions is silently ignored — bun:sqlite +// offers no parameter-name introspection to close that last gap. +// - SQL time and randomness are NOT pinned — bun:sqlite exposes no VFS +// hook. The spec already forbids golden-tested apps from depending on +// them; tests/db.test.ts stays on deterministic SQL. +// +// Inject via bootWorld's extraGlobals: { db: host.ns }, the way a device +// host mounts the namespace beside `ui`. + +import { Database as BunDatabase } from "bun:sqlite"; +import { + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MAX_SAFE_INTEGER, + DB_MEMORY, + DB_NAME_PATTERN, + DB_BLOB_KEY, +} from "../../contracts/spec/db.ts"; + +interface SimDb { + bun: BunDatabase; + name: string; + lastError: string; +} + +export interface SimDbHost { + /** The `globalThis.db` namespace (one method per DB_OP). */ + ns: Record; + /** Every op call in order (for trace assertions). */ + log: string[]; + /** Close every underlying database (end of scenario). */ + dispose(): void; +} + +function encodeCell(v: unknown): unknown { + if (v === null || typeof v === "string") return v; + if (typeof v === "bigint") { + if (v > BigInt(DB_MAX_SAFE_INTEGER) || v < -BigInt(DB_MAX_SAFE_INTEGER)) { + throw new Error("integer result exceeds DB_MAX_SAFE_INTEGER"); + } + return Number(v); + } + if (typeof v === "number") { + if (!Number.isFinite(v)) throw new Error("non-finite REAL result"); + return v; + } + if (v instanceof Uint8Array) return { [DB_BLOB_KEY]: Buffer.from(v).toString("base64") }; + throw new Error(`unencodable result value: ${typeof v}`); +} + +function decodeParam(v: unknown): unknown { + if (v !== null && typeof v === "object" && !Array.isArray(v)) { + const b64 = (v as Record)[DB_BLOB_KEY]; + if (typeof b64 !== "string") throw new Error("malformed blob parameter"); + return new Uint8Array(Buffer.from(b64, "base64")); + } + if (typeof v === "number" && !Number.isFinite(v)) { + throw new Error("cannot bind a non-finite number"); + } + return v; +} + +const ATTACH = /\battach\b/i; + +export function createSimDbHost(): SimDbHost { + const dbs = new Map(); + const byName = new Map(); + // Serialized images of closed named databases — reopening restores them, + // the way Storage::Dir keeps the file after close on a device host. + const closedImages = new Map(); + const log: string[] = []; + let nextHandle = 1; + + function live(handle: number): SimDb | null { + return dbs.get(handle) ?? null; + } + + const ns = { + open(name: string): number { + log.push(`op open ${name}`); + if (name !== DB_MEMORY) { + if (!DB_NAME_PATTERN.test(name)) return -1; + const existing = byName.get(name); + if (existing !== undefined) return existing; + } + if (dbs.size >= DB_MAX_DATABASES) return -1; + const image = name !== DB_MEMORY ? closedImages.get(name) : undefined; + const bun = image + ? BunDatabase.deserialize(image, { safeIntegers: true }) + : new BunDatabase(":memory:", { safeIntegers: true }); + const handle = nextHandle++; + dbs.set(handle, { bun, name, lastError: "" }); + if (name !== DB_MEMORY) { + closedImages.delete(name); + byName.set(name, handle); + } + return handle; + }, + close(handle: number): void { + log.push(`op close ${handle}`); + const db = live(handle); + if (!db) return; + if (db.name !== DB_MEMORY) { + closedImages.set(db.name, db.bun.serialize()); + byName.delete(db.name); + } + db.bun.close(); + dbs.delete(handle); + }, + exec(handle: number, sql: string): number { + log.push(`op exec ${handle}`); + const db = live(handle); + if (!db) return 1; + if (ATTACH.test(sql)) { + db.lastError = "ATTACH is refused (contracts/spec/db.ts storage rule)"; + return 1; + } + try { + db.bun.exec(sql); + db.lastError = ""; + return 0; + } catch (error) { + db.lastError = error instanceof Error ? error.message : String(error); + return 1; + } + }, + query(handle: number, sql: string, args: string): string { + log.push(`op query ${handle} ${sql}`); + const db = live(handle); + if (!db) return JSON.stringify({ error: "database is closed" }); + if (ATTACH.test(sql)) { + db.lastError = "ATTACH is refused (contracts/spec/db.ts storage rule)"; + return JSON.stringify({ error: db.lastError }); + } + try { + const parsed = JSON.parse(args) as unknown[] | Record; + if (!Array.isArray(parsed)) { + // The spec spells named parameters $x / :x / @x; bun would bind a + // bare key anyway, the reference core fails it — fail like the core. + for (const key of Object.keys(parsed)) { + if (!/^[$:@]/.test(key)) throw new Error(`unknown parameter: ${key}`); + } + } + const params = Array.isArray(parsed) + ? parsed.map(decodeParam) + : Object.fromEntries(Object.entries(parsed).map(([k, v]) => [k, decodeParam(v)])); + const statement = db.bun.query(sql); + // bun returns null (not []) for statements that produce no rows. + const rows = ((Array.isArray(params) + ? statement.values(...(params as never[])) + : statement.values(params as never)) ?? []) as unknown[][]; + if (rows.length > DB_MAX_RESULT_ROWS) { + throw new Error("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate"); + } + const counters = db.bun + .query("SELECT changes() AS c, last_insert_rowid() AS r") + .get() as { c: bigint; r: bigint }; + db.lastError = ""; + return JSON.stringify({ + cols: statement.columnNames, + rows: rows.map((r) => r.map(encodeCell)), + changes: Number(counters.c), + lastInsertRowid: Number(counters.r), + }); + } catch (error) { + db.lastError = error instanceof Error ? error.message : String(error); + return JSON.stringify({ error: db.lastError }); + } + }, + lastError(handle: number): string { + const db = live(handle); + return db ? db.lastError : "database is closed"; + }, + }; + + return { + ns, + log, + dispose(): void { + for (const db of dbs.values()) db.bun.close(); + dbs.clear(); + byName.clear(); + closedImages.clear(); + }, + }; +} diff --git a/hosts/sim/sim.ts b/hosts/sim/sim.ts index 30c7bc6a..b986b8e5 100644 --- a/hosts/sim/sim.ts +++ b/hosts/sim/sim.ts @@ -241,6 +241,7 @@ export async function bootWorld( : undefined; g.frame = undefined; g.audio = undefined; // audio module namespace: absent unless extraGlobals mounts one + g.db = undefined; // db module namespace: absent unless extraGlobals mounts one g.__pocketApp = app; g.__simHz = hz; g.__pocketEffectTrace = (e: EffectEvent) => effects.push(e); diff --git a/package.json b/package.json index 6a15b57e..6a21826d 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "./audio": "./framework/src/audio-api.ts", "./clock": "./framework/src/clock.ts", "./config": "./framework/src/config.ts", + "./db": "./framework/src/db-api.ts", "./components": "./framework/src/components.ts", "./devtools": "./framework/src/devtools.ts", "./effects": "./framework/src/effects.ts", @@ -100,6 +101,7 @@ "./vue-vapor/animation": "./framework/src/animation.ts", "./vue-vapor/audio": "./framework/src/audio-api.ts", "./vue-vapor/clock": "./framework/src/clock.ts", + "./vue-vapor/db": "./framework/src/db-api.ts", "./vue-vapor/components": "./framework/src/components-vue-vapor.ts", "./vue-vapor/effects": "./framework/src/effects.ts", "./vue-vapor/lifecycle": "./framework/src/lifecycle-vue-vapor.ts", @@ -109,6 +111,7 @@ "./octane/animation": "./framework/src/animation.ts", "./octane/audio": "./framework/src/audio-api.ts", "./octane/clock": "./framework/src/clock.ts", + "./octane/db": "./framework/src/db-api.ts", "./octane/components": "./framework/src/components-octane.tsx", "./octane/effects": "./framework/src/effects.ts", "./octane/lifecycle": "./framework/src/lifecycle-octane.ts", diff --git a/tests/db.test.ts b/tests/db.test.ts new file mode 100644 index 00000000..9db44922 --- /dev/null +++ b/tests/db.test.ts @@ -0,0 +1,294 @@ +// Db module unit tests: the sim host against the pinned op contract, the +// bun:sqlite-shaped SDK over it, and Bun's own SQLite as the oracle for row +// results. Runs entirely in-process; no built bundle, no disk. + +import { afterEach, describe, expect, test } from "bun:test"; +import { Database as BunDatabase } from "bun:sqlite"; +import { + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MEMORY, + DB_NAME_PATTERN, +} from "../contracts/spec/db.ts"; +import { Database, dbHost } from "../framework/src/db-api.ts"; +import { createSimDbHost, type SimDbHost } from "../hosts/sim/db.ts"; + +const g = globalThis as { db?: unknown }; +let host: SimDbHost | null = null; + +/** Mount a fresh sim host as globalThis.db, the way bootWorld's + * extraGlobals does for a scenario. */ +function mount(): SimDbHost { + host = createSimDbHost(); + g.db = host.ns; + return host; +} + +afterEach(() => { + host?.dispose(); + host = null; + g.db = undefined; +}); + +// --- the namespace contract (ops, straight through) ------------------------- + +describe("sim host ops", () => { + test("open refuses bad names, path traversal, and over-limit opens", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + expect(open("../escape")).toBe(-1); + expect(open("a/b")).toBe(-1); + expect(open("")).toBe(-1); + expect(open(".hidden")).toBe(-1); + const handles = []; + for (let i = 0; i < DB_MAX_DATABASES; i++) handles.push(open(`app-${i}`)); + for (const h of handles) expect(h).toBeGreaterThan(0); + expect(open("one-too-many")).toBe(-1); + }); + + test("the same persistent name returns the same handle; :memory: never does", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const a = open("app"); + expect(open("app")).toBe(a); + expect(open(DB_MEMORY)).not.toBe(open(DB_MEMORY)); + }); + + test("ATTACH is refused on both exec and query", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const lastError = ns.lastError as (h: number) => string; + const h = open(DB_MEMORY); + expect(exec(h, "ATTACH DATABASE '/tmp/x' AS other")).toBe(1); + expect(lastError(h)).toContain("ATTACH"); + const result = JSON.parse(query(h, "attach database ':memory:' as other", "[]")); + expect(result.error).toContain("ATTACH"); + // Every spelling SQLite accepts, not just the DATABASE-keyword form: + // bare string, and an expression filename (which on the reference core + // reaches the authorizer as NULL and is caught by SQLITE_LIMIT_ATTACHED). + expect(exec(h, "ATTACH ':memory:' AS o1")).toBe(1); + expect(exec(h, "ATTACH hex('2f746d702f78') AS o2")).toBe(1); + }); + + test("a named parameter without the $/:/@ prefix fails like the reference core", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + exec(h, "CREATE TABLE t (a, b)"); + const result = JSON.parse(query(h, "INSERT INTO t VALUES ($a, $b)", '{"a":1,"$b":2}')); + expect(result.error).toBe("unknown parameter: a"); + }); + + test("ops on a closed handle fail with 'database is closed'", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const close = ns.close as (h: number) => void; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + close(h); + expect(JSON.parse(query(h, "SELECT 1", "[]")).error).toBe("database is closed"); + }); + + test("query result carries cols, rows, changes and lastInsertRowid", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + expect(exec(h, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")).toBe(0); + const ins = JSON.parse(query(h, "INSERT INTO t (v) VALUES (?)", '["hello"]')); + expect(ins.changes).toBe(1); + expect(ins.lastInsertRowid).toBe(1); + const sel = JSON.parse(query(h, "SELECT id, v FROM t", "[]")); + expect(sel.cols).toEqual(["id", "v"]); + expect(sel.rows).toEqual([[1, "hello"]]); + }); + + test("a result beyond DB_MAX_RESULT_ROWS fails loudly", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + exec(h, "CREATE TABLE n (v INTEGER)"); + exec( + h, + `WITH RECURSIVE seq(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM seq LIMIT ${DB_MAX_RESULT_ROWS + 1}) + INSERT INTO n SELECT x FROM seq`, + ); + const over = JSON.parse(query(h, "SELECT v FROM n", "[]")); + expect(over.error).toContain("DB_MAX_RESULT_ROWS"); + const capped = JSON.parse(query(h, `SELECT v FROM n LIMIT ${DB_MAX_RESULT_ROWS}`, "[]")); + expect(capped.rows.length).toBe(DB_MAX_RESULT_ROWS); + }); + + test("an integer beyond 2^53-1 fails instead of losing precision", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + const over = JSON.parse(query(h, "SELECT 9007199254740993", "[]")); + expect(over.error).toContain("DB_MAX_SAFE_INTEGER"); + const ok = JSON.parse(query(h, "SELECT 9007199254740991", "[]")); + expect(ok.rows).toEqual([[9007199254740991]]); + }); +}); + +// --- the SDK (the bun:sqlite shape over the mounted namespace) --------------- + +describe("Database SDK", () => { + test("throws where the module is unmounted", () => { + expect(dbHost()).toBeNull(); + expect(() => new Database()).toThrow("data.sqlite"); + }); + + test("CRUD round-trip with objects, values and named parameters", () => { + mount(); + const db = new Database(); + db.exec(`CREATE TABLE positions ( + symbol TEXT PRIMARY KEY, qty REAL NOT NULL, cost_cents INTEGER NOT NULL + )`); + const insert = db.query("INSERT INTO positions VALUES ($s, $q, $c)"); + insert.run({ $s: "AAPL", $q: 10, $c: 190_00 }); + insert.run({ $s: "NVDA", $q: 2.5, $c: 121_50 }); + expect(db.query("SELECT count(*) AS n FROM positions").get()).toEqual({ n: 2 }); + expect(db.query("SELECT symbol FROM positions ORDER BY symbol").values()).toEqual([ + ["AAPL"], + ["NVDA"], + ]); + const row = db.query("SELECT * FROM positions WHERE symbol = ?").get("NVDA"); + expect(row).toEqual({ symbol: "NVDA", qty: 2.5, cost_cents: 12150 }); + expect(db.query("SELECT * FROM positions WHERE symbol = ?").get("MSFT")).toBeNull(); + const del = db.run("DELETE FROM positions WHERE qty < ?", [5]); + expect(del.changes).toBe(1); + }); + + test("statement caching: query() reuses, prepare() does not", () => { + mount(); + const db = new Database(); + expect(db.query("SELECT 1")).toBe(db.query("SELECT 1")); + expect(db.prepare("SELECT 1")).not.toBe(db.prepare("SELECT 1")); + }); + + test("columnNames populate on execution", () => { + mount(); + const db = new Database(); + const q = db.query("SELECT 1 AS one, 2 AS two"); + expect(q.columnNames).toEqual([]); + q.get(); + expect(q.columnNames).toEqual(["one", "two"]); + }); + + test("blobs round-trip as Uint8Array", () => { + mount(); + const db = new Database(); + db.exec("CREATE TABLE b (data BLOB)"); + const bytes = new Uint8Array([0, 1, 2, 250, 251, 252, 253, 254, 255]); + db.run("INSERT INTO b VALUES (?)", [bytes]); + const out = db.query("SELECT data FROM b").get(); + expect(out?.data).toBeInstanceOf(Uint8Array); + expect(Array.from(out?.data as Uint8Array)).toEqual(Array.from(bytes)); + }); + + test("transaction commits, rolls back on throw, and nests as savepoints", () => { + mount(); + const db = new Database(); + db.exec("CREATE TABLE t (v INTEGER)"); + const insertBoth = db.transaction((a: number, b: number) => { + db.run("INSERT INTO t VALUES (?)", [a]); + db.run("INSERT INTO t VALUES (?)", [b]); + return a + b; + }); + expect(insertBoth(1, 2)).toBe(3); + expect(db.query("SELECT count(*) AS n FROM t").get()).toEqual({ n: 2 }); + + const failing = db.transaction(() => { + db.run("INSERT INTO t VALUES (99)"); + throw new Error("boom"); + }); + expect(() => failing()).toThrow("boom"); + expect(db.query("SELECT count(*) AS n FROM t").get()).toEqual({ n: 2 }); + + const outer = db.transaction(() => { + db.run("INSERT INTO t VALUES (10)"); + const inner = db.transaction(() => { + db.run("INSERT INTO t VALUES (11)"); + throw new Error("inner"); + }); + expect(() => inner()).toThrow("inner"); + }); + outer(); + const values = db.query("SELECT v FROM t ORDER BY v").values(); + expect(values).toEqual([[1], [2], [10]]); + }); + + test("a persistent name survives close/reopen inside one host", () => { + mount(); + const first = new Database("ledger"); + first.exec("CREATE TABLE snap (v INTEGER)"); + first.run("INSERT INTO snap VALUES (42)"); + const second = new Database("ledger"); + expect(second.query("SELECT v FROM snap").get()).toEqual({ v: 42 }); + // Through an actual close(), too — the way Storage::Dir keeps the file + // on a device host. + second.close(); + const third = new Database("ledger"); + expect(third.query("SELECT v FROM snap").get()).toEqual({ v: 42 }); + }); + + test("SQL errors surface as thrown Errors with SQLite's message", () => { + mount(); + const db = new Database(); + expect(() => db.query("SELECT * FROM missing").all()).toThrow("missing"); + expect(() => db.exec("NOT SQL AT ALL")).toThrow(); + }); +}); + +// --- oracle: the SDK over the sim host agrees with bun:sqlite directly ------- + +describe("bun:sqlite oracle", () => { + test("identical statements produce identical rows", () => { + mount(); + const ours = new Database(); + const oracle = new BunDatabase(":memory:"); + const ddl = `CREATE TABLE history ( + bucket TEXT PRIMARY KEY, total_cents INTEGER, day_pnl_cents INTEGER + )`; + const rows: [string, number, number][] = [ + ["2026-08-06T14:30", 1_532_042, 1824], + ["2026-08-06T14:35", 1_531_010, 792], + ["2026-08-06T14:40", 1_540_500, 10_282], + ]; + ours.exec(ddl); + oracle.exec(ddl); + for (const r of rows) { + ours.run("INSERT INTO history VALUES (?, ?, ?)", r); + oracle.query("INSERT INTO history VALUES (?, ?, ?)").run(...r); + } + const sql = `SELECT bucket, total_cents FROM history + WHERE day_pnl_cents > ? ORDER BY bucket DESC`; + expect(ours.query(sql).values(1000)).toEqual( + oracle.query(sql).values(1000) as never, + ); + }); +}); + +// --- spec sanity -------------------------------------------------------------- + +describe("spec constants", () => { + test("DB_NAME_PATTERN accepts tokens and refuses paths", () => { + // 57 chars is the ceiling: `.sqlite` (+7 bytes) stays within the + // fs module's 64-byte segment ceiling, keeping the database file + // addressable by a co-mounted fs module. + for (const good of ["app", "portfolio-history", "a.b_c-1", "A", "a".repeat(57)]) { + expect(DB_NAME_PATTERN.test(good)).toBe(true); + } + for (const bad of ["", ".hidden", "-lead", "a/b", "a\\b", "..", "a".repeat(58)]) { + expect(DB_NAME_PATTERN.test(bad)).toBe(false); + } + }); +}); diff --git a/tools/test.ts b/tools/test.ts index 12b251d9..e2799740 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -55,6 +55,7 @@ const SUITE: readonly Stage[] = [ "tests/kinetics.test.ts", "tests/osk-controller.test.ts", "tests/audio.test.ts", + "tests/db.test.ts", "tests/vita-package.test.ts", "tests/psp-toolchain.test.ts", "tests/symbian-data.test.ts",