diff --git a/contracts/spec/db.ts b/contracts/spec/db.ts new file mode 100644 index 00000000..0c613ef5 --- /dev/null +++ b/contracts/spec/db.ts @@ -0,0 +1,142 @@ +// PocketJS db spec — the boundary of the DB module (`globalThis.db`). +// +// This is a MODULE spec in the docs/RUNTIMES.md §5 sense: a vertical slice +// with its own vocabulary, mounted as its own namespace, pinned here as data. +// It is deliberately NOT part of the `ui` op table — db evolves append-only +// in its own op space, and a host adopts it independently of the UI surface +// (capability id `data.sqlite` in contracts/spec/platforms.ts). +// +// The module is SQLite behind a five-op namespace. The engine, the SQL +// dialect, and the file format are SQLite's — the spec pins only what +// crosses the boundary: op codes, the JSON row encoding, the resource +// ceilings, and the storage/clock rules a host must follow. +// +// The four parts of the boundary: +// +// ops guest -> core intent (numeric codes below, append-only) +// events none — every op is synchronous; the module owns no clock +// data contract the JSON value encoding + database name rules (this file) +// frame contract every op completes inside the guest's single per-tick +// turn (law 3 holds unchanged). SQL time and randomness +// resolve host-side: deterministic hosts pin them, and a +// golden-tested app must not depend on `random()` or +// 'now'-relative SQL (same rule as Date.now in guest code). +// +// Storage rule: `open(name)` is the ONLY path to a database. Names are +// logical (below); the HOST maps them to real files under the app's own +// data root (or memory). The guest never sees a path, and a database file +// is never shared between apps. Hosts MUST refuse `ATTACH` — it is the one +// SQL statement that names a file — so the app's data root stays the +// sandbox boundary. `sqlite3_load_extension` stays disabled (SQLite's +// default). +// +// If you change ANY value here: run `bun contracts/spec/gen-rust.ts`, commit +// the regenerated engine/core/src/spec.rs (tests/contract.ts byte-compares). + +// --------------------------------------------------------------------------- +// Db ops (the `db.*` native contract) +// --------------------------------------------------------------------------- +// Numeric codes are the FFI ABI identity of each op. 0 is reserved +// (invalid/nop). Codes are append-only: never renumber, never reuse. +// +// Signatures (authoritative; hosts marshal them however they like): +// open(name:string) -> handle | -1 +// [name is DB_MEMORY or matches DB_NAME_PATTERN; -1 = +// refused (bad name or DB_MAX_DATABASES already open). +// Opening the same persistent name twice returns the +// SAME handle; DB_MEMORY always opens a fresh private +// database] +// close(handle) [idempotent; a closed handle is +// dead and every later op on it +// fails with "database is closed"] +// exec(handle, sql:string) -> 0 | 1 [run one or more statements, no +// result rows — the schema and +// migration path. 1 = failed; the +// detail is lastError()] +// query(handle, sql:string, args:string) -> string +// [run ONE statement with bound parameters and return +// the complete result as one JSON line (shape below). +// `args` is a JSON array (positional) or object (named +// $x / :x / @x) of encoded values. Statement caching is +// HOST-side, keyed by the sql string — the guest holds +// no statement handles] +// lastError(handle) -> string [detail for the last failed +// exec/query on this handle; "" +// when none] +// +// query() result, one JSON object per call (fields append-only): +// +// { "cols": ["id","name"], "rows": [[1,"a"],[2,"b"]], +// "changes": 0, "lastInsertRowid": 0 } +// `cols` are the statement's column names ([] for non-readers), +// `rows` are arrays in column order using the value encoding below. +// `changes` / `lastInsertRowid` are SQLite's counters after the call. +// { "error": "no such table: t" } +// The statement failed (parse, bind, step, or a ceiling below). +// lastError() returns the same string. + +export const DB_OP = { + open: 1, + close: 2, + exec: 3, + query: 4, + lastError: 5, +} as const; + +// --------------------------------------------------------------------------- +// Data contract — value encoding (rows out, parameters in) +// --------------------------------------------------------------------------- +// SQLite value -> JSON value, both directions: +// +// NULL <-> null +// INTEGER <-> number [|v| must be <= 2^53 - 1. A larger stored +// integer makes the op FAIL — a loud error +// beats silent precision loss. Store money +// in cents and ids under 2^53.] +// REAL <-> number [non-finite REAL fails the same way] +// TEXT <-> string +// BLOB <-> { "$b": "" } +// +// Binding accepts additionally: true/false bind as INTEGER 1/0, and a JSON +// number binds as INTEGER when integer-valued, else REAL (the bun:sqlite / +// better-sqlite3 convention the SDK mirrors). + +/** Marker key for a BLOB value inside a row or a parameter list. */ +export const DB_BLOB_KEY = "$b"; + +/** Largest integer magnitude that crosses the boundary losslessly. */ +export const DB_MAX_SAFE_INTEGER = 9007199254740991; + +// --------------------------------------------------------------------------- +// Data contract — database names +// --------------------------------------------------------------------------- + +/** The in-memory database name (private to the handle, gone on close). */ +export const DB_MEMORY = ":memory:"; + +/** + * Logical persistent-database names: a filename-safe token, no paths, no + * extensions games. The host maps a name to a real file under the app's own + * data root; the mapping is host policy and never guest-visible. The 57-char + * ceiling keeps the reference mapping `.sqlite` (+7 bytes) within the + * fs module's 64-byte segment ceiling, so a co-mounted fs module can always + * address the database file the docs call "visible like any of its files". + */ +export const DB_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,56}$/; + +// --------------------------------------------------------------------------- +// Data contract — resource ceilings +// --------------------------------------------------------------------------- + +/** Open databases per guest. An app has its own db plus room for a scratch + * or migration companion. Deliberately tiny — more simultaneous databases + * is a schema smell, not a bigger constant. */ +export const DB_MAX_DATABASES = 4; + +/** + * Result-row ceiling per query() call. A query producing more rows FAILS + * ("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate") — the row + * budget of a 480x272..720x1280 UI is far below this, and an unbounded + * SELECT on a device heap is a bug surfaced early, not a workload. + */ +export const DB_MAX_RESULT_ROWS = 4096; diff --git a/contracts/spec/fs.ts b/contracts/spec/fs.ts new file mode 100644 index 00000000..df4bc896 --- /dev/null +++ b/contracts/spec/fs.ts @@ -0,0 +1,247 @@ +// PocketJS fs spec — the boundary of the FS module (`globalThis.fs`). +// +// This is a MODULE spec in the docs/RUNTIMES.md §5 sense: a vertical slice +// with its own vocabulary, mounted as its own namespace, pinned here as data. +// It is deliberately NOT part of the `ui` op table — fs evolves append-only +// in its own op space, and a host adopts it independently of the UI surface +// (capability id `data.fs` in contracts/spec/platforms.ts). +// +// The module is a per-app file tree behind nine synchronous ops. The SDK +// (@pocketjs/framework/fs) is the Bun shape — `file()`/`write()` plus the +// node:fs sync subset Bun implements — so file code written against Bun runs +// against the mounted module with the async wrappers dropped. +// +// The four parts of the boundary: +// +// ops guest -> core intent (numeric codes below, append-only) +// events none — every op is synchronous; the module owns no clock. +// There is no watch(): watching needs events and a clock, +// and a per-tick guest can poll stat() when it must. +// data contract the path grammar + payload encoding below +// frame contract every op completes inside the guest's single per-tick +// turn (law 3 holds unchanged). stat() carries NO mtime — +// a timestamp is the fs spelling of Date.now, and a +// golden-tested app must not depend on one. An app that +// needs a timestamp writes it into content it controls. +// +// Storage rule: every path is RELATIVE and resolves under the app's own +// data root; the host binds that root when it mounts the module, and the +// guest never sees a real path. There is no op that names another app's +// tree — isolation is by construction, not by permission check (the same +// principle as db's "open(name) is the only path to a database" and its +// ATTACH refusal). Hosts MUST NOT follow a symlink out of the root: the +// guest cannot create symlinks through this API, but a host-side actor may +// have (on Pocket Pi the device agent owns the whole workspace and every +// app root under it — that asymmetry is host layout policy, above this +// boundary, see docs/FS.md), so the reference core lstat-checks every +// segment. +// +// If you change ANY value here: run `bun contracts/spec/gen-rust.ts`, commit +// the regenerated engine/core/src/spec.rs (tests/contract.ts byte-compares). + +// --------------------------------------------------------------------------- +// Fs ops (the `fs.*` native contract) +// --------------------------------------------------------------------------- +// Numeric codes are the FFI ABI identity of each op. 0 is reserved +// (invalid/nop). Codes are append-only: never renumber, never reuse. +// +// Signatures (authoritative; hosts marshal them however they like). Ops +// returning 0 | 1 report detail through lastError(); ops returning a JSON +// line carry their own {"error": "..."} shape (and set lastError too). +// +// read(path, offset:number, maxBytes:number) -> string +// [one JSON line: {"data":{"$b":""},"size":N, +// "eof":bool} or {"error":...}. Reads up to maxBytes +// bytes at byte offset; maxBytes must be 1..FS_MAX_IO_BYTES +// or the op fails. `size` is the file's total byte size, +// `eof` is true when offset+data reaches it. Reading a +// directory fails] +// write(path, data:string, mode:number) -> 0 | 1 +// [data is the payload encoding below, decoded byte length +// <= FS_MAX_IO_BYTES per call (the SDK chunks larger +// writes). mode FS_WRITE_TRUNCATE replaces the file +// ATOMICALLY — the old content or the new, never a torn +// middle (temp + rename; the power-loss contract device +// hosts inherit from LittleFS's atomic rename. Temps +// live in a host directory outside the bound root, so +// the app's tree never shows host machinery). +// FS_WRITE_APPEND appends and is not atomic. Parent +// directories are created automatically (Bun.write +// semantics). Writing over a directory fails] +// remove(path, recursive:number) -> 0 | 1 +// [removes a file, or a directory when empty; recursive=1 +// removes a directory tree. A missing path fails with +// "not found" (the SDK's rmSync force option swallows +// that one). remove("") — the root — always fails] +// list(path, offset:number) -> string +// [{"entries":[{"name":"a.txt","kind":"file","size":N}, +// {"name":"sub","kind":"dir","size":0}],"eof":bool} or +// {"error":...}. Entries sort by name in Unicode code +// point order (= UTF-8 byte order; NOT UTF-16 code unit +// order — hosts written in JS must sort by code point) — +// deterministic across hosts — and one call returns at +// most FS_MAX_DIR_ENTRIES of them starting at `offset` +// in that order; `eof` false means page again. list("") +// lists the root] +// stat(path) -> string +// [{"kind":"file","size":N} | {"kind":"dir","size":0} or +// {"error":"not found"}. stat("") is the root: always +// {"kind":"dir","size":0}. No mtime — see the frame +// contract above] +// mkdir(path) -> 0 | 1 +// [recursive (every missing ancestor is created) and +// idempotent (an existing directory is success). A file +// anywhere on the path fails] +// rename(from, to) -> 0 | 1 +// [moves a file or directory within the root. An existing +// file at `to` is replaced atomically; an existing +// directory at `to` fails; a missing parent of `to` +// fails (mkdir first — rename does not create). Renaming +// a directory into its own subtree fails] +// usage() -> string +// [{"usedBytes":N,"quotaBytes":N} — usedBytes sums every +// file's size under the root; quotaBytes is the host's +// configured budget for this app, 0 = unmetered. When a +// quota is set, a write/append that would exceed it +// fails with "quota exceeded"] +// lastError() -> string +// [detail for the last failed op on this module; "" when +// the last op succeeded. Module-scoped — there are no +// handles in this vocabulary] + +export const FS_OP = { + read: 1, + write: 2, + remove: 3, + list: 4, + stat: 5, + mkdir: 6, + rename: 7, + usage: 8, + lastError: 9, +} as const; + +/** write() modes. */ +export const FS_WRITE_TRUNCATE = 0; +export const FS_WRITE_APPEND = 1; + +// --------------------------------------------------------------------------- +// Data contract — payload encoding (write data in, read data out) +// --------------------------------------------------------------------------- +// A payload crossing the boundary is one JSON value: +// +// text <-> a JSON string [stored as its UTF-8 bytes] +// bytes <-> { "$b": "" } [the db module's blob spelling] +// +// write() accepts either; read() always returns bytes — the file does not +// remember which spelling wrote it, and the SDK's .text() decodes UTF-8 +// guest-side (QuickJS has no TextDecoder; the SDK carries the codec). +// +// A text payload must be well-formed Unicode, like a path segment: an +// unpaired surrogate has no UTF-8 spelling, so what happens to one is +// host-dependent (a JS host lossily encodes U+FFFD where a JSON-parsing +// native core fails the op). Arbitrary byte data belongs in the bytes +// spelling, never in a string. + +/** Marker key for a bytes payload (same spelling as db's DB_BLOB_KEY). */ +export const FS_BLOB_KEY = "$b"; + +// --------------------------------------------------------------------------- +// Data contract — the path grammar +// --------------------------------------------------------------------------- +// A path is 1..FS_MAX_DEPTH segments joined by "/": no leading or trailing +// slash, no empty segment. A segment is ANY well-formed Unicode string — +// Chinese names, dot-prefixed names, whatever the app wants — except the +// four things no filesystem can or this sandbox may allow: +// +// "." and ".." the escape hatches (this is the security rule); +// "/" in a name unrepresentable — it IS the separator, on every +// filesystem on earth; +// control chars C0 (U+0000..U+001F) and DEL (U+007F); +// oversize a segment > FS_MAX_SEGMENT_BYTES of UTF-8; +// lone surrogates ill-formed Unicode has no UTF-8 spelling — a JS host +// could store one byte-exactly while the QuickJS-to- +// native bridge mangles it into a DIFFERENT name, so +// the shared predicate refuses it on every host. +// +// No name is reserved to the host. "" names the root and is valid only +// where an op says so (list, stat). Total path <= FS_MAX_PATH_BYTES of +// UTF-8. +// +// Identity: segments are byte-for-byte identities (compared as UTF-8, no +// case folding, no Unicode normalization). Some host filesystems fold case +// or normalize (macOS APFS); two sibling names differing only by case or +// normalization form are therefore NOT portable — an app must never create +// both. The deterministic hosts (sim, the reference core's Memory storage) +// are byte-exact, so a golden test catches the collision early. + +/** Maximum UTF-8 bytes in one segment. */ +export const FS_MAX_SEGMENT_BYTES = 64; + +/** Maximum segments in a path (root = depth 0). */ +export const FS_MAX_DEPTH = 8; + +/** Maximum total path length in UTF-8 bytes (segments + separators). */ +export const FS_MAX_PATH_BYTES = 160; + +/** UTF-8 byte length of a JS string (QuickJS has no TextEncoder). */ +function utf8Bytes(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++) { + const c = s.codePointAt(i)!; + n += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4; + if (c >= 0x10000) i++; + } + return n; +} + +/** True when `s` is well-formed Unicode (every surrogate is paired). */ +function wellFormed(s: string): boolean { + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i); + if (c >= 0xdc00 && c <= 0xdfff) return false; // low with no high before it + if (c >= 0xd800 && c <= 0xdbff) { + const next = s.charCodeAt(i + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return false; + i++; + } + } + return true; +} + +/** True when `segment` is one valid path segment under the grammar above. */ +export function fsValidSegment(segment: string): boolean { + if (segment.length === 0 || segment === "." || segment === "..") return false; + if (!wellFormed(segment)) return false; + // eslint-disable-next-line no-control-regex + if (/[\u0000-\u001f\u007f]/.test(segment)) return false; + return utf8Bytes(segment) <= FS_MAX_SEGMENT_BYTES; +} + +/** True when `path` is a valid non-root path under the grammar above. + * The SAME predicate every host implements; exported so hosts and tests + * share one spelling. */ +export function fsValidPath(path: string): boolean { + if (path.length === 0 || utf8Bytes(path) > FS_MAX_PATH_BYTES) return false; + const segments = path.split("/"); + if (segments.length > FS_MAX_DEPTH) return false; + return segments.every(fsValidSegment); +} + +// --------------------------------------------------------------------------- +// Data contract — resource ceilings +// --------------------------------------------------------------------------- + +/** + * Payload ceiling per read()/write() call, in bytes. One call's payload + * must fit a device heap comfortably; the SDK loops for larger files, so + * the ceiling bounds marshaling, not file size. + */ +export const FS_MAX_IO_BYTES = 65536; + +/** + * Entries per list() call. list() pages (offset + eof), so a big directory + * is slower to enumerate, never impossible — a ceiling an app cannot get + * stuck behind, unlike an unpaged cap. + */ +export const FS_MAX_DIR_ENTRIES = 256; diff --git a/contracts/spec/gen-rust.ts b/contracts/spec/gen-rust.ts index b19bd9f6..8bb065d8 100644 --- a/contracts/spec/gen-rust.ts +++ b/contracts/spec/gen-rust.ts @@ -1,4 +1,4 @@ -// Deterministic codegen: contracts/spec/{spec,audio}.ts -> engine/core/src/spec.rs. +// Deterministic codegen: contracts/spec/{spec,audio,db}.ts -> engine/core/src/spec.rs. // // Run from PocketJS/: bun contracts/spec/gen-rust.ts // @@ -16,6 +16,25 @@ import { AUDIO_RING_FRAMES, audioFramesForTick, } from "./audio.ts"; +import { + DB_BLOB_KEY, + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MAX_SAFE_INTEGER, + DB_MEMORY, + DB_OP, +} from "./db.ts"; +import { + FS_BLOB_KEY, + FS_MAX_DEPTH, + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_MAX_PATH_BYTES, + FS_MAX_SEGMENT_BYTES, + FS_OP, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, +} from "./fs.ts"; import { ANALOG_CENTER, ANIMATABLE, @@ -462,6 +481,60 @@ export function generateRust(): string { put(` pub const EVENT_${screaming(name)}: &str = ${JSON.stringify(v)};`); } put("}"); + put(""); + + // --- db module ----------------------------------------------------------- + // The db MODULE's boundary (contracts/spec/db.ts): SQLite behind five + // synchronous ops, mounted as `globalThis.db`, independent of the ui + // surface. A native host implementing it reads these constants; the + // reference implementation is engine/crates/pocket-db. + put("/// DB module boundary (contracts/spec/db.ts — `globalThis.db`)."); + put("/// SQLite behind five synchronous ops; rows cross as one JSON line per"); + put("/// query() call. The module owns no clock and emits no events."); + put("pub mod db {"); + for (const [name, v] of Object.entries(DB_OP)) { + put(` pub const OP_${screaming(name)}: u8 = ${v};`); + } + put(` /// The in-memory database name (private to the handle).`); + put(` pub const MEMORY: &str = ${JSON.stringify(DB_MEMORY)};`); + put(` /// Marker key for a BLOB value inside a row or a parameter list.`); + put(` pub const BLOB_KEY: &str = ${JSON.stringify(DB_BLOB_KEY)};`); + put(` /// Largest integer magnitude that crosses the boundary losslessly.`); + put(` pub const MAX_SAFE_INTEGER: i64 = ${DB_MAX_SAFE_INTEGER};`); + put(` pub const MAX_DATABASES: usize = ${DB_MAX_DATABASES};`); + put(` /// Result-row ceiling per query() call (exceeding it fails the op).`); + put(` pub const MAX_RESULT_ROWS: usize = ${DB_MAX_RESULT_ROWS};`); + put("}"); + put(""); + + // --- fs module ----------------------------------------------------------- + // The fs MODULE's boundary (contracts/spec/fs.ts): a per-app file tree + // behind nine synchronous ops, mounted as `globalThis.fs`, independent of + // the ui surface. A native host implementing it reads these constants; the + // reference implementation is engine/crates/pocket-fs. + put("/// FS module boundary (contracts/spec/fs.ts — `globalThis.fs`)."); + put("/// A per-app file tree behind nine synchronous ops; every path resolves"); + put("/// under the app's own data root. No clock, no events, no mtime."); + put("pub mod fs {"); + for (const [name, v] of Object.entries(FS_OP)) { + put(` pub const OP_${screaming(name)}: u8 = ${v};`); + } + put(` /// write() modes.`); + put(` pub const WRITE_TRUNCATE: u32 = ${FS_WRITE_TRUNCATE};`); + put(` pub const WRITE_APPEND: u32 = ${FS_WRITE_APPEND};`); + put(` /// Marker key for a bytes payload (db's blob spelling).`); + put(` pub const BLOB_KEY: &str = ${JSON.stringify(FS_BLOB_KEY)};`); + put(` /// Maximum UTF-8 bytes in one path segment.`); + put(` pub const MAX_SEGMENT_BYTES: usize = ${FS_MAX_SEGMENT_BYTES};`); + put(` /// Maximum segments in a path.`); + put(` pub const MAX_DEPTH: usize = ${FS_MAX_DEPTH};`); + put(` /// Maximum total path length in bytes.`); + put(` pub const MAX_PATH_BYTES: usize = ${FS_MAX_PATH_BYTES};`); + put(` /// Payload ceiling per read()/write() call, in bytes.`); + put(` pub const MAX_IO_BYTES: usize = ${FS_MAX_IO_BYTES};`); + put(` /// Entries per list() call (paged via offset + eof).`); + put(` pub const MAX_DIR_ENTRIES: usize = ${FS_MAX_DIR_ENTRIES};`); + put("}"); return L.join("\n") + "\n"; } diff --git a/contracts/spec/platforms.ts b/contracts/spec/platforms.ts index 88ece438..3284b6bc 100644 --- a/contracts/spec/platforms.ts +++ b/contracts/spec/platforms.ts @@ -144,6 +144,25 @@ export const POCKET_CAPABILITIES = defineCapabilityRegistry([ // appends the id to its profile only when its native host ships the module // (the ring/thread discipline to copy is hosts/psp/src/audio.rs). "audio.pcm", + // SQLite behind the db module's own namespace (`globalThis.db`, + // contracts/spec/db.ts): five synchronous ops, rows as one JSON line per + // query() call, per-app storage the host confines. Registered ahead of any + // stock TARGET advertising it: the sim host and the engine/crates/pocket-db + // reference core implement and test the whole contract, so apps can already + // declare the requirement and fail admission where the module is absent. A + // device target appends the id to its profile only when its native host + // ships the module. + "data.sqlite", + // A per-app file tree behind the fs module's own namespace + // (`globalThis.fs`, contracts/spec/fs.ts): nine synchronous ops, every + // path confined to the app's own data root — apps cannot name, let alone + // reach, each other's trees. Registered ahead of any stock TARGET + // advertising it: the sim host and the engine/crates/pocket-fs reference + // core implement and test the whole contract, so apps can already declare + // the requirement and fail admission where the module is absent. A device + // target appends the id to its profile only when its native host ships + // the module. + "data.fs", // Copy/cut/paste round-trips with the OS clipboard. "host.clipboard", // The logical viewport is runtime-mutable: the app is told about live diff --git a/docs/DB.md b/docs/DB.md new file mode 100644 index 00000000..1076975e --- /dev/null +++ b/docs/DB.md @@ -0,0 +1,162 @@ +# The DB Module + +DB is PocketJS's fourth module (after `ui`, `strike` and `audio`): SQLite +mounted as `globalThis.db` behind five synchronous ops. Like audio it was +written spec-first — the boundary existed before any host code, every host +implements the same pinned protocol, and a developer adding a data feature +extends the spec instead of forking a host. `contracts/spec/db.ts` is +normative; this page is the map. + +``` +platform storage (POSIX file · LittleFS · memory) Host / substrate + ↑ SQLite's own VFS is the port point +db core: SQLite + handle table + statement cache the module +db spec: ops (open, close, exec, query, lastError) + events (none — every op is synchronous) + data contract (JSON value encoding · name rules · ceilings) + frame contract (no module clock; ops complete in the guest turn) +SDK: @pocketjs/framework/db (Database, Statement — the bun:sqlite shape) + ↓ +app: schema in exec(), rows out of query(), state in tables Guest +``` + +## The boundary in one page + +**Mount.** The module is its own namespace: `globalThis.db`, one method per +op (`DB_OP` codes are the ABI identity, append-only). Capability id +`data.sqlite`. Unlike audio, absence does **not** degrade to a no-op — data +code that silently drops writes is a corruption bug, so the SDK throws where +the namespace is unmounted, and an app declares `data.sqlite` in +`pocket.json` `requires` so admission catches the gap before eval does. + +**Ops** (guest → core, all synchronous): `open(name)` → handle, +`close(handle)`, `exec(handle, sql)` → 0/1 for schema and migrations, +`query(handle, sql, argsJson)` → one JSON line +(`{cols, rows, changes, lastInsertRowid}` or `{error}`), and +`lastError(handle)`. Statement caching is **host-side**, keyed by the sql +string — the guest holds no statement handles, so there is nothing to +finalize and nothing to leak. + +**Values** cross as JSON: NULL ↔ `null`, INTEGER ↔ number, REAL ↔ number, +TEXT ↔ string, BLOB ↔ `{"$b": ""}`. Integers beyond 2^53 − 1 and +non-finite REALs **fail the op** instead of losing precision silently — +store money in cents. Booleans bind as 1/0; integer-valued numbers bind as +INTEGER, fractional as REAL (the bun:sqlite convention). + +**Storage rule.** `open(name)` is the only path to a database: names are +filename-safe tokens (`DB_NAME_PATTERN`, ≤ 57 chars so `.sqlite` +stays within the fs module's 64-byte segment ceiling) or `:memory:`, and +the host maps a name to a real file under the app's own data root — the reference core +spells that mapping `/.sqlite`. The database is an +ORDINARY file in the app's home, deliberately visible to a co-mounted fs +module: it is the app's own asset (backup = a file copy), and an app that +overwrites its own database corrupts its own data — the same trust class +as deleting its own files (SQLite fails loudly on a corrupt image). The +guest never sees a path. `ATTACH` — the one SQL statement that names a +file — is refused twice over (engine/crates/pocket-db uses a real SQLite +authorizer for the literal spelling and **`SQLITE_LIMIT_ATTACHED=0`** for +the expression spelling that reaches an authorizer with a NULL filename), +and `load_extension` stays disabled, so the data root stays the sandbox +boundary. + +**Ceilings.** `DB_MAX_DATABASES` (4) open handles; `DB_MAX_RESULT_ROWS` +(4096) rows per `query()` call, above which the op fails with "add LIMIT or +aggregate" — an unbounded SELECT on a device heap is a bug surfaced early, +not a workload. + +**Frame contract.** The module owns no clock and emits no events: every op +completes inside the guest's single per-tick turn (law 3 unchanged). SQL +time and randomness resolve host-side — deterministic hosts pin them, and a +golden-tested app must not depend on `random()` or 'now'-relative SQL, the +same rule as `Date.now` in guest code. + +## The SDK + +`@pocketjs/framework/db` is the bun:sqlite shape, so code written against +Bun's built-in SQLite runs against the mounted module unchanged: + +```ts +import { Database } from "@pocketjs/framework/db"; + +const db = new Database("portfolio"); +db.exec(`CREATE TABLE IF NOT EXISTS history ( + bucket TEXT PRIMARY KEY, total_cents INTEGER NOT NULL +)`); + +const insert = db.query("INSERT OR REPLACE INTO history VALUES ($bucket, $cents)"); +const record = db.transaction((bucket: string, cents: number) => { + insert.run({ $bucket: bucket, $cents: cents }); +}); +record("2026-08-06T14:35", 1_532_042); + +db.query("SELECT * FROM history ORDER BY bucket DESC LIMIT 288").all(); +``` + +`query()` returns a cached `Statement` (`.get/.all/.values/.run`); +`prepare()` skips the cache; `transaction(fn)` wraps BEGIN/COMMIT with +ROLLBACK on throw and nests as savepoints — batching writes into one +transaction is also the flash-wear discipline on device hosts. + +## Host status + +| Host | Implementation | Status | +|---|---|---| +| sim (`hosts/sim/db.ts`) | bun:sqlite behind the op namespace, injected via `bootWorld` `extraGlobals` | ships with the test host; `tests/db.test.ts` runs the contract and an oracle comparison against bun:sqlite directly | +| reference core (`engine/crates/pocket-db`) | rusqlite (bundled SQLite) + the ATTACH authorizer, mountable on any `pocket-mod` guest as `globalThis.db` | tested including a live QuickJS guest round-trip | +| consoles / devices | — | a target appends `data.sqlite` to its profile when its native host ships the module; the port point is SQLite's VFS (POSIX on desktop, LittleFS-backed on MCU hosts). ESP-IDF support ships in the crate — see below | + +## Adoption path + +A device host that wants the module makes three moves, none of which touch +the spec, the SDK, or any app: + +1. compile SQLite with the platform toolchain and register a VFS for the + platform filesystem (the desktop default VFS already serves + `Storage::Dir`); +2. mount the namespace beside `ui` — `pocket_db::mount(&guest, module)` on + `pocket-mod` hosts, or the raw-QuickJS spelling of the same five + functions elsewhere; +3. append `data.sqlite` to the target's profile in + `contracts/spec/platforms.ts`. + +## ESP32 / ESP-IDF + +`pocket-db` carries its own ESP-IDF support behind +`cfg(target_os = "espidf")` — desktop builds never see it: + +- **newlib shims** for the POSIX symbols SQLite's syscall table references + but newlib lacks (`geteuid`/`fchmod`/`fchown`/`utimes`/`readlink` no-ops + that are honest on a filesystem with no users or symlinks, and + `nanosleep` routed through `usleep` for the busy handler); +- **the `unix-none` VFS** on open — LittleFS has no fcntl file locks, and + a module instance is its files' only writer — plus flash-friendly + pragmas (`journal_mode=TRUNCATE`, `synchronous=NORMAL`, + `cache_size=-32`). + +What the crate cannot carry is the build environment; a firmware adds, in +its `.cargo/config.toml` (values validated on an ESP32-P4, ESP-IDF v5.5.x, +LittleFS workspace — where a 288-row transaction landed in ~0.4 s on +~70–80 KB of heap and survived power cycling): + +```toml +[env] +# The vendored sqlite3.c, tuned for the device: temp tables in memory, no +# mmap, no WAL (shared memory), no dynamic extension loading; lstat does +# not exist on newlib and the VFS never follows symlinks anyway. +LIBSQLITE3_FLAGS = "-DSQLITE_TEMP_STORE=3 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_MAX_MMAP_SIZE=0 -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -Dlstat=stat" +# CC_/AR_/CFLAGS_riscv32imafc_esp_espidf point at the ESP-IDF gcc as usual. +# newlib has no sys/ioctl.h: put an EMPTY sys/ioctl.h in a shim directory +# and add `-isystem ` to CFLAGS. Setting CFLAGS in [env] replaces +# cargo's derived flags, so repeat the arch flags (-mabi/-march) alongside. +``` + +`sqlite3_os_init` for the `unix-none` VFS, journal-file creation, and +power-loss recovery all run against ESP-IDF's VFS layer over LittleFS — +no SQLite source patches, no custom VFS to write. + +A firmware that brings its own QuickJS embedding depends with +`default-features = false`: that drops the `mount` helper and its +pocket-mod/rquickjs dependency, so the MCU build compiles only the module +core plus SQLite. Verified: `cargo check` for `riscv32imafc-esp-espidf` +compiles the crate and the bundled `libsqlite3.a` clean under this +recipe. diff --git a/docs/FS.md b/docs/FS.md new file mode 100644 index 00000000..df162e77 --- /dev/null +++ b/docs/FS.md @@ -0,0 +1,165 @@ +# The FS Module + +FS is PocketJS's fifth module (after `ui`, `strike`, `audio` and `db`): a +per-app file tree mounted as `globalThis.fs` behind nine synchronous ops. +Like db it was written spec-first — the boundary existed before any host +code, every host implements the same pinned protocol, and a developer +adding a storage feature extends the spec instead of forking a host. +`contracts/spec/fs.ts` is normative; this page is the map. + +``` +platform storage (POSIX dir · LittleFS · memory) Host / substrate + ↑ the app's own data root is the port point +fs core: path grammar + confinement + atomic writes the module +fs spec: ops (read, write, remove, list, stat, mkdir, + rename, usage, lastError) + events (none — every op is synchronous) + data contract (path grammar · payload encoding · ceilings) + frame contract (no module clock; no mtime; ops complete in the turn) +SDK: @pocketjs/framework/fs (file/write + the node:fs sync subset — the Bun shape) + ↓ +app: notes in files, assets in dirs, config in json Guest +``` + +## The boundary in one page + +**Mount.** The module is its own namespace: `globalThis.fs`, one method per +op (`FS_OP` codes are the ABI identity, append-only). Capability id +`data.fs`. Like db, absence does **not** degrade to a no-op — file code +that silently drops writes is a corruption bug, so the SDK throws where the +namespace is unmounted, and an app declares `data.fs` in `pocket.json` +`requires` so admission catches the gap before eval does. + +**Ops** (guest → core, all synchronous): `read(path, offset, maxBytes)` → +one JSON line (`{data:{"$b":…}, size, eof}`), `write(path, data, mode)` +with truncate/append modes, `remove(path, recursive)`, `list(path, offset)` +→ name-sorted, paged entries, `stat(path)` → `{kind, size}`, +`mkdir(path)` (recursive, idempotent), `rename(from, to)`, `usage()` → +`{usedBytes, quotaBytes}`, and `lastError()`. + +**Payloads** cross as one JSON value: text as a JSON string (stored as its +UTF-8 bytes), bytes as `{"$b": ""}` — the db module's blob +spelling. `read` always returns bytes; the SDK's `.text()` decodes UTF-8 +guest-side (QuickJS has no TextDecoder; the SDK carries the codec). + +**The storage rule — isolation by construction.** Every path is relative +and resolves under the app's own data root, bound by the host at mount. +Names are universal — any well-formed Unicode a filesystem can hold, +dot-prefixed included; nothing in the app's tree is reserved to the host. +Isolation never depended on names: `..`, absolute paths, and `/` inside a +name are unrepresentable, so there is no way to *spell* another app's +tree — the same principle as db's "open(name) is the only path to a +database" and its ATTACH refusal. Hosts must not follow a symlink out of +the root; the reference core lstat-checks every segment and treats any +symlink as absent. + +The confinement binds the **guest**, not the host. On Pocket Pi the device +agent's home is the whole workspace — with every app root laid out under +it (`/workspace/apps//data/`), the agent reads and writes every +app's tree through the same module, bound wider, while apps still cannot +reach each other. Privilege is the binding, not the code. + +One data root serves both data modules: a database is an ordinary file +(`/.sqlite`) in the app's home — its own asset, visible +like any of its files (backup = a file copy). Overwriting it corrupts the +app's own data, the same trust class as deleting its own files; SQLite +fails loudly on a corrupt image. + +**Atomicity.** A truncate `write` lands completely or not at all: the +payload lands in the module's own temp directory — outside the bound +root, same filesystem — then renames over the target, so after power +loss the file holds the old content or the new, never a torn middle, and +the app's tree never shows host machinery (the module owns the temp +directory and clears it on construction, so a crash orphan cannot +outlive the next boot). Append is not atomic. LittleFS's rename is +atomic, so device hosts inherit the contract by the same moves. + +**The op is the atomic unit.** A file larger than `FS_MAX_IO_BYTES` +crosses as one truncate plus appends (the SDK's chunking), so power loss +between chunks can leave the leading chunks only. An app that needs +whole-file atomicity above 64 KiB writes to a sibling name and +`rename`s over the target — the same move the module itself makes. + +**Ceilings.** `FS_MAX_IO_BYTES` (64 KiB) per read/write payload — the SDK +chunks larger files, so the ceiling bounds marshaling, not file size. +`FS_MAX_DIR_ENTRIES` (256) per `list()` call, paged via offset + eof — a +big directory is slower to enumerate, never impossible. Paths: +`FS_MAX_DEPTH` (8) segments of `FS_MAX_SEGMENT_BYTES` (64) each, +`FS_MAX_PATH_BYTES` (160) total. A per-app byte quota is host policy, +reported by `usage()` (0 = unmetered) and enforced on write. + +**Frame contract.** The module owns no clock and emits no events: every op +completes inside the guest's single per-tick turn (law 3 unchanged). There +is no `watch()` — watching needs events and a clock; a per-tick guest +polls `stat()` when it must. `stat` carries **no mtime**: a timestamp is +the fs spelling of `Date.now`, and a golden-tested app must not depend on +one. An app that needs a timestamp writes it into content it controls. + +**Identity.** Segments are byte-for-byte identities (UTF-8, no case +folding, no Unicode normalization), but some host filesystems fold or +normalize (macOS APFS). Two sibling names differing only by case or +normalization form are not portable — never create both. The +deterministic hosts (sim, the reference core's Memory storage) are +byte-exact, so a golden test catches the collision before a folding +device filesystem hides it. + +## The SDK + +`@pocketjs/framework/fs` is the Bun shape — `file()`/`write()` plus the +node:fs sync subset Bun implements — so file code written against Bun runs +against the mounted module unchanged. Methods return values synchronously +(the frame contract), and `await` unwraps a plain value, so Bun-idiomatic +`await file(p).text()` needs no edits: + +```ts +import { file, write, readdirSync, mkdirSync, rmSync, usage } from "@pocketjs/framework/fs"; + +write("notes/today.md", "# Today\n- ship the fs module"); // atomic, mkdir -p +const f = file("notes/today.md"); +f.exists(); // true +f.size; // bytes +f.text(); // the string (await f.text() works too) +f.bytes(); // Uint8Array +f.json(); // parsed JSON (for config files) + +mkdirSync("assets/img"); +readdirSync("notes", { withFileTypes: true }); // name-sorted entries +rmSync("notes", { recursive: true }); +usage(); // { usedBytes, quotaBytes } +``` + +Also exported: `readFileSync`, `writeFileSync`, `appendFileSync`, +`renameSync`, `statSync`, `existsSync` — each the node spelling Bun also +serves. Files larger than one payload chunk transparently +(`FS_MAX_IO_BYTES` per op crossing). + +Choosing between fs and db: rows, queries and transactions belong in +`data.sqlite`; documents, assets and configs belong here. A key-value need +is one db table, not a third module. + +## Host status + +| Host | Implementation | Status | +|---|---|---| +| sim (`hosts/sim/fs.ts`) | in-memory tree behind the op namespace, injected via `bootWorld` `extraGlobals` | ships with the test host; `tests/fs.test.ts` runs the contract, the SDK, and an oracle comparison against Bun's real fs | +| reference core (`engine/crates/pocket-fs`) | `Storage::Memory`/`Storage::Dir` over std::fs — grammar confinement, symlink refusal, atomic truncate writes, quota — mountable on any `pocket-mod` guest as `globalThis.fs` | tested including a live QuickJS guest round-trip | +| consoles / devices | — | a target appends `data.fs` to its profile when its native host ships the module; the port point is the data root (POSIX dir on desktop, a LittleFS directory on MCU hosts) | + +## Adoption path + +A device host that wants the module makes three moves, none of which touch +the spec, the SDK, or any app: + +1. pick the app's data root and a sibling temp directory on the platform + filesystem (Pocket Pi: `/workspace/apps//data/` — the SAME + root the db module binds — and `/workspace/apps//tmp/`) and + construct the module bound to them — + `pocket_fs::FsModule::new(Storage::Dir { root, tmp })`, one instance + per app; +2. mount the namespace beside `ui` — `pocket_fs::mount(&guest, module)` on + `pocket-mod` hosts, or the raw-QuickJS spelling of the same nine + functions elsewhere (depend with `default-features = false` to drop the + pocket-mod dependency; verified to `cargo check` clean for + `riscv32imafc-esp-espidf`); +3. append `data.fs` to the target's profile in + `contracts/spec/platforms.ts`. diff --git a/engine/Cargo.lock b/engine/Cargo.lock index 4138dfe9..876a42c7 100644 --- a/engine/Cargo.lock +++ b/engine/Cargo.lock @@ -187,6 +187,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bit-set" version = "0.8.0" @@ -535,6 +541,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fdeflate" version = "0.3.7" @@ -689,7 +707,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3ce1918195723ce6ac74e80542c5a96a40c2b26162c1957a5cd70799b8cacf7" dependencies = [ - "base64", + "base64 0.13.1", "byteorder", "gltf-json", "image", @@ -809,6 +827,15 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -820,6 +847,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -1042,6 +1078,17 @@ dependencies = [ "redox_syscall 0.9.0", ] +[[package]] +name = "libsqlite3-sys" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1587,6 +1634,29 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "pocket-db" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.23.1", + "pocket-mod", + "pocketjs-core", + "rusqlite", + "serde_json", +] + +[[package]] +name = "pocket-fs" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64 0.23.1", + "pocket-mod", + "pocketjs-core", + "serde_json", +] + [[package]] name = "pocket-mod" version = "0.1.0" @@ -1915,6 +1985,31 @@ dependencies = [ "cc", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + +[[package]] +name = "rusqlite" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11438310b19e3109b6446c33d1ed5e889428cf2e278407bc7896bc4aaea43323" +dependencies = [ + "bitflags 2.13.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2137,6 +2232,18 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2367,6 +2474,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/engine/Cargo.toml b/engine/Cargo.toml index d4a627f8..cb23d421 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -11,6 +11,8 @@ [workspace] resolver = "2" members = [ + "crates/pocket-db", + "crates/pocket-fs", "crates/pocket-mod", "crates/pocket-ui-surface", "crates/pocket-ui-wgpu", @@ -42,6 +44,8 @@ repository = "https://github.com/pocket-stack/pocketjs" [workspace.dependencies] pocket3d = { path = "pocket3d/crates/pocket3d" } pocket3d-bsp = { path = "pocket3d/crates/pocket3d-bsp" } +pocket-db = { path = "crates/pocket-db" } +pocket-fs = { path = "crates/pocket-fs" } pocket-mod = { path = "crates/pocket-mod" } pocket-ui-surface = { path = "crates/pocket-ui-surface" } pocket-ui-wgpu = { path = "crates/pocket-ui-wgpu" } @@ -65,6 +69,12 @@ ab_glyph = "0.2" memmap2 = "0.9" serde = { version = "1", features = ["derive"] } serde_json = "1" +# bundled: compile the vendored sqlite3.c — the same source a device host +# compiles with its own toolchain; hooks: the authorizer that refuses ATTACH; +# limits: SQLITE_LIMIT_ATTACHED=0, the backstop that refuses ATTACH spellings +# the authorizer cannot see (an expression filename reaches it as NULL). +rusqlite = { version = "0.40", features = ["bundled", "hooks", "limits"] } +base64 = "0.23" env_logger = "0.11" font8x8 = "0.3" diff --git a/engine/core/src/spec.rs b/engine/core/src/spec.rs index f5d6ac2e..8ac86e9a 100644 --- a/engine/core/src/spec.rs +++ b/engine/core/src/spec.rs @@ -498,3 +498,53 @@ pub mod audio { pub const EVENT_UNDERRUN: &str = "underrun"; pub const EVENT_ENDED: &str = "ended"; } + +/// DB module boundary (contracts/spec/db.ts — `globalThis.db`). +/// SQLite behind five synchronous ops; rows cross as one JSON line per +/// query() call. The module owns no clock and emits no events. +pub mod db { + pub const OP_OPEN: u8 = 1; + pub const OP_CLOSE: u8 = 2; + pub const OP_EXEC: u8 = 3; + pub const OP_QUERY: u8 = 4; + pub const OP_LAST_ERROR: u8 = 5; + /// The in-memory database name (private to the handle). + pub const MEMORY: &str = ":memory:"; + /// Marker key for a BLOB value inside a row or a parameter list. + pub const BLOB_KEY: &str = "$b"; + /// Largest integer magnitude that crosses the boundary losslessly. + pub const MAX_SAFE_INTEGER: i64 = 9007199254740991; + pub const MAX_DATABASES: usize = 4; + /// Result-row ceiling per query() call (exceeding it fails the op). + pub const MAX_RESULT_ROWS: usize = 4096; +} + +/// FS module boundary (contracts/spec/fs.ts — `globalThis.fs`). +/// A per-app file tree behind nine synchronous ops; every path resolves +/// under the app's own data root. No clock, no events, no mtime. +pub mod fs { + pub const OP_READ: u8 = 1; + pub const OP_WRITE: u8 = 2; + pub const OP_REMOVE: u8 = 3; + pub const OP_LIST: u8 = 4; + pub const OP_STAT: u8 = 5; + pub const OP_MKDIR: u8 = 6; + pub const OP_RENAME: u8 = 7; + pub const OP_USAGE: u8 = 8; + pub const OP_LAST_ERROR: u8 = 9; + /// write() modes. + pub const WRITE_TRUNCATE: u32 = 0; + pub const WRITE_APPEND: u32 = 1; + /// Marker key for a bytes payload (db's blob spelling). + pub const BLOB_KEY: &str = "$b"; + /// Maximum UTF-8 bytes in one path segment. + pub const MAX_SEGMENT_BYTES: usize = 64; + /// Maximum segments in a path. + pub const MAX_DEPTH: usize = 8; + /// Maximum total path length in bytes. + pub const MAX_PATH_BYTES: usize = 160; + /// Payload ceiling per read()/write() call, in bytes. + pub const MAX_IO_BYTES: usize = 65536; + /// Entries per list() call (paged via offset + eof). + pub const MAX_DIR_ENTRIES: usize = 256; +} diff --git a/engine/crates/pocket-db/Cargo.toml b/engine/crates/pocket-db/Cargo.toml new file mode 100644 index 00000000..85386356 --- /dev/null +++ b/engine/crates/pocket-db/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "pocket-db" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "The db module's reference core: SQLite behind the five-op contracts/spec/db.ts boundary, mountable as globalThis.db via pocket-mod" + +[features] +# `mount` brings pocket-mod (and its QuickJS embedding) for the one-line +# globalThis.db install. A device host with its own QuickJS wiring turns +# it off (`default-features = false`) and drives DbModule directly — the +# MCU build then never compiles an engine it doesn't use. +default = ["mount"] +mount = ["dep:pocket-mod", "dep:anyhow"] + +[dependencies] +pocketjs-core = { workspace = true } +pocket-mod = { workspace = true, optional = true } +anyhow = { workspace = true, optional = true } +rusqlite = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } diff --git a/engine/crates/pocket-db/src/lib.rs b/engine/crates/pocket-db/src/lib.rs new file mode 100644 index 00000000..bcf6d1a0 --- /dev/null +++ b/engine/crates/pocket-db/src/lib.rs @@ -0,0 +1,657 @@ +//! pocket-db — the db module's reference core. +//! +//! SQLite behind the five-op boundary pinned in contracts/spec/db.ts +//! (`pocketjs_core::spec::db` is the generated mirror): open / close / exec / +//! query / lastError, mounted as `globalThis.db` through [`mount`]. Rows +//! cross as one JSON line per `query()` call using the spec's value +//! encoding; statement caching is host-side (rusqlite's prepared-statement +//! cache, keyed by the sql string), so the guest holds no statement handles. +//! +//! Storage policy is the host's: [`Storage::Memory`] for tests and +//! throwaway guests, [`Storage::Dir`] to map each logical database name to +//! `/.sqlite` — an ORDINARY file in the app's own data root, +//! the same root the fs module is typically bound to. That is deliberate: +//! the database is the app's own asset, visible and touchable like any of +//! its files (backup = a file copy). An app that overwrites its own +//! database corrupts its own data — the same trust class as deleting its +//! own files, and SQLite fails loudly (SQLITE_CORRUPT), not unsafely. +//! `ATTACH` is refused twice over — a real SQLite authorizer for the +//! literal spelling, and `SQLITE_LIMIT_ATTACHED=0` for the expression +//! spelling the authorizer cannot see — which keeps that root the sandbox +//! boundary; `load_extension` stays off (rusqlite's default). +//! +//! ESP32/LittleFS: this crate carries its own ESP-IDF support (the +//! `espidf` module below — newlib symbol shims, the `unix-none` VFS, the +//! flash-friendly pragmas), all behind `cfg(target_os = "espidf")`; +//! desktop builds never see it. The build-environment recipe a firmware +//! needs (C flags, header shim) is documented in docs/DB.md — validated +//! on an ESP32-P4 with a LittleFS workspace, where data survived reopen +//! and power cycling. + +#[cfg(feature = "mount")] +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::PathBuf; +#[cfg(feature = "mount")] +use std::rc::Rc; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use pocketjs_core::spec::db as spec; +use rusqlite::hooks::Authorization; +use rusqlite::types::{Value, ValueRef}; +use rusqlite::Connection; +use serde_json::{json, Map as JsonMap, Value as Json}; + +/// Where logical database names live. +pub enum Storage { + /// Every database, named or `:memory:`, is in-memory (tests, previews). + /// Named databases still share a handle for the module's lifetime. + Memory, + /// A named database maps to `/.sqlite` — an ordinary file + /// in the app's own data root (created on first open), the same root + /// the fs module is typically bound to. The guest never sees the path. + Dir(PathBuf), +} + +struct Db { + conn: Connection, + name: String, + last_error: String, +} + +/// The db module: every op as a method, [`mount`] to install the namespace. +pub struct DbModule { + storage: Storage, + dbs: HashMap, + by_name: HashMap, + next_handle: i32, +} + +impl DbModule { + pub fn new(storage: Storage) -> DbModule { + DbModule { + storage, + dbs: HashMap::new(), + by_name: HashMap::new(), + next_handle: 1, + } + } + + /// `open(name) -> handle | -1` (spec OP_OPEN). + pub fn open(&mut self, name: &str) -> i32 { + let memory = name == spec::MEMORY; + if !memory { + if !valid_name(name) { + return -1; + } + if let Some(handle) = self.by_name.get(name) { + return *handle; + } + } + if self.dbs.len() >= spec::MAX_DATABASES { + return -1; + } + let conn = match &self.storage { + Storage::Memory => Connection::open_in_memory(), + Storage::Dir(dir) if !memory => { + if std::fs::create_dir_all(dir).is_err() { + return -1; + } + open_file(&dir.join(format!("{name}.sqlite"))) + } + Storage::Dir(_) => Connection::open_in_memory(), + }; + let conn = match conn { + Ok(conn) => conn, + Err(_) => return -1, + }; + // The storage rule's teeth: ATTACH names a file, so it is denied at + // the engine level. A database the authorizer cannot guard is a + // database this module refuses to open. + if conn + .authorizer(Some(|ctx: rusqlite::hooks::AuthContext<'_>| { + match ctx.action { + rusqlite::hooks::AuthAction::Attach { .. } => Authorization::Deny, + _ => Authorization::Allow, + } + })) + .is_err() + { + return -1; + } + // The authorizer only sees a FILENAME literal: `ATTACH AS x` + // reaches it with a NULL filename, which rusqlite maps to + // AuthAction::Unknown — allowed by the catch-all above. The engine + // attach limit closes every spelling; the authorizer stays for the + // clearer "not authorized" on the literal form. + if conn + .set_limit(rusqlite::limits::Limit::SQLITE_LIMIT_ATTACHED, 0) + .is_err() + { + return -1; + } + let handle = self.next_handle; + self.next_handle += 1; + self.dbs.insert( + handle, + Db { + conn, + name: name.to_owned(), + last_error: String::new(), + }, + ); + if !memory { + self.by_name.insert(name.to_owned(), handle); + } + handle + } + + /// `close(handle)` (spec OP_CLOSE) — idempotent. + pub fn close(&mut self, handle: i32) { + if let Some(db) = self.dbs.remove(&handle) + && db.name != spec::MEMORY + { + self.by_name.remove(&db.name); + } + } + + /// `exec(handle, sql) -> 0 | 1` (spec OP_EXEC). + pub fn exec(&mut self, handle: i32, sql: &str) -> i32 { + let Some(db) = self.dbs.get_mut(&handle) else { + return 1; + }; + match db.conn.execute_batch(sql) { + Ok(()) => { + db.last_error.clear(); + 0 + } + Err(error) => { + db.last_error = error.to_string(); + 1 + } + } + } + + /// `query(handle, sql, args) -> json line` (spec OP_QUERY). + pub fn query(&mut self, handle: i32, sql: &str, args: &str) -> String { + let Some(db) = self.dbs.get_mut(&handle) else { + return json!({ "error": "database is closed" }).to_string(); + }; + match run_query(&db.conn, sql, args) { + Ok(line) => { + db.last_error.clear(); + line + } + Err(message) => { + db.last_error = message.clone(); + json!({ "error": message }).to_string() + } + } + } + + /// `lastError(handle) -> string` (spec OP_LAST_ERROR). + pub fn last_error(&self, handle: i32) -> String { + match self.dbs.get(&handle) { + Some(db) => db.last_error.clone(), + None => "database is closed".to_owned(), + } + } +} + +/// Open a persistent database file the platform way. Desktop: the default +/// VFS. ESP-IDF: the `unix-none` VFS — LittleFS has no fcntl file locks, +/// and a Pocket guest's module instance is the file's only writer — plus +/// the flash-friendly pragmas the ESP32-P4 probe validated (TRUNCATE +/// journal: WAL is compiled out on MCU builds; NORMAL sync; a 32 KiB page +/// cache sized for a device heap). +fn open_file(path: &std::path::Path) -> rusqlite::Result { + #[cfg(not(target_os = "espidf"))] + { + Connection::open(path) + } + #[cfg(target_os = "espidf")] + { + let conn = Connection::open_with_flags_and_vfs( + path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE + | rusqlite::OpenFlags::SQLITE_OPEN_CREATE + | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX, + "unix-none", + )?; + conn.query_row("PRAGMA journal_mode=TRUNCATE", [], |_| Ok(()))?; + conn.execute_batch("PRAGMA synchronous=NORMAL; PRAGMA cache_size=-32;")?; + Ok(conn) + } +} + +// --------------------------------------------------------------------------- +// ESP-IDF (ESP32) support +// --------------------------------------------------------------------------- +// SQLite's unix VFS keeps a syscall table referencing a handful of POSIX +// symbols newlib does not provide. Under this crate's configuration +// (unix-none VFS, `-Dlstat=stat`, no symlinks, no dotlock files) the first +// five are never actually called — no-op successes are the honest +// implementations for a filesystem with no users, permissions or symlinks. +// nanosleep IS called (the busy handler sleeps); it routes through +// ESP-IDF's usleep. Compiled only for espidf; a desktop build never sees +// these. The build-environment recipe these link against is in docs/DB.md. +#[cfg(target_os = "espidf")] +mod espidf { + #[repr(C)] + pub struct Timespec { + tv_sec: i64, // espidf_time64: 64-bit time_t + tv_nsec: i32, + } + + unsafe extern "C" { + fn usleep(microseconds: u32) -> i32; + } + + #[unsafe(no_mangle)] + extern "C" fn geteuid() -> u32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn fchmod(_fd: i32, _mode: u32) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn fchown(_fd: i32, _owner: u32, _group: u32) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn utimes( + _path: *const core::ffi::c_char, + _times: *const core::ffi::c_void, + ) -> i32 { + 0 + } + + #[unsafe(no_mangle)] + extern "C" fn readlink( + _path: *const core::ffi::c_char, + _buf: *mut core::ffi::c_char, + _len: usize, + ) -> isize { + -1 // never a symlink on LittleFS + } + + #[unsafe(no_mangle)] + extern "C" fn nanosleep(request: *const Timespec, _remain: *mut Timespec) -> i32 { + let request = unsafe { &*request }; + let micros = (request.tv_sec as u64) + .saturating_mul(1_000_000) + .saturating_add((request.tv_nsec as u64) / 1_000); + unsafe { usleep(micros.min(u32::MAX as u64) as u32) } + } +} + +/// Logical persistent-database names (spec DB_NAME_PATTERN): +/// `^[A-Za-z0-9][A-Za-z0-9._-]{0,56}$`, spelled out to keep regex out of +/// the dependency tree. 57 chars keeps `.sqlite` within the fs +/// module's 64-byte segment ceiling. +fn valid_name(name: &str) -> bool { + let bytes = name.as_bytes(); + if bytes.is_empty() || bytes.len() > 57 { + return false; + } + if !bytes[0].is_ascii_alphanumeric() { + return false; + } + bytes[1..] + .iter() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +fn run_query(conn: &Connection, sql: &str, args: &str) -> Result { + let params: Json = serde_json::from_str(args).map_err(|e| format!("malformed args: {e}"))?; + let mut statement = conn + .prepare_cached(sql) + .map_err(|e| e.to_string())?; + + match ¶ms { + Json::Array(list) => { + if list.len() != statement.parameter_count() { + return Err(format!( + "expected {} parameters, got {}", + statement.parameter_count(), + list.len() + )); + } + for (i, value) in list.iter().enumerate() { + statement + .raw_bind_parameter(i + 1, decode_param(value)?) + .map_err(|e| e.to_string())?; + } + } + Json::Object(named) => { + for (key, value) in named { + let index = statement + .parameter_index(key) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("unknown parameter: {key}"))?; + statement + .raw_bind_parameter(index, decode_param(value)?) + .map_err(|e| e.to_string())?; + } + } + _ => return Err("args must be a JSON array or object".to_owned()), + } + + let cols: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let column_count = cols.len(); + + let mut rows_out: Vec = Vec::new(); + let mut rows = statement.raw_query(); + loop { + let row = rows.next().map_err(|e| e.to_string())?; + let Some(row) = row else { break }; + if rows_out.len() >= spec::MAX_RESULT_ROWS { + return Err("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate".to_owned()); + } + let mut cells: Vec = Vec::with_capacity(column_count); + for i in 0..column_count { + cells.push(encode_cell(row.get_ref(i).map_err(|e| e.to_string())?)?); + } + rows_out.push(Json::Array(cells)); + } + drop(rows); + + Ok(json!({ + "cols": cols, + "rows": rows_out, + "changes": conn.changes(), + "lastInsertRowid": conn.last_insert_rowid(), + }) + .to_string()) +} + +/// JSON parameter -> SQLite value (the spec's binding rules). +fn decode_param(value: &Json) -> Result { + Ok(match value { + Json::Null => Value::Null, + Json::Bool(b) => Value::Integer(i64::from(*b)), + Json::Number(n) => { + if let Some(i) = n.as_i64() { + if i.unsigned_abs() > spec::MAX_SAFE_INTEGER as u64 { + return Err("integer exceeds DB_MAX_SAFE_INTEGER".to_owned()); + } + Value::Integer(i) + } else { + let f = n.as_f64().ok_or("unrepresentable number")?; + if !f.is_finite() { + return Err("cannot bind a non-finite number".to_owned()); + } + Value::Real(f) + } + } + Json::String(s) => Value::Text(s.clone()), + Json::Object(map) => Value::Blob(decode_blob(map)?), + Json::Array(_) => return Err("cannot bind an array value".to_owned()), + }) +} + +fn decode_blob(map: &JsonMap) -> Result, String> { + let Some(Json::String(b64)) = map.get(spec::BLOB_KEY) else { + return Err("malformed blob parameter".to_owned()); + }; + BASE64 + .decode(b64) + .map_err(|e| format!("malformed blob parameter: {e}")) +} + +/// SQLite cell -> JSON value (the spec's row encoding; loud on lossy). +fn encode_cell(cell: ValueRef<'_>) -> Result { + Ok(match cell { + ValueRef::Null => Json::Null, + ValueRef::Integer(i) => { + if i.unsigned_abs() > spec::MAX_SAFE_INTEGER as u64 { + return Err("integer result exceeds DB_MAX_SAFE_INTEGER".to_owned()); + } + json!(i) + } + ValueRef::Real(f) => { + if !f.is_finite() { + return Err("non-finite REAL result".to_owned()); + } + json!(f) + } + ValueRef::Text(t) => { + Json::String(String::from_utf8(t.to_vec()).map_err(|_| "non-UTF-8 TEXT result")?) + } + ValueRef::Blob(b) => json!({ spec::BLOB_KEY: BASE64.encode(b) }), + }) +} + +/// Mount the module as `globalThis.db` on a pocket-mod [`Guest`] — one JS +/// function per spec op, marshaled as (i32, String) -> i32/String. +/// Feature `mount` (default); a host with its own QuickJS wiring turns it +/// off and spells these five functions itself. +#[cfg(feature = "mount")] +pub fn mount(guest: &pocket_mod::Guest, module: Rc>) -> anyhow::Result<()> { + use pocket_mod::qjs::Function; + guest.mount("db", |ctx, ns| { + let m = module.clone(); + ns.set( + "open", + Function::new(ctx.clone(), move |name: String| -> i32 { + m.borrow_mut().open(&name) + })?, + )?; + let m = module.clone(); + ns.set( + "close", + Function::new(ctx.clone(), move |handle: i32| { + m.borrow_mut().close(handle); + })?, + )?; + let m = module.clone(); + ns.set( + "exec", + Function::new(ctx.clone(), move |handle: i32, sql: String| -> i32 { + m.borrow_mut().exec(handle, &sql) + })?, + )?; + let m = module.clone(); + ns.set( + "query", + Function::new( + ctx.clone(), + move |handle: i32, sql: String, args: String| -> String { + m.borrow_mut().query(handle, &sql, &args) + }, + )?, + )?; + let m = module.clone(); + ns.set( + "lastError", + Function::new(ctx.clone(), move |handle: i32| -> String { + m.borrow().last_error(handle) + })?, + )?; + Ok(()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn module() -> DbModule { + DbModule::new(Storage::Memory) + } + + fn rows(line: &str) -> Json { + serde_json::from_str(line).unwrap() + } + + #[test] + fn crud_round_trip_with_positional_and_named_parameters() { + let mut m = module(); + let h = m.open(spec::MEMORY); + assert!(h > 0); + assert_eq!(m.exec(h, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"), 0); + let ins = rows(&m.query(h, "INSERT INTO t (v) VALUES ($v)", r#"{"$v":"hello"}"#)); + assert_eq!(ins["changes"], 1); + assert_eq!(ins["lastInsertRowid"], 1); + let sel = rows(&m.query(h, "SELECT id, v FROM t WHERE id = ?", "[1]")); + assert_eq!(sel["cols"], json!(["id", "v"])); + assert_eq!(sel["rows"], json!([[1, "hello"]])); + } + + #[test] + fn open_refuses_bad_names_and_over_limit() { + let mut m = module(); + assert_eq!(m.open("../escape"), -1); + assert_eq!(m.open(".hidden"), -1); + assert_eq!(m.open(""), -1); + // 57 is the ceiling: `.sqlite` stays a valid fs segment. + assert_eq!(m.open(&"a".repeat(58)), -1); + let longest = m.open(&"a".repeat(57)); + assert!(longest > 0); + m.close(longest); + for i in 0..spec::MAX_DATABASES { + assert!(m.open(&format!("app-{i}")) > 0); + } + assert_eq!(m.open("one-too-many"), -1); + } + + #[test] + fn same_persistent_name_shares_a_handle_and_memory_never_does() { + let mut m = module(); + let a = m.open("app"); + assert_eq!(m.open("app"), a); + assert_ne!(m.open(spec::MEMORY), m.open(spec::MEMORY)); + } + + #[test] + fn attach_is_denied_by_the_authorizer() { + let mut m = module(); + let h = m.open(spec::MEMORY); + assert_eq!(m.exec(h, "ATTACH DATABASE ':memory:' AS other"), 1); + assert!(m.last_error(h).contains("not authorized"), "{}", m.last_error(h)); + let line = m.query(h, "ATTACH DATABASE ':memory:' AS other", "[]"); + assert!(rows(&line)["error"].as_str().unwrap().contains("not authorized")); + } + + #[test] + fn attach_with_an_expression_filename_is_refused_by_the_attach_limit() { + // `ATTACH AS x` reaches the authorizer with a NULL filename + // (AuthAction::Unknown), so only SQLITE_LIMIT_ATTACHED=0 refuses it. + let mut m = module(); + let h = m.open(spec::MEMORY); + assert_eq!(m.exec(h, "ATTACH hex('2f746d702f78') AS other"), 1); + assert!( + m.last_error(h).contains("attached databases"), + "{}", + m.last_error(h) + ); + let line = m.query(h, "ATTACH ':memory:' AS other", "[]"); + let error = rows(&line)["error"].as_str().unwrap().to_owned(); + assert!( + error.contains("not authorized") || error.contains("attached databases"), + "{error}" + ); + } + + #[test] + fn closed_handles_fail_loudly_and_close_is_idempotent() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.close(h); + m.close(h); + assert_eq!(m.exec(h, "SELECT 1"), 1); + assert_eq!(rows(&m.query(h, "SELECT 1", "[]"))["error"], "database is closed"); + assert_eq!(m.last_error(h), "database is closed"); + } + + #[test] + fn blobs_round_trip_and_big_integers_fail_loudly() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.exec(h, "CREATE TABLE b (data BLOB)"); + let ins = m.query(h, "INSERT INTO b VALUES (?)", r#"[{"$b":"AAEC+vv8/f7/"}]"#); + assert_eq!(rows(&ins)["changes"], 1); + let sel = rows(&m.query(h, "SELECT data FROM b", "[]")); + assert_eq!(sel["rows"][0][0][spec::BLOB_KEY], "AAEC+vv8/f7/"); + + let over = rows(&m.query(h, "SELECT 9007199254740993", "[]")); + assert!(over["error"].as_str().unwrap().contains("DB_MAX_SAFE_INTEGER")); + let ok = rows(&m.query(h, "SELECT 9007199254740991", "[]")); + assert_eq!(ok["rows"], json!([[9007199254740991i64]])); + } + + #[test] + fn result_rows_beyond_the_ceiling_fail() { + let mut m = module(); + let h = m.open(spec::MEMORY); + m.exec(h, "CREATE TABLE n (v INTEGER)"); + let fill = format!( + "WITH RECURSIVE seq(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM seq LIMIT {}) + INSERT INTO n SELECT x FROM seq", + spec::MAX_RESULT_ROWS + 1 + ); + assert_eq!(m.exec(h, &fill), 0); + let over = rows(&m.query(h, "SELECT v FROM n", "[]")); + assert!(over["error"].as_str().unwrap().contains("DB_MAX_RESULT_ROWS")); + let capped = rows(&m.query( + h, + &format!("SELECT v FROM n LIMIT {}", spec::MAX_RESULT_ROWS), + "[]", + )); + assert_eq!(capped["rows"].as_array().unwrap().len(), spec::MAX_RESULT_ROWS); + } + + #[test] + fn persistent_dir_storage_survives_reopen() { + let dir = std::env::temp_dir().join(format!("pocket-db-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + { + let mut m = DbModule::new(Storage::Dir(dir.clone())); + let h = m.open("ledger"); + m.exec(h, "CREATE TABLE snap (v INTEGER); INSERT INTO snap VALUES (42)"); + } + { + let mut m = DbModule::new(Storage::Dir(dir.clone())); + let h = m.open("ledger"); + let sel = rows(&m.query(h, "SELECT v FROM snap", "[]")); + assert_eq!(sel["rows"], json!([[42]])); + } + // The database is an ordinary file in the data root — the app's own + // asset, visible to a co-mounted fs module like any of its files. + assert!(dir.join("ledger.sqlite").is_file()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[cfg(feature = "mount")] + #[test] + fn mounted_namespace_serves_a_quickjs_guest() { + let guest = pocket_mod::Guest::new().unwrap(); + let module = Rc::new(RefCell::new(module())); + mount(&guest, module).unwrap(); + guest + .eval( + "boot", + r#" + const h = db.open(":memory:"); + if (h < 0) throw new Error("open failed"); + if (db.exec(h, "CREATE TABLE t (v TEXT)") !== 0) throw new Error(db.lastError(h)); + const ins = JSON.parse(db.query(h, "INSERT INTO t VALUES (?)", '["from-guest"]')); + if (ins.changes !== 1) throw new Error("insert failed"); + const sel = JSON.parse(db.query(h, "SELECT v FROM t", "[]")); + globalThis.result = sel.rows[0][0]; + "#, + ) + .unwrap(); + let result: String = guest.with(|ctx| ctx.globals().get("result").unwrap()); + assert_eq!(result, "from-guest"); + } +} diff --git a/engine/crates/pocket-fs/Cargo.toml b/engine/crates/pocket-fs/Cargo.toml new file mode 100644 index 00000000..b9c4f2c7 --- /dev/null +++ b/engine/crates/pocket-fs/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "pocket-fs" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +description = "The fs module's reference core: a per-app file tree behind the nine-op contracts/spec/fs.ts boundary, mountable as globalThis.fs via pocket-mod" + +[features] +# `mount` brings pocket-mod (and its QuickJS embedding) for the one-line +# globalThis.fs install. A device host with its own QuickJS wiring turns +# it off (`default-features = false`) and drives FsModule directly — the +# MCU build then never compiles an engine it doesn't use. +default = ["mount"] +mount = ["dep:pocket-mod", "dep:anyhow"] + +[dependencies] +pocketjs-core = { workspace = true } +pocket-mod = { workspace = true, optional = true } +anyhow = { workspace = true, optional = true } +serde_json = { workspace = true } +base64 = { workspace = true } diff --git a/engine/crates/pocket-fs/src/lib.rs b/engine/crates/pocket-fs/src/lib.rs new file mode 100644 index 00000000..6e847590 --- /dev/null +++ b/engine/crates/pocket-fs/src/lib.rs @@ -0,0 +1,1044 @@ +//! pocket-fs — the fs module's reference core. +//! +//! A per-app file tree behind the nine-op boundary pinned in +//! contracts/spec/fs.ts (`pocketjs_core::spec::fs` is the generated +//! mirror): read / write / remove / list / stat / mkdir / rename / usage / +//! lastError, mounted as `globalThis.fs` through [`mount`]. Payloads cross +//! as one JSON value (a string for text, `{"$b": base64}` for bytes); +//! results cross as one JSON line. +//! +//! Storage policy is the host's: [`Storage::Memory`] for tests and +//! throwaway guests, [`Storage::Dir`] to bind the module to the app's own +//! data root on a real filesystem. Names are universal — any well-formed +//! Unicode, dot-prefixed included; nothing in the app's tree is reserved +//! to the host. Isolation is by construction and never depended on names: +//! every path is relative, `..`/absolute/`/`-in-segment are +//! unrepresentable, so the bound root is the sandbox boundary the way +//! db's ATTACH refusal keeps its data root one. The guest cannot create +//! symlinks through this API, but a host-side actor may have (on Pocket +//! Pi the device agent owns the whole workspace), so the Dir backend +//! lstat-checks every segment and treats any symlink as absent. +//! +//! Truncate writes are ATOMIC (temp + rename): after power loss a file +//! holds the old content or the new, never a torn middle. Temps land in +//! the module's own `tmp` directory — OUTSIDE the bound root, on the +//! same filesystem (cross-directory rename stays atomic) — so the app's +//! tree never shows host machinery, and a crash orphan cannot outlive +//! the next construction: the module OWNS `tmp` and clears it on +//! construction, which is provably safe precisely because nothing else +//! may live there. Porting note (the ESP32/LittleFS path): LittleFS's +//! rename is atomic, so a device host keeps the same contract by the +//! same moves. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Read as _, Seek as _, SeekFrom, Write as _}; +use std::path::{Path, PathBuf}; + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use pocketjs_core::spec::fs as spec; +use serde_json::{json, Value as Json}; + +/// Where the app's file tree lives. +pub enum Storage { + /// The whole tree in memory (tests, previews). Byte-exact names — the + /// deterministic twin of the sim host. + Memory, + /// The tree under `root` — the app's own data root; the guest never + /// sees the path. `tmp` is a host-private directory for atomic-write + /// temps: same filesystem, outside `root` (Pocket Pi layout: + /// `apps//data` and `apps//tmp`). The module OWNS `tmp` and + /// clears it on construction. + Dir { root: PathBuf, tmp: PathBuf }, +} + +enum Backend { + Memory { + files: BTreeMap>, + dirs: BTreeSet, + }, + Dir { + root: PathBuf, + tmp: PathBuf, + tmp_counter: u64, + }, +} + +/// The fs module: every op as a method, [`mount`] to install the namespace. +pub struct FsModule { + backend: Backend, + /// Byte budget for the tree; 0 = unmetered. Enforced on write. + quota_bytes: u64, + last_error: String, +} + +impl FsModule { + pub fn new(storage: Storage) -> FsModule { + FsModule::with_quota(storage, 0) + } + + pub fn with_quota(storage: Storage, quota_bytes: u64) -> FsModule { + FsModule { + backend: match storage { + Storage::Memory => Backend::Memory { + files: BTreeMap::new(), + dirs: BTreeSet::new(), + }, + Storage::Dir { root, tmp } => { + // Best-effort sweep: any leftover temp is an orphan + // from a crash mid-write; nothing else lives here. + let _ = std::fs::remove_dir_all(&tmp); + Backend::Dir { + root, + tmp, + tmp_counter: 0, + } + } + }, + quota_bytes, + last_error: String::new(), + } + } + + fn ok_line(&mut self, line: String) -> String { + self.last_error.clear(); + line + } + + fn err_line(&mut self, message: &str) -> String { + self.last_error = message.to_owned(); + json!({ "error": message }).to_string() + } + + fn status(&mut self, result: Result<(), String>) -> i32 { + match result { + Ok(()) => { + self.last_error.clear(); + 0 + } + Err(message) => { + self.last_error = message; + 1 + } + } + } + + /// `read(path, offset, maxBytes) -> json line` (spec OP_READ). + pub fn read(&mut self, path: &str, offset: i64, max_bytes: i64) -> String { + if !valid_path(path) { + return self.err_line("invalid path"); + } + if max_bytes < 1 || max_bytes as usize > spec::MAX_IO_BYTES { + return self.err_line("read maxBytes out of range"); + } + if offset < 0 { + return self.err_line("read offset out of range"); + } + let result = match &mut self.backend { + Backend::Memory { files, dirs } => { + match files.get(path) { + Some(bytes) => { + let start = (offset as usize).min(bytes.len()); + let end = (start + max_bytes as usize).min(bytes.len()); + Ok((bytes[start..end].to_vec(), bytes.len() as u64, end >= bytes.len())) + } + None if dirs.contains(path) => Err("is a directory".to_owned()), + None => Err("not found".to_owned()), + } + } + Backend::Dir { root, .. } => dir_read(root, path, offset as u64, max_bytes as usize), + }; + match result { + Ok((chunk, size, eof)) => self.ok_line( + json!({ + "data": { spec::BLOB_KEY: BASE64.encode(&chunk) }, + "size": size, + "eof": eof, + }) + .to_string(), + ), + Err(message) => self.err_line(&message), + } + } + + /// `write(path, data, mode) -> 0 | 1` (spec OP_WRITE). + pub fn write(&mut self, path: &str, data: &str, mode: u32) -> i32 { + let result = self.write_inner(path, data, mode); + self.status(result) + } + + fn write_inner(&mut self, path: &str, data: &str, mode: u32) -> Result<(), String> { + if !valid_path(path) { + return Err("invalid path".to_owned()); + } + if mode != spec::WRITE_TRUNCATE && mode != spec::WRITE_APPEND { + return Err("invalid write mode".to_owned()); + } + let payload = decode_payload(data)?; + if payload.len() > spec::MAX_IO_BYTES { + return Err("write exceeds FS_MAX_IO_BYTES".to_owned()); + } + let quota = self.quota_bytes; + match &mut self.backend { + Backend::Memory { files, dirs } => { + if dirs.contains(path) { + return Err("is a directory".to_owned()); + } + for ancestor in ancestors_of(path) { + if files.contains_key(&ancestor) { + return Err(format!("not a directory: {ancestor}")); + } + dirs.insert(ancestor); + } + let existing = files.get(path).map(Vec::len).unwrap_or(0) as u64; + let next = if mode == spec::WRITE_APPEND { + existing + payload.len() as u64 + } else { + payload.len() as u64 + }; + let used: u64 = files.values().map(|b| b.len() as u64).sum(); + if quota > 0 && used - existing + next > quota { + return Err("quota exceeded".to_owned()); + } + if mode == spec::WRITE_APPEND { + files.entry(path.to_owned()).or_default().extend_from_slice(&payload); + } else { + files.insert(path.to_owned(), payload); + } + Ok(()) + } + Backend::Dir { + root, + tmp, + tmp_counter, + } => { + *tmp_counter += 1; + dir_write(root, tmp, path, &payload, mode, quota, *tmp_counter) + } + } + } + + /// `remove(path, recursive) -> 0 | 1` (spec OP_REMOVE). + pub fn remove(&mut self, path: &str, recursive: u32) -> i32 { + let result = (|| { + if !valid_path(path) { + return Err("invalid path".to_owned()); + } + match &mut self.backend { + Backend::Memory { files, dirs } => { + if files.remove(path).is_some() { + return Ok(()); + } + if !dirs.contains(path) { + return Err("not found".to_owned()); + } + let prefix = format!("{path}/"); + let occupied = files.keys().any(|k| k.starts_with(&prefix)) + || dirs.iter().any(|k| k.starts_with(&prefix)); + if occupied && recursive != 1 { + return Err("directory not empty".to_owned()); + } + files.retain(|k, _| !k.starts_with(&prefix)); + dirs.retain(|k| !k.starts_with(&prefix)); + dirs.remove(path); + Ok(()) + } + Backend::Dir { root, .. } => dir_remove(root, path, recursive == 1), + } + })(); + self.status(result) + } + + /// `list(path, offset) -> json line` (spec OP_LIST). + pub fn list(&mut self, path: &str, offset: i64) -> String { + if !path.is_empty() && !valid_path(path) { + return self.err_line("invalid path"); + } + let offset = offset.max(0) as usize; + let result = match &mut self.backend { + Backend::Memory { files, dirs } => { + if files.contains_key(path) { + Err("not a directory".to_owned()) + } else if !path.is_empty() && !dirs.contains(path) { + Err("not found".to_owned()) + } else { + let mut names: BTreeSet = BTreeSet::new(); + let prefix = if path.is_empty() { String::new() } else { format!("{path}/") }; + for key in files.keys().chain(dirs.iter()) { + if let Some(rest) = key.strip_prefix(&prefix) { + if key == path || rest.is_empty() { + continue; + } + names.insert(rest.split('/').next().unwrap().to_owned()); + } + } + Ok(names + .into_iter() + .map(|name| { + let full = + if path.is_empty() { name.clone() } else { format!("{path}/{name}") }; + match files.get(&full) { + Some(bytes) => (name, "file", bytes.len() as u64), + None => (name, "dir", 0), + } + }) + .collect::>()) + } + } + Backend::Dir { root, .. } => dir_list(root, path), + }; + match result { + Ok(all) => { + let page: Vec = all + .iter() + .skip(offset) + .take(spec::MAX_DIR_ENTRIES) + .map(|(name, kind, size)| json!({ "name": name, "kind": kind, "size": size })) + .collect(); + let eof = offset + page.len() >= all.len(); + self.ok_line(json!({ "entries": page, "eof": eof }).to_string()) + } + Err(message) => self.err_line(&message), + } + } + + /// `stat(path) -> json line` (spec OP_STAT). + pub fn stat(&mut self, path: &str) -> String { + if path.is_empty() { + return self.ok_line(json!({ "kind": "dir", "size": 0 }).to_string()); + } + if !valid_path(path) { + return self.err_line("invalid path"); + } + let result = match &mut self.backend { + Backend::Memory { files, dirs } => match files.get(path) { + Some(bytes) => Some(("file", bytes.len() as u64)), + None if dirs.contains(path) => Some(("dir", 0)), + None => None, + }, + Backend::Dir { root, .. } => dir_stat(root, path), + }; + match result { + Some((kind, size)) => self.ok_line(json!({ "kind": kind, "size": size }).to_string()), + None => self.err_line("not found"), + } + } + + /// `mkdir(path) -> 0 | 1` (spec OP_MKDIR) — recursive, idempotent. + pub fn mkdir(&mut self, path: &str) -> i32 { + let result = (|| { + if !valid_path(path) { + return Err("invalid path".to_owned()); + } + match &mut self.backend { + Backend::Memory { files, dirs } => { + if files.contains_key(path) { + return Err(format!("not a directory: {path}")); + } + for ancestor in ancestors_of(path) { + if files.contains_key(&ancestor) { + return Err(format!("not a directory: {ancestor}")); + } + dirs.insert(ancestor); + } + dirs.insert(path.to_owned()); + Ok(()) + } + Backend::Dir { root, .. } => dir_mkdir(root, path), + } + })(); + self.status(result) + } + + /// `rename(from, to) -> 0 | 1` (spec OP_RENAME). + pub fn rename(&mut self, from: &str, to: &str) -> i32 { + let result = (|| { + if !valid_path(from) || !valid_path(to) { + return Err("invalid path".to_owned()); + } + if from == to { + return Ok(()); + } + if to.starts_with(&format!("{from}/")) { + return Err("cannot rename into own subtree".to_owned()); + } + match &mut self.backend { + Backend::Memory { files, dirs } => { + let to_parent = parent_of(to); + if !to_parent.is_empty() && !dirs.contains(to_parent) { + return Err("not found".to_owned()); + } + if dirs.contains(to) { + return Err("destination exists".to_owned()); + } + if let Some(bytes) = files.remove(from) { + files.insert(to.to_owned(), bytes); + return Ok(()); + } + if !dirs.contains(from) { + return Err("not found".to_owned()); + } + if files.contains_key(to) { + return Err("destination exists".to_owned()); + } + let prefix = format!("{from}/"); + let moved_files: Vec<(String, Vec)> = files + .iter() + .filter(|(k, _)| k.starts_with(&prefix)) + .map(|(k, v)| (format!("{to}/{}", &k[prefix.len()..]), v.clone())) + .collect(); + files.retain(|k, _| !k.starts_with(&prefix)); + files.extend(moved_files); + let moved_dirs: Vec = dirs + .iter() + .filter(|k| k.starts_with(&prefix)) + .map(|k| format!("{to}/{}", &k[prefix.len()..])) + .collect(); + dirs.retain(|k| !k.starts_with(&prefix)); + dirs.extend(moved_dirs); + dirs.remove(from); + dirs.insert(to.to_owned()); + Ok(()) + } + Backend::Dir { root, .. } => dir_rename(root, from, to), + } + })(); + self.status(result) + } + + /// `usage() -> json line` (spec OP_USAGE). + pub fn usage(&mut self) -> String { + let used: u64 = match &self.backend { + Backend::Memory { files, .. } => files.values().map(|b| b.len() as u64).sum(), + Backend::Dir { root, .. } => dir_used_bytes(root), + }; + let quota = self.quota_bytes; + self.ok_line(json!({ "usedBytes": used, "quotaBytes": quota }).to_string()) + } + + /// `lastError() -> string` (spec OP_LAST_ERROR) — module-scoped. + pub fn last_error(&self) -> String { + self.last_error.clone() + } +} + +// --------------------------------------------------------------------------- +// The path grammar (contracts/spec/fs.ts, spelled out — no regex dependency) +// --------------------------------------------------------------------------- + +/// Universal names: any well-formed Unicode (a Rust `&str` already is) +/// except the escape hatches ("." and ".."), control characters, and +/// oversize segments. "/" inside a name is unrepresentable — the caller +/// split on it. +fn valid_segment(segment: &str) -> bool { + if segment.is_empty() || segment.len() > spec::MAX_SEGMENT_BYTES { + return false; + } + if segment == "." || segment == ".." { + return false; + } + !segment.bytes().any(|b| b < 0x20 || b == 0x7f) +} + +/// fsValidSegment / FS_MAX_DEPTH / FS_MAX_PATH_BYTES, one predicate. +fn valid_path(path: &str) -> bool { + if path.is_empty() || path.len() > spec::MAX_PATH_BYTES { + return false; + } + let segments: Vec<&str> = path.split('/').collect(); + segments.len() <= spec::MAX_DEPTH && segments.iter().all(|s| valid_segment(s)) +} + +/// Ancestor paths of a valid path, nearest last ("a/b/c" -> ["a", "a/b"]). +fn ancestors_of(path: &str) -> Vec { + let mut out = Vec::new(); + for (i, b) in path.bytes().enumerate() { + if b == b'/' { + out.push(path[..i].to_owned()); + } + } + out +} + +fn parent_of(path: &str) -> &str { + match path.rfind('/') { + Some(i) => &path[..i], + None => "", + } +} + +/// JSON payload -> bytes (a string stores as UTF-8; {"$b": base64} as-is). +fn decode_payload(data: &str) -> Result, String> { + let parsed: Json = + serde_json::from_str(data).map_err(|e| format!("malformed payload: {e}"))?; + match parsed { + Json::String(text) => Ok(text.into_bytes()), + Json::Object(map) => match map.get(spec::BLOB_KEY) { + Some(Json::String(b64)) => { + BASE64.decode(b64).map_err(|e| format!("malformed payload: {e}")) + } + _ => Err("malformed payload: a JSON string or {\"$b\": base64}".to_owned()), + }, + _ => Err("malformed payload: a JSON string or {\"$b\": base64}".to_owned()), + } +} + +// --------------------------------------------------------------------------- +// The Dir backend — std::fs under the app root, symlinks treated as absent +// --------------------------------------------------------------------------- + +/// Resolve `path` under `root`, refusing any symlink component. The grammar +/// already forbids `..`/absolute paths; this guards against a HOST-side +/// actor having planted a link inside the root. +fn resolve(root: &Path, path: &str) -> Result { + let mut current = root.to_path_buf(); + for segment in path.split('/') { + current.push(segment); + if std::fs::symlink_metadata(¤t).is_ok_and(|md| md.file_type().is_symlink()) { + return Err("not found".to_owned()); + } + } + Ok(current) +} + +fn dir_read(root: &Path, path: &str, offset: u64, max_bytes: usize) -> Result<(Vec, u64, bool), String> { + let full = resolve(root, path)?; + let md = std::fs::metadata(&full).map_err(|_| "not found".to_owned())?; + if md.is_dir() { + return Err("is a directory".to_owned()); + } + let size = md.len(); + let mut file = std::fs::File::open(&full).map_err(|e| e.to_string())?; + file.seek(SeekFrom::Start(offset.min(size))).map_err(|e| e.to_string())?; + let mut chunk = vec![0u8; max_bytes]; + let mut filled = 0; + while filled < max_bytes { + let n = file.read(&mut chunk[filled..]).map_err(|e| e.to_string())?; + if n == 0 { + break; + } + filled += n; + } + chunk.truncate(filled); + let eof = offset.min(size) + filled as u64 >= size; + Ok((chunk, size, eof)) +} + +fn dir_write( + root: &Path, + tmp_dir: &Path, + path: &str, + payload: &[u8], + mode: u32, + quota: u64, + tmp_counter: u64, +) -> Result<(), String> { + let full = resolve(root, path)?; + if full.is_dir() { + return Err("is a directory".to_owned()); + } + // Refuse a file on the ancestor chain with the memory backend's message. + for ancestor in ancestors_of(path) { + let p = resolve(root, &ancestor)?; + if p.is_file() { + return Err(format!("not a directory: {ancestor}")); + } + } + let parent = full.parent().expect("resolved path always has a parent"); + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + if quota > 0 { + let existing = std::fs::metadata(&full).map(|m| m.len()).unwrap_or(0); + let next = if mode == spec::WRITE_APPEND { + existing + payload.len() as u64 + } else { + payload.len() as u64 + }; + if dir_used_bytes(root) - existing + next > quota { + return Err("quota exceeded".to_owned()); + } + } + if mode == spec::WRITE_APPEND { + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&full) + .map_err(|e| e.to_string())?; + file.write_all(payload).map_err(|e| e.to_string())?; + file.sync_all().map_err(|e| e.to_string())?; + return Ok(()); + } + // The atomicity contract: land the payload in the host-owned temp + // directory, sync, then rename over the target (same filesystem — + // cross-directory rename is atomic). + std::fs::create_dir_all(tmp_dir).map_err(|e| e.to_string())?; + let mut suffix = tmp_counter; + let (tmp, mut file) = loop { + let candidate = tmp_dir.join(suffix.to_string()); + match std::fs::OpenOptions::new().write(true).create_new(true).open(&candidate) { + Ok(file) => break (candidate, file), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => suffix += 1, + Err(e) => return Err(e.to_string()), + } + }; + let landed = file + .write_all(payload) + .and_then(|()| file.sync_all()) + .map_err(|e| e.to_string()); + drop(file); + landed + .and_then(|()| std::fs::rename(&tmp, &full).map_err(|e| e.to_string())) + .inspect_err(|_| { + let _ = std::fs::remove_file(&tmp); + }) +} + +fn dir_remove(root: &Path, path: &str, recursive: bool) -> Result<(), String> { + let full = resolve(root, path)?; + let md = std::fs::symlink_metadata(&full).map_err(|_| "not found".to_owned())?; + if md.is_file() { + return std::fs::remove_file(&full).map_err(|e| e.to_string()); + } + if !recursive { + return match std::fs::remove_dir(&full) { + Ok(()) => Ok(()), + Err(_) if std::fs::read_dir(&full).map(|mut d| d.next().is_some()).unwrap_or(false) => { + Err("directory not empty".to_owned()) + } + Err(e) => Err(e.to_string()), + }; + } + std::fs::remove_dir_all(&full).map_err(|e| e.to_string()) +} + +fn dir_list(root: &Path, path: &str) -> Result, String> { + let full = if path.is_empty() { root.to_path_buf() } else { resolve(root, path)? }; + let md = std::fs::metadata(&full).map_err(|_| "not found".to_owned())?; + if md.is_file() { + return Err("not a directory".to_owned()); + } + let mut out: Vec<(String, &'static str, u64)> = Vec::new(); + for entry in std::fs::read_dir(&full).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = match entry.file_name().into_string() { + Ok(name) => name, + Err(_) => continue, + }; + // A name the vocabulary cannot address (control chars, oversize) + // does not exist to the guest — it could be listed but never read. + if !valid_segment(&name) { + continue; + } + let emd = entry.metadata().map_err(|e| e.to_string())?; + if emd.file_type().is_symlink() { + continue; + } + if emd.is_dir() { + out.push((name, "dir", 0)); + } else { + out.push((name, "file", emd.len())); + } + } + out.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(out) +} + +fn dir_stat(root: &Path, path: &str) -> Option<(&'static str, u64)> { + let full = resolve(root, path).ok()?; + let md = std::fs::symlink_metadata(&full).ok()?; + if md.file_type().is_symlink() { + return None; + } + if md.is_dir() { + Some(("dir", 0)) + } else { + Some(("file", md.len())) + } +} + +fn dir_mkdir(root: &Path, path: &str) -> Result<(), String> { + for ancestor in ancestors_of(path).into_iter().chain([path.to_owned()]) { + let p = resolve(root, &ancestor)?; + if p.is_file() { + return Err(format!("not a directory: {ancestor}")); + } + } + let full = resolve(root, path)?; + std::fs::create_dir_all(&full).map_err(|e| e.to_string()) +} + +fn dir_rename(root: &Path, from: &str, to: &str) -> Result<(), String> { + let from_full = resolve(root, from)?; + let from_md = std::fs::symlink_metadata(&from_full).map_err(|_| "not found".to_owned())?; + let to_full = resolve(root, to)?; + let to_parent = to_full.parent().expect("resolved path always has a parent"); + if !to_parent.is_dir() { + return Err("not found".to_owned()); + } + if std::fs::symlink_metadata(&to_full).is_ok_and(|to_md| to_md.is_dir() || from_md.is_dir()) { + return Err("destination exists".to_owned()); + } + std::fs::rename(&from_full, &to_full).map_err(|e| e.to_string()) +} + +fn dir_used_bytes(root: &Path) -> u64 { + fn walk(dir: &Path) -> u64 { + let Ok(entries) = std::fs::read_dir(dir) else { + return 0; + }; + let mut total = 0; + for entry in entries.flatten() { + let Ok(md) = entry.metadata() else { continue }; + if md.file_type().is_symlink() { + continue; + } + if md.is_dir() { + total += walk(&entry.path()); + } else { + total += md.len(); + } + } + total + } + walk(root) +} + +// --------------------------------------------------------------------------- +// Mount +// --------------------------------------------------------------------------- + +#[cfg(feature = "mount")] +use std::cell::RefCell; +#[cfg(feature = "mount")] +use std::rc::Rc; + +/// Mount the module as `globalThis.fs` on a pocket-mod [`Guest`] — one JS +/// function per spec op, marshaled as (String, f64) -> i32/String. +/// Feature `mount` (default); a host with its own QuickJS wiring turns it +/// off and spells these nine functions itself. +#[cfg(feature = "mount")] +pub fn mount(guest: &pocket_mod::Guest, module: Rc>) -> anyhow::Result<()> { + use pocket_mod::qjs::Function; + guest.mount("fs", |ctx, ns| { + let m = module.clone(); + ns.set( + "read", + Function::new( + ctx.clone(), + move |path: String, offset: f64, max_bytes: f64| -> String { + m.borrow_mut().read(&path, offset as i64, max_bytes as i64) + }, + )?, + )?; + let m = module.clone(); + ns.set( + "write", + Function::new( + ctx.clone(), + move |path: String, data: String, mode: f64| -> i32 { + m.borrow_mut().write(&path, &data, mode as u32) + }, + )?, + )?; + let m = module.clone(); + ns.set( + "remove", + Function::new(ctx.clone(), move |path: String, recursive: f64| -> i32 { + m.borrow_mut().remove(&path, recursive as u32) + })?, + )?; + let m = module.clone(); + ns.set( + "list", + Function::new(ctx.clone(), move |path: String, offset: f64| -> String { + m.borrow_mut().list(&path, offset as i64) + })?, + )?; + let m = module.clone(); + ns.set( + "stat", + Function::new(ctx.clone(), move |path: String| -> String { + m.borrow_mut().stat(&path) + })?, + )?; + let m = module.clone(); + ns.set( + "mkdir", + Function::new(ctx.clone(), move |path: String| -> i32 { + m.borrow_mut().mkdir(&path) + })?, + )?; + let m = module.clone(); + ns.set( + "rename", + Function::new(ctx.clone(), move |from: String, to: String| -> i32 { + m.borrow_mut().rename(&from, &to) + })?, + )?; + let m = module.clone(); + ns.set( + "usage", + Function::new(ctx.clone(), move || -> String { m.borrow_mut().usage() })?, + )?; + let m = module.clone(); + ns.set( + "lastError", + Function::new(ctx.clone(), move || -> String { m.borrow().last_error() })?, + )?; + Ok(()) + }) +} + +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn module() -> FsModule { + FsModule::new(Storage::Memory) + } + + fn line(s: &str) -> Json { + serde_json::from_str(s).unwrap() + } + + fn text(s: &str) -> String { + json!(s).to_string() + } + + #[test] + fn path_grammar_refuses_escapes_and_nothing_else() { + // The security rule: escapes and malformed shapes. + for bad in ["", "/abs", "a//b", "a/", "../up", "a/../b", "a/.", "a\x07b"] { + assert!(!valid_path(bad), "{bad:?} should be invalid"); + } + assert!(valid_path(&vec!["a"; spec::MAX_DEPTH].join("/"))); + assert!(!valid_path(&vec!["a"; spec::MAX_DEPTH + 1].join("/"))); + assert!(!valid_path(&format!("{}x", "a".repeat(spec::MAX_PATH_BYTES)))); + assert!(!valid_path(&"名".repeat(22)), "22 CJK chars = 66 bytes > segment cap"); + // Universal names: anything an app wants to call its own files. + for good in [ + "a", + "notes/today.md", + "A1._-x", + ".config", + "notes/.drafts/今日笔记.md", + "-lead", + "a\\b", + "space in name.txt", + ] { + assert!(valid_path(good), "{good:?} should be valid"); + } + } + + #[test] + fn universal_names_round_trip() { + let mut m = module(); + assert_eq!(m.write("笔记/今天.md", &text("你好"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write(".config", &text("k=v"), spec::WRITE_TRUNCATE), 0); + assert_eq!(line(&m.stat("笔记/今天.md"))["size"], 6); + let listing = line(&m.list("", 0)); + let names: Vec<&str> = listing["entries"] + .as_array() + .unwrap() + .iter() + .map(|e| e["name"].as_str().unwrap()) + .collect(); + // Code point order: "." (U+002E) < "笔" (U+7B14). + assert_eq!(names, [".config", "笔记"]); + } + + #[test] + fn write_read_round_trip_text_and_bytes() { + let mut m = module(); + assert_eq!(m.write("notes/today.md", &text("# 今天"), spec::WRITE_TRUNCATE), 0); + let read = line(&m.read("notes/today.md", 0, spec::MAX_IO_BYTES as i64)); + let bytes = BASE64.decode(read["data"][spec::BLOB_KEY].as_str().unwrap()).unwrap(); + assert_eq!(String::from_utf8(bytes).unwrap(), "# 今天"); + assert_eq!(read["eof"], true); + + let payload = json!({ spec::BLOB_KEY: BASE64.encode([0u8, 1, 255]) }).to_string(); + assert_eq!(m.write("raw.bin", &payload, spec::WRITE_TRUNCATE), 0); + assert_eq!(line(&m.stat("raw.bin"))["size"], 3); + } + + #[test] + fn append_and_chunked_read() { + let mut m = module(); + assert_eq!(m.write("log.txt", &text("aaa"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write("log.txt", &text("bbb"), spec::WRITE_APPEND), 0); + let first = line(&m.read("log.txt", 0, 4)); + assert_eq!(first["size"], 6); + assert_eq!(first["eof"], false); + let rest = line(&m.read("log.txt", 4, 4)); + assert_eq!(rest["eof"], true); + let bytes = BASE64.decode(rest["data"][spec::BLOB_KEY].as_str().unwrap()).unwrap(); + assert_eq!(bytes, b"bb"); + } + + #[test] + fn write_creates_parents_and_refuses_file_ancestors() { + let mut m = module(); + assert_eq!(m.write("a/b/c.txt", &text("x"), spec::WRITE_TRUNCATE), 0); + assert_eq!(line(&m.stat("a/b"))["kind"], "dir"); + assert_eq!(m.write("a/b/c.txt/d.txt", &text("x"), spec::WRITE_TRUNCATE), 1); + assert!(m.last_error().contains("not a directory")); + } + + #[test] + fn remove_semantics() { + let mut m = module(); + m.write("dir/f.txt", &text("x"), spec::WRITE_TRUNCATE); + assert_eq!(m.remove("missing.txt", 0), 1); + assert_eq!(m.last_error(), "not found"); + assert_eq!(m.remove("dir", 0), 1); + assert_eq!(m.last_error(), "directory not empty"); + assert_eq!(m.remove("dir", 1), 0); + assert!(line(&m.stat("dir"))["error"].as_str().is_some()); + } + + #[test] + fn list_is_sorted_and_pages() { + let mut m = module(); + for i in 0..(spec::MAX_DIR_ENTRIES + 3) { + m.write(&format!("d/f{i:04}.txt"), &text("x"), spec::WRITE_TRUNCATE); + } + let first = line(&m.list("d", 0)); + assert_eq!(first["entries"].as_array().unwrap().len(), spec::MAX_DIR_ENTRIES); + assert_eq!(first["eof"], false); + assert_eq!(first["entries"][0]["name"], "f0000.txt"); + let second = line(&m.list("d", spec::MAX_DIR_ENTRIES as i64)); + assert_eq!(second["entries"].as_array().unwrap().len(), 3); + assert_eq!(second["eof"], true); + } + + #[test] + fn rename_semantics() { + let mut m = module(); + m.write("a.txt", &text("A"), spec::WRITE_TRUNCATE); + m.write("b.txt", &text("B"), spec::WRITE_TRUNCATE); + assert_eq!(m.rename("a.txt", "b.txt"), 0, "file over file replaces"); + assert_eq!(line(&m.stat("a.txt"))["error"], "not found"); + + m.mkdir("sub"); + assert_eq!(m.rename("b.txt", "sub"), 1); + assert_eq!(m.last_error(), "destination exists"); + assert_eq!(m.rename("b.txt", "ghost/x.txt"), 1, "missing parent fails"); + assert_eq!(m.rename("sub", "sub/inner"), 1); + assert_eq!(m.last_error(), "cannot rename into own subtree"); + + m.write("sub/deep/f.txt", &text("x"), spec::WRITE_TRUNCATE); + assert_eq!(m.rename("sub", "moved"), 0); + assert_eq!(line(&m.stat("moved/deep/f.txt"))["kind"], "file"); + } + + #[test] + fn quota_is_enforced_and_usage_reports() { + let mut m = FsModule::with_quota(Storage::Memory, 10); + assert_eq!(m.write("a.txt", &text("12345678"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write("b.txt", &text("123"), spec::WRITE_TRUNCATE), 1); + assert_eq!(m.last_error(), "quota exceeded"); + assert_eq!(m.write("a.txt", &text("1"), spec::WRITE_TRUNCATE), 0, "shrink fits"); + let usage = line(&m.usage()); + assert_eq!(usage["usedBytes"], 1); + assert_eq!(usage["quotaBytes"], 10); + } + + #[test] + fn io_ceiling_fails_loudly() { + let mut m = module(); + let too_big = "x".repeat(spec::MAX_IO_BYTES + 1); + assert_eq!(m.write("big.txt", &text(&too_big), spec::WRITE_TRUNCATE), 1); + assert!(m.last_error().contains("FS_MAX_IO_BYTES")); + m.write("ok.txt", &text("x"), spec::WRITE_TRUNCATE); + let over = line(&m.read("ok.txt", 0, spec::MAX_IO_BYTES as i64 + 1)); + assert!(over["error"].as_str().unwrap().contains("maxBytes")); + } + + #[test] + fn dir_storage_round_trip_atomicity_and_symlink_refusal() { + let base = std::env::temp_dir().join(format!("pocket-fs-test-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + let root = base.join("data"); + let tmp = base.join("tmp"); + let dir = || Storage::Dir { root: root.clone(), tmp: tmp.clone() }; + std::fs::create_dir_all(&root).unwrap(); + // A leftover orphan from a "crash" is swept on construction. + std::fs::create_dir_all(&tmp).unwrap(); + std::fs::write(tmp.join("7"), b"orphan").unwrap(); + { + let mut m = FsModule::new(dir()); + assert!(!tmp.join("7").exists(), "orphan swept on construction"); + assert_eq!(m.write("notes/a.md", &text("hello"), spec::WRITE_TRUNCATE), 0); + assert_eq!(m.write("notes/a.md", &text(" world"), spec::WRITE_APPEND), 0); + m.mkdir("empty"); + let listing = line(&m.list("", 0)); + let names: Vec<&str> = listing["entries"] + .as_array() + .unwrap() + .iter() + .map(|e| e["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, ["empty", "notes"]); + } + { + // A fresh module over the same root sees the persisted tree. + let mut m = FsModule::new(dir()); + let read = line(&m.read("notes/a.md", 0, 64)); + let bytes = BASE64.decode(read["data"][spec::BLOB_KEY].as_str().unwrap()).unwrap(); + assert_eq!(bytes, b"hello world"); + // The app tree holds ONLY app names — temps live in `tmp`, + // outside the bound root. + let names: Vec = std::fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert!(names.iter().all(|n| n == "notes" || n == "empty"), "{names:?}"); + } + #[cfg(unix)] + { + let outside = root.parent().unwrap().join("pocket-fs-outside.txt"); + std::fs::write(&outside, b"secret").unwrap(); + std::os::unix::fs::symlink(&outside, root.join("link.txt")).unwrap(); + let mut m = FsModule::new(dir()); + let read = line(&m.read("link.txt", 0, 64)); + assert_eq!(read["error"], "not found", "a symlink is invisible"); + assert_eq!(line(&m.stat("link.txt"))["error"], "not found"); + let listing = line(&m.list("", 0)); + assert!(!listing["entries"] + .as_array() + .unwrap() + .iter() + .any(|e| e["name"] == "link.txt")); + std::fs::remove_file(&outside).unwrap(); + } + std::fs::remove_dir_all(&base).unwrap(); + } + + #[cfg(feature = "mount")] + #[test] + fn mounted_namespace_serves_a_quickjs_guest() { + let guest = pocket_mod::Guest::new().unwrap(); + let module = Rc::new(RefCell::new(module())); + mount(&guest, module).unwrap(); + guest + .eval( + "boot", + r#" + if (fs.write("notes/hi.txt", JSON.stringify("from-guest"), 0) !== 0) { + throw new Error(fs.lastError()); + } + const stat = JSON.parse(fs.stat("notes/hi.txt")); + if (stat.kind !== "file" || stat.size !== 10) throw new Error("bad stat"); + const read = JSON.parse(fs.read("notes/hi.txt", 0, 64)); + if (!read.eof) throw new Error("expected eof"); + const escape = JSON.parse(fs.read("../../etc/passwd", 0, 64)); + if (escape.error !== "invalid path") throw new Error("traversal not refused"); + globalThis.result = read.data["$b"]; + "#, + ) + .unwrap(); + let result: String = guest.with(|ctx| ctx.globals().get("result").unwrap()); + assert_eq!(BASE64.decode(result).unwrap(), b"from-guest"); + } +} diff --git a/framework/compiler/subpaths.ts b/framework/compiler/subpaths.ts index 451dad23..bcbb84a9 100644 --- a/framework/compiler/subpaths.ts +++ b/framework/compiler/subpaths.ts @@ -55,6 +55,7 @@ export const SUBPATHS: Record = { audio: { file: "framework/src/audio-api.ts", aliases: TWINS }, clock: { file: "framework/src/clock.ts", aliases: TWINS }, config: { file: "framework/src/config.ts" }, + db: { file: "framework/src/db-api.ts", aliases: TWINS }, components: { file: { solid: "framework/src/components.ts", @@ -65,6 +66,7 @@ export const SUBPATHS: Record = { }, devtools: { file: "framework/src/devtools.ts" }, effects: { file: "framework/src/effects.ts", aliases: TWINS }, + fs: { file: "framework/src/fs-api.ts", aliases: TWINS }, gesture: { file: { solid: "framework/src/gesture.ts" } }, host: { file: "framework/src/host.ts" }, lifecycle: { diff --git a/framework/src/bytes.ts b/framework/src/bytes.ts new file mode 100644 index 00000000..681d64d7 --- /dev/null +++ b/framework/src/bytes.ts @@ -0,0 +1,90 @@ +// Byte codecs shared by the data-module SDKs (db, fs). Internal — not a +// framework subpath. QuickJS has no btoa/Buffer/TextEncoder/TextDecoder, so +// the codecs are spelled out; every caller is a cold path (payloads cross +// the boundary far less often than draw ops). + +const B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +export function bytesToBase64(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += 3) { + const a = bytes[i]; + const b = i + 1 < bytes.length ? bytes[i + 1] : 0; + const c = i + 2 < bytes.length ? bytes[i + 2] : 0; + out += B64[a >> 2] + B64[((a & 3) << 4) | (b >> 4)]; + out += i + 1 < bytes.length ? B64[((b & 15) << 2) | (c >> 6)] : "="; + out += i + 2 < bytes.length ? B64[c & 63] : "="; + } + return out; +} + +const B64_INV: Record = {}; +for (let i = 0; i < B64.length; i++) B64_INV[B64[i]] = i; + +export function base64ToBytes(s: string): Uint8Array { + while (s.endsWith("=")) s = s.slice(0, -1); + const out = new Uint8Array(Math.floor((s.length * 3) / 4)); + let o = 0; + for (let i = 0; i < s.length; i += 4) { + const n = + (B64_INV[s[i]] << 18) | + ((B64_INV[s[i + 1]] ?? 0) << 12) | + ((B64_INV[s[i + 2]] ?? 0) << 6) | + (B64_INV[s[i + 3]] ?? 0); + out[o++] = n >> 16; + if (o < out.length) out[o++] = (n >> 8) & 0xff; + if (o < out.length) out[o++] = n & 0xff; + } + return out; +} + +/** UTF-8 decode, strict: malformed sequences throw (a file that fails + * .text() is a bytes file — read it with .bytes()). */ +export function utf8ToString(bytes: Uint8Array): string { + let out = ""; + let i = 0; + while (i < bytes.length) { + const a = bytes[i++]; + if (a < 0x80) { + out += String.fromCharCode(a); + continue; + } + let n: number; + let extra: number; + if ((a & 0xe0) === 0xc0) { + n = a & 0x1f; + extra = 1; + } else if ((a & 0xf0) === 0xe0) { + n = a & 0x0f; + extra = 2; + } else if ((a & 0xf8) === 0xf0) { + n = a & 0x07; + extra = 3; + } else { + throw new Error("invalid UTF-8"); + } + if (i + extra > bytes.length) throw new Error("invalid UTF-8"); + for (let k = 0; k < extra; k++) { + const b = bytes[i++]; + if ((b & 0xc0) !== 0x80) throw new Error("invalid UTF-8"); + n = (n << 6) | (b & 0x3f); + } + // Reject overlong encodings and surrogate-range codepoints. + if ( + n > 0x10ffff || + (n >= 0xd800 && n <= 0xdfff) || + (extra === 1 && n < 0x80) || + (extra === 2 && n < 0x800) || + (extra === 3 && n < 0x10000) + ) { + throw new Error("invalid UTF-8"); + } + if (n < 0x10000) { + out += String.fromCharCode(n); + } else { + n -= 0x10000; + out += String.fromCharCode(0xd800 + (n >> 10), 0xdc00 + (n & 0x3ff)); + } + } + return out; +} diff --git a/framework/src/db-api.ts b/framework/src/db-api.ts new file mode 100644 index 00000000..04a2bc92 --- /dev/null +++ b/framework/src/db-api.ts @@ -0,0 +1,254 @@ +// Db module SDK — the thin guest-side algebra over the `db` spec +// (contracts/spec/db.ts). Framework-agnostic (no solid-js, no JSX): the +// same file serves ./db, ./vue-vapor/db and ./octane/db. +// +// The API is the bun:sqlite shape — `new Database(name)`, `db.query(sql)` +// returning a cached Statement with `.get/.all/.values/.run`, `db.exec`, +// `db.transaction(fn)` — so code written against Bun's built-in SQLite runs +// against the mounted module unchanged, and the model of "SQL in, plain +// objects out" carries across hosts. Two deliberate deviations from +// bun:sqlite, both from the spec: +// +// - a Statement is a guest-side (db, sql) pair — statement caching is +// HOST-side, keyed by the sql string, so there is nothing to finalize +// and no handle to leak. `columnNames` is populated by execution +// (empty before the first run). +// - integers beyond 2^53 - 1 and non-finite REALs fail loudly instead of +// losing precision (DB_MAX_SAFE_INTEGER; store money in cents). +// +// Unlike the audio SDK, absence does NOT degrade to a no-op: data code that +// silently drops writes is a corruption bug, not a missing enhancement. +// `new Database(...)` throws where `globalThis.db` is unmounted — declare +// `data.sqlite` in pocket.json `requires` so admission catches it first. + +import { DB_BLOB_KEY, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; +// QuickJS has no btoa/Buffer; the codec lives in bytes.ts (cold path), +// shared with the fs SDK. +import { base64ToBytes, bytesToBase64 } from "./bytes.ts"; + +export { DB_MAX_RESULT_ROWS, DB_MAX_SAFE_INTEGER, DB_MEMORY } from "../../contracts/spec/db.ts"; + +/** The mounted db namespace — one method per spec op (DB_OP codes). */ +export interface DbOps { + open(name: string): number; + close(handle: number): void; + exec(handle: number, sql: string): number; + query(handle: number, sql: string, args: string): string; + lastError(handle: number): string; +} + +/** The db module namespace, or null where the host doesn't mount one. + * A live lookup (not cached): hosts install `globalThis.db` before eval + * and reset it per app load, exactly like `globalThis.ui`. */ +export function dbHost(): DbOps | null { + const ns = (globalThis as { db?: unknown }).db; + if (!ns || typeof ns !== "object") return null; + return typeof (ns as DbOps).open === "function" ? (ns as DbOps) : null; +} + +// --------------------------------------------------------------------------- +// Value encoding (the contracts/spec/db.ts data contract, both directions) +// --------------------------------------------------------------------------- + +/** A value crossing the boundary: what a row cell or a bound parameter is. */ +export type SqlValue = null | number | string | boolean | Uint8Array; +export type SqlParams = readonly SqlValue[] | Readonly>; + +function encodeValue(v: SqlValue): unknown { + if (v instanceof Uint8Array) return { [DB_BLOB_KEY]: bytesToBase64(v) }; + if (typeof v === "number" && !Number.isFinite(v)) { + throw new Error("db: cannot bind a non-finite number"); + } + if (typeof v === "number" && Number.isInteger(v) && Math.abs(v) > DB_MAX_SAFE_INTEGER) { + throw new Error("db: integer exceeds DB_MAX_SAFE_INTEGER"); + } + return v; +} + +function decodeValue(v: unknown): SqlValue { + if (v !== null && typeof v === "object") { + return base64ToBytes((v as Record)[DB_BLOB_KEY]); + } + return v as SqlValue; +} + +function encodeParams(params: SqlParams): string { + if (Array.isArray(params)) return JSON.stringify(params.map(encodeValue)); + const out: Record = {}; + for (const [k, v] of Object.entries(params)) out[k] = encodeValue(v as SqlValue); + return JSON.stringify(out); +} + +// --------------------------------------------------------------------------- +// Database / Statement (the bun:sqlite shape) +// --------------------------------------------------------------------------- + +interface QueryResult { + cols?: string[]; + rows?: unknown[][]; + changes?: number; + lastInsertRowid?: number; + error?: string; +} + +export interface RunResult { + changes: number; + lastInsertRowid: number; +} + +export class Statement { + private cols: string[] = []; + + constructor( + private readonly ops: DbOps, + private readonly handle: number, + private readonly sql: string, + ) {} + + /** Column names of the last execution ([] before the first run). */ + get columnNames(): readonly string[] { + return this.cols; + } + + private execute(params: SqlParams): QueryResult { + const line = this.ops.query(this.handle, this.sql, encodeParams(params)); + const result = JSON.parse(line) as QueryResult; + if (result.error !== undefined) throw new Error(`db: ${result.error}`); + this.cols = result.cols ?? []; + return result; + } + + /** First row as a column-name keyed object, or null. */ + get(...params: SqlValue[]): Record | null; + get(params: SqlParams): Record | null; + get(...args: unknown[]): Record | null { + const rows = this.values(...(args as SqlValue[])); + if (rows.length === 0) return null; + const out: Record = {}; + this.cols.forEach((c, i) => (out[c] = rows[0][i])); + return out; + } + + /** Every row as a column-name keyed object. */ + all(...params: SqlValue[]): Record[]; + all(params: SqlParams): Record[]; + all(...args: unknown[]): Record[] { + const rows = this.values(...(args as SqlValue[])); + return rows.map((r) => { + const out: Record = {}; + this.cols.forEach((c, i) => (out[c] = r[i])); + return out; + }); + } + + /** Every row as an array in column order. */ + values(...params: SqlValue[]): SqlValue[][]; + values(params: SqlParams): SqlValue[][]; + values(...args: unknown[]): SqlValue[][] { + const params = normalizeArgs(args); + const result = this.execute(params); + return (result.rows ?? []).map((r) => r.map(decodeValue)); + } + + /** Execute for effect; rows (if any) are discarded. */ + run(...params: SqlValue[]): RunResult; + run(params: SqlParams): RunResult; + run(...args: unknown[]): RunResult { + const result = this.execute(normalizeArgs(args)); + return { changes: result.changes ?? 0, lastInsertRowid: result.lastInsertRowid ?? 0 }; + } +} + +/** Spread positional values, one array, or one named-parameter object. */ +function normalizeArgs(args: unknown[]): SqlParams { + if (args.length === 1 && Array.isArray(args[0])) return args[0] as SqlValue[]; + if ( + args.length === 1 && + args[0] !== null && + typeof args[0] === "object" && + !(args[0] instanceof Uint8Array) + ) { + return args[0] as Record; + } + return args as SqlValue[]; +} + +export class Database { + private readonly ops: DbOps; + private readonly handle: number; + private readonly statements = new Map(); + private txDepth = 0; + + constructor(name: string = DB_MEMORY) { + const ops = dbHost(); + if (!ops) { + throw new Error( + "db: globalThis.db is not mounted — declare `data.sqlite` in pocket.json requires", + ); + } + const handle = ops.open(name); + if (handle < 0) throw new Error(`db: open(${JSON.stringify(name)}) refused`); + this.ops = ops; + this.handle = handle; + } + + /** Cached statement for `sql` (host-side prepare cache backs it). */ + query(sql: string): Statement { + let statement = this.statements.get(sql); + if (!statement) { + statement = new Statement(this.ops, this.handle, sql); + this.statements.set(sql, statement); + } + return statement; + } + + /** Uncached statement (the bun:sqlite `prepare` spelling). */ + prepare(sql: string): Statement { + return new Statement(this.ops, this.handle, sql); + } + + /** Run one statement with parameters, for effect. */ + run(sql: string, params: SqlParams = []): RunResult { + return this.query(sql).run(params); + } + + /** Run one or more statements with no parameters and no result rows — + * the schema/migration path. */ + exec(sql: string): void { + if (this.ops.exec(this.handle, sql) !== 0) { + throw new Error(`db: ${this.ops.lastError(this.handle)}`); + } + } + + /** + * Wrap `fn` in BEGIN/COMMIT with ROLLBACK on throw; nested calls become + * savepoints (the bun:sqlite convention). Batching writes into one + * transaction is also the flash-wear discipline on device hosts. + */ + transaction(fn: (...args: A) => R): (...args: A) => R { + return (...args: A): R => { + const name = `pocket_tx_${this.txDepth}`; + const [begin, commit, rollback] = + this.txDepth === 0 + ? ["BEGIN", "COMMIT", "ROLLBACK"] + : [`SAVEPOINT ${name}`, `RELEASE ${name}`, `ROLLBACK TO ${name}; RELEASE ${name}`]; + this.exec(begin); + this.txDepth++; + try { + const result = fn(...args); + this.txDepth--; + this.exec(commit); + return result; + } catch (error) { + this.txDepth--; + this.exec(rollback); + throw error; + } + }; + } + + close(): void { + this.statements.clear(); + this.ops.close(this.handle); + } +} diff --git a/framework/src/fs-api.ts b/framework/src/fs-api.ts new file mode 100644 index 00000000..3b9a1730 --- /dev/null +++ b/framework/src/fs-api.ts @@ -0,0 +1,325 @@ +// Fs module SDK — the thin guest-side algebra over the `fs` spec +// (contracts/spec/fs.ts). Framework-agnostic (no solid-js, no JSX): the +// same file serves ./fs, ./vue-vapor/fs and ./octane/fs. +// +// The API is the Bun shape — `file(path)` returning a lazy handle with +// `.text()/.bytes()/.json()/.size/.exists()`, `write(path, data)` — plus +// the node:fs sync subset Bun implements (readFileSync, writeFileSync, +// appendFileSync, mkdirSync, readdirSync, rmSync, renameSync, statSync, +// existsSync) — so file code written against Bun runs against the mounted +// module with the async wrappers dropped. One deliberate deviation, from +// the module family's frame contract: everything is synchronous (every op +// completes inside the guest's per-tick turn), so `.text()` returns the +// string, not a Promise. Migration stays painless anyway: `await` unwraps +// a plain value, so Bun-idiomatic code — `await Bun.file(p).text()`, +// `await Bun.write(p, data)` — runs against this SDK unchanged. +// +// Paths are RELATIVE to the app's own data root — the host binds the root +// at mount; there is no way to spell another app's tree (or an absolute +// path) in this vocabulary. The SDK chunks payloads larger than +// FS_MAX_IO_BYTES, so file size is bounded by storage (and any host +// quota), not by the marshaling ceiling. +// +// Like db (and unlike audio), absence does NOT degrade to a no-op: file +// code that silently drops writes is a corruption bug, not a missing +// enhancement. Every entry point throws where `globalThis.fs` is +// unmounted — declare `data.fs` in pocket.json `requires` so admission +// catches it first. + +import { + FS_BLOB_KEY, + FS_MAX_IO_BYTES, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, +} from "../../contracts/spec/fs.ts"; +import { base64ToBytes, bytesToBase64, utf8ToString } from "./bytes.ts"; + +export { + FS_MAX_DEPTH, + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_MAX_PATH_BYTES, + fsValidPath, +} from "../../contracts/spec/fs.ts"; + +/** The mounted fs namespace — one method per spec op (FS_OP codes). */ +export interface FsOps { + read(path: string, offset: number, maxBytes: number): string; + write(path: string, data: string, mode: number): number; + remove(path: string, recursive: number): number; + list(path: string, offset: number): string; + stat(path: string): string; + mkdir(path: string): number; + rename(from: string, to: string): number; + usage(): string; + lastError(): string; +} + +/** The fs module namespace, or null where the host doesn't mount one. + * A live lookup (not cached): hosts install `globalThis.fs` before eval + * and reset it per app load, exactly like `globalThis.ui`. */ +export function fsHost(): FsOps | null { + const ns = (globalThis as { fs?: unknown }).fs; + if (!ns || typeof ns !== "object") return null; + return typeof (ns as FsOps).read === "function" ? (ns as FsOps) : null; +} + +function host(): FsOps { + const ops = fsHost(); + if (!ops) { + throw new Error("fs: globalThis.fs is not mounted — declare `data.fs` in pocket.json requires"); + } + return ops; +} + +function fail(ops: FsOps, op: string): never { + throw new Error(`fs: ${op}: ${ops.lastError()}`); +} + +// --------------------------------------------------------------------------- +// Payload encoding (the contracts/spec/fs.ts data contract) +// --------------------------------------------------------------------------- + +interface ReadResult { + data?: { [FS_BLOB_KEY]: string }; + size?: number; + eof?: boolean; + error?: string; +} + +/** Read the whole file as bytes, chunking past FS_MAX_IO_BYTES. */ +function readAll(ops: FsOps, path: string): Uint8Array { + const chunks: Uint8Array[] = []; + let offset = 0; + for (;;) { + const result = JSON.parse(ops.read(path, offset, FS_MAX_IO_BYTES)) as ReadResult; + if (result.error !== undefined) throw new Error(`fs: read ${path}: ${result.error}`); + const chunk = base64ToBytes(result.data![FS_BLOB_KEY]); + chunks.push(chunk); + offset += chunk.length; + if (result.eof) break; + } + if (chunks.length === 1) return chunks[0]; + const out = new Uint8Array(offset); + let o = 0; + for (const c of chunks) { + out.set(c, o); + o += c.length; + } + return out; +} + +/** Write `data` in <= FS_MAX_IO_BYTES payloads: one truncate, then appends. + * A string payload crosses as the JSON string itself (stored as UTF-8); + * bytes cross base64. Returns bytes written. */ +function writeAll(ops: FsOps, path: string, data: string | Uint8Array, mode: number): number { + if (typeof data === "string") { + // JS string length bounds UTF-8 length only within 3x; slice by + // codepoint-safe chunks conservatively sized so the encoded payload + // stays under the ceiling. + const step = Math.floor(FS_MAX_IO_BYTES / 3); + if (data.length <= step && mode === FS_WRITE_TRUNCATE) { + if (ops.write(path, JSON.stringify(data), mode) !== 0) fail(ops, `write ${path}`); + return utf8Length(data); + } + let m = mode; + let i = 0; + do { + let end = Math.min(i + step, data.length); + // Never split a surrogate pair across payloads. + if (end < data.length && isHighSurrogate(data.charCodeAt(end - 1))) end--; + if (ops.write(path, JSON.stringify(data.slice(i, end)), m) !== 0) { + fail(ops, `write ${path}`); + } + i = end; + m = FS_WRITE_APPEND; + } while (i < data.length); + return utf8Length(data); + } + let m = mode; + let i = 0; + do { + const chunk = data.subarray(i, Math.min(i + FS_MAX_IO_BYTES, data.length)); + const payload = JSON.stringify({ [FS_BLOB_KEY]: bytesToBase64(chunk) }); + if (ops.write(path, payload, m) !== 0) fail(ops, `write ${path}`); + i += chunk.length; + m = FS_WRITE_APPEND; + } while (i < data.length); + return data.length; +} + +function isHighSurrogate(code: number): boolean { + return code >= 0xd800 && code <= 0xdbff; +} + +function utf8Length(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++) { + const c = s.codePointAt(i)!; + n += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4; + if (c >= 0x10000) i++; + } + return n; +} + +// --------------------------------------------------------------------------- +// file / write (the Bun shape) +// --------------------------------------------------------------------------- + +interface StatResult { + kind?: "file" | "dir"; + size?: number; + error?: string; +} + +function statOf(ops: FsOps, path: string): StatResult { + return JSON.parse(ops.stat(path)) as StatResult; +} + +/** A lazy handle on one path — the Bun.file shape, synchronous. */ +export class PocketFile { + constructor(readonly path: string) {} + + /** File size in bytes; 0 when the file does not exist (Bun's behavior). */ + get size(): number { + const s = statOf(host(), this.path); + return s.kind === "file" ? s.size! : 0; + } + + exists(): boolean { + return statOf(host(), this.path).kind === "file"; + } + + bytes(): Uint8Array { + return readAll(host(), this.path); + } + + text(): string { + return utf8ToString(this.bytes()); + } + + json(): unknown { + return JSON.parse(this.text()); + } + + /** Delete the file (Bun.file(...).delete()). */ + delete(): void { + const ops = host(); + if (ops.remove(this.path, 0) !== 0) fail(ops, `remove ${this.path}`); + } +} + +/** `file(path)` — a lazy handle; nothing is read until a method call. */ +export function file(path: string): PocketFile { + return new PocketFile(path); +} + +/** `write(path, data)` — replace the file atomically, creating parent + * directories (Bun.write semantics). Returns bytes written. */ +export function write(path: string, data: string | Uint8Array): number { + return writeAll(host(), path, data, FS_WRITE_TRUNCATE); +} + +/** `usage()` — the app's storage footprint and budget (0 = unmetered). */ +export function usage(): { usedBytes: number; quotaBytes: number } { + return JSON.parse(host().usage()) as { usedBytes: number; quotaBytes: number }; +} + +// --------------------------------------------------------------------------- +// The node:fs sync subset (the spelling Bun also implements) +// --------------------------------------------------------------------------- + +export function readFileSync(path: string): Uint8Array; +export function readFileSync(path: string, encoding: "utf8" | "utf-8"): string; +export function readFileSync(path: string, encoding?: string): Uint8Array | string { + const bytes = readAll(host(), path); + return encoding === "utf8" || encoding === "utf-8" ? utf8ToString(bytes) : bytes; +} + +export function writeFileSync(path: string, data: string | Uint8Array): void { + writeAll(host(), path, data, FS_WRITE_TRUNCATE); +} + +export function appendFileSync(path: string, data: string | Uint8Array): void { + writeAll(host(), path, data, FS_WRITE_APPEND); +} + +/** Always recursive (every missing ancestor is created), idempotent. */ +export function mkdirSync(path: string): void { + const ops = host(); + if (ops.mkdir(path) !== 0) fail(ops, `mkdir ${path}`); +} + +export interface DirEntry { + name: string; + kind: "file" | "dir"; + size: number; + isFile(): boolean; + isDirectory(): boolean; +} + +interface ListResult { + entries?: { name: string; kind: "file" | "dir"; size: number }[]; + eof?: boolean; + error?: string; +} + +export function readdirSync(path: string): string[]; +export function readdirSync(path: string, options: { withFileTypes: true }): DirEntry[]; +export function readdirSync( + path: string, + options?: { withFileTypes?: boolean }, +): string[] | DirEntry[] { + const ops = host(); + const entries: DirEntry[] = []; + let offset = 0; + for (;;) { + const result = JSON.parse(ops.list(path, offset)) as ListResult; + if (result.error !== undefined) throw new Error(`fs: readdir ${path}: ${result.error}`); + for (const e of result.entries!) { + entries.push({ + ...e, + isFile: () => e.kind === "file", + isDirectory: () => e.kind === "dir", + }); + } + offset += result.entries!.length; + if (result.eof) break; + } + return options?.withFileTypes ? entries : entries.map((e) => e.name); +} + +/** `force` swallows "not found" (node semantics); `recursive` removes a + * directory tree. */ +export function rmSync(path: string, options?: { recursive?: boolean; force?: boolean }): void { + const ops = host(); + if (ops.remove(path, options?.recursive ? 1 : 0) !== 0) { + if (options?.force && ops.lastError() === "not found") return; + fail(ops, `rm ${path}`); + } +} + +export function renameSync(from: string, to: string): void { + const ops = host(); + if (ops.rename(from, to) !== 0) fail(ops, `rename ${from} -> ${to}`); +} + +export interface Stats { + size: number; + isFile(): boolean; + isDirectory(): boolean; +} + +export function statSync(path: string): Stats { + const s = statOf(host(), path); + if (s.error !== undefined) throw new Error(`fs: stat ${path}: ${s.error}`); + return { + size: s.size!, + isFile: () => s.kind === "file", + isDirectory: () => s.kind === "dir", + }; +} + +export function existsSync(path: string): boolean { + return statOf(host(), path).kind !== undefined; +} diff --git a/hosts/esp32p4/examples/data-smoke/.cargo/config.toml b/hosts/esp32p4/examples/data-smoke/.cargo/config.toml new file mode 100644 index 00000000..17f32957 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/.cargo/config.toml @@ -0,0 +1,22 @@ +[build] +target = "riscv32imafc-esp-espidf" + +[target.'cfg(target_os = "espidf")'] +linker = "ldproxy" +runner = "espflash flash --monitor" +rustflags = ["--cfg", "espidf_time64"] + +[unstable] +build-std = ["std", "panic_abort"] + +[env] +MCU = "esp32p4" +ESP_IDF_VERSION = "v5.5.3" +# Tools install into this example's .embuild by default; set IDF_TOOLS_PATH +# to reuse an existing espressif tools dir (the cc/ar wrappers honor it). +ESP_IDF_TOOLS_INSTALL_DIR = "workspace" +CC_riscv32imafc_esp_espidf = { value = "tools/data-smoke-cc", relative = true } +AR_riscv32imafc_esp_espidf = { value = "tools/data-smoke-ar", relative = true } +CFLAGS_riscv32imafc_esp_espidf = "-mabi=ilp32f -march=rv32imafc_zicsr_zifencei_xesppie -fno-pic -fno-PIC -Wno-error=incompatible-pointer-types" +# The vendored sqlite3.c, tuned for the device (docs/DB.md "ESP32 / ESP-IDF"). +LIBSQLITE3_FLAGS = "-DSQLITE_TEMP_STORE=3 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_MAX_MMAP_SIZE=0 -DSQLITE_OMIT_WAL -DSQLITE_OMIT_LOAD_EXTENSION -Dlstat=stat" diff --git a/hosts/esp32p4/examples/data-smoke/.gitignore b/hosts/esp32p4/examples/data-smoke/.gitignore new file mode 100644 index 00000000..dfaa254a --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/.gitignore @@ -0,0 +1,4 @@ +target/ +.embuild/ +sdkconfig +Cargo.lock diff --git a/hosts/esp32p4/examples/data-smoke/Cargo.toml b/hosts/esp32p4/examples/data-smoke/Cargo.toml new file mode 100644 index 00000000..51654e15 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/Cargo.toml @@ -0,0 +1,39 @@ +# On-device conformance smoke for the data modules (pocket-db, pocket-fs) +# over a LittleFS partition — the hardware half of the verification story +# (the contract half lives in tests/{db,fs}.test.ts and the crates' own +# tests). Build with the local toolchain this directory pins, flash, and +# watch UART for "DATA-SMOKE: PASS". Power-cycle and run again: the boot +# counter proves persistence through real power loss. +[package] +name = "data-smoke" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "data-smoke" +harness = false + +[workspace] + +[profile.release] +opt-level = "s" +lto = "thin" +codegen-units = 1 + +[dependencies] +anyhow = "1" +log = "0.4" +serde_json = "1" +esp-idf-svc = { version = "0.52.1", features = ["critical-section"] } +# default-features = false: the firmware brings no QuickJS here — the smoke +# drives the module cores directly, exactly like a device host that has its +# own guest wiring. +pocket-db = { path = "../../../../engine/crates/pocket-db", default-features = false } +pocket-fs = { path = "../../../../engine/crates/pocket-fs", default-features = false } + +[build-dependencies] +embuild = "0.33" + +[[package.metadata.esp-idf-sys.extra_components]] +remote_component = { name = "joltwallet/littlefs", version = "1.14.*" } diff --git a/hosts/esp32p4/examples/data-smoke/README.md b/hosts/esp32p4/examples/data-smoke/README.md new file mode 100644 index 00000000..780dd0f1 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/README.md @@ -0,0 +1,42 @@ +# data-smoke + +On-device conformance smoke for the data modules — `pocket-db` and +`pocket-fs` driven directly (no `mount` feature, the way a device host +with its own guest wiring consumes them) against a LittleFS partition on +an ESP32-P4. + +The contract semantics are verified host-side (`tests/{db,fs}.test.ts`, +the crates' unit tests). This binary asks only the questions hardware can +answer: does everything compile and link here, does LittleFS behave +(atomic rename, persistence), and what does it cost (heap, timings). + +## Run + +```sh +cargo build --release +espflash flash --port --partition-table partitions.csv --monitor \ + target/riscv32imafc-esp-espidf/release/data-smoke +``` + +The first build bootstraps ESP-IDF v5.5.3 into `.embuild/` (set +`IDF_TOOLS_PATH` to reuse an existing espressif tools directory — the +`tools/data-smoke-cc` wrapper honors it). The SQLite build recipe this +directory pins (`LIBSQLITE3_FLAGS`, the empty `sys/ioctl.h` shim) is +documented in docs/DB.md "ESP32 / ESP-IDF". + +Watch UART for: + +``` +DATA-SMOKE: fs ok in ...; boot N; usedBytes ... +DATA-SMOKE: db ok in ... (288-row tx ...) +DATA-SMOKE: PASS boot=N +``` + +A boot counter (an fs truncate-write) and a per-boot 288-row transaction +persist across runs; power-cycle the board and `boot` increments while the +row count is verified as `boots × 288` — flash persistence proven through +both modules, not just asserted. + +Measured on a Waveshare ESP32-P4 (rev 1.3, LittleFS on 8 MB partition): +fs contract pass ~0.5 s, 288-row insert transaction ~0.5–0.7 s, steady +heap delta ~1 KB across the whole run. diff --git a/hosts/esp32p4/examples/data-smoke/build.rs b/hosts/esp32p4/examples/data-smoke/build.rs new file mode 100644 index 00000000..1ef09435 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/build.rs @@ -0,0 +1,4 @@ +fn main() { + println!("cargo:rerun-if-changed=Cargo.toml"); + embuild::espidf::sysenv::output(); +} diff --git a/hosts/esp32p4/examples/data-smoke/components_esp32p4.lock b/hosts/esp32p4/examples/data-smoke/components_esp32p4.lock new file mode 100644 index 00000000..71aba26d --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/components_esp32p4.lock @@ -0,0 +1,20 @@ +dependencies: + idf: + source: + type: idf + version: 5.5.3 + joltwallet/littlefs: + component_hash: 362f1f5beb5087b0c60169aff82676d2d0ffc991ead975212b0cba95959181c5 + dependencies: + - name: idf + require: private + version: '>=4.3' + source: + registry_url: https://components.espressif.com/ + type: service + version: 1.14.8 +direct_dependencies: +- joltwallet/littlefs +manifest_hash: 63c4596bdefd7c81424d6f1f243201e4c3ff7245cb13401c292c520c662859c9 +target: esp32p4 +version: 2.0.0 diff --git a/hosts/esp32p4/examples/data-smoke/partitions.csv b/hosts/esp32p4/examples/data-smoke/partitions.csv new file mode 100644 index 00000000..f1ae316f --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 0xfa0000, +workspace, data, littlefs,0xfb0000, 0x800000, diff --git a/hosts/esp32p4/examples/data-smoke/rust-toolchain.toml b/hosts/esp32p4/examples/data-smoke/rust-toolchain.toml new file mode 100644 index 00000000..d772578a --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +# Pin the cross-compiled std ABI (nightly for -Zbuild-std). Newer nightlies +# broke on ESP-IDF in 2026-08 (std began requiring libc::AT_FDCWD). +channel = "nightly-2026-05-01" +components = ["rust-src"] diff --git a/hosts/esp32p4/examples/data-smoke/sdkconfig.defaults b/hosts/esp32p4/examples/data-smoke/sdkconfig.defaults new file mode 100644 index 00000000..ada13f43 --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/sdkconfig.defaults @@ -0,0 +1,8 @@ +# Waveshare ESP32-P4 (ESP32-P4NRW32) baseline — the vendor's flash/PSRAM +# values, trimmed to what the data smoke needs (no display, no Wi-Fi). +CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y +CONFIG_ESP32P4_REV_MIN_1=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +CONFIG_ESPTOOLPY_FLASHSIZE_32MB=y +CONFIG_ESPTOOLPY_FLASHSIZE="32MB" +CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384 diff --git a/hosts/esp32p4/examples/data-smoke/shim/sys/ioctl.h b/hosts/esp32p4/examples/data-smoke/shim/sys/ioctl.h new file mode 100644 index 00000000..e69de29b diff --git a/hosts/esp32p4/examples/data-smoke/src/main.rs b/hosts/esp32p4/examples/data-smoke/src/main.rs new file mode 100644 index 00000000..361180ff --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/src/main.rs @@ -0,0 +1,267 @@ +//! data-smoke — on-device conformance check for the data modules. +//! +//! Runs pocket-fs and pocket-db (no `mount` feature — the module cores +//! directly, the way a device host with its own guest wiring drives them) +//! against a LittleFS partition on real hardware, and reports over UART: +//! +//! DATA-SMOKE: PASS boot= +//! +//! A boot counter persists across runs, so flashing once and power-cycling +//! twice proves both modules keep data through real power loss. The +//! contract semantics themselves are verified host-side (tests/*.test.ts, +//! the crates' unit tests); this binary only asks the questions hardware +//! can answer: does it compile here, does LittleFS behave, what does it +//! cost. + +use std::time::Instant; + +use esp_idf_svc::fs::littlefs::Littlefs; +use esp_idf_svc::io::vfs::MountedLittlefs; +use pocket_db::{DbModule, Storage as DbStorage}; +use pocket_fs::{FsModule, Storage as FsStorage}; +use serde_json::Value as Json; + +const WORKSPACE_ROOT: &str = "/workspace"; +const DATA_ROOT: &str = "/workspace/apps/smoke/data"; +const TMP_DIR: &str = "/workspace/apps/smoke/tmp"; + +fn main() { + esp_idf_svc::sys::link_patches(); + esp_idf_svc::log::EspLogger::initialize_default(); + match run() { + Ok(boot) => log::info!("DATA-SMOKE: PASS boot={boot}"), + Err(error) => log::error!("DATA-SMOKE: FAIL: {error:#}"), + } + loop { + std::thread::sleep(std::time::Duration::from_secs(10)); + log::info!("DATA-SMOKE: idle"); + } +} + +fn run() -> anyhow::Result { + let _mount = mount_workspace()?; + std::fs::create_dir_all(DATA_ROOT)?; + let heap_before = unsafe { esp_idf_svc::sys::esp_get_free_heap_size() }; + + let boot = fs_smoke()?; + db_smoke()?; + + let heap_after = unsafe { esp_idf_svc::sys::esp_get_free_heap_size() }; + log::info!( + "DATA-SMOKE: heap before {heap_before} after {heap_after} (delta {})", + heap_before as i64 - heap_after as i64 + ); + Ok(boot) +} + +fn expect(condition: bool, what: &str) -> anyhow::Result<()> { + anyhow::ensure!(condition, "expectation failed: {what}"); + Ok(()) +} + +// --- fs: the nine-op contract against real LittleFS ------------------------ + +fn fs_smoke() -> anyhow::Result { + let started = Instant::now(); + let mut fs = FsModule::new(FsStorage::Dir { + root: DATA_ROOT.into(), + tmp: TMP_DIR.into(), + }); + + // Boot counter: truncate-write on every boot; its value is the proof + // that atomic writes and LittleFS persistence survive power cycling. + let boot = match parse(&fs.read("boot.txt", 0, 64)) { + Ok(line) => { + let b64 = line["data"]["$b"].as_str().unwrap_or_default(); + String::from_utf8(base64_decode(b64))?.trim().parse::()? + 1 + } + Err(_) => 0, // first boot on a fresh partition + }; + let write = fs.write("boot.txt", &format!("{:?}", boot.to_string()), 0); + expect(write == 0, "boot counter write")?; + + // Text + append round-trip. + expect(fs.write("notes/hello.md", "\"# hi\"", 0) == 0, "write text")?; + expect(fs.write("notes/hello.md", "\" there\"", 1) == 0, "append text")?; + let read = parse(&fs.read("notes/hello.md", 0, 64))?; + expect(read["size"].as_i64() == Some(10), "size after append")?; + expect(read["eof"].as_bool() == Some(true), "eof")?; + + // Bytes round-trip via the {"$b": base64} spelling. + expect( + fs.write("raw.bin", r#"{"$b":"AAEC/w=="}"#, 0) == 0, + "write bytes", + )?; + let stat = parse(&fs.stat("raw.bin"))?; + expect(stat["size"].as_i64() == Some(4), "bytes size")?; + + // list is name-sorted; mkdir/rename/remove behave. (Listing a fresh + // subdirectory, not the root — the root also holds the db module's + // ordinary files, main.sqlite and a transient journal.) + expect(fs.mkdir("assets/img") == 0, "mkdir -p")?; + expect(fs.rename("raw.bin", "assets/raw.bin") == 0, "rename")?; + let listing = parse(&fs.list("assets", 0))?; + let names: Vec<&str> = listing["entries"] + .as_array() + .map(|entries| entries.iter().filter_map(|e| e["name"].as_str()).collect()) + .unwrap_or_default(); + anyhow::ensure!(names == ["img", "raw.bin"], "listing sorted: got {names:?}"); + expect(fs.remove("assets", 0) == 1, "non-recursive remove of full dir refused")?; + expect(fs.remove("assets", 1) == 0, "recursive remove")?; + + // The sandbox refusal holds on-device exactly as in the goldens, and + // universal names (dot-prefixed, CJK) round-trip on real LittleFS. + expect( + parse(&fs.read("../../etc/passwd", 0, 16)).is_err(), + "traversal refused", + )?; + expect(fs.write(".config", "\"k=v\"", 0) == 0, "dot name allowed")?; + expect(fs.write("笔记/今天.md", "\"你好\"", 0) == 0, "CJK name allowed")?; + expect(fs.remove("笔记", 1) == 0 && fs.remove(".config", 0) == 0, "cleanup")?; + + let usage = parse(&fs.usage())?; + log::info!( + "DATA-SMOKE: fs ok in {:?}; boot {boot}; usedBytes {}", + started.elapsed(), + usage["usedBytes"] + ); + Ok(boot) +} + +// --- db: SQLite through the module core over the same data root ------------ + +fn db_smoke() -> anyhow::Result<()> { + let started = Instant::now(); + let mut db = DbModule::new(DbStorage::Dir(DATA_ROOT.into())); + let handle = db.open("main"); + anyhow::ensure!(handle > 0, "db open failed"); + + expect( + db.exec( + handle, + "CREATE TABLE IF NOT EXISTS samples ( + captured_at INTEGER PRIMARY KEY, + total_cents INTEGER NOT NULL + );", + ) == 0, + "ddl", + )?; + + // Prior completed runs' rows must still be there, and ONLY whole + // transactions: a run interrupted mid-transaction (reset, power loss) + // contributes exactly zero rows. The %288 invariant is SQLite's + // atomicity witnessed across power cycles, through the module. + let prior = parse(&db.query(handle, "SELECT COUNT(*) FROM samples", "[]"))?; + let prior_rows = prior["rows"][0][0].as_i64().unwrap_or(-1); + expect(prior_rows >= 0 && prior_rows % 288 == 0, "whole transactions only")?; + + // One day of 5-minute samples in one transaction — the flash-wear shape. + let tx_started = Instant::now(); + expect(db.exec(handle, "BEGIN") == 0, "begin")?; + for i in 0..288i64 { + let at = (prior_rows + i) * 300; + let cents = 1_500_000 + (i % 97) * 137; + let line = db.query( + handle, + "INSERT INTO samples (captured_at, total_cents) VALUES (?, ?)", + &format!("[{at}, {cents}]"), + ); + parse(&line)?; + } + expect(db.exec(handle, "COMMIT") == 0, "commit")?; + let tx_elapsed = tx_started.elapsed(); + + let agg = parse(&db.query(handle, "SELECT COUNT(*) FROM samples", "[]"))?; + expect( + agg["rows"][0][0].as_i64() == Some(prior_rows + 288), + "aggregate row count", + )?; + + // The ATTACH refusal holds on-device; the database is an ordinary + // file in the app's data root. + expect( + db.exec(handle, "ATTACH DATABASE '/workspace/x' AS other") == 1, + "attach refused", + )?; + expect( + std::path::Path::new(DATA_ROOT).join("main.sqlite").is_file(), + "db is an ordinary file in the data root", + )?; + + log::info!( + "DATA-SMOKE: db ok in {:?} (288-row tx {tx_elapsed:?})", + started.elapsed() + ); + Ok(()) +} + +// --- small helpers ---------------------------------------------------------- + +/// Parse one op result line; an {"error": ...} shape becomes an Err. +fn parse(line: &str) -> anyhow::Result { + let value: Json = serde_json::from_str(line)?; + match value.get("error").and_then(Json::as_str) { + Some(error) => anyhow::bail!("op error: {error}"), + None => Ok(value), + } +} + +/// Minimal base64 decode (standard alphabet, padded) — enough for the boot +/// counter without pulling a crate into the example. +fn base64_decode(s: &str) -> Vec { + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let value = |c: u8| ALPHABET.iter().position(|&a| a == c).unwrap_or(0) as u32; + let s = s.trim_end_matches('=').as_bytes(); + let mut out = Vec::with_capacity(s.len() * 3 / 4); + for chunk in s.chunks(4) { + let mut n = 0u32; + for (i, &c) in chunk.iter().enumerate() { + n |= value(c) << (18 - 6 * i); + } + for i in 0..chunk.len().saturating_sub(1) { + out.push((n >> (16 - 8 * i)) as u8); + } + } + out +} + +// --- LittleFS mount (the pocket-pi firmware's semantics: format only a +// blank partition, never a corrupted one) ----------------------------------- + +type WorkspaceMount = MountedLittlefs>; + +fn mount_workspace() -> anyhow::Result { + let fs = unsafe { Littlefs::<()>::new_partition("workspace")? }; + match MountedLittlefs::mount(fs, WORKSPACE_ROOT) { + Ok(mounted) => Ok(mounted), + Err(_mount_error) if partition_is_blank()? => { + let mut fs = unsafe { Littlefs::<()>::new_partition("workspace")? }; + fs.format()?; + MountedLittlefs::mount(fs, WORKSPACE_ROOT).map_err(Into::into) + } + Err(mount_error) => Err(anyhow::anyhow!( + "LittleFS workspace mount failed; preserving non-blank partition: {mount_error}" + )), + } +} + +fn partition_is_blank() -> anyhow::Result { + let partition = unsafe { + esp_idf_svc::sys::esp_partition_find_first( + esp_idf_svc::sys::esp_partition_type_t_ESP_PARTITION_TYPE_DATA, + esp_idf_svc::sys::esp_partition_subtype_t_ESP_PARTITION_SUBTYPE_DATA_LITTLEFS, + c"workspace".as_ptr(), + ) + }; + if partition.is_null() { + anyhow::bail!("LittleFS workspace partition is missing"); + } + let mut prefix = [0u8; 4096]; + let status = unsafe { + esp_idf_svc::sys::esp_partition_read(partition, 0, prefix.as_mut_ptr().cast(), prefix.len()) + }; + if status != esp_idf_svc::sys::ESP_OK { + anyhow::bail!("read LittleFS workspace partition: ESP error {status}"); + } + Ok(prefix.iter().all(|byte| *byte == 0xff)) +} diff --git a/hosts/esp32p4/examples/data-smoke/tools/data-smoke-ar b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-ar new file mode 100755 index 00000000..d552c91f --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-ar @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +example_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +tools_root=${IDF_TOOLS_PATH:-$example_root/.embuild/espressif} +for ar in \ + "$tools_root"/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-ar +do + if [ -x "$ar" ]; then + exec "$ar" "$@" + fi +done +echo "ESP32 ar is missing; build once so esp-idf-sys installs the toolchain (or set IDF_TOOLS_PATH)" >&2 +exit 1 diff --git a/hosts/esp32p4/examples/data-smoke/tools/data-smoke-cc b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-cc new file mode 100755 index 00000000..49c466ef --- /dev/null +++ b/hosts/esp32p4/examples/data-smoke/tools/data-smoke-cc @@ -0,0 +1,17 @@ +#!/bin/sh +# ESP32 gcc wrapper for the C sources this example vendors (SQLite via +# libsqlite3-sys). Finds the toolchain esp-idf-sys installed — locally in +# .embuild by default, or wherever IDF_TOOLS_PATH points — and injects the +# empty sys/ioctl.h shim (newlib has no such header; SQLite includes it). +set -eu +example_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +tools_root=${IDF_TOOLS_PATH:-$example_root/.embuild/espressif} +for compiler in \ + "$tools_root"/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-gcc +do + if [ -x "$compiler" ]; then + exec "$compiler" -isystem "$example_root/shim" "$@" + fi +done +echo "ESP32 gcc is missing; build once so esp-idf-sys installs the toolchain (or set IDF_TOOLS_PATH)" >&2 +exit 1 diff --git a/hosts/sim/db.ts b/hosts/sim/db.ts new file mode 100644 index 00000000..11fb8898 --- /dev/null +++ b/hosts/sim/db.ts @@ -0,0 +1,206 @@ +// hosts/sim/db.ts — the bun:sqlite-backed implementation of the db module +// (contracts/spec/db.ts) for the headless sim host. +// +// Bun's built-in SQLite is the same engine a device host links, so the sim +// runs the real dialect — only the storage policy is sim-shaped: every +// database, named or DB_MEMORY, lives in memory (no disk, no cleanup), and +// named databases persist for the life of the host object — across close() +// and reopen (the image is stashed with serialize()) and across an app +// reload inside one scenario — the way a device keeps its files. +// +// Three dev-host caveats, all spec-permitted: +// - ATTACH is refused by matching the word anywhere in the sql — the only +// net that catches every spelling SQLite accepts (`ATTACH DATABASE f`, +// `ATTACH 'f'`, `ATTACH hex(...)`) without an authorizer, which +// bun:sqlite does not expose (engine/crates/pocket-db carries the +// authoritative refusal: authorizer + SQLITE_LIMIT_ATTACHED=0). A string +// literal containing the word "attach" is a false positive a test can +// spell around. +// - a named parameter whose key lacks the $ / : / @ prefix is refused here +// (the reference core fails it with "unknown parameter"), but a PREFIXED +// key the statement never mentions is silently ignored — bun:sqlite +// offers no parameter-name introspection to close that last gap. +// - SQL time and randomness are NOT pinned — bun:sqlite exposes no VFS +// hook. The spec already forbids golden-tested apps from depending on +// them; tests/db.test.ts stays on deterministic SQL. +// +// Inject via bootWorld's extraGlobals: { db: host.ns }, the way a device +// host mounts the namespace beside `ui`. + +import { Database as BunDatabase } from "bun:sqlite"; +import { + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MAX_SAFE_INTEGER, + DB_MEMORY, + DB_NAME_PATTERN, + DB_BLOB_KEY, +} from "../../contracts/spec/db.ts"; + +interface SimDb { + bun: BunDatabase; + name: string; + lastError: string; +} + +export interface SimDbHost { + /** The `globalThis.db` namespace (one method per DB_OP). */ + ns: Record; + /** Every op call in order (for trace assertions). */ + log: string[]; + /** Close every underlying database (end of scenario). */ + dispose(): void; +} + +function encodeCell(v: unknown): unknown { + if (v === null || typeof v === "string") return v; + if (typeof v === "bigint") { + if (v > BigInt(DB_MAX_SAFE_INTEGER) || v < -BigInt(DB_MAX_SAFE_INTEGER)) { + throw new Error("integer result exceeds DB_MAX_SAFE_INTEGER"); + } + return Number(v); + } + if (typeof v === "number") { + if (!Number.isFinite(v)) throw new Error("non-finite REAL result"); + return v; + } + if (v instanceof Uint8Array) return { [DB_BLOB_KEY]: Buffer.from(v).toString("base64") }; + throw new Error(`unencodable result value: ${typeof v}`); +} + +function decodeParam(v: unknown): unknown { + if (v !== null && typeof v === "object" && !Array.isArray(v)) { + const b64 = (v as Record)[DB_BLOB_KEY]; + if (typeof b64 !== "string") throw new Error("malformed blob parameter"); + return new Uint8Array(Buffer.from(b64, "base64")); + } + if (typeof v === "number" && !Number.isFinite(v)) { + throw new Error("cannot bind a non-finite number"); + } + return v; +} + +const ATTACH = /\battach\b/i; + +export function createSimDbHost(): SimDbHost { + const dbs = new Map(); + const byName = new Map(); + // Serialized images of closed named databases — reopening restores them, + // the way Storage::Dir keeps the file after close on a device host. + const closedImages = new Map(); + const log: string[] = []; + let nextHandle = 1; + + function live(handle: number): SimDb | null { + return dbs.get(handle) ?? null; + } + + const ns = { + open(name: string): number { + log.push(`op open ${name}`); + if (name !== DB_MEMORY) { + if (!DB_NAME_PATTERN.test(name)) return -1; + const existing = byName.get(name); + if (existing !== undefined) return existing; + } + if (dbs.size >= DB_MAX_DATABASES) return -1; + const image = name !== DB_MEMORY ? closedImages.get(name) : undefined; + const bun = image + ? BunDatabase.deserialize(image, { safeIntegers: true }) + : new BunDatabase(":memory:", { safeIntegers: true }); + const handle = nextHandle++; + dbs.set(handle, { bun, name, lastError: "" }); + if (name !== DB_MEMORY) { + closedImages.delete(name); + byName.set(name, handle); + } + return handle; + }, + close(handle: number): void { + log.push(`op close ${handle}`); + const db = live(handle); + if (!db) return; + if (db.name !== DB_MEMORY) { + closedImages.set(db.name, db.bun.serialize()); + byName.delete(db.name); + } + db.bun.close(); + dbs.delete(handle); + }, + exec(handle: number, sql: string): number { + log.push(`op exec ${handle}`); + const db = live(handle); + if (!db) return 1; + if (ATTACH.test(sql)) { + db.lastError = "ATTACH is refused (contracts/spec/db.ts storage rule)"; + return 1; + } + try { + db.bun.exec(sql); + db.lastError = ""; + return 0; + } catch (error) { + db.lastError = error instanceof Error ? error.message : String(error); + return 1; + } + }, + query(handle: number, sql: string, args: string): string { + log.push(`op query ${handle} ${sql}`); + const db = live(handle); + if (!db) return JSON.stringify({ error: "database is closed" }); + if (ATTACH.test(sql)) { + db.lastError = "ATTACH is refused (contracts/spec/db.ts storage rule)"; + return JSON.stringify({ error: db.lastError }); + } + try { + const parsed = JSON.parse(args) as unknown[] | Record; + if (!Array.isArray(parsed)) { + // The spec spells named parameters $x / :x / @x; bun would bind a + // bare key anyway, the reference core fails it — fail like the core. + for (const key of Object.keys(parsed)) { + if (!/^[$:@]/.test(key)) throw new Error(`unknown parameter: ${key}`); + } + } + const params = Array.isArray(parsed) + ? parsed.map(decodeParam) + : Object.fromEntries(Object.entries(parsed).map(([k, v]) => [k, decodeParam(v)])); + const statement = db.bun.query(sql); + // bun returns null (not []) for statements that produce no rows. + const rows = ((Array.isArray(params) + ? statement.values(...(params as never[])) + : statement.values(params as never)) ?? []) as unknown[][]; + if (rows.length > DB_MAX_RESULT_ROWS) { + throw new Error("query exceeds DB_MAX_RESULT_ROWS; add LIMIT or aggregate"); + } + const counters = db.bun + .query("SELECT changes() AS c, last_insert_rowid() AS r") + .get() as { c: bigint; r: bigint }; + db.lastError = ""; + return JSON.stringify({ + cols: statement.columnNames, + rows: rows.map((r) => r.map(encodeCell)), + changes: Number(counters.c), + lastInsertRowid: Number(counters.r), + }); + } catch (error) { + db.lastError = error instanceof Error ? error.message : String(error); + return JSON.stringify({ error: db.lastError }); + } + }, + lastError(handle: number): string { + const db = live(handle); + return db ? db.lastError : "database is closed"; + }, + }; + + return { + ns, + log, + dispose(): void { + for (const db of dbs.values()) db.bun.close(); + dbs.clear(); + byName.clear(); + closedImages.clear(); + }, + }; +} diff --git a/hosts/sim/fs.ts b/hosts/sim/fs.ts new file mode 100644 index 00000000..d60da0ef --- /dev/null +++ b/hosts/sim/fs.ts @@ -0,0 +1,256 @@ +// hosts/sim/fs.ts — the in-memory implementation of the fs module +// (contracts/spec/fs.ts) for the headless sim host. +// +// Storage policy is sim-shaped: the whole tree lives in memory (no disk, +// no cleanup) and persists for the life of the host object, so an app +// reload inside one scenario keeps its files, the way a device keeps its +// flash. The tree is case-sensitive — the deterministic host that catches +// a case-only collision before a case-folding device filesystem hides it. +// +// Inject via bootWorld's extraGlobals: { fs: host.ns }, the way a device +// host mounts the namespace beside `ui`. One host per guest = one app's +// data root, which is the isolation model: a second app gets a second +// host object, and neither vocabulary can name the other's tree. + +import { + FS_BLOB_KEY, + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, + fsValidPath, +} from "../../contracts/spec/fs.ts"; + +export interface SimFsHost { + /** The `globalThis.fs` namespace (one method per FS_OP). */ + ns: Record; + /** Every op call in order (for trace assertions). */ + log: string[]; + /** Drop the whole tree (end of scenario). */ + dispose(): void; +} + +/** Parent path of a valid path ("" = the root). */ +function parentOf(path: string): string { + const i = path.lastIndexOf("/"); + return i < 0 ? "" : path.slice(0, i); +} + +function decodePayload(data: string): Uint8Array | string { + const parsed = JSON.parse(data) as unknown; + if (typeof parsed === "string") return new Uint8Array(Buffer.from(parsed, "utf8")); + if (parsed !== null && typeof parsed === "object") { + const b64 = (parsed as Record)[FS_BLOB_KEY]; + if (typeof b64 === "string") return new Uint8Array(Buffer.from(b64, "base64")); + } + return "malformed payload: a JSON string or {\"$b\": base64}"; +} + +export function createSimFsHost(options?: { quotaBytes?: number }): SimFsHost { + const files = new Map(); + const dirs = new Set(); // the root "" is implicit + const log: string[] = []; + const quota = options?.quotaBytes ?? 0; + let lastError = ""; + + const ok = (value: T): T => { + lastError = ""; + return value; + }; + const err = (message: string): 1 => { + lastError = message; + return 1; + }; + const errLine = (message: string): string => { + lastError = message; + return JSON.stringify({ error: message }); + }; + + const isDir = (path: string): boolean => path === "" || dirs.has(path); + + /** Sorted child names of a directory — Unicode CODE POINT order (= UTF-8 + * byte order, the spec's order). JS default sort compares UTF-16 code + * units, which disagrees for astral-plane names, hence the comparator. */ + function childrenOf(path: string): string[] { + const prefix = path === "" ? "" : `${path}/`; + const names = new Set(); + for (const key of [...files.keys(), ...dirs]) { + if (!key.startsWith(prefix) || key === path) continue; + const rest = key.slice(prefix.length); + const slash = rest.indexOf("/"); + names.add(slash < 0 ? rest : rest.slice(0, slash)); + } + return [...names].sort((a, b) => { + const as = [...a]; + const bs = [...b]; + for (let i = 0; i < Math.min(as.length, bs.length); i++) { + const d = as[i].codePointAt(0)! - bs[i].codePointAt(0)!; + if (d !== 0) return d; + } + return as.length - bs.length; + }); + } + + /** Create every missing ancestor of `path`; error string if one is a file. */ + function ensureParents(path: string): string | null { + for (let p = parentOf(path); p !== ""; p = parentOf(p)) { + if (files.has(p)) return `not a directory: ${p}`; + dirs.add(p); + } + return null; + } + + function usedBytes(): number { + let n = 0; + for (const bytes of files.values()) n += bytes.length; + return n; + } + + const ns = { + read(path: string, offset: number, maxBytes: number): string { + log.push(`op read ${path} ${offset} ${maxBytes}`); + if (!fsValidPath(path)) return errLine("invalid path"); + if (maxBytes < 1 || maxBytes > FS_MAX_IO_BYTES) { + return errLine("read maxBytes out of range"); + } + if (offset < 0) return errLine("read offset out of range"); + const bytes = files.get(path); + if (!bytes) return errLine(isDir(path) ? "is a directory" : "not found"); + const chunk = bytes.subarray(offset, offset + maxBytes); + return ok( + JSON.stringify({ + data: { [FS_BLOB_KEY]: Buffer.from(chunk).toString("base64") }, + size: bytes.length, + eof: offset + chunk.length >= bytes.length, + }), + ); + }, + write(path: string, data: string, mode: number): number { + log.push(`op write ${path} ${mode}`); + if (!fsValidPath(path)) return err("invalid path"); + if (mode !== FS_WRITE_TRUNCATE && mode !== FS_WRITE_APPEND) { + return err("invalid write mode"); + } + const payload = decodePayload(data); + if (typeof payload === "string") return err(payload); + if (payload.length > FS_MAX_IO_BYTES) return err("write exceeds FS_MAX_IO_BYTES"); + if (isDir(path)) return err("is a directory"); + const parentProblem = ensureParents(path); + if (parentProblem) return err(parentProblem); + const existing = mode === FS_WRITE_APPEND ? files.get(path) : undefined; + const nextSize = (existing?.length ?? 0) + payload.length; + if (quota > 0 && usedBytes() - (files.get(path)?.length ?? 0) + nextSize > quota) { + return err("quota exceeded"); + } + if (existing) { + const joined = new Uint8Array(nextSize); + joined.set(existing, 0); + joined.set(payload, existing.length); + files.set(path, joined); + } else { + files.set(path, payload.slice()); + } + return ok(0); + }, + remove(path: string, recursive: number): number { + log.push(`op remove ${path} ${recursive}`); + if (!fsValidPath(path)) return err("invalid path"); + if (files.delete(path)) return ok(0); + if (!dirs.has(path)) return err("not found"); + if (childrenOf(path).length > 0 && recursive !== 1) return err("directory not empty"); + const prefix = `${path}/`; + for (const key of [...files.keys()]) if (key.startsWith(prefix)) files.delete(key); + for (const key of [...dirs]) if (key.startsWith(prefix)) dirs.delete(key); + dirs.delete(path); + return ok(0); + }, + list(path: string, offset: number): string { + log.push(`op list ${path} ${offset}`); + if (path !== "" && !fsValidPath(path)) return errLine("invalid path"); + if (files.has(path)) return errLine("not a directory"); + if (!isDir(path)) return errLine("not found"); + const names = childrenOf(path); + // Clamp like the reference core: a negative offset must not wrap to + // slice-from-the-end. + offset = Math.max(0, offset); + const page = names.slice(offset, offset + FS_MAX_DIR_ENTRIES); + return ok( + JSON.stringify({ + entries: page.map((name) => { + const full = path === "" ? name : `${path}/${name}`; + const bytes = files.get(full); + return bytes + ? { name, kind: "file", size: bytes.length } + : { name, kind: "dir", size: 0 }; + }), + eof: offset + page.length >= names.length, + }), + ); + }, + stat(path: string): string { + log.push(`op stat ${path}`); + if (path === "") return ok(JSON.stringify({ kind: "dir", size: 0 })); + if (!fsValidPath(path)) return errLine("invalid path"); + const bytes = files.get(path); + if (bytes) return ok(JSON.stringify({ kind: "file", size: bytes.length })); + if (dirs.has(path)) return ok(JSON.stringify({ kind: "dir", size: 0 })); + return errLine("not found"); + }, + mkdir(path: string): number { + log.push(`op mkdir ${path}`); + if (!fsValidPath(path)) return err("invalid path"); + if (files.has(path)) return err(`not a directory: ${path}`); + const parentProblem = ensureParents(path); + if (parentProblem) return err(parentProblem); + dirs.add(path); + return ok(0); + }, + rename(from: string, to: string): number { + log.push(`op rename ${from} ${to}`); + if (!fsValidPath(from) || !fsValidPath(to)) return err("invalid path"); + if (from === to) return ok(0); + if (!isDir(parentOf(to))) return err("not found"); + if (dirs.has(to)) return err("destination exists"); + const fromFile = files.get(from); + if (fromFile) { + if (files.has(to)) files.delete(to); + files.delete(from); + files.set(to, fromFile); + return ok(0); + } + if (!dirs.has(from)) return err("not found"); + if (files.has(to)) return err("destination exists"); + if (to.startsWith(`${from}/`)) return err("cannot rename into own subtree"); + const prefix = `${from}/`; + for (const [key, bytes] of [...files.entries()]) { + if (!key.startsWith(prefix)) continue; + files.delete(key); + files.set(`${to}/${key.slice(prefix.length)}`, bytes); + } + for (const key of [...dirs]) { + if (!key.startsWith(prefix)) continue; + dirs.delete(key); + dirs.add(`${to}/${key.slice(prefix.length)}`); + } + dirs.delete(from); + dirs.add(to); + return ok(0); + }, + usage(): string { + log.push("op usage"); + return ok(JSON.stringify({ usedBytes: usedBytes(), quotaBytes: quota })); + }, + lastError(): string { + return lastError; + }, + }; + + return { + ns, + log, + dispose(): void { + files.clear(); + dirs.clear(); + }, + }; +} diff --git a/hosts/sim/sim.ts b/hosts/sim/sim.ts index 30c7bc6a..f05e6e45 100644 --- a/hosts/sim/sim.ts +++ b/hosts/sim/sim.ts @@ -241,6 +241,8 @@ export async function bootWorld( : undefined; g.frame = undefined; g.audio = undefined; // audio module namespace: absent unless extraGlobals mounts one + g.db = undefined; // db module namespace: absent unless extraGlobals mounts one + g.fs = undefined; // fs module namespace: absent unless extraGlobals mounts one g.__pocketApp = app; g.__simHz = hz; g.__pocketEffectTrace = (e: EffectEvent) => effects.push(e); diff --git a/package.json b/package.json index 6a15b57e..c1317605 100644 --- a/package.json +++ b/package.json @@ -74,9 +74,11 @@ "./audio": "./framework/src/audio-api.ts", "./clock": "./framework/src/clock.ts", "./config": "./framework/src/config.ts", + "./db": "./framework/src/db-api.ts", "./components": "./framework/src/components.ts", "./devtools": "./framework/src/devtools.ts", "./effects": "./framework/src/effects.ts", + "./fs": "./framework/src/fs-api.ts", "./gesture": "./framework/src/gesture.ts", "./host": "./framework/src/host.ts", "./lifecycle": "./framework/src/lifecycle.ts", @@ -100,8 +102,10 @@ "./vue-vapor/animation": "./framework/src/animation.ts", "./vue-vapor/audio": "./framework/src/audio-api.ts", "./vue-vapor/clock": "./framework/src/clock.ts", + "./vue-vapor/db": "./framework/src/db-api.ts", "./vue-vapor/components": "./framework/src/components-vue-vapor.ts", "./vue-vapor/effects": "./framework/src/effects.ts", + "./vue-vapor/fs": "./framework/src/fs-api.ts", "./vue-vapor/lifecycle": "./framework/src/lifecycle-vue-vapor.ts", "./vue-vapor/input": "./framework/src/input-api.ts", "./vue-vapor/renderer": "./framework/src/renderer-vue-vapor.ts", @@ -109,8 +113,10 @@ "./octane/animation": "./framework/src/animation.ts", "./octane/audio": "./framework/src/audio-api.ts", "./octane/clock": "./framework/src/clock.ts", + "./octane/db": "./framework/src/db-api.ts", "./octane/components": "./framework/src/components-octane.tsx", "./octane/effects": "./framework/src/effects.ts", + "./octane/fs": "./framework/src/fs-api.ts", "./octane/lifecycle": "./framework/src/lifecycle-octane.ts", "./octane/input": "./framework/src/input-api.ts", "./octane/renderer": "./framework/src/renderer-octane.ts" diff --git a/site/content/docs/concepts.md b/site/content/docs/concepts.md index 4dff9d66..2b0cf51a 100644 --- a/site/content/docs/concepts.md +++ b/site/content/docs/concepts.md @@ -16,8 +16,8 @@ Runtime = Host + mounted Modules + Guest ┌────────────────────────── Runtime ──────────────────────────┐ │ Guest product code (QuickJS bundle / wasm host eval) │ │ ───────── one namespace per mounted module ───────────── │ -│ Modules ui audio strike (OpenStrike) │ -│ core+spec core+spec core+spec │ +│ Modules ui · audio · db · fs · strike (OpenStrike) │ +│ core+spec, one per module │ │ Substrate pocket3d · platform drivers (no guest API) │ │ Host PSP EBOOT · Vita · browser · headless sim │ └──────────────────────────────────────────────────────────────┘ diff --git a/tests/db.test.ts b/tests/db.test.ts new file mode 100644 index 00000000..9db44922 --- /dev/null +++ b/tests/db.test.ts @@ -0,0 +1,294 @@ +// Db module unit tests: the sim host against the pinned op contract, the +// bun:sqlite-shaped SDK over it, and Bun's own SQLite as the oracle for row +// results. Runs entirely in-process; no built bundle, no disk. + +import { afterEach, describe, expect, test } from "bun:test"; +import { Database as BunDatabase } from "bun:sqlite"; +import { + DB_MAX_DATABASES, + DB_MAX_RESULT_ROWS, + DB_MEMORY, + DB_NAME_PATTERN, +} from "../contracts/spec/db.ts"; +import { Database, dbHost } from "../framework/src/db-api.ts"; +import { createSimDbHost, type SimDbHost } from "../hosts/sim/db.ts"; + +const g = globalThis as { db?: unknown }; +let host: SimDbHost | null = null; + +/** Mount a fresh sim host as globalThis.db, the way bootWorld's + * extraGlobals does for a scenario. */ +function mount(): SimDbHost { + host = createSimDbHost(); + g.db = host.ns; + return host; +} + +afterEach(() => { + host?.dispose(); + host = null; + g.db = undefined; +}); + +// --- the namespace contract (ops, straight through) ------------------------- + +describe("sim host ops", () => { + test("open refuses bad names, path traversal, and over-limit opens", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + expect(open("../escape")).toBe(-1); + expect(open("a/b")).toBe(-1); + expect(open("")).toBe(-1); + expect(open(".hidden")).toBe(-1); + const handles = []; + for (let i = 0; i < DB_MAX_DATABASES; i++) handles.push(open(`app-${i}`)); + for (const h of handles) expect(h).toBeGreaterThan(0); + expect(open("one-too-many")).toBe(-1); + }); + + test("the same persistent name returns the same handle; :memory: never does", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const a = open("app"); + expect(open("app")).toBe(a); + expect(open(DB_MEMORY)).not.toBe(open(DB_MEMORY)); + }); + + test("ATTACH is refused on both exec and query", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const lastError = ns.lastError as (h: number) => string; + const h = open(DB_MEMORY); + expect(exec(h, "ATTACH DATABASE '/tmp/x' AS other")).toBe(1); + expect(lastError(h)).toContain("ATTACH"); + const result = JSON.parse(query(h, "attach database ':memory:' as other", "[]")); + expect(result.error).toContain("ATTACH"); + // Every spelling SQLite accepts, not just the DATABASE-keyword form: + // bare string, and an expression filename (which on the reference core + // reaches the authorizer as NULL and is caught by SQLITE_LIMIT_ATTACHED). + expect(exec(h, "ATTACH ':memory:' AS o1")).toBe(1); + expect(exec(h, "ATTACH hex('2f746d702f78') AS o2")).toBe(1); + }); + + test("a named parameter without the $/:/@ prefix fails like the reference core", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + exec(h, "CREATE TABLE t (a, b)"); + const result = JSON.parse(query(h, "INSERT INTO t VALUES ($a, $b)", '{"a":1,"$b":2}')); + expect(result.error).toBe("unknown parameter: a"); + }); + + test("ops on a closed handle fail with 'database is closed'", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const close = ns.close as (h: number) => void; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + close(h); + expect(JSON.parse(query(h, "SELECT 1", "[]")).error).toBe("database is closed"); + }); + + test("query result carries cols, rows, changes and lastInsertRowid", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + expect(exec(h, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")).toBe(0); + const ins = JSON.parse(query(h, "INSERT INTO t (v) VALUES (?)", '["hello"]')); + expect(ins.changes).toBe(1); + expect(ins.lastInsertRowid).toBe(1); + const sel = JSON.parse(query(h, "SELECT id, v FROM t", "[]")); + expect(sel.cols).toEqual(["id", "v"]); + expect(sel.rows).toEqual([[1, "hello"]]); + }); + + test("a result beyond DB_MAX_RESULT_ROWS fails loudly", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const exec = ns.exec as (h: number, sql: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + exec(h, "CREATE TABLE n (v INTEGER)"); + exec( + h, + `WITH RECURSIVE seq(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM seq LIMIT ${DB_MAX_RESULT_ROWS + 1}) + INSERT INTO n SELECT x FROM seq`, + ); + const over = JSON.parse(query(h, "SELECT v FROM n", "[]")); + expect(over.error).toContain("DB_MAX_RESULT_ROWS"); + const capped = JSON.parse(query(h, `SELECT v FROM n LIMIT ${DB_MAX_RESULT_ROWS}`, "[]")); + expect(capped.rows.length).toBe(DB_MAX_RESULT_ROWS); + }); + + test("an integer beyond 2^53-1 fails instead of losing precision", () => { + const { ns } = mount(); + const open = ns.open as (name: string) => number; + const query = ns.query as (h: number, sql: string, args: string) => string; + const h = open(DB_MEMORY); + const over = JSON.parse(query(h, "SELECT 9007199254740993", "[]")); + expect(over.error).toContain("DB_MAX_SAFE_INTEGER"); + const ok = JSON.parse(query(h, "SELECT 9007199254740991", "[]")); + expect(ok.rows).toEqual([[9007199254740991]]); + }); +}); + +// --- the SDK (the bun:sqlite shape over the mounted namespace) --------------- + +describe("Database SDK", () => { + test("throws where the module is unmounted", () => { + expect(dbHost()).toBeNull(); + expect(() => new Database()).toThrow("data.sqlite"); + }); + + test("CRUD round-trip with objects, values and named parameters", () => { + mount(); + const db = new Database(); + db.exec(`CREATE TABLE positions ( + symbol TEXT PRIMARY KEY, qty REAL NOT NULL, cost_cents INTEGER NOT NULL + )`); + const insert = db.query("INSERT INTO positions VALUES ($s, $q, $c)"); + insert.run({ $s: "AAPL", $q: 10, $c: 190_00 }); + insert.run({ $s: "NVDA", $q: 2.5, $c: 121_50 }); + expect(db.query("SELECT count(*) AS n FROM positions").get()).toEqual({ n: 2 }); + expect(db.query("SELECT symbol FROM positions ORDER BY symbol").values()).toEqual([ + ["AAPL"], + ["NVDA"], + ]); + const row = db.query("SELECT * FROM positions WHERE symbol = ?").get("NVDA"); + expect(row).toEqual({ symbol: "NVDA", qty: 2.5, cost_cents: 12150 }); + expect(db.query("SELECT * FROM positions WHERE symbol = ?").get("MSFT")).toBeNull(); + const del = db.run("DELETE FROM positions WHERE qty < ?", [5]); + expect(del.changes).toBe(1); + }); + + test("statement caching: query() reuses, prepare() does not", () => { + mount(); + const db = new Database(); + expect(db.query("SELECT 1")).toBe(db.query("SELECT 1")); + expect(db.prepare("SELECT 1")).not.toBe(db.prepare("SELECT 1")); + }); + + test("columnNames populate on execution", () => { + mount(); + const db = new Database(); + const q = db.query("SELECT 1 AS one, 2 AS two"); + expect(q.columnNames).toEqual([]); + q.get(); + expect(q.columnNames).toEqual(["one", "two"]); + }); + + test("blobs round-trip as Uint8Array", () => { + mount(); + const db = new Database(); + db.exec("CREATE TABLE b (data BLOB)"); + const bytes = new Uint8Array([0, 1, 2, 250, 251, 252, 253, 254, 255]); + db.run("INSERT INTO b VALUES (?)", [bytes]); + const out = db.query("SELECT data FROM b").get(); + expect(out?.data).toBeInstanceOf(Uint8Array); + expect(Array.from(out?.data as Uint8Array)).toEqual(Array.from(bytes)); + }); + + test("transaction commits, rolls back on throw, and nests as savepoints", () => { + mount(); + const db = new Database(); + db.exec("CREATE TABLE t (v INTEGER)"); + const insertBoth = db.transaction((a: number, b: number) => { + db.run("INSERT INTO t VALUES (?)", [a]); + db.run("INSERT INTO t VALUES (?)", [b]); + return a + b; + }); + expect(insertBoth(1, 2)).toBe(3); + expect(db.query("SELECT count(*) AS n FROM t").get()).toEqual({ n: 2 }); + + const failing = db.transaction(() => { + db.run("INSERT INTO t VALUES (99)"); + throw new Error("boom"); + }); + expect(() => failing()).toThrow("boom"); + expect(db.query("SELECT count(*) AS n FROM t").get()).toEqual({ n: 2 }); + + const outer = db.transaction(() => { + db.run("INSERT INTO t VALUES (10)"); + const inner = db.transaction(() => { + db.run("INSERT INTO t VALUES (11)"); + throw new Error("inner"); + }); + expect(() => inner()).toThrow("inner"); + }); + outer(); + const values = db.query("SELECT v FROM t ORDER BY v").values(); + expect(values).toEqual([[1], [2], [10]]); + }); + + test("a persistent name survives close/reopen inside one host", () => { + mount(); + const first = new Database("ledger"); + first.exec("CREATE TABLE snap (v INTEGER)"); + first.run("INSERT INTO snap VALUES (42)"); + const second = new Database("ledger"); + expect(second.query("SELECT v FROM snap").get()).toEqual({ v: 42 }); + // Through an actual close(), too — the way Storage::Dir keeps the file + // on a device host. + second.close(); + const third = new Database("ledger"); + expect(third.query("SELECT v FROM snap").get()).toEqual({ v: 42 }); + }); + + test("SQL errors surface as thrown Errors with SQLite's message", () => { + mount(); + const db = new Database(); + expect(() => db.query("SELECT * FROM missing").all()).toThrow("missing"); + expect(() => db.exec("NOT SQL AT ALL")).toThrow(); + }); +}); + +// --- oracle: the SDK over the sim host agrees with bun:sqlite directly ------- + +describe("bun:sqlite oracle", () => { + test("identical statements produce identical rows", () => { + mount(); + const ours = new Database(); + const oracle = new BunDatabase(":memory:"); + const ddl = `CREATE TABLE history ( + bucket TEXT PRIMARY KEY, total_cents INTEGER, day_pnl_cents INTEGER + )`; + const rows: [string, number, number][] = [ + ["2026-08-06T14:30", 1_532_042, 1824], + ["2026-08-06T14:35", 1_531_010, 792], + ["2026-08-06T14:40", 1_540_500, 10_282], + ]; + ours.exec(ddl); + oracle.exec(ddl); + for (const r of rows) { + ours.run("INSERT INTO history VALUES (?, ?, ?)", r); + oracle.query("INSERT INTO history VALUES (?, ?, ?)").run(...r); + } + const sql = `SELECT bucket, total_cents FROM history + WHERE day_pnl_cents > ? ORDER BY bucket DESC`; + expect(ours.query(sql).values(1000)).toEqual( + oracle.query(sql).values(1000) as never, + ); + }); +}); + +// --- spec sanity -------------------------------------------------------------- + +describe("spec constants", () => { + test("DB_NAME_PATTERN accepts tokens and refuses paths", () => { + // 57 chars is the ceiling: `.sqlite` (+7 bytes) stays within the + // fs module's 64-byte segment ceiling, keeping the database file + // addressable by a co-mounted fs module. + for (const good of ["app", "portfolio-history", "a.b_c-1", "A", "a".repeat(57)]) { + expect(DB_NAME_PATTERN.test(good)).toBe(true); + } + for (const bad of ["", ".hidden", "-lead", "a/b", "a\\b", "..", "a".repeat(58)]) { + expect(DB_NAME_PATTERN.test(bad)).toBe(false); + } + }); +}); diff --git a/tests/fs.test.ts b/tests/fs.test.ts new file mode 100644 index 00000000..257f460f --- /dev/null +++ b/tests/fs.test.ts @@ -0,0 +1,277 @@ +// Fs module unit tests: the sim host against the pinned op contract, and +// the Bun-shaped SDK over it. Runs entirely in-process; no disk. + +import { afterEach, describe, expect, test } from "bun:test"; +import { + FS_MAX_DIR_ENTRIES, + FS_MAX_IO_BYTES, + FS_WRITE_APPEND, + FS_WRITE_TRUNCATE, + fsValidPath, +} from "../contracts/spec/fs.ts"; +import { + appendFileSync, + existsSync, + file, + fsHost, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + statSync, + usage, + write, + writeFileSync, +} from "../framework/src/fs-api.ts"; +import { createSimFsHost, type SimFsHost } from "../hosts/sim/fs.ts"; + +const g = globalThis as { fs?: unknown }; +let host: SimFsHost | null = null; + +/** Mount a fresh sim host as globalThis.fs, the way bootWorld's + * extraGlobals does for a scenario. */ +function mount(options?: { quotaBytes?: number }): SimFsHost { + host = createSimFsHost(options); + g.fs = host.ns; + return host; +} + +afterEach(() => { + host?.dispose(); + host = null; + g.fs = undefined; +}); + +type Ns = { + read(path: string, offset: number, maxBytes: number): string; + write(path: string, data: string, mode: number): number; + remove(path: string, recursive: number): number; + list(path: string, offset: number): string; + stat(path: string): string; + mkdir(path: string): number; + rename(from: string, to: string): number; + usage(): string; + lastError(): string; +}; + +const text = (s: string) => JSON.stringify(s); + +// --- the namespace contract (ops, straight through) ------------------------- + +describe("sim host ops", () => { + test("the path grammar refuses traversal and escapes — and nothing else", () => { + const ns = mount().ns as Ns; + // The predicate itself is covered in the spec-constants block; here the + // point is that the HOST enforces it on every op. + for (const bad of ["", "/etc/passwd", "../up", "a/../b", "a//b", "a/", "a/.."]) { + expect(JSON.parse(ns.read(bad, 0, 16)).error).toBe("invalid path"); + expect(ns.write(bad, text("x"), FS_WRITE_TRUNCATE)).toBe(1); + } + // Universal names: an app calls its files whatever it wants. + for (const ok of ["notes/today.md", ".config", "笔记/今日笔记.md", "space in name.txt"]) { + expect(ns.write(ok, text("ok"), FS_WRITE_TRUNCATE)).toBe(0); + expect(JSON.parse(ns.stat(ok)).kind).toBe("file"); + } + }); + + test("write creates parents; truncate replaces; append appends", () => { + const ns = mount().ns as Ns; + expect(ns.write("a/b/c.txt", text("one"), FS_WRITE_TRUNCATE)).toBe(0); + expect(JSON.parse(ns.stat("a/b")).kind).toBe("dir"); + expect(ns.write("a/b/c.txt", text("two"), FS_WRITE_TRUNCATE)).toBe(0); + expect(ns.write("a/b/c.txt", text("+"), FS_WRITE_APPEND)).toBe(0); + const read = JSON.parse(ns.read("a/b/c.txt", 0, FS_MAX_IO_BYTES)); + expect(read.size).toBe(4); + expect(read.eof).toBe(true); + }); + + test("read pages with offset/eof and refuses out-of-range maxBytes", () => { + const ns = mount().ns as Ns; + ns.write("f.txt", text("abcdef"), FS_WRITE_TRUNCATE); + const first = JSON.parse(ns.read("f.txt", 0, 4)); + expect(first.eof).toBe(false); + const rest = JSON.parse(ns.read("f.txt", 4, 4)); + expect(rest.eof).toBe(true); + expect(JSON.parse(ns.read("f.txt", 0, 0)).error).toContain("maxBytes"); + expect(JSON.parse(ns.read("f.txt", 0, FS_MAX_IO_BYTES + 1)).error).toContain("maxBytes"); + }); + + test("a payload beyond FS_MAX_IO_BYTES fails loudly", () => { + const ns = mount().ns as Ns; + expect(ns.write("big.bin", text("x".repeat(FS_MAX_IO_BYTES + 1)), FS_WRITE_TRUNCATE)).toBe(1); + expect(ns.lastError()).toContain("FS_MAX_IO_BYTES"); + }); + + test("list is name-sorted, pages, and stats carry kind/size", () => { + const ns = mount().ns as Ns; + ns.write("d/b.txt", text("xx"), FS_WRITE_TRUNCATE); + ns.write("d/a.txt", text("x"), FS_WRITE_TRUNCATE); + ns.mkdir("d/sub"); + const listing = JSON.parse(ns.list("d", 0)); + expect(listing.entries).toEqual([ + { name: "a.txt", kind: "file", size: 1 }, + { name: "b.txt", kind: "file", size: 2 }, + { name: "sub", kind: "dir", size: 0 }, + ]); + expect(listing.eof).toBe(true); + + for (let i = 0; i < FS_MAX_DIR_ENTRIES + 2; i++) { + ns.write(`many/f${String(i).padStart(4, "0")}`, text("x"), FS_WRITE_TRUNCATE); + } + const page1 = JSON.parse(ns.list("many", 0)); + expect(page1.entries.length).toBe(FS_MAX_DIR_ENTRIES); + expect(page1.eof).toBe(false); + const page2 = JSON.parse(ns.list("many", FS_MAX_DIR_ENTRIES)); + expect(page2.entries.length).toBe(2); + expect(page2.eof).toBe(true); + }); + + test("stat('') is the root; a missing path is 'not found'", () => { + const ns = mount().ns as Ns; + expect(JSON.parse(ns.stat(""))).toEqual({ kind: "dir", size: 0 }); + expect(JSON.parse(ns.stat("ghost.txt")).error).toBe("not found"); + expect(JSON.parse(ns.list("", 0)).entries).toEqual([]); + }); + + test("remove: files, empty dirs, recursive trees; root refused", () => { + const ns = mount().ns as Ns; + ns.write("tree/deep/f.txt", text("x"), FS_WRITE_TRUNCATE); + expect(ns.remove("tree", 0)).toBe(1); + expect(ns.lastError()).toBe("directory not empty"); + expect(ns.remove("tree", 1)).toBe(0); + expect(JSON.parse(ns.stat("tree")).error).toBe("not found"); + expect(ns.remove("", 0)).toBe(1); + expect(ns.remove("ghost", 0)).toBe(1); + expect(ns.lastError()).toBe("not found"); + }); + + test("rename: atomic file replace, dir moves, guarded destinations", () => { + const ns = mount().ns as Ns; + ns.write("a.txt", text("A"), FS_WRITE_TRUNCATE); + ns.write("b.txt", text("B"), FS_WRITE_TRUNCATE); + expect(ns.rename("a.txt", "b.txt")).toBe(0); + expect(JSON.parse(ns.stat("a.txt")).error).toBe("not found"); + ns.mkdir("sub"); + expect(ns.rename("b.txt", "sub")).toBe(1); + expect(ns.lastError()).toBe("destination exists"); + expect(ns.rename("b.txt", "ghost/c.txt")).toBe(1); + ns.write("sub/deep/f.txt", text("x"), FS_WRITE_TRUNCATE); + expect(ns.rename("sub", "sub/inner")).toBe(1); + expect(ns.rename("sub", "moved")).toBe(0); + expect(JSON.parse(ns.stat("moved/deep/f.txt")).kind).toBe("file"); + }); + + test("quota: writes beyond the budget fail; usage() reports", () => { + const ns = mount({ quotaBytes: 10 }).ns as Ns; + expect(ns.write("a.txt", text("12345678"), FS_WRITE_TRUNCATE)).toBe(0); + expect(ns.write("b.txt", text("123"), FS_WRITE_TRUNCATE)).toBe(1); + expect(ns.lastError()).toBe("quota exceeded"); + expect(ns.write("a.txt", text("1"), FS_WRITE_TRUNCATE)).toBe(0); + expect(JSON.parse(ns.usage())).toEqual({ usedBytes: 1, quotaBytes: 10 }); + }); +}); + +// --- the SDK (the Bun shape over the mounted namespace) ---------------------- + +describe("fs SDK", () => { + test("throws where the module is unmounted", () => { + expect(fsHost()).toBeNull(); + expect(() => write("a.txt", "x")).toThrow("data.fs"); + expect(() => file("a.txt").text()).toThrow("data.fs"); + }); + + test("file()/write() round-trip text, bytes and json", () => { + mount(); + expect(write("notes/today.md", "# 今天 🚀")).toBe(Buffer.byteLength("# 今天 🚀")); + const f = file("notes/today.md"); + expect(f.exists()).toBe(true); + expect(f.size).toBe(Buffer.byteLength("# 今天 🚀")); + expect(f.text()).toBe("# 今天 🚀"); + + const bytes = new Uint8Array([0, 1, 2, 253, 254, 255]); + write("raw.bin", bytes); + expect(Array.from(file("raw.bin").bytes())).toEqual(Array.from(bytes)); + + write("config.json", JSON.stringify({ theme: "dark", volume: 7 })); + expect(file("config.json").json()).toEqual({ theme: "dark", volume: 7 }); + + file("raw.bin").delete(); + expect(file("raw.bin").exists()).toBe(false); + }); + + test("payloads larger than FS_MAX_IO_BYTES chunk transparently", () => { + mount(); + const big = new Uint8Array(FS_MAX_IO_BYTES * 2 + 123); + for (let i = 0; i < big.length; i++) big[i] = i % 251; + write("big.bin", big); + expect(file("big.bin").size).toBe(big.length); + const back = file("big.bin").bytes(); + expect(back.length).toBe(big.length); + expect(back[FS_MAX_IO_BYTES + 7]).toBe((FS_MAX_IO_BYTES + 7) % 251); + + const bigText = "样🚀x".repeat(40_000); // multi-byte, crosses chunk seams + write("big.txt", bigText); + expect(file("big.txt").text()).toBe(bigText); + }); + + test("the node:fs sync subset behaves like node", () => { + mount(); + mkdirSync("a/b"); + writeFileSync("a/b/f.txt", "one"); + appendFileSync("a/b/f.txt", "+two"); + expect(readFileSync("a/b/f.txt", "utf8")).toBe("one+two"); + expect(readFileSync("a/b/f.txt")).toBeInstanceOf(Uint8Array); + + expect(readdirSync("a")).toEqual(["b"]); + const entries = readdirSync("a/b", { withFileTypes: true }); + expect(entries[0].name).toBe("f.txt"); + expect(entries[0].isFile()).toBe(true); + expect(statSync("a/b/f.txt").size).toBe(7); + expect(statSync("a").isDirectory()).toBe(true); + expect(existsSync("a/b/f.txt")).toBe(true); + + renameSync("a/b/f.txt", "a/g.txt"); + expect(existsSync("a/b/f.txt")).toBe(false); + + expect(() => rmSync("ghost.txt")).toThrow("not found"); + rmSync("ghost.txt", { force: true }); // node semantics: force swallows + rmSync("a", { recursive: true }); + expect(existsSync("a")).toBe(false); + + expect(usage().usedBytes).toBe(0); + }); + + test("errors surface as thrown Errors with the op detail", () => { + mount(); + expect(() => readFileSync("missing.txt")).toThrow("not found"); + expect(() => readdirSync("missing")).toThrow("not found"); + expect(() => statSync("missing")).toThrow("not found"); + mkdirSync("d"); + expect(() => writeFileSync("d", "x")).toThrow("is a directory"); + }); +}); + +// --- spec sanity -------------------------------------------------------------- + +describe("spec constants", () => { + test("fsValidPath allows universal names and refuses only escapes", () => { + for (const good of ["a", "notes/today.md", ".config", "笔记/今天.md", "a b", "a\\b"]) { + expect(fsValidPath(good)).toBe(true); + } + for (const bad of ["", "/a", "a//b", "../x", "a/..", "a/.", "a/", "a\u0007b"]) { + expect(fsValidPath(bad)).toBe(false); + } + expect(fsValidPath(Array(9).fill("a").join("/"))).toBe(false); + expect(fsValidPath(Array(8).fill("a").join("/"))).toBe(true); + expect(fsValidPath("名".repeat(22))).toBe(false); // 66 UTF-8 bytes > segment cap + expect(fsValidPath("名".repeat(21))).toBe(true); + // Ill-formed Unicode has no UTF-8 spelling: a lone surrogate would be + // byte-exact on a JS host but mangled by the QuickJS-to-native bridge, + // so the shared predicate refuses it; the paired form stays valid. + expect(fsValidPath("a\uD800b")).toBe(false); + expect(fsValidPath("a\uDC00b")).toBe(false); + expect(fsValidPath("tail\uDBFF")).toBe(false); + expect(fsValidPath("😀.txt")).toBe(true); // a real surrogate pair + }); +}); diff --git a/tools/test.ts b/tools/test.ts index 12b251d9..eff3e3b1 100644 --- a/tools/test.ts +++ b/tools/test.ts @@ -55,6 +55,8 @@ const SUITE: readonly Stage[] = [ "tests/kinetics.test.ts", "tests/osk-controller.test.ts", "tests/audio.test.ts", + "tests/db.test.ts", + "tests/fs.test.ts", "tests/vita-package.test.ts", "tests/psp-toolchain.test.ts", "tests/symbian-data.test.ts",