diff --git a/.gitignore b/.gitignore index f43331fe..d99042ec 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ npm/ # deno task build:web output — built into the package at release time, # never a repository source file (specs/release-process-spec.md) packages/web/generated/ + +# spike #349 working artifacts — vendored build output and databases stay out +spikes/349-dofs/vendor-build/ diff --git a/deno.json b/deno.json index 926a8eb4..eda03298 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "workspace": ["packages/*", "site"], - "exclude": ["scripts/tests/fixtures"], + "exclude": ["scripts/tests/fixtures", "spikes"], "nodeModulesDir": "auto", "lock": { "frozen": true @@ -57,6 +57,9 @@ "review:local": "deno run --allow-all packages/cli/src/deno.ts run .reviews/ReviewPR.local.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.local.jsonl", "analyze": "deno run --allow-all packages/cli/src/deno.ts run .reviews/AnalyzeRepo.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.analyze.jsonl", "analyze:ci": "deno run --allow-all packages/cli/src/deno.ts run .reviews/AnalyzeRepoCI.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.analyze.ci.jsonl", - "analyze:dispatch": "deno run --allow-all packages/cli/src/deno.ts run .reviews/DispatchRepoAnalysis.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.dispatch.jsonl" + "analyze:dispatch": "deno run --allow-all packages/cli/src/deno.ts run .reviews/DispatchRepoAnalysis.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir packages/core/components -j .reviews/journal.dispatch.jsonl", + "spike:349": "deno task --cwd spikes/349-dofs all", + "spike:349:build": "deno task --cwd spikes/349-dofs build", + "spike:349:test": "deno task --cwd spikes/349-dofs test" } } diff --git a/spikes/349-dofs/README.md b/spikes/349-dofs/README.md new file mode 100644 index 00000000..a2c56214 --- /dev/null +++ b/spikes/349-dofs/README.md @@ -0,0 +1,55 @@ +# Spike 349: Cloudflare DOFS directly in Deno + +Proves that Executable.md can host a persistent SQLite Workspace directly in +the Deno process by reusing Cloudflare Computer's DOFS filesystem layer over +a file-backed `node:sqlite` adapter — no `workerd`, no Node, no Wrangler. +Compares this topology with the bundled-`workerd` evidence in #347 / PR #348 +for the decision in #346. Findings live in +[evidence/EVIDENCE.md](evidence/EVIDENCE.md). + +## Run it + +From the repository root: + +```bash +deno task spike:349 # vendor-build + compile + test +``` + +Stepwise: `deno task --cwd spikes/349-dofs vendor` installs the vendored +package's build toolchain and compiles it with `tsc`; +`deno task spike:349:build` compiles `dist/proof`; +`deno task spike:349:test` runs the scenario suite. One prerequisite the +tasks assume: `npm install --install-links --no-audit --no-fund` in this +directory (dependencies are npm-owned here — see Layout). + +The proof executable performs one filesystem op per invocation against a +workspace database file, so consecutive invocations are full restart cycles: + +```bash +dist/proof /tmp/ws.db write /notes/a.md "alpha" +dist/proof /tmp/ws.db read /notes/a.md +dist/proof /tmp/ws.db ls /notes +``` + +## Layout + +- `vendor/dofs/` — pinned vendored copy of `packages/dofs` from + cloudflare/computer `v0.1.1` (`63d3636`), MIT; provenance, the two manifest + edits, and the upgrade procedure are in + [vendor/dofs/PROVENANCE.md](vendor/dofs/PROVENANCE.md). Source files are + unmodified. +- `host/file-storage.ts` — the entire adapter: a file-backed + `DurableObjectStorageLike` over Deno's `node:sqlite` (~100 lines, + mirroring upstream's own in-memory test fixture). +- `host/main.ts` — the proof CLI; `host/types/` — typed facade for the + consumed surface (Deno pairs `.js` with `.d.ts` for registry packages but + not `file:`-resolved ones). +- `tests/spike.test.ts` — the scenario suite backing the evidence. +- `evidence/probes/` — raw probe ledgers with every command and error. + +This spike uses npm-owned dependencies (`nodeModulesDir: "manual"`, +`package.json` + `package-lock.json`) because the vendored package is +consumed as a `file:` dependency through its exports map — itself part of +the reuse-boundary evidence. The directory stays outside the workspace via +the root `deno.json` `exclude`, so the four verification gates keep their +scope. diff --git a/spikes/349-dofs/build.ts b/spikes/349-dofs/build.ts new file mode 100644 index 00000000..7f634d11 --- /dev/null +++ b/spikes/349-dofs/build.ts @@ -0,0 +1,32 @@ +import { main } from "effection"; +import { exec } from "@effectionx/process"; + +const here = new URL("./", import.meta.url).pathname; + +main(function* () { + yield* exec(Deno.execPath(), { + arguments: [ + "compile", + "--allow-all", + "--frozen", + "--node-modules-dir=manual", + "--output", + "dist/proof", + "host/main.ts", + ], + cwd: here, + }).expect(); + yield* exec(Deno.execPath(), { + arguments: [ + "compile", + "--allow-all", + "--frozen", + "--node-modules-dir=manual", + "--output", + "dist/proof-shim", + "host/shim-main.ts", + ], + cwd: here, + }).expect(); + console.log("built dist/proof and dist/proof-shim"); +}); diff --git a/spikes/349-dofs/deno.json b/spikes/349-dofs/deno.json new file mode 100644 index 00000000..555dabb7 --- /dev/null +++ b/spikes/349-dofs/deno.json @@ -0,0 +1,15 @@ +{ + "exclude": [ + "vendor", + "vendor-build", + "dist" + ], + "nodeModulesDir": "manual", + "tasks": { + "all": "deno task vendor && deno task build && deno task test", + "vendor": "deno run --allow-all vendor.ts", + "build": "deno run --allow-all build.ts", + "test": "deno test --allow-all tests/", + "check": "deno check ." + } +} \ No newline at end of file diff --git a/spikes/349-dofs/evidence/COMPARISON.md b/spikes/349-dofs/evidence/COMPARISON.md new file mode 100644 index 00000000..a1944eab --- /dev/null +++ b/spikes/349-dofs/evidence/COMPARISON.md @@ -0,0 +1,60 @@ +# Comparison: Deno-local DOFS (#349) vs bundled workerd (#347 / PR #348) + +Both spikes ran on the same host (macOS 15 arm64, Deno 2.9.1) against the +same pinned `cloudflare/computer` v0.1.1. #347 numbers come from +`spikes/347-workerd/evidence/EVIDENCE.md` on PR #348; #349 numbers from +[EVIDENCE.md](EVIDENCE.md). + +| Axis | #349 Deno-local DOFS | #347 bundled workerd | +| --- | --- | --- | +| Runtime processes | one (Deno; plus a Node sidecar only if real FUSE is required) | two: xmd host + supervised workerd child | +| Artifact size | 110 MB proof (no extra runtime) | 191 MB proof (109 MB embedded workerd) | +| Startup / attach | 0.08 s full op cycle; no materialization step | 0.25 s serve-to-ready warm + 1.46 s first-run materialization of the 109 MB binary | +| Filesystem semantics | the DOFS filesystem API in-process; native subprocess access via shim (dev-only) or FUSE (Node sidecar; release-only durability, kernel-cache staleness ≤1 s) | the same DOFS filesystem behind the Worker boundary; native subprocess access only via the Container backend (Docker) | +| Execution backends | none of Computer's backends — exec is XMD's own process capability against the mount | all three Computer backends work (worker-shell, worker-javascript; container with Docker) | +| Platforms | linux-x64 proven; darwin-arm64: filesystem+shim only (FUSE dead end at this pin); linux-arm64 no addon prebuild; Windows: no FUSE at all | all five release targets have pinned workerd binaries; Linux glibc 2.35+, no musl; Windows lacks SIGTERM drain | +| Host prerequisites | none for filesystem+shim; `/dev/fuse` + libfuse2 (+ Node sidecar) for real FUSE | none for core; Docker for the container backend | +| Sandbox / security | none — DOFS code and subprocesses run with the host process's privileges | workerd isolates Worker JS but explicitly disclaims hardened sandboxing; native exec only inside Docker containers | +| Schema ownership / upgrades | identical schema, consumed at source (vendored 6.5k LOC, MIT provenance; 3-line upstream diff drafted as the exit) | identical schema, consumed through the published package (exact pin; refuse-on-downgrade) | +| Supervision | no child processes for the core path; FUSE adds mount lifecycle (auto_unmount proven; deadlock runbook recorded) | workerd child supervision (clean stop proven; SIGKILL orphans; container leak on SIGTERM) | +| Streaming / cancellation / retained exec handles | XMD's existing process capability (its own semantics) | Computer's `runtime.exec` handles (status/stdout/value round-trip proven) | +| Hosted-Cloudflare compatibility | filesystem layer byte-compatible; **no Durable Object identity, no Workers surface, no backends** — a local SQLite file is not a Durable Object | runs the actual published Computer package inside the actual Workers runtime — behaviorally closest to hosted | +| Migration / replication path | dofs ships its sync protocol helpers (`applyChanges`, manifests, watermarks) — the same protocol computerd speaks over capnweb; unexercised in this spike | same protocol, exercised end-to-end by the container backend's sync in #347 | + +## What each topology is best at + +**#349 wins on weight and directness**: a persistent SQLite Workspace with +Cloudflare's exact schema, in-process, 40% smaller artifact, ~3× faster +per operation, no child processes, no materialization cache, and identity +as simple as a database file path. The costs: no Computer execution +backends, no Workers isolation of any kind, a dev-only shim as the only +portable subprocess bridge today, and real FUSE gated on a Deno uv-polyfill +gap (upstream-ready repro committed) plus a Node sidecar in the interim. + +**#347 wins on fidelity**: it runs the real published package in the real +runtime with all three backends, so anything proven there transfers to +hosted Cloudflare almost by construction. The costs: 81 MB of extra +artifact, a supervised child process with the recorded teardown gaps, and +Docker for anything native. + +## Recommendation for #346 + +**Limit, not select-or-reject.** The evidence supports a split by concern: + +1. For the *filesystem-only Workspace* — the substrate `` needs + first — select the #349 topology: same schema, dramatically cheaper, + no supervision surface, and the vendoring path is proven with a small + upstream diff as its exit. +2. For *Computer execution backends* (worker-shell, worker-javascript, + containers) — if and when #346 wants them locally — the #347 topology + is the only one that provides them; keep it as the documented option + for that subset rather than the default local host. +3. Treat native-subprocess workspace access under #349 as explicitly + limited today: shim = development-only, FUSE = Linux with a Node + sidecar until the Deno N-API uv gap closes (file it upstream) — + and note that XMD's own exec against a mount is *not* behaviorally + equivalent to Computer's backends. + +Both spikes leave the door open to hosted Cloudflare through the same +sync protocol and identical schema; neither forecloses the other. The +final selection is recorded on #346. diff --git a/spikes/349-dofs/evidence/EVIDENCE.md b/spikes/349-dofs/evidence/EVIDENCE.md new file mode 100644 index 00000000..4b5272ee --- /dev/null +++ b/spikes/349-dofs/evidence/EVIDENCE.md @@ -0,0 +1,193 @@ +# Evidence: Deno-local DOFS (#349) + +Answers to the spike's eight questions, measured on macOS 15 (Darwin +25.5.0, arm64), Deno 2.9.1, cloudflare/computer `v0.1.1` +(`63d363632e558f7e077794988d36ed75017c2a62`), 2026-08-06. Claims marked +**[test]** are asserted by `tests/spike.test.ts`; **[measured]** were +observed and are reproducible from the commands shown; **[probe]** carry +their full command-by-command ledger in `probes/`. + +## 1. Does a file-backed `node:sqlite` adapter satisfy DOFS's contract? + +Yes, with zero changes to Cloudflare's schema or filesystem primitives. +The whole contract is `{ sql.exec(query, ...bindings) → { toArray() }, +transactionSync }`; `host/file-storage.ts` implements it over +`new DatabaseSync(path)` in ~100 lines, mirroring upstream's own +`SQLiteTestStorage` fixture (which is `:memory:`-only). **[test]** + +- `mkdir`, `writeFile`, `readFile`, `readdir`, `stat`, `lstat`, `rename`, + `symlink`, `readlink`, `rm` all work through Cloudflare's unmodified + `Database` → `initializeSchema` → `WorkspaceFilesystem` stack; `readFile` + follows symlinks, `lstat` reports the link. **[test]** +- The filesystem frontier survives full process restarts — every test op is + a separate compiled-binary invocation — including deletion persistence and + a create → delete → create sequence. **[test]** +- Separate database paths are separate, initially-empty workspaces. There is + no Durable Object identity layer in this topology: the database file path + *is* the workspace identity. **[test]** +- Schema safety: `initializeSchema` is idempotent (IF-NOT-EXISTS DDL) and is + itself the version gate — a database stamped with a newer + `vfs_meta.schema_version` is refused loudly + (`Unsupported workspace filesystem schema version 99`), not recreated; + the failed open leaves the database untouched. `SCHEMA_VERSION` is 5. + **[test]** +- WAL mode: `-wal`/`-shm` exist only while open; `close()` checkpoints and + removes them, leaving a single-file artifact. **[test]** +- `node:sqlite` under Deno needed no workarounds: `DatabaseSync`, prepared + statements, and transactions all behave; dofs's own row normalization + handles the null-prototype rows. **[probe]** +- Sizes and timings **[measured]**: compiled proof 110 MB (no workerd, no + Worker bundle, no materialization step); full op cycle + (process start → op → close) 0.08 s wall, 14 ms in-process on a cold + database; upstream build of the vendored package is a clean plain `tsc`. + +## 2. Which reuse boundary works? + +Tested independently **[probe: probes/slice2-reuse.md]**: + +| Approach | Works? | Upgrades | MIT notices | Cloudflare code owned | +| --- | --- | --- | --- | --- | +| 1. Direct import of published `@cloudflare/computer@0.1.1` | **No** — `dist/index.js` unconditionally imports `cloudflare:workers` (`ERR_UNSUPPORTED_ESM_URL_SCHEME`); deep imports blocked by the exports map; no `cloudflare:`-free DOFS chunk exists in dist | — | none | zero | +| 2. Tree-shaken bundle of the published package | **No (structurally)** — one stub silences `cloudflare:workers`, but the published surface never exports `Database`/`initializeSchema`/`WorkspaceFilesystem`, so the schema DDL is not even present in the bundle | re-bundle + re-prove stubs per release | bundle must carry LICENSE | stubs + reimplemented schema DDL | +| 3. Pinned vendored `packages/dofs` | **Yes, fully** — clean `tsc` build, zero runtime deps, source unmodified; consumed as a `file:` npm dependency through its exports map | re-vendor from tag, re-apply 3 manifest edits, rebuild, re-run tests | copy root LICENSE + provenance note (no per-package license upstream) | 48 files / ~6.5k LOC (src minus tests) | +| 4. Upstream export change | **Proven mechanically** — a 3-line upstream diff (re-export module + bundler input + exports entry) yields a DOFS chunk whose module graph imports only `node:crypto`/`node:events`; verified under Deno | `npm i` once accepted | none | zero after acceptance | + +The spike ships approach 3 (`vendor/dofs/` + `PROVENANCE.md`), with +approach 4 drafted in the probe ledger as the exit path — the vendored copy +keeps the `@cloudflare/dofs` specifier, so landing the upstream change +reduces to a dependency swap. This slice satisfies "the exact supported +code-reuse mechanism is demonstrated": vendoring is the only mechanism that +works today, and its obligations are notice retention plus a +three-edit-manifest upgrade procedure. + +## 7. Userspace shim — verdict: development-only fallback + +Separate from the real-FUSE verdict (§3-6), per the spike's terms. Upstream's +`shim.ts` runs under Deno **byte-identical, zero runtime failures** — it is +vendored as a subset package (`vendor/computerd-shim/`, provenance + +sha1), and a compiled `proof-shim` executes a native subprocess that reads +an API-written file through the mount and writes back into SQLite, and +rematerializes an emptied mount directory from the persisted frontier. +**[test]** Full ledger: `probes/slice6-shim.md`. + +Measured (medians over 10 rounds, default 100 ms provider watch / 250 ms +shim poll) **[probe]**: + +| Direction | Median | +| --- | --- | +| VFS API write → visible on disk | 80.8 ms | +| external disk write → visible via API | 108.3 ms (worst 205.7 ms) | +| external disk write → committed in SQLite | 109.4 ms | +| rematerialize 100 files / 10 MB | 78.9 ms | +| 50 MB file, 1-byte change, reconcile | ~600 ms each way (full re-read) | + +Demonstrated losses: conflicts within a poll window resolve **VFS-wins** +(3/3 both orders); symlinks degrade to content copies on both sides +(dangling links dropped); chmod is invisible; after SIGKILL the WAL +recovers and convergence completes in ~1 s, but up to one poll window +(250 ms) of external disk writes is silently clobbered by the VFS copy. +Two integration facts a host must honor: the mount path is embedded in the +workspace namespace (the same database mounted at a different absolute path +materializes nothing), and `@platformatic/vfs`'s `create()` silently falls +back to a MemoryProvider unless the prototype splice is verified +(`host/vfs-wiring.ts` guards this explicitly). + +Verdict: viable as a **development-only fallback** — sub-poll-window +durability, symlink/metadata fidelity, and large-file costs disqualify it +as a supported production path. On darwin-arm64 it is currently the *only* +path (see §4/§5 platform record). + +## 3.-6. Real FUSE — verdict: works, but not in-process under Deno today + +Measured in a Linux/amd64 container with `/dev/fuse` (the same environment +upstream's own FUSE tests use); full ledger +`probes/slice3-5-fuse-linux.md`, key artifacts (container recipe, minimal +Deno repro, mount harness) in `fuse-linux/`. **[probe]** + +**The gating fact.** Deno 2.9.1/2.9.4 loads the fuse-native N-API addon +cleanly (both `import` through node-gyp-build and raw `process.dlopen`), +but any actual `fuse.mount()` aborts the whole Deno process uncatchably: +`bad result in uv polyfill: 1` (SIGABRT, exit 134) — Deno's +`uv_default_loop` polyfill returns null and fuse-native's mount path +exercises it. Reproduced under `deno run` and `deno compile`, under both +Rosetta and qemu; Node 22 mounts the identical script in 27 ms. A minimal +upstream-issue-ready repro is committed (`fuse-linux/g2-min.mjs`). Until +that Deno gap closes, real FUSE requires a **Node sidecar process**. + +**The stack itself is sound.** Under a Node sidecar, the full chain — +file-backed SQLite → dofs → spliced provider → `@platformatic/vfs` → +upstream `driver.ts` compiled verbatim — mounts in 118-133 ms cold, and a +second process (Deno) reads **and writes** the same WAL database with +changes visible through the mount immediately: the +Deno-main + Node-FUSE-sidecar topology over one shared database is proven +live, not hypothesized. All exercised ops pass: create, overwrite, +positional write (`dd seek`), truncate, rename, symlink, delete, +traversal, concurrent readers. Two semantic caveats: API-side overwrites +of kernel-cached files can read stale through the mount for ≤1 s +(`auto_cache` + `attr_timeout=1`), and `writeFileRangesSync` is confirmed +unreachable (probed by the driver, never forwarded by upstream's wiring — +dead code by omission). + +**Durability (slice 5 matrix).** The write-commit boundary is *release*, +observed strictly: + +| Scenario | Result | +| --- | --- | +| write + close, then read from a separate process | committed | +| write + `fsync`, no close, SIGKILL | **lost — the inode never reaches the database before release** | +| write + close, SIGKILL immediately after | **lost** (RELEASE is async; +500 ms → committed) | +| SIGTERM / SIGKILL of the mount host | `auto_unmount` clears the mount; no ENOTCONN hang; WAL recovers on next open | +| second process opens the live database | reads and writes concurrently (WAL); visible through the mount | + +`close()` is not a durability barrier and `fsync` is a durability no-op: +a host needs release-observed barriers (or explicit provider-level flush) +before treating a file as persisted. + +**The forbidden-spawn hazard, demonstrated.** Spawning with `cwd` inside +the mount from the process serving FUSE deadlocked the mount on the first +try — parent blocked in `pipe_read`, forked child in +`request_wait_answer`, readers unkillable in D state — and SIGKILL of the +host does **not** recover it (the forked child inherits the `/dev/fuse` +fd); recovery required `fusermount -z` plus a fusectl connection abort. +Upstream's `cd`-prefix technique works and is mandatory. + +**Packaging (slice 4).** `deno compile` can carry and load the addon both +ways (static import of the embedded npm graph, and `--include` → +materialize → `process.dlopen`), 142 MB binary — the packaging story is +proven even though the in-process mount then hits the same uv-polyfill +abort. Platform matrix, tested rather than inferred: linux-x64 works (via +sidecar); darwin-arm64 is a dead end at this pin (no prebuild; a source +build loads but SIGSEGVs on mount — no arm64 slice in the bundled +osxfuse-era dylib; macFUSE absent and would need kernel approval); +darwin-x64 has a prebuild but was not tested on real hardware; linux-arm64 +has no addon prebuild; Windows has no fuse-native support at all. One +tooling landmine recorded: Docker's Rosetta runner cannot execute +deno-compiled amd64 binaries (ld.so assert) — qemu-user was required. + +Blueprint facts established from source and confirmed by the probes +(cloudflare/computer@v0.1.1): + +- The FUSE adapter is `makeFUSEOps(vfs, mountPoint)` — one 1092-line file + whose only dependency is a `@platformatic/vfs` instance; `fuse-native@2.2.6` + (libfuse 2.9 API) with prebuilds for linux-x64 and darwin-x64 only — no + darwin-arm64, no linux-arm64 addon prebuild. +- Wiring `SQLiteWorkspaceProvider` into `@platformatic/vfs` requires two + upstream hacks replicated verbatim: a prototype splice (vfs's `create()` + silently falls back to a MemoryProvider on an instanceof failure) and + explicit forwarding of ten dofs-specific sync methods onto the vfs facade. +- The FUSE write-commit boundary is **release-only**: dofs's write buffer + commits when the open count reaches zero; `flush` and `fsync` are + durability no-ops in the production configuration. +- The darwin-arm64 platform record **[probe: probes/slice6-shim.md]**: + `fuse-native@2.2.6` ships no darwin-arm64 prebuild; a manual source build + produces an arm64 addon that loads under Deno and Node, but its bundled + `libosxfuse.dylib` carries no arm64 slice (kext-era osxfuse 3.x) and an + actual mount SIGSEGVs under both runtimes; macFUSE is not installed on + this host (`/Library/Filesystems/macfuse.fs` absent). Real FUSE on Apple + Silicon is a dead end at this pinned version even before the + macFUSE-install/kernel-approval prerequisite. + +## 8. Comparison with #347 + +Recorded in [COMPARISON.md](COMPARISON.md), including the +select/reject/limit recommendation for #346. diff --git a/spikes/349-dofs/evidence/fuse-linux/Dockerfile b/spikes/349-dofs/evidence/fuse-linux/Dockerfile new file mode 100644 index 00000000..7733f78a --- /dev/null +++ b/spikes/349-dofs/evidence/fuse-linux/Dockerfile @@ -0,0 +1,10 @@ +FROM --platform=linux/amd64 node:22-slim +RUN apt-get update && apt-get install -y --no-install-recommends \ + fuse libfuse2 procps curl unzip ca-certificates \ + && rm -rf /var/lib/apt/lists/* +RUN curl -fsSL -o /tmp/deno.zip https://dl.deno.land/release/v2.9.1/deno-x86_64-unknown-linux-gnu.zip \ + && unzip /tmp/deno.zip -d /usr/local/bin \ + && rm /tmp/deno.zip \ + && chmod +x /usr/local/bin/deno +RUN mkdir -p /mnt/ws +CMD ["sleep", "infinity"] diff --git a/spikes/349-dofs/evidence/fuse-linux/g2-min.mjs b/spikes/349-dofs/evidence/fuse-linux/g2-min.mjs new file mode 100644 index 00000000..0adb72aa --- /dev/null +++ b/spikes/349-dofs/evidence/fuse-linux/g2-min.mjs @@ -0,0 +1,44 @@ +// Minimal mount repro, runtime-agnostic (node g2-min.mjs / deno run -A g2-min.mjs). +// Mounts a trivial one-file FS, stats it, unmounts. Isolates the FUSE binding +// from the dofs/vfs stack. +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); +const Fuse = require("fuse-native"); + +const mountPoint = process.argv[2] ?? "/mnt/ws"; +const stat = (mode, size) => ({ + mtime: new Date(), atime: new Date(), ctime: new Date(), + size, mode, uid: 0, gid: 0, +}); +const ops = { + readdir: (path, cb) => cb(0, path === "/" ? ["hello.txt"] : []), + getattr: (path, cb) => { + if (path === "/") return cb(0, stat(0o40755, 4096)); + if (path === "/hello.txt") return cb(0, stat(0o100644, 6)); + return cb(Fuse.ENOENT); + }, + open: (path, flags, cb) => cb(0, 42), + read: (path, fd, buf, len, pos, cb) => { + const data = Buffer.from("hello\n").subarray(pos, pos + len); + data.copy(buf); + cb(data.length); + }, + release: (path, fd, cb) => cb(0), +}; + +console.log(JSON.stringify({ event: "constructing", mountPoint })); +const fuse = new Fuse(mountPoint, ops, { autoUnmount: true, debug: false }); +const t0 = performance.now(); +fuse.mount((err) => { + if (err) { + console.log(JSON.stringify({ event: "mount-error", error: String(err) })); + process.exit(1); + } + console.log(JSON.stringify({ event: "mounted", ms: Math.round(performance.now() - t0) })); + setTimeout(() => { + fuse.unmount((err2) => { + console.log(JSON.stringify({ event: "unmounted", error: err2 ? String(err2) : null })); + process.exit(0); + }); + }, 10_000); +}); diff --git a/spikes/349-dofs/evidence/fuse-linux/mount-host.mjs b/spikes/349-dofs/evidence/fuse-linux/mount-host.mjs new file mode 100644 index 00000000..9dd8b21a --- /dev/null +++ b/spikes/349-dofs/evidence/fuse-linux/mount-host.mjs @@ -0,0 +1,148 @@ +// Goal 2 host: full stack. +// FileSQLiteStorage(file db) -> Database -> initializeSchema +// -> SQLiteWorkspaceProvider -> prototype splice -> vfs create (+forwarding) +// -> makeFUSEOps/mountFuse (compiled CJS driver) -> /mnt/ws +// Then serves an HTTP control API on 127.0.0.1:9976 so separate processes +// (curl in docker exec) can drive the dofs API side while shells poke the mount. +// +// Runtime-agnostic on purpose: `deno run --allow-all mount-host.mjs ...` +// aborts in the uv polyfill at mount() (goal 1/2 finding); `node +// mount-host.mjs ...` is the working sidecar topology. +import { createRequire } from "node:module"; +import { createServer } from "node:http"; +import process from "node:process"; +import { createFileVfs, verifySqliteBacked } from "./vfs-wiring.mjs"; + +const [dbPath = "/probe/data/ws.db", mountPoint = "/mnt/ws"] = process.argv.slice(2); +const require = createRequire(import.meta.url); + +const t0 = performance.now(); +const handle = createFileVfs(dbPath); +const { vfs, wfs, storage } = handle; +// makeFUSEOps uses mountPoint as the VFS namespace prefix: kernel "/" +// maps to vfs "". The backing dir chain must exist in the vfs +// or every op (including getattr of the mount root) returns ENOENT. +{ + const segments = mountPoint.split("/").filter(Boolean); + let acc = ""; + for (const seg of segments) { + acc += `/${seg}`; + if (!vfs.existsSync(acc)) vfs.mkdirSync(acc, { mode: 0o755 }); + } +} +if (!vfs.existsSync("/workspace")) vfs.mkdirSync("/workspace", { mode: 0o755 }); +const sqliteProof = await verifySqliteBacked(handle, dbPath); +const tVfs = performance.now(); + +// Record whether writeFileRangesSync reached the facade (upstream omits it +// from the forward list even though driver.ts probes for it). +const forwarding = Object.fromEntries( + [ + "linkSync", "createFileSync", "writeRangeSync", "truncateFileSync", + "chmodSync", "readRangeSync", "openWriteBufferSync", + "openWriteBufferForCreateSync", "releaseWriteBufferSync", + "writeFileRangesSync", + ].map((name) => [name, typeof vfs[name]]), +); + +const { mountFuse } = require("./dist-cjs/driver.js"); +const tRequire = performance.now(); +const mount = await mountFuse({ mountPoint, vfs }); +const tMounted = performance.now(); + +console.log(JSON.stringify({ + event: "mounted", + runtime: typeof Deno === "undefined" ? `node ${process.version}` : `deno ${Deno.version.deno}`, + pid: process.pid, + dbPath, + mountPoint, + sqliteProof, + forwarding, + timingsMs: { + vfsUp: r(tVfs - t0), + driverRequire: r(tRequire - tVfs), + fuseMount: r(tMounted - tRequire), + total: r(tMounted - t0), + }, +})); + +function r(x) { return Math.round(x * 100) / 100; } + +const server = createServer(async (req, res) => { + const url = new URL(req.url, "http://127.0.0.1:9976"); + const q = (k) => url.searchParams.get(k); + const json = (obj, status = 200) => { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(obj)); + }; + try { + switch (url.pathname) { + case "/write": { + const t = performance.now(); + await wfs.writeFile(q("path"), q("body") ?? ""); + return json({ ok: true, ms: r(performance.now() - t) }); + } + case "/read": { + const t = performance.now(); + const body = await wfs.readFile(q("path"), "utf8"); + return json({ ok: true, body, ms: r(performance.now() - t) }); + } + case "/ls": { + const entries = (await wfs.readdir(q("path"))).map((e) => e.name).sort(); + return json({ ok: true, entries }); + } + case "/stats": + return json({ ok: true, bufferStats: mount.getBufferStats?.() ?? null }); + case "/bench-direct": { + const n = Number(q("n") ?? 100); + const t = performance.now(); + for (let i = 0; i < n; i++) { + await wfs.writeFile(`${mountPoint}/bench-api-${i}.txt`, `payload-${i}\n`); + } + const total = r(performance.now() - t); + return json({ ok: true, n, totalMs: total, perOpMs: r(total / n) }); + } + case "/exec": { + // Upstream exec/runner.ts technique: NEVER pass cwd: to spawn from the process serving FUSE (uv_spawn forks then + // chdirs pre-exec; the chdir triggers a FUSE request only this + // process can serve -> deadlock). Instead prefix `cd &&` so the + // chdir happens in the child shell after exec. + const dir = q("cwd") ?? "/"; + const cmd = q("cmd") ?? "pwd"; + const { execFile } = await import("node:child_process"); + const t = performance.now(); + const out = await new Promise((resolve) => { + execFile("/bin/sh", ["-c", `cd ${dir} && ${cmd}`], { timeout: 10_000 }, + (error, stdout, stderr) => resolve({ error: error ? String(error) : null, stdout, stderr })); + }); + return json({ ok: out.error === null, technique: "cd-prefix", ...out, ms: r(performance.now() - t) }); + } + case "/exec-cwd": { + // The hazardous variant, for characterization only: spawn with the + // cwd option pointing inside our own mount. + const dir = q("cwd") ?? "/mnt/ws"; + const { execFile } = await import("node:child_process"); + const t = performance.now(); + const out = await new Promise((resolve) => { + execFile("/bin/sh", ["-c", q("cmd") ?? "pwd"], { cwd: dir, timeout: 10_000 }, + (error, stdout, stderr) => resolve({ error: error ? String(error) : null, stdout, stderr })); + }); + return json({ ok: out.error === null, technique: "spawn-cwd-option", ...out, ms: r(performance.now() - t) }); + } + case "/unmount": { + await mount.unmount(); + storage.close(); + setTimeout(() => process.exit(0), 100); + return json({ ok: true, unmounted: true }); + } + default: + return json({ ok: false, error: "unknown endpoint" }, 404); + } + } catch (error) { + return json({ ok: false, error: String(error?.message ?? error), code: error?.code }, 500); + } +}); +server.listen(9976, "127.0.0.1", () => { + console.log(JSON.stringify({ event: "control-api", port: 9976 })); +}); diff --git a/spikes/349-dofs/evidence/fuse-linux/vfs-wiring.mjs b/spikes/349-dofs/evidence/fuse-linux/vfs-wiring.mjs new file mode 100644 index 00000000..a0f57e5b --- /dev/null +++ b/spikes/349-dofs/evidence/fuse-linux/vfs-wiring.mjs @@ -0,0 +1,107 @@ +// Re-implementation of computerd's src/fuse/vfs.ts wiring in plain JS, +// minus the @cloudflare/computer-rpc sync loop, plus a file-backed +// SQLite storage instead of SQLiteTestStorage(:memory:). +// +// Wiring steps (mirroring vfs.ts): +// 1. prototype-splice SQLiteWorkspaceProvider -> VirtualProvider +// (else @platformatic/vfs's create() SILENTLY falls back to MemoryProvider) +// 2. create(provider, { moduleHooks: false }) +// 3. Object.defineProperty-forward EXTRA_VFS_METHODS onto the vfs facade. +// NOTE: writeFileRangesSync is probed by driver.ts but deliberately +// NOT in the upstream forward list; we replicate that exactly and +// record whether it reaches the vfs. + +import { + Database, + initializeSchema, + SQLiteWorkspaceProvider, + WorkspaceFilesystem, +} from "./dofs/dist/index.js"; +import { FileSQLiteStorage } from "./file-storage.mjs"; +import { create, VirtualProvider } from "@platformatic/vfs"; + +// Exactly the upstream list from packages/computerd/src/fuse/vfs.ts. +const EXTRA_VFS_METHODS = [ + "linkSync", + "createFileSync", + "writeRangeSync", + "truncateFileSync", + "chmodSync", + "readRangeSync", + "openWriteBufferSync", + "openWriteBufferForCreateSync", + "releaseWriteBufferSync", +]; + +let prototypePatched = false; +function ensureVirtualProviderPrototype() { + if (prototypePatched) return; + const proto = SQLiteWorkspaceProvider.prototype; + const parent = Object.getPrototypeOf(proto); + if (parent === VirtualProvider.prototype) { + prototypePatched = true; + return; + } + Object.setPrototypeOf(proto, VirtualProvider.prototype); + prototypePatched = true; +} + +export function createFileVfs(dbPath) { + ensureVirtualProviderPrototype(); + const storage = new FileSQLiteStorage(dbPath); + const db = new Database(storage); + initializeSchema(db, () => Date.now()); + + const provider = new SQLiteWorkspaceProvider(db); + const vfs = create(provider, { moduleHooks: false }); + for (const name of EXTRA_VFS_METHODS) { + const fn = provider[name]; + if (typeof fn !== "function") continue; + Object.defineProperty(vfs, name, { + value: (...args) => fn.apply(provider, args), + writable: true, + configurable: true, + }); + } + + // WorkspaceFilesystem over the same Database = the "API side". + const wfs = new WorkspaceFilesystem(db); + return { vfs, db, wfs, storage, provider }; +} + +// Prove we're on SQLite, not the silent MemoryProvider fallback: +// write through the vfs facade, read back through a FRESH node:sqlite +// connection over the same db file. +export async function verifySqliteBacked(vfsHandle, dbPath) { + const marker = `sqlite-proof-${Date.now()}`; + if (!vfsHandle.vfs.existsSync("/workspace")) { + vfsHandle.vfs.mkdirSync("/workspace", { mode: 0o755 }); + } + vfsHandle.vfs.writeFileSync("/workspace/.sqlite-proof", marker); + const { DatabaseSync } = await import("node:sqlite"); + const fresh = new DatabaseSync(dbPath); + try { + const rows = fresh + .prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + .all(); + const tables = rows.map((r) => r.name); + // Find the inode row for the proof file. + let proofRow = null; + for (const t of tables) { + try { + const hit = fresh + .prepare(`SELECT * FROM ${t} WHERE CAST(name AS TEXT) = ? LIMIT 1`) + .all(".sqlite-proof"); + if (hit.length > 0) { + proofRow = { table: t }; + break; + } + } catch { + // table without a name column; ignore + } + } + return { tables, proofFileFoundInTable: proofRow?.table ?? null, marker }; + } finally { + fresh.close(); + } +} diff --git a/spikes/349-dofs/evidence/probes/slice1-adapter.md b/spikes/349-dofs/evidence/probes/slice1-adapter.md new file mode 100644 index 00000000..5900219f --- /dev/null +++ b/spikes/349-dofs/evidence/probes/slice1-adapter.md @@ -0,0 +1,259 @@ +# Probe: file-backed node:sqlite adapter for Cloudflare DOFS under Deno (issue #349, slice 1) + +Runtime under test: Deno 2.9.1 (aarch64-apple-darwin). Reference: cloudflare/computer v0.1.1 +(63d363632e558f7e077794988d36ed75017c2a62), `packages/dofs`, read-only clone at `../cf-computer`. +Working vendored copy: `./dofs` (modified only in `package.json` devDeps + `tsconfig.build.json` +types array — see Step 1). + +## Verdict (condensed) + +PASS on every axis. A ~70-line file-backed `DurableObjectStorageLike` adapter over +`new DatabaseSync(path)` from `node:sqlite` satisfies the whole contract with ZERO changes to +dofs source: the filesystem frontier (writes, deletes, renames, symlinks, delete-then-recreate) +survives full Deno process restarts; a second db file is a fully isolated empty workspace; the +schema-version gate refuses a database stamped by a newer binary with a loud +`WorkspaceFsError: Unsupported workspace filesystem schema version 99` and exit code 1. +Warm per-op process wall time ~35 ms (in-process op time 1.5–4 ms). + +## Step 1 — vendor + build + +``` +cp -R cf-computer/packages/dofs probe-dofs/dofs +cd probe-dofs/dofs && npm i -D typescript @cloudflare/workers-types @types/node +``` + +First install FAILED (verbatim core): + +``` +npm error code ERESOLVE +npm error While resolving: wrangler@4.119.0 +npm error Found: @cloudflare/workers-types@4.20260702.1 +npm error peerOptional @cloudflare/workers-types@"^5.20260801.1" from wrangler@4.119.0 +npm error Conflicting peer dependency: @cloudflare/workers-types@5.20260804.1 +``` + +Cause: the package's own devDeps pin `wrangler@^4.107.1`, and the wrangler line has since moved +to a `workers-types@^5` peer — the conflict is between two TEST-ONLY devDeps (wrangler, +@cloudflare/vitest-pool-workers), not anything the build needs. + +Fix (in the vendored copy only): +1. `package.json` devDependencies trimmed to `typescript@^6.0.3`, + `@cloudflare/workers-types@^4.20260616.1`, `@types/node@^24`. +2. `tsconfig.build.json` overrides `compilerOptions.types` to + `["@cloudflare/workers-types", "node"]` (the base tsconfig demands `vitest/globals` and + `@cloudflare/vitest-pool-workers/types`, again test-only). + +Then: + +``` +npm i # clean, 0 vulnerabilities +./node_modules/.bin/tsc -p tsconfig.build.json # exit 0, no diagnostics +``` + +Build is CLEAN. Emits `dist/` as ESM `.js` + `.d.ts` mirroring `src/` (21 top-level entries incl. +`index.js`, `storage.js`, `schema/`, `fs/` with every op — `fs/rename.js` included). No +`--unstable-sloppy-imports` fallback needed; the tsc-build vendoring story works as-is. (Caveat: +`npx tsc` without a local install grabs the squatter `tsc@2.0.4` npm package — use +`./node_modules/.bin/tsc` or install typescript first.) + +## Step 2 — adapter: `file-storage.mjs` + +Mirrors `src/testing.ts` `SQLiteTestStorage` exactly (statement cache, `toSQLiteValue` +normalization: undefined/null→null, boolean→1/0, Uint8Array passthrough, string/number/bigint +passthrough, TypeError otherwise; BEGIN/COMMIT/ROLLBACK `transactionSync`) but opens a real path. +Pragmas chosen: `journal_mode = WAL`, `foreign_keys = ON`, `synchronous = NORMAL`. Exposes +`close()` and a `pragma()` helper for the probe. + +```js +// FileSQLiteStorage — file-backed DurableObjectStorageLike over +// node:sqlite under Deno. Mirrors dofs's SQLiteTestStorage +// (src/testing.ts) exactly — prepared-statement cache, binding +// normalization, BEGIN/COMMIT/ROLLBACK transactionSync — but opens +// a real database file instead of ":memory:". + +import { DatabaseSync } from "node:sqlite"; + +class Cursor { + #rows; + constructor(rows) { + this.#rows = rows; + } + toArray() { + return this.#rows; + } +} + +export class FileSQLiteStorage { + #db; + #cache = new Map(); + sql; + + constructor(path, { journalMode = "WAL" } = {}) { + this.#db = new DatabaseSync(path); + this.#db.exec(`PRAGMA journal_mode = ${journalMode}`); + this.#db.exec("PRAGMA foreign_keys = ON"); + this.#db.exec("PRAGMA synchronous = NORMAL"); + this.sql = { + exec: (query, ...bindings) => { + let stmt = this.#cache.get(query); + if (stmt === undefined) { + stmt = this.#db.prepare(query); + this.#cache.set(query, stmt); + } + const normalized = bindings.map(toSQLiteValue); + const rows = stmt.all(...normalized) ?? []; + return new Cursor(rows); + }, + }; + } + + transactionSync(closure) { + this.#db.exec("BEGIN"); + try { + const result = closure(); + this.#db.exec("COMMIT"); + return result; + } catch (error) { + this.#db.exec("ROLLBACK"); + throw error; + } + } + + pragma(name) { + return this.#db.prepare(`PRAGMA ${name}`).all(); + } + + close() { + this.#cache.clear(); + this.#db.close(); + } +} + +// Same normalization as SQLiteTestStorage.toSQLiteValue. +function toSQLiteValue(value) { + if (value === undefined || value === null) return null; + if (typeof value === "boolean") return value ? 1 : 0; + if (value instanceof Uint8Array) return value; + if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") { + return value; + } + throw new TypeError(`FileSQLiteStorage cannot bind value of type ${typeof value}`); +} +``` + +## Step 3 — probe harness: `probe.mjs` + +One op per invocation → one Deno process per op. Construction recipe (canonical order taken +from `src/fs/filesystem.test.ts` `withFs`): + +```js +import { Database, initializeSchema, SCHEMA_VERSION, WorkspaceFilesystem } from "./dofs/dist/index.js"; +import { rename } from "./dofs/dist/fs/rename.js"; // NOT on the class or index.ts — free function only +import { FileSQLiteStorage } from "./file-storage.mjs"; + +const storage = new FileSQLiteStorage(dbPath); // 1. adapter +const db = new Database(storage); // 2. wrap +initializeSchema(db, Date.now); // 3. idempotent DDL + migration gate + root inode +const fs = new WorkspaceFilesystem(db); // 4. ops surface ({ now } optional) +// ... run op ... +storage.close(); // 5. checkpoints WAL, removes -wal/-shm +``` + +`initializeSchema(db, now)` is safe to call on EVERY process start: all DDL is +`CREATE ... IF NOT EXISTS`, version stamping is insert-or-ignore + update, and it is exactly +where the too-new-schema gate lives. + +Surprise: `WorkspaceFilesystem` has no `rename` method and `index.ts` does not export the free +function; `rename(db, oldPath, newPath)` must be imported from `dist/fs/rename.js` directly. + +## Step 4 — frontier across fresh processes (every line = a separate `deno run -A probe.mjs ws.db ...`) + +``` +init → {"op":"init","schemaVersion":5,"ms":2.92} (cold; 0.050s wall) +mkdir /notes → ok ms:2.7 +write /notes/a.md alpha → ok ms:4.18 +read /notes/a.md → "alpha" +write /notes/b.md beta → ok +rm /notes/a.md → ok +ls /notes → ["b.md"] +write /f.txt v1 → ok +rm /f.txt → ok +write /f.txt v2 → ok +read /f.txt → "v2" (delete-then-recreate survives) +stat /notes/b.md → {"name":"b.md","inode":4,"mode":420,"mtime":1785988833560, + "size":4,"isFile":true,"isDirectory":false,"isSymbolicLink":false} +rename /notes/b.md /notes/c.md → ok +ls /notes → ["c.md"]; lsflat / → ["/f.txt","/notes/c.md"] +symlink /notes/c.md /link → ok (signature: fs.symlink(target, path)) +readlink /link → "/notes/c.md" +read /link → "beta" → YES, readFile FOLLOWS symlinks +lstat /link → {"isSymbolicLink":true,"isFile":false,"mode":511,"size":11} +read /notes/c.md → "beta" (final frontier proof after ALL of the above) +``` + +VERDICT: PASS. Every mutation made in one process is observed by the next; nothing leaked from +deleted files. + +## Step 5 — isolation & identity + +`deno run -A probe.mjs ws2.db lsflat /` → `{"paths":[]}`; `ls /` → `[]`. A second db path is a +fresh, empty workspace. Identity model: ONE DB FILE = ONE WORKSPACE. There is no Durable Object +identity layer here — in Cloudflare's runtime the DO id names the storage; file-backed, the +filesystem path of the .db file IS the workspace identity, and nothing inside the schema names +or authenticates the workspace. Callers own the path→workspace mapping (and any locking between +concurrent writers — SQLite WAL allows one writer at a time across processes). + +## Step 6 — schema guard + +`meta` on the live db: binary `SCHEMA_VERSION = 5`; `vfs_meta` = `[{k:"rev",v:10}, +{k:"schema_version",v:5}]`; `journal_mode` = `wal`. (Version lives in table `vfs_meta`, +key `schema_version`.) + +Guard test: `cp ws.db ws-future.db`, then `bump-version.mjs` sets stored `schema_version = 99`. +Opening it: + +``` +error: Uncaught (in promise) WorkspaceFsError: Unsupported workspace filesystem schema version 99 + at createWorkspaceError (dofs/dist/errors.js:2:19) + at dofs/dist/schema/index.js:25:19 + at FileSQLiteStorage.transactionSync (file-storage.mjs:46:22) + at Database.transactionSync (dofs/dist/storage.js:39:36) + at initializeSchema (dofs/dist/schema/index.js:7:8) +``` + +Exit code 1. The throw happens inside `transactionSync`, so the adapter's ROLLBACK ran and the +future-versioned db is untouched. VERDICT: the "binary older than on-disk schema_version" +gate works loudly and exactly as in the DO runtime. + +## Step 7 — timings (M-series mac, warm OS cache) + +- Cold (fresh db file, full init): 0.050 s process wall; 2.92 ms in-process (open → init done). +- Warm (already-initialized db): 0.035 s process wall; 1.5–4.2 ms in-process per op + (reads ~1.5–2 ms, writes ~3.5–4.2 ms — writes pay the transaction fsync at + `synchronous = NORMAL`). +- Per-process overhead is dominated by Deno startup (~30 ms), not dofs or SQLite. + +## Step 8 — node:sqlite under Deno 2.9.1 + +- `DatabaseSync(path)`, `prepare()`, `StatementSync.all()/run()/get()`, `exec()` for + BEGIN/COMMIT/ROLLBACK, and SAVEPOINT/RELEASE (used by `Database`'s reentrant + `transactionSync`) all work. `PRAGMA journal_mode/foreign_keys/synchronous` work. +- No Deno-specific failures were hit anywhere in the probe. +- node:sqlite quirks dofs ALREADY handles: rows come back with a null prototype + (`storage.ts` `normalizeRow` re-keys into plain objects) and BLOBs come back as Uint8Array + (also normalized there). `bump-version.mjs` output showed the raw + `[Object: null prototype] { v: 99 }` shape, confirming the normalization is doing real work. +- WAL sidecars: `wal-check.mjs` proves `ws.db-wal` and `ws.db-shm` EXIST while the database is + open with uncommitted-to-main-file writes, and `close()` checkpoints and REMOVES both + (`{"during":{"wal":true,"shm":true},"after":{"wal":false,"shm":false}}`). After every probe + run only the bare `.db` file remains, so a process that closes cleanly leaves a + single-file artifact; a killed process would leave `-wal`/`-shm` behind and the next open + replays them (standard SQLite WAL recovery). + +## Files + +- `file-storage.mjs` — the adapter (final code inline above) +- `probe.mjs` — CLI probe harness +- `bump-version.mjs`, `wal-check.mjs` — step 6 / step 8 helpers +- `dofs/` — vendored package (2 build-config edits, zero source edits), `dofs/dist/` — tsc output +- `ws.db` — the surviving workspace; `ws2.db` — isolation check; `ws-future.db` — guard fixture diff --git a/spikes/349-dofs/evidence/probes/slice2-reuse.md b/spikes/349-dofs/evidence/probes/slice2-reuse.md new file mode 100644 index 00000000..0f01c13c --- /dev/null +++ b/spikes/349-dofs/evidence/probes/slice2-reuse.md @@ -0,0 +1,318 @@ +# Probe: DOFS reuse boundaries (issue #349 slice 2) + +Reference clone: `scratchpad/cf-computer` @ tag v0.1.1 (63d3636). Deno 2.9.1. + +## Verdict table + +| Approach | Works? | Upgrade path | MIT obligations | Cloudflare code XMD owns | +|---|---|---|---|---| +| 1. Direct import of published `@cloudflare/computer@0.1.1` | **No** — every entry/deep path either hits `ERR_UNSUPPORTED_ESM_URL_SCHEME` (`cloudflare:`) or `ERR_PACKAGE_PATH_NOT_EXPORTED`; and 3 of the 4 needed symbols aren't exported at all | `npm i` (moot) | None beyond upstream's own tarball | Zero | +| 2. Tree-shaken esbuild bundle of the published package | **Half** — bundle loads under Deno with exactly 1 trivial stub, but only `SQLiteWorkspaceProvider` is reachable; `Database`/`initializeSchema`/`WorkspaceFilesystem` are unexported, so the schema can never be created | `npm i` + re-bundle + re-prove the stub-is-dead-code fact each release | LICENSE text must ship next to the bundle (embeds CF source) | Stub + build script + a hand-rolled `Database` duck-type + reimplemented schema DDL (DOFS internals) | +| 3. Pinned vendored copy of `packages/dofs` | **Yes** — tsc build clean; all 4 symbols import under Deno; schema v5 initializes over in-memory `node:sqlite`; `npm pack` tarball + bare-specifier consumer also works | Re-vendor from upstream tag + rebuild + re-run tests (manual, diffable — src is unmodified) | Copy root LICENSE into the vendored dir + provenance note (VENDOR.txt) | 48 files / 6,554 LOC of unmodified src + ~40 lines of package/tsconfig scaffolding | +| 4. Upstream `./dofs` subpath export (or publishing `@cloudflare/dofs`) | **Yes (proven mechanically)** — a 3-line rolldown entry + package.json exports entry yields a `dofs.js` whose chunk graph is 100% cloudflare:-free; runs under Deno, schema initializes | `npm i @cloudflare/computer` once accepted; Approach 3 vendored copy in the interim | None beyond upstream's tarball | Zero (after acceptance) | + +## License facts + +Single `LICENSE` at the monorepo root; **no per-package LICENSE files** anywhere +(`find cf-computer -iname 'LICENSE*'` returns only the root file, and +`packages/dofs/package.json` has no `license` field — it is `"private": true`, +version `0.0.0`). The published `@cloudflare/computer` package.json declares +`"license": "MIT"` but the npm tarball ships **no LICENSE file** (files: dist, +README.md only). + +Root LICENSE text: `MIT License Copyright (c) 2026 Cloudflare, Inc.` followed by +the standard MIT grant. Its condition: + +> The above copyright notice and this permission notice (including the next +> paragraph) shall be included in all copies or substantial portions of the +> Software. + +So any redistribution of DOFS code (vendored source OR a bundle containing it) +must carry, verbatim: the line `MIT License Copyright (c) 2026 Cloudflare, Inc.`, +the permission notice, **and** the warranty-disclaimer paragraph (the "next +paragraph" is explicitly pulled into the retention requirement). Placing a copy +of the root LICENSE file next to the vendored/bundled code plus a provenance +header (upstream repo URL, tag, commit) satisfies this. No copyleft, no +source-disclosure duty; modifications are allowed. + +## Approach 1 — direct import of the published package under Deno + +Setup: + +```sh +# approach1/package.json: { "type": "module" } +npm install --prefix approach1 @cloudflare/computer@0.1.1 @platformatic/vfs zod +# added 86 packages +``` + +### Dist inspection + +`node_modules/@cloudflare/computer/dist` layout: `index.js`, `git.js`, +`artifacts/`, `assets/`, `backends/{container,worker-javascript,worker-shell}/`, +`observe/`, `tools/`, and six `shared-*.js` chunks (rolldown build). + +- Chunks containing DOFS symbols (`initializeSchema`, `SQLiteWorkspaceProvider`, + `vfs_nodes`): **only `dist/index.js` and `dist/git.js`**. +- `dist/git.js` merely *proxies* method calls to a `SQLiteWorkspaceProvider` + instance; the class/schema definitions live **solely in `dist/index.js`** + (regions `../dofs/src/errors.ts`, `../dofs/src/path.ts`, … are inlined there). +- `dist/index.js` line 6, top-level and unconditional: + `import { RpcTarget as RpcTarget$1, WorkerEntrypoint } from "cloudflare:workers";` +- Files with `cloudflare:` scheme imports: `index.js`, + `backends/container/index.js`, `backends/worker-javascript/index.js`, + `backends/worker-shell/index.js`. +- No `shared-*` chunk carries the DOFS layer, so **no cloudflare:-free chunk + containing DOFS exists in the published dist**. +- `dist/index.js` exports include `SQLiteWorkspaceProvider`, `Workspace`, + `getWorkspace`, `withWorkspace` — but NOT `Database`, `initializeSchema`, or + `WorkspaceFilesystem` (those are internal to the chunk). + +### Runs + +`import { Workspace, SQLiteWorkspaceProvider } from "@cloudflare/computer"`: + +``` +$ deno run --allow-all --node-modules-dir=manual approach1/import-main.ts +error: [ERR_UNSUPPORTED_ESM_URL_SCHEME] Only file and data URLs are supported by the default ESM loader. Received protocol 'cloudflare' +``` + +Deep import `@cloudflare/computer/dist/index.js`: + +``` +error: [ERR_PACKAGE_PATH_NOT_EXPORTED] Package subpath './dist/index.js' is not defined by "exports" in .../@cloudflare/computer/package.json +``` + +Direct file-path import `./node_modules/@cloudflare/computer/dist/index.js` +(bypasses the exports map): + +``` +error: [ERR_UNSUPPORTED_ESM_URL_SCHEME] Only file and data URLs are supported by the default ESM loader. Received protocol 'cloudflare' +``` + +**Verdict: does not work.** The only chunk that defines the DOFS layer +statically imports `cloudflare:workers`, and Deno's ESM loader rejects the +`cloudflare:` protocol. No published entrypoint or deep import yields the DOFS +layer under Deno without patching. (Upgrades would have been `npm i`-simple and +MIT notice duty would sit with Cloudflare's tarball, but the approach is moot.) + +Additional blocker independent of the scheme problem: the published export +surface contains `SQLiteWorkspaceProvider` but **not** `Database`, +`initializeSchema`, or `WorkspaceFilesystem` — those never left `packages/dofs` +(`dist/index.js`'s export statement ends with `... SQLiteWorkspaceProvider, +TestBackend, Workspace, ... getWorkspace, noopObserver, sh, shellQuote, +withWorkspace`). + +## Approach 2 — tree-shaken esbuild bundle of the published package + +Setup: `approach2/package.json` with `@cloudflare/computer@0.1.1`, +`@platformatic/vfs`, `zod`, devDep `esbuild@0.28.1`; `npm install --prefix +approach2`. Facade (`facade.ts`): + +```ts +export { SQLiteWorkspaceProvider } from "@cloudflare/computer"; +``` + +(Only symbol of the four we need that the package exports — see Approach 1.) + +### Round 1 — externals only + +```sh +./node_modules/.bin/esbuild facade.ts --bundle --format=esm --platform=neutral \ + --main-fields=module,main --conditions=import \ + '--external:cloudflare:*' '--external:node:*' --outfile=bundle.js +# bundle.js 150.2kb +``` + +Output still contains (line 2501): +`import { RpcTarget as RpcTarget$1, WorkerEntrypoint } from "cloudflare:workers";` +— but grep shows **neither `RpcTarget$1` nor `WorkerEntrypoint` is referenced +anywhere else in the bundle**: esbuild keeps external imports for potential side +effects; the symbols themselves are fully tree-shaken. + +### Round 2 — one stub via alias + +Stub `stub-cloudflare-workers.ts` (empty `RpcTarget`, `WorkerEntrypoint`, +`DurableObject` classes) + `'--alias:cloudflare:workers=./stub-cloudflare-workers.ts'` +instead of the external. Result: `grep -c "cloudflare:" bundle.js` → **0**. +Exactly **one stub**, and it does not touch DOFS internals (its symbols are +dead code in the bundle). + +### Run under Deno + +`run-bundle.ts` constructs `SQLiteWorkspaceProvider` over a hand-rolled +duck-type of the (unexported) `Database` interface backed by +`node:sqlite` `DatabaseSync`: + +``` +$ deno run --allow-all approach2/run-bundle.ts +provider constructed: SQLiteWorkspaceProvider { supportsSymlinks: true, supportsWatch: true } +statSync without schema fails as expected: no such table: vfs_nodes +``` + +**Verdict: half-works, not viable as the sole source.** The bundle is +cloudflare:-free with a single trivial stub and loads under Deno — but +tree-shaking follows the *export surface*, and the published surface exposes +only 1 of the 4 needed symbols. `initializeSchema`'s body is not even present +in the bundle (only an error-message string mentioning it), so the schema can +never be created through this path; `Database` and `WorkspaceFilesystem` are +likewise unreachable. XMD would have to reimplement the Database wrapper and +the entire schema DDL from DOFS internals — at which point it owns the very +code it tried to reuse. Also drags in ~2.5k lines of capnweb RPC runtime the +DOFS layer doesn't need. Upgrades: `npm i` + re-bundle + re-verify stub +assumptions each release (the "unused import" fact must be re-proven per +version — brittle). MIT: the bundle embeds Cloudflare source, so the LICENSE +text must ship next to `bundle.js`. + +## Approach 3 — pinned vendored copy of `packages/dofs` + +### Build steps + +```sh +cp -R cf-computer/packages/dofs approach3/dofs +cp cf-computer/LICENSE approach3/dofs/LICENSE # root LICENSE, verbatim +# + VENDOR.txt (upstream URL, tag v0.1.1, commit 63d3636, list of local changes) +# package.json edits: drop "private", version 0.0.0-vendored.63d3636, +# license: MIT, files: [dist, LICENSE, VENDOR.txt, README.md], +# devDeps: typescript ^5.9.2 + @types/node (upstream wants typescript ^6 + +# workers-types + vitest — none needed for the build) +# tsconfig.vendored.json = upstream tsconfig.build.json merged with base, +# with types: ["node"] instead of [workers-types, vitest, node, pool-workers] +npm install --prefix approach3/dofs # 3 packages +approach3/dofs/node_modules/.bin/tsc -p tsconfig.vendored.json # clean, no edits to src/ +``` + +Source is fully self-contained: zero runtime deps, no `cloudflare:` imports, +storage abstracted behind its own `DurableObjectStorageLike` interface +(src/types.ts); the only Cloudflare types mentioned are in comments. +`src/testing.ts` (part of the `./testing` export) is backed by `node:sqlite` — +usable as-is under Deno. + +### Import-and-construct under Deno (direct dist path) + +``` +$ deno run --allow-all approach3/run-vendored.ts +schema initialized: version 5 (SCHEMA_VERSION = 5) +tables: _vfs_fetch_cursor, _vfs_mounts, _vfs_watermark, sqlite_sequence, vfs_blob_bytes, vfs_blobs, vfs_changes, vfs_chunks, vfs_dirents, vfs_manifests, vfs_meta, vfs_nodes +provider constructed: SQLiteWorkspaceProvider +WorkspaceFilesystem loaded: function +``` + +All four symbols reachable: `Database`, `initializeSchema`, +`SQLiteWorkspaceProvider` from the root export; `WorkspaceFilesystem` too +(root export, src/index.ts line 5). One API gotcha hit while consuming the +untyped dist: `initializeSchema(db, now)` requires the `now: () => number` +argument (first run: `TypeError: now is not a function` from +dist/schema/index.js line 45); the .d.ts files catch this in typed consumers. + +### Packaging variant (npm pack → tarball consumer) + +```sh +(cd approach3/dofs && npm pack --pack-destination ..) +# cloudflare-dofs-0.0.0-vendored.63d3636.tgz — 100 files +# consumer/package.json: "@cloudflare/dofs": "file:../cloudflare-dofs-....tgz" +npm install --prefix approach3/consumer # 1 package +deno run --allow-all --node-modules-dir=manual approach3/consumer/main.ts +# → bare-specifier import OK: SQLiteWorkspaceProvider function 5 +``` + +Friction: only the package.json fixes already listed (name kept as +`@cloudflare/dofs` so import specifiers match upstream; `private` removed; +`files` added). Bare specifiers `@cloudflare/dofs` and +`@cloudflare/dofs/testing` both resolve under Deno with a node_modules dir. + +### Upgrade consumption / owned surface + +Upgrade = re-copy `packages/dofs/src` from the new upstream tag, re-apply the +two scaffold files (package.json, tsconfig.vendored.json — src itself is +unmodified so `diff -r` against upstream is clean), rebuild, re-run XMD's +adapter tests. Owned surface: **48 source files, 6,554 LOC** (src minus +`*.test.ts`, `bench/`, `with-db.workers.ts`) — owned in the "must re-vendor and +re-verify" sense, not forked. MIT: LICENSE copy + provenance note as above. + +## Approach 4 — upstream package/export change + +### What the published dist shows (from Approach 1/2) + +A pure package.json `exports` addition is **not** sufficient today: the v0.1.1 +dist has no cloudflare:-free chunk containing DOFS (the layer is inlined into +`dist/index.js`, which imports `cloudflare:workers` at top level). A build +change is required — one new rolldown entry. + +### Mechanical proof (COPY of packages/computer, clone untouched) + +Copied `packages/{computer,dofs,rpc}` into `approach4/`. Added +`src/dofs-entry.ts` (`export * from "@cloudflare/dofs";`) and a probe rolldown +config = the original with `dofs: "src/dofs-entry.ts"` added to `input` +(entries trimmed to index+git+dofs; dts plugin dropped — it needs the monorepo +type env, irrelevant to the chunk-graph question; also added two alias lines +for `@cloudflare/computer-rpc/{client,debug}` that the trimmed install needed). + +``` +$ ./node_modules/.bin/rolldown -c rolldown.probe.config.ts +dofs.js 2.67 kB · shared-DtkNvqpY.js 107.79 kB · index.js 111.29 kB · git.js 102.42 kB ... +``` + +Rolldown hoists the entire DOFS layer into `shared-DtkNvqpY.js`; +`dofs.js` re-exports the **full** `@cloudflare/dofs` surface (Database, +initializeSchema, SQLiteWorkspaceProvider, WorkspaceFilesystem, +RecordingStorage, sync/*, …). `grep -c "cloudflare:"` per chunk in the dofs +graph: `dofs.js` 0, `shared-DtkNvqpY.js` 0 (its only imports are `node:crypto`, +`node:events`); `cloudflare:workers` remains confined to `index.js`. Run proof: + +``` +$ deno run --allow-all approach4/run-dofs-entry.ts +dofs entry under Deno OK: SQLiteWorkspaceProvider function function schema v5 +``` + +### Draft upstream diff (what we'd ask Cloudflare for) + +```diff +--- a/packages/computer/src/dofs.ts ++++ b/packages/computer/src/dofs.ts (new file) +@@ -0,0 +1,4 @@ ++// Runtime-neutral re-export of the DOFS layer. This entry's module ++// graph must stay free of cloudflare:workers so non-workerd runtimes ++// (Node, Deno, Bun) can consume the SQLite filesystem directly. ++export * from "@cloudflare/dofs"; + +--- a/packages/computer/rolldown.config.ts ++++ b/packages/computer/rolldown.config.ts +@@ export default defineConfig({ + input: { + index: "src/index.ts", ++ dofs: "src/dofs.ts", + git: "src/git/index.ts", + +--- a/packages/computer/package.json ++++ b/packages/computer/package.json +@@ "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, ++ "./dofs": { ++ "types": "./dist/dofs.d.ts", ++ "import": "./dist/dofs.js" ++ }, +``` + +(Optionally a CI guard: `grep -L cloudflare: dist/dofs.js` + its chunk +imports.) The simpler alternative ask — publish `@cloudflare/dofs` itself — +is equally proven: Approach 3 showed the package builds standalone with plain +tsc and zero runtime deps; upstream would only delete `"private": true` and add +`license`/`repository`/`files` fields. + +### Interim story + +Until Cloudflare accepts either change, ship Approach 3's vendored copy; the +vendored import surface (`@cloudflare/dofs`) is specifier-identical to the +published-package future, so the eventual migration is a dependency swap, not a +code change. (If they add `./dofs` instead of publishing the package, migration +is a one-line import-path change to `@cloudflare/computer/dofs`.) + +## Files in this probe + +- `approach1/` — npm consumer of the published package + 3 failing Deno import scripts +- `approach2/` — facade, stub, esbuild bundle, Deno runner +- `approach3/dofs/` — vendored build; `approach3/consumer/` — tarball consumer; `run-vendored.ts` +- `approach4/computer-copy/` — probe rolldown build (`dist-probe/`); `run-dofs-entry.ts` diff --git a/spikes/349-dofs/evidence/probes/slice3-5-fuse-linux.md b/spikes/349-dofs/evidence/probes/slice3-5-fuse-linux.md new file mode 100644 index 00000000..89782033 --- /dev/null +++ b/spikes/349-dofs/evidence/probes/slice3-5-fuse-linux.md @@ -0,0 +1,263 @@ +# probe-fuse-linux — real FUSE under Deno (issue #349 slices 3-5) + +Date: 2026-08-06. Host: macOS (Apple Silicon), Docker Desktop. +Container: `fuse-probe:latest` = node:22-slim (amd64 under Rosetta) + `fuse` + +`libfuse2` (2.9.9-6+b1) + procps + curl + unzip + Deno 2.9.1 (linux x64). +Run flags: `--platform linux/amd64 --device /dev/fuse --cap-add SYS_ADMIN +--security-opt apparmor:unconfined -v :/probe`. +Second container `qemuprobe2`: debian:stable-slim arm64 (native) + qemu-user + +fuse, same FUSE flags — used to run amd64 binaries under qemu-x86_64 instead of +Rosetta (differential, and because Rosetta cannot run deno-compiled binaries at +all; see Goal 5). + +Stack under test: `FileSQLiteStorage(file db, WAL)` → dofs `Database` → +`initializeSchema` → `SQLiteWorkspaceProvider` → prototype-splice onto +`@platformatic/vfs` `VirtualProvider` → `create(provider, { moduleHooks: +false })` + `EXTRA_VFS_METHODS` forwarding (re-implemented from computerd +`src/fuse/vfs.ts` in `vfs-wiring.mjs`) → upstream `driver.ts`/`options.ts` +compiled verbatim with tsc → `mountFuse` → `/mnt/ws`. + +## Per-goal verdicts + +| # | Goal | Verdict | +|---|------|---------| +| 1 | Deno loads fuse-native N-API addon | **YES — loads perfectly. But `mount()` SIGABRTs the whole Deno process** (`bad result in uv polyfill: 1`), uncatchable, on deno run AND deno compile, v2.9.1 AND v2.9.4, under Rosetta AND qemu. Node 22 control mounts fine. Deno cannot *serve* FUSE via fuse-native today. | +| 2 | Full-stack mount | **PASS under Node 22 sidecar** (the only viable host). fuseMount 57 ms, whole stack up 118-133 ms. SQLite-backed proven (no silent MemoryProvider). `writeFileRangesSync` confirmed NOT forwarded. | +| 3 | Bidirectional + subprocess | **PASS**, with a ≤1 s staleness window for API overwrites of kernel-cached files (auto_cache + attr_timeout=1). cd-prefix exec works from the FUSE host; the spawn-cwd variant deadlocks the entire mount (demonstrated + forensics + recovery runbook). | +| 4 | Durability matrix | **Release-only commit CONFIRMED and sharpened**: fsync is a durability no-op AND the inode doesn't exist in the db until release; write+close is only durable once the *async* RELEASE op lands (kill 0 ms after close() → LOST; +500 ms → committed). | +| 5 | Compiled executable | **Addon loading from `deno compile` binary: YES both ways** (embedded npm graph require, and `--include` → materialize → `process.dlopen`). Mount: same uv-polyfill abort. Binary 142 MB. Rosetta cannot execute deno-compiled binaries at all (even hello-world) — measured under qemu-user. | +| 6 | Sizes/latencies | Table below. | + +--- + +## Goal 1 — Deno + fuse-native N-API addon + +``` +docker exec fuseprobe sh -c 'cd /probe && deno run --allow-all g1-load.mjs' +``` +→ `import("fuse-native")` through node-gyp-build resolves the prebuild and +returns the Fuse class: `ok: true, ms: 297.14, typeofDefault: "function"`, +full errno table + `beforeMount/beforeUnmount/configure/...` statics present. + +``` +deno run --allow-all g1-dlopen.mjs # process.dlopen on prebuilds/linux-x64/node.napi.node +``` +→ `ok: true`, 70 exports (`fuse_native_mount`, 35 signal fns, 35 op ids). +No LD_LIBRARY_PATH needed in-container: the prebuild's only non-libc DT_NEEDED +is `libfuse.so.2`, satisfied by apt `libfuse2`. (`fuse-shared-library-linux` +also ships its own `libfuse/lib/libfuse.so`, 939,064 B, for hosts without it.) + +**But**: any actual `fuse.mount()` under Deno: + +``` +{"event":"constructing","mountPoint":"/mnt/ws"} +bad result in uv polyfill: 1 +Aborted # exit code 134 (SIGABRT) +``` + +- Reproduced: `deno run` 2.9.1, `deno run` 2.9.4, `deno compile` binary; under + Docker Rosetta and under qemu-x86_64 → not an emulation artifact. +- Node 22.23.2 control with the identical script (`g2-min.mjs`): mounts in + 27 ms, `cat` works, clean unmount, exit 0. +- Attribution: the message comes from `assert_ok` in Deno `ext/napi/uv.rs` — + it `std::process::abort()`s whenever a polyfilled uv init fn returns + non-zero (`uv_mutex_init`, `uv_async_init`, ...). Deno's `uv_default_loop` + polyfill **returns a null pointer**; fuse-native's `fuse_native_mount` uses + `uv_default_loop` + `uv_async_init` + `uv_mutex_init` + `uv_sem_init/wait/post` + (symbol scrape of the prebuild). A null loop into `uv_async_init` is the + prime suspect for the non-zero result. Upstream issue material: minimal + repro is `g2-min.mjs`; abort is pre-kernel (happens with or without a + usable /dev/fuse once fusermount succeeds; with fusermount missing the + mount fails earlier with a *catchable* "fuse: failed to exec fusermount"). + +## Goal 2 — full-stack mount (Node 22 sidecar) + +``` +node mount-host.mjs /probe/data/ws.db /mnt/ws +{"event":"mounted","runtime":"node v22.23.2", ... + "sqliteProof":{"tables":["_vfs_fetch_cursor","_vfs_mounts","_vfs_watermark","sqlite_sequence", + "vfs_blob_bytes","vfs_blobs","vfs_changes","vfs_chunks","vfs_dirents","vfs_manifests", + "vfs_meta","vfs_nodes"],"proofFileFoundInTable":"vfs_dirents"}, + "forwarding":{"linkSync":"function","createFileSync":"function","writeRangeSync":"function", + "truncateFileSync":"function","chmodSync":"function","readRangeSync":"function", + "openWriteBufferSync":"function","openWriteBufferForCreateSync":"function", + "releaseWriteBufferSync":"function","writeFileRangesSync":"undefined"}, + "timingsMs":{"vfsUp":57.73,"driverRequire":3.52,"fuseMount":56.59,"total":117.85}} +mount | grep fuse +/dev/fuse on /mnt/ws type fuse (rw,nosuid,nodev,user_id=0,group_id=0,max_read=524288) +``` + +- SQLite proof: vfs-facade write found in `vfs_dirents` via a **fresh** + `node:sqlite` connection on the db file → prototype splice worked, no silent + MemoryProvider fallback. +- `max_read=524288` in the mount table → the `_fuseOptions()` monkeypatch + (options.ts profile) reached the kernel. +- **`writeFileRangesSync`: `undefined` at the driver boundary — confirmed.** + driver.ts probes it but vfs.ts's EXTRA_VFS_METHODS omits it, so the ranged + spill path is dead code. Harmless in practice: all direct-write probes + (`createFileSync`/`writeRangeSync`/`truncateFileSync` + buffered-write trio) + are forwarded, so `hasDirectWrites`/`hasBufferedWrites`/`hasDeferredCreate` + are all true and `flushEntry` (the only `writeFileRangesSync` consumer) is + bypassed for driver-created files. +- Gotcha found live: `makeFUSEOps(vfs, mountPoint)` uses the **mountPoint as + the VFS namespace prefix** (kernel `/` → vfs `/mnt/ws`). The dir chain for + the mountPoint must pre-exist in the vfs or every op — including getattr of + the mount root — is ENOENT (`ls: cannot access '/mnt/ws'` while the mount + table shows the mount). Fix: mkdir the chain through the vfs before mounting. +- node:sqlite on Node 22.23: works, prints `ExperimentalWarning: SQLite is an + experimental feature`. + +## Goal 3 — bidirectional + subprocess + +| Test | Result | +|---|---| +| API write → `cat` via mount, new file | immediate (`from-api` readable right after `/write` returns; negative_timeout=0) | +| shell `echo > /mnt/ws/sub.txt` → API read | immediate (`from-shell` visible via WorkspaceFilesystem right after the redirect closes: close→flush… commit landed before the next process turn) | +| API **overwrite** of a file the kernel already cached | **stale ≤ ~1 s**: immediate `cat` returned old content, `cat` after 1.2 s returned new (auto_cache + attr_timeout=1) | +| exec with cwd inside mount, cd-prefix technique (from FUSE host itself) | works: `/exec?cwd=/mnt/ws&cmd=pwd && echo…` → stdout `/mnt/ws`, file visible via API in 0.21 ms | +| create / overwrite / `dd seek=100` / `truncate -s 10` / `mv` / `ln -s` (+readlink+read-through) / `mkdir -p` / `find` / `rm` | all pass (sparse.bin 104 B, ops.txt truncated to `v2-overwri`) | +| 2 × 200 concurrent `cat` loops | no failures | + +### spawn-cwd-inside-mount deadlock (upstream runner.ts rationale) — demonstrated + +`/exec-cwd?cwd=/mnt/ws` (execFile with `cwd:` option from the FUSE-serving +process) wedged the entire mount. Forensic snapshot (`ps -eo pid,stat,wchan`): + +``` +1178 Sl pipe_read node mount-host.mjs … <- parent: event loop blocked in uv_spawn's exec-report pipe read +1313 S request_wait_answer node mount-host.mjs … <- forked child, pre-exec chdir into own mount, waiting on a FUSE answer only the blocked parent can serve +1315 D request_wait_answer ls /mnt/ws <- every other client: uninterruptible sleep, TERM-immune +``` + +- `timeout(1)`'s SIGTERM cannot kill the D-state readers; the docker execs hung. +- **SIGKILL of the host is NOT full recovery**: the forked child inherited the + `/dev/fuse` fd, so the connection (and the wedge) outlived the host; + `fusermount -u` → "Device or resource busy"; `fusermount -u -z` detached the + mount table entry but left the D-state processes. +- Actual recovery: `mount -t fusectl fusectl /sys/fs/fuse/connections` then + `echo 1 > /sys/fs/fuse/connections//abort` → all wedged processes + released instantly. This is the runbook the spike should ship. + +## Goal 4 — durability matrix + +All checks by a separate process (`deno run -A probe.mjs read `, +fresh `FileSQLiteStorage` + `Database` on the db file). FUSE host: Node. + +| # | Scenario | Result | Evidence | +|---|----------|--------|----------| +| a | write+close through mount → fresh-process read | **COMMITTED** | `a-file.txt` → `"a-committed\n"` | +| b | open, write, `fsync(fd)`, **no close**, kill -9 host | **LOST** — and the inode never existed in the db even *before* the kill (separate-process read: `no such file` while held open; same-process API *did* see the bytes via the in-process write buffer — deceptive!). fsync/flush are durability no-ops; `openWriteBufferForCreateSync` defers the INSERT to release. | `b-file.txt` | +| c | write+close, then kill -9 | **COMMITTED — IF the async RELEASE op landed.** close(2) returns after FLUSH; RELEASE is async. kill -9 **0 ms** after close → **LOST** (`race0.txt`, also `term.txt` with SIGTERM); kill **500 ms** after close → committed (`race500.txt`). | race harness | +| d | SIGTERM / SIGKILL mid-mount | Both: `auto_unmount` removes the mount within ~1 s; `ls /mnt/ws` on the (empty) underlying dir returns exit 0 — **no hang, no "transport endpoint is not connected"**. Caveat: not true when a forked child holds the fuse fd (see deadlock above). No unmount-time flush → WAL left. | mount table empty after each | +| e | second Deno process opens same db while mounted | **Works, read AND write** (WAL): `probe.mjs write` from a second process committed in 45 ms, immediately visible through the mount and the host API. No SQLITE_BUSY observed (short transactions). | `second-writer.txt` | +| f | kill -9 leaves `-wal`(1.1 MB)/`-shm`; next open? | **Recovers cleanly**: fresh open reads meta (rev 25, schema 5, journal_mode wal); after clean close the WAL is checkpointed (db 4 KB→106 KB, sidecars gone). | `probe.mjs meta` | + +Net durability contract for the spike: **a write is durable only after +open-count→0 release AND its dofs transaction commit; close() returning is +not a durability barrier, and fsync lies.** A sync point must observe the db +(or drain releases), not the POSIX fd lifecycle. + +## Goal 5 — `deno compile` binary + +``` +deno compile --allow-all --no-check \ + --include node_modules/fuse-native/prebuilds/linux-x64/node.napi.node \ + --include node_modules/fuse-shared-library-linux/libfuse/lib/libfuse.so \ + --output g5-bin g5-compiled.mjs # Files: 32.28MB → binary 142,191,712 B +``` + +- **Environment landmine**: under Docker Desktop **Rosetta**, deno-compiled + binaries do not run at all — `Inconsistency detected by ld.so: rtld.c: 1293: + rtld_setup_main_map: Assertion 'GL(dl_rtld_map).l_libname' failed!` (and a + hello-world compile segfaults, exit 139). Plain `deno` runs fine. Workaround + used: run the amd64 binary in a native arm64 container via `qemu-x86_64 -L + ` (sysroot = container's /lib64 + /lib/x86_64-linux-gnu with + ld.so symlink dereferenced, else qemu resolves the symlink against the wrong + root). +- **Step A — embedded npm graph**: static `import Fuse from "fuse-native"` in + the compiled binary **works**: the addon loads from the embedded graph, + `typeof Fuse === "function"` at 0.16 ms into main. +- **Step B — `--include` + materialize + dlopen**: reading the two included + files out of the binary VFS (`Deno.readFile(new URL(...))` → 68,072 B addon, + 939,064 B libfuse), writing to `Deno.makeTempDir()`, `process.dlopen` on the + real-path copy: **works**, 70 exports, 34.7 ms into main. +- **Mount from the compiled binary**: with fusermount absent → catchable + `Error: fuse failed` (never reached the uv path). With fusermount + /dev/fuse + present → same `bad result in uv polyfill: 1` abort as `deno run`. +- Cold start: 1205 ms wall under qemu-user emulation for start → both load + steps done (includes qemu startup; not representative of native — expect + well under 200 ms native given `deno run` import was 297 ms un-warmed). + +## Goal 6 — sizes & latencies + +All timings under emulation (Rosetta for fuseprobe, qemu for qemuprobe2); +treat as relative, not absolute. + +| Metric | Value | +|---|---| +| Image fuse-probe (node:22-slim + fuse + deno) | 503 MB | +| fuse-native package (with prebuilds) | 1.6 MB; addon `node.napi.node` 68,072 B | +| fuse-shared-library-linux | 3.1 MB; `libfuse.so` 939,064 B | +| node_modules total (incl. typescript+@types) | 34 MB | +| deno-compiled binary (2 includes) | 142 MB (embedded files 32.28 MB) | +| Deno `import("fuse-native")` | 297 ms | +| Full stack cold → mounted (Node) | 118-133 ms (vfsUp 58-72, require 3-4, fuseMount 57-58) | +| Minimal fuse-native mount (Node) | 27 ms | +| 100 small writes THROUGH mount | 225 ms → **2.25 ms/op** | +| 100 small writes direct dofs API (same process) | 110 ms → **1.10 ms/op** (mount overhead ≈ 2.0×) | +| 100 small reads through mount | 101 ms → 1.01 ms/op | +| Separate-process Deno probe (cold start + db open + 1 op) | 24-45 ms script time | + +## Compile & environment fights (verbatim) + +1. tsc on the upstream driver: **zero fights.** `driver.ts`, `options.ts`, + `tracer.ts`, `backend.ts` compiled unmodified, first try, with computerd's + own settings (`module nodenext`, CJS via package.json, `skipLibCheck`, + `@types/node`). Sole accommodation: a hand-written `src/vfs.ts` type shim + (driver.ts's import of it is type-only; the real vfs.ts drags in + `@cloudflare/computer-rpc`). `fuse-native.d.ts` used as-is beside driver.ts. +2. `WorkspaceFsError: path exists: /workspace` — my wiring double-mkdir'd; + guard with existsSync. (Proved en route that the facade dispatches to + SQLiteWorkspaceProvider.) +3. `ls: cannot access '/mnt/ws': No such file or directory` *while mounted* — + the mountPoint-as-vfs-prefix gotcha (Goal 2 above). +4. `sh: 1: time: not found` (slim image) — use `date +%s%N`. +5. `qemu-x86_64: Could not open '/lib64/ld-linux-x86-64.so.2'` — twice: once + for missing sysroot (`-L`), once because `cp -a` preserved the ld.so + symlink whose absolute target qemu resolved against the arm64 root; fix + `cp -L`. +6. `pgrep -f "node mount-host.mjs"` matches the `sh -c` wrapper and the exec's + own shell → killed the wrong processes twice (one `docker exec` died exit + 137). Match on the concrete node child instead. +7. After the fusectl mass-abort + ~15 zombies (pid 1 is `sleep infinity`, + never reaps), `docker exec` on fuseprobe began failing (rc 255, empty) with + the container still "running" — final size measurements taken from the host + side of the bind mount instead. + +## What the committed spike must replicate + +1. **Topology**: FUSE service must live in a **Node sidecar process**; + Deno cannot host fuse-native until the ext/napi uv polyfill gap is fixed + (file upstream with the `g2-min.mjs` repro + `assert_ok`/null + `uv_default_loop` attribution). Everything else — dofs API side, probes, + supervisors — runs fine under Deno, and matrix (e) proves the shared-WAL + two-process design: Deno main + Node FUSE sidecar over one db file works + with immediate cross-visibility. +2. **The vfs wiring invariants**: prototype splice before `create()` (verify + with a fresh-connection read of `vfs_dirents`, or you may be silently on + MemoryProvider); forward the nine methods; pre-create the mountPoint dir + chain in the vfs namespace. Decide whether to also forward + `writeFileRangesSync` (today it's dead code by omission). +3. **Durability rules**: commit is at release only; `fsync`/`close` are not + barriers; a durability-sensitive caller must wait for release + commit + (observe the db) — and any "write then kill" path (worker recycling!) needs + ≥ the close→release async window or an explicit drain. +4. **Exec rules**: `cd &&` prefix, never `cwd:` into the own mount; ship + the recovery runbook (`fusermount -z`, then fusectl `abort` — SIGKILL alone + leaves an fd-inherited wedge if any child was mid-spawn). +5. **Ops expectations**: ≤1 s staleness for API overwrites under the + production option profile; auto_unmount reliably clears the mount on + host death; WAL recovery on next open is clean. +6. **Packaging**: `deno compile` CAN carry and load the addon (both embedded + npm graph and `--include`+dlopen) — the blocker is only the mount call, not + distribution. Don't test deno-compiled binaries under Docker Rosetta. diff --git a/spikes/349-dofs/evidence/probes/slice6-shim.md b/spikes/349-dofs/evidence/probes/slice6-shim.md new file mode 100644 index 00000000..1dbfccb8 --- /dev/null +++ b/spikes/349-dofs/evidence/probes/slice6-shim.md @@ -0,0 +1,177 @@ +# Probe: issue #349 slice 6 — userspace shim (cf-computer `mountShim`) under Deno + +Host: darwin-arm64 (Darwin 25.5.0), Deno 2.9.1, Node v-homebrew, 2026-08-06. +Working dir: `scratchpad/probe-shim`. Reuses `scratchpad/probe-dofs` (built dofs dist + +`file-storage.mjs` FileSQLiteStorage adapter). cf-computer v0.1.1 clone read-only. + +## Setup / provenance + +- `src/shim/shim.ts` — byte-identical copy of `cf-computer/packages/computerd/src/shim/shim.ts` + (shasum `05e6a2ac…` matches). Compiled with `npx tsc` (typescript from npm, target ES2022, + module NodeNext, ESM output) → `dist/shim/shim.js`. **Compilation did not fight: zero errors, + zero edits to upstream source.** +- `src/fuse/vfs.ts` — type-only stub (`export type { VirtualFileSystem as NodeVirtualFileSystem } + from "@platformatic/vfs"`), matching upstream vfs.ts's type alias. shim.ts imports only the type. +- `wiring.mjs` — hand-port of upstream `fuse/vfs.ts` runtime wiring. Deviations (full list): + 1. RPC surface dropped (SyncRPC / pullOnce / tick / startSyncLoop — not needed, imports + unavailable). + 2. `SQLiteTestStorage` (in-memory) → `FileSQLiteStorage(dbPath)` (probe-dofs adapter, WAL). + 3. Provider options pass-through (upstream hardcodes defaults). + 4. ESM instead of CommonJS. + `ensureVirtualProviderPrototype` (the prototype splice) and the `EXTRA_VFS_METHODS` + defineProperty loop are ported verbatim. +- Blueprint correction: `EXTRA_VFS_METHODS` in v0.1.1 has **nine** entries, not ten + (linkSync, createFileSync, writeRangeSync, truncateFileSync, chmodSync, readRangeSync, + openWriteBufferSync, openWriteBufferForCreateSync, releaseWriteBufferSync). +- Confirmed the silent-fallback trap in `@platformatic/vfs@0.4.0` `index.js` `create()`: + a provider that fails `instanceof VirtualProvider` and is a plain object is **reassigned as + the options argument** and provider becomes undefined → `new VirtualFileSystem(undefined, …)` + → `provider ?? new MemoryProvider()`. No error, no warning. + +Deps installed in probe-shim: `npm i @platformatic/vfs@0.4.0 typescript @types/node` (clean). + +## Q1 — does the shim run under Deno at all? + +Command: `deno run -A exp1.mjs` + +Result: **yes, first try, zero runtime failures.** No error output at any point. + +- `vfs.provider instanceof MemoryProvider` → false; `vfs.provider === provider` → true; + constructor name `SQLiteWorkspaceProvider`. +- Cross-process verification: wrote `/verify/marker.txt` via the vfs facade, read it back with + a **separate Deno process** opening a fresh Database over the same db file + (`probe-dofs/probe.mjs read`) → `{"op":"read","body":"sqlite-not-memory"}`. Not a + MemoryProvider. +- `mountShim({ vfs, mountPoint })` (default pollIntervalMs 250) returned; watchAsync loop, + setInterval poll, and node:crypto sha1 all work under Deno. +- First-shot latencies: VFS→disk 106 ms (provider's default 100 ms rev-poll watcher), + disk→VFS 149 ms (within the 250 ms reconcile poll). + +## Q2 — bidirectional visibility + latency + +Command: `deno run -A exp2.mjs` (defaults: provider watchIntervalMs 100, shim pollIntervalMs 250; +2 ms detection poll; ~120–130 ms decorrelation sleep between rounds). + +| direction | rounds (ms) | median (ms) | bound | +|---|---|---|---| +| VFS write → visible on disk | 104.7, 82.6, 81.0, 81.1, 78.6, 80.6, 81.6, 78.3, 78.9, 80.2 | **80.8** | provider rev-poll (100 ms) | +| disk write (separate `/bin/sh` process) → readable via vfs API | 205.7, 105.4, 106.4, 109.2, 107.5, 110.4, 104.3, 109.8, 110.3, 105.6 | **108.3** | shim poll (250 ms), phase-dependent | +| disk write → visible via fresh Database over the db file | 207.8, 106.7, 107.0, 110.4, 108.4, 111.7, 105.2, 111.3, 111.3, 106.8 | **109.4** | ≈ vfs visibility + ~1 ms (writes commit synchronously) | + +Both directions are phase-dependent polling: worst case ≈ interval + walk cost. db-visibility +tracks vfs-visibility within ~1 ms because the reconcile's `vfs.writeFileSync` commits to SQLite +synchronously (fresh `FileSQLiteStorage` connection per check, WAL). + +Native subprocess round-trip (`/bin/sh -c 'echo hi > sub.txt && cat other.txt'`, cwd = mount dir, +after `vfs.writeFileSync(other.txt)` + `shim.flush()`): success — `cat` printed the API-written +content, and `sub.txt` appeared in the VFS 88.5 ms later. The upstream flush()-before-exec +contract works as documented. + +## Q3 — restart persistence + +Upstream context: computerd's mountPoint is a **stable configured path** +(`MOUNT_POINT` env / `DEFAULT_MOUNT_POINT`, cli/computerd.ts:457,499) — the VFS namespace embeds +it, so restart re-mounts at the same path. + +- Phase A (`exp3a.mjs`): fresh db, 100 files × 100 KB = 10,000,000 bytes across 10 dirs written + via vfs (56 ms), `mountShim` boot materialisation 77 ms, disk tree verified (100 files/10 MB), + unmount, close, **delete the whole on-disk mount dir**. +- Phase B (`exp3b.mjs`, new process): reopen db — frontier intact (100 files / 10,000,000 bytes + readable via vfs before any mount). Re-mount at the same now-empty path: + **materialisation of ~100 files / 10 MB took 78.9 ms**; disk tree reproduced exactly + (100 files, 10 MB, 5/5 sampled sha1s match db content). `initializeSchema` on reopen does not + wipe (idempotent). +- Trap demonstrated: mounting the same vfs at a **different** fresh mkdtemp path materialises + **nothing** (`[]`) — VFS paths embed the mount prefix. A committed spike must keep the mount + path stable across restarts (or migrate paths in the db). + +## Q4 — documented limitations, demonstrated (`deno run -A exp4.mjs`) + +| probe | result | +|---|---| +| symlink via vfs API (live target) | stays a symlink **in** the VFS (`lstatSync` → symlink); on disk it materialises as a **regular file containing the target's content** (`safeVfsStat` uses `statSync`, which follows the link) — not "absent" as naively expected, but the link-ness is lost | +| symlink via vfs API (dangling) | **absent on disk** (`statSync` throws → entry skipped) | +| symlink created on disk (live, `ln -s`) | reconciled into the VFS as a **regular file** with duplicated content (`walkDisk` uses `fsStat`, follows) | +| symlink created on disk (dangling) | **ignored** — never enters the VFS (ENOENT) | +| `chmod 755` on a synced disk file | **not reflected**: vfs mode stays 644 while disk shows 755; the shadow diff keys on (size, mtime) only | +| conflict, both sides write same path in one poll window (3 rounds, both orders) | **VFS wins 3/3** — both sides converge to the vfs-side content, including when the vfs write happened *first* and the disk write *last* | +| 50 MB file, new, disk→VFS | 283 ms | +| 50 MB file, 1-byte change, disk→VFS | **594 ms** (full re-read + sha1 + full SQLite rewrite) | +| 50 MB file, 1-byte change, VFS→disk | **654 ms** (full read from SQLite + full disk write) | +| 1 KB file, new / change, disk→VFS | 56 ms / 48 ms | +| steady-state `reconcileNow()` with unchanged 50 MB present | 0.7–2.2 ms (shadow (size, mtime) short-circuit works) | + +## Q5 — kill -9 mid-activity (`deno run -A exp5.mjs`) + +Worker (`exp5-worker.mjs`) mounted the shim and wrote on BOTH sides every 5 ms +(20 vfs-side + 20 disk-side files, round-robin). Parent SIGKILLed it at iteration 200. + +- Post-mortem: WAL file 4,169,472 bytes pending; fresh connection `PRAGMA integrity_check` → ok + (standard SQLite WAL crash recovery — no corruption). +- Divergence at death: 40 files present on both sides, **29 of 40 content-diverged** + (writes in flight across the two poll seams). +- Restart + re-mount into the SAME dir: boot `materialiseVfsToDisk` overwrites disk with VFS + content, first poll ticks pull disk-side extras; after 1.2 s **disk and VFS fully converged, + 0 mismatches**. +- Loss window (by design, demonstrated by the convergence direction): disk-side writes not yet + polled into the VFS at crash time are **overwritten by the VFS copy on next boot** — up to + `pollIntervalMs` (250 ms) of external writes can be silently lost across a crash. VFS-side + writes are never lost (committed to SQLite synchronously before the watcher even fires). + +## Q6 — darwin-arm64 real-FUSE record (`fuse-probe/`) + +Host: darwin-arm64 (Apple Silicon), Darwin 25.5.0. + +- `/Library/Filesystems/macfuse.fs` — **does not exist**. +- `/Library/Filesystems/osxfuse.fs` — exists (stale osxfuse 3.11.2 remnant) but has **no + `configured` marker**, so fuse-shared-library treats it as unconfigured. +- `npm i fuse-native@2.2.6` → installs in 3 s, **no darwin-arm64 prebuild** (`prebuilds/` ships + only `darwin-x64` and `linux-x64`). npm's allow-scripts policy on this host additionally + blocked the `node-gyp-build` install script (`npm warn allow-scripts fuse-native@2.2.6`). +- Running `npx node-gyp-build` manually: **compiles from source successfully** (exit 0) — + produces an arm64 `build/Release/fuse.node` linking only `/usr/lib/libc++.1.dylib` and + `libSystem.B.dylib`. +- Loading: `require("fuse-native")` succeeds under **both Node and Deno** (createRequire). + `Fuse.isConfigured` → **false**. +- The bundled userspace library `fuse-shared-library-darwin/osxfuse/libosxfuse.dylib` is a + ppc_7400/ppc64/i386/x86_64 universal binary — **no arm64 slice**. Its `configure()` path + untars an osxfuse 3.x bundle (kext-based; Apple Silicon requires reduced-security mode and + macFUSE ≥ 4, so this can never work on this platform). +- Actual `fuse.mount()` attempt: **SIGSEGV (exit 139), no error surfaced**, identically under + Node and Deno. +- Verdict for macOS-arm64: real FUSE via cf-computer's stack is a dead end — even with macFUSE + installed, fuse-native 2.2.6's darwin support predates Apple Silicon. The shim is the only + viable path on this platform. (macFUSE was NOT installed, per instructions.) + +## Q7 — verdict + +**Development-only fallback — suitable as exactly that, and it is the ONLY option on +darwin-arm64.** + +For it (measured): +- Runs under Deno unmodified: upstream shim.ts compiled with zero edits, zero runtime failures. +- Latency is fine for interactive/dev use: ~81 ms VFS→disk, ~108 ms disk→VFS medians; + subprocesses inside the mount see API-written files (after `flush()`) and their writes land + back in ~90 ms. +- Restart persistence is real: SQLite frontier survives close + dir deletion; 100 files / 10 MB + re-materialise in ~79 ms at the same mount path. +- Crash-safe at the storage layer: SIGKILL mid-activity → WAL recovery ok, remount converges + fully in ~1 s. + +Against production use (demonstrated, not speculative): +- Up to one poll window (250 ms) of **external disk writes silently lost** on crash, and VFS + unconditionally clobbers concurrent disk writes (3/3 conflicts, both orders). +- Symlinks are silently degraded to content copies (or dropped when dangling) in BOTH + directions; chmod invisible; no watch fan-out. +- Large files pay full re-read + full rewrite per change: ~0.6 s per touch of a 50 MB file on + either side; scales linearly with size, not delta. +- The VFS namespace embeds the absolute mount path: a moved/renamed mount dir orphans the + entire tree (materialises nothing). +- Everything is polling (provider rev-poll 100 ms + shim walk 250 ms): steady-state cost is + fine (~1–2 ms/tick at this tree size) but grows with tree breadth, and correctness windows + are timing-defined, not event-defined. + +What a committed spike must replicate: the prototype splice + MemoryProvider-fallback +verification (the failure is SILENT), a stable absolute mount path, `flush()` before any exec +inside the mount, and the FileSQLiteStorage(WAL) adapter; and it must NOT rely on symlinks, +permissions, or sub-poll-window external-write durability. diff --git a/spikes/349-dofs/host/file-storage.ts b/spikes/349-dofs/host/file-storage.ts new file mode 100644 index 00000000..d073d352 --- /dev/null +++ b/spikes/349-dofs/host/file-storage.ts @@ -0,0 +1,103 @@ +import { DatabaseSync, type StatementSync } from "node:sqlite"; +// @ts-types="./types/dofs.d.ts" +import type { + DurableObjectStorageLike, + SQLCursorLike, +} from "@cloudflare/dofs"; + +class Cursor implements SQLCursorLike { + #rows: Row[]; + constructor(rows: Row[]) { + this.#rows = rows; + } + toArray(): Row[] { + return this.#rows; + } +} + +// File-backed DurableObjectStorageLike over node:sqlite. Mirrors dofs's own +// SQLiteTestStorage (vendor/dofs/src/testing.ts) — prepared-statement cache, +// binding normalization, BEGIN/COMMIT/ROLLBACK — but opens a real database +// file, which is the entire difference between a unit-test fixture and a +// persistent local workspace. +export class FileSQLiteStorage implements DurableObjectStorageLike { + #db: DatabaseSync; + #cache = new Map(); + readonly sql: { + exec: ( + query: string, + ...bindings: unknown[] + ) => SQLCursorLike; + }; + + constructor(path: string) { + this.#db = new DatabaseSync(path); + this.#db.exec("PRAGMA journal_mode = WAL"); + this.#db.exec("PRAGMA foreign_keys = ON"); + this.#db.exec("PRAGMA synchronous = NORMAL"); + this.sql = { + exec: ( + query: string, + ...bindings: unknown[] + ): SQLCursorLike => { + let statement = this.#cache.get(query); + if (statement === undefined) { + statement = this.#db.prepare(query); + this.#cache.set(query, statement); + } + const normalized = bindings.map(toSQLiteValue); + const rows: Row[] = []; + for (const row of statement.all(...normalized)) { + if (isRow(row)) { + rows.push(row); + } + } + return new Cursor(rows); + }, + }; + } + + transactionSync(closure: () => T): T { + this.#db.exec("BEGIN"); + try { + const result = closure(); + this.#db.exec("COMMIT"); + return result; + } catch (error) { + this.#db.exec("ROLLBACK"); + throw error; + } + } + + close(): void { + this.#cache.clear(); + this.#db.close(); + } +} + +type SQLiteValue = null | number | bigint | string | Uint8Array; + +function toSQLiteValue(value: unknown): SQLiteValue { + if (value === undefined || value === null) { + return null; + } + if (typeof value === "boolean") { + return value ? 1 : 0; + } + if (value instanceof Uint8Array) { + return value; + } + if ( + typeof value === "string" || typeof value === "number" || + typeof value === "bigint" + ) { + return value; + } + throw new TypeError( + `FileSQLiteStorage cannot bind value of type ${typeof value}`, + ); +} + +function isRow(value: unknown): value is Row { + return typeof value === "object" && value !== null; +} diff --git a/spikes/349-dofs/host/main.ts b/spikes/349-dofs/host/main.ts new file mode 100644 index 00000000..359531e6 --- /dev/null +++ b/spikes/349-dofs/host/main.ts @@ -0,0 +1,92 @@ +import { main, until } from "effection"; +// @ts-types="./types/dofs.d.ts" +import { + Database, + initializeSchema, + SCHEMA_VERSION, + WorkspaceFilesystem, +} from "@cloudflare/dofs"; +// @ts-types="./types/dofs-rename.d.ts" +import { rename } from "@cloudflare/dofs/fs/rename"; +import { FileSQLiteStorage } from "./file-storage.ts"; + +function usage(): never { + console.error( + [ + "usage: proof [args...]", + "ops: init | mkdir

| write

| read

| ls

| rm

[-r]", + " stat

| lstat

| rename | symlink ", + " readlink

| meta", + ].join("\n"), + ); + Deno.exit(2); +} + +main(function* () { + const started = performance.now(); + const [dbPath, op, ...args] = Deno.args; + if (dbPath === undefined || op === undefined) { + usage(); + } + + const storage = new FileSQLiteStorage(dbPath); + const db = new Database(storage); + initializeSchema(db, Date.now); + const fs = new WorkspaceFilesystem(db); + + let result: Record; + switch (op) { + case "init": + result = { schemaVersion: SCHEMA_VERSION }; + break; + case "mkdir": + yield* until(fs.mkdir(args[0], { recursive: true })); + result = { ok: true }; + break; + case "write": + yield* until(fs.writeFile(args[0], args[1])); + result = { ok: true }; + break; + case "read": + result = { body: yield* until(fs.readFile(args[0], "utf8")) }; + break; + case "ls": { + const entries = yield* until(fs.readdir(args[0])); + result = { entries: entries.map((entry) => entry.name).sort() }; + break; + } + case "rm": + yield* until(fs.rm(args[0], { recursive: args.includes("-r") })); + result = { ok: true }; + break; + case "stat": + result = { stat: yield* until(fs.stat(args[0])) }; + break; + case "lstat": + result = { lstat: yield* until(fs.lstat(args[0])) }; + break; + case "rename": + rename(db, args[0], args[1]); + result = { ok: true }; + break; + case "symlink": + yield* until(fs.symlink(args[0], args[1])); + result = { ok: true }; + break; + case "readlink": + result = { target: yield* until(fs.readlink(args[0])) }; + break; + case "meta": + result = { + binarySchemaVersion: SCHEMA_VERSION, + vfsMeta: db.all("SELECT k, v FROM vfs_meta ORDER BY k"), + }; + break; + default: + usage(); + } + + storage.close(); + const ms = Math.round((performance.now() - started) * 100) / 100; + console.log(JSON.stringify({ op, ...result, ms })); +}); diff --git a/spikes/349-dofs/host/shim-main.ts b/spikes/349-dofs/host/shim-main.ts new file mode 100644 index 00000000..3e5c21cc --- /dev/null +++ b/spikes/349-dofs/host/shim-main.ts @@ -0,0 +1,63 @@ +import { main, until } from "effection"; +import { exec } from "@effectionx/process"; +// @ts-types="./types/computerd-shim.d.ts" +import { mountShim } from "@xmd-spike/computerd-shim"; +import { createFileBackedVfs } from "./vfs-wiring.ts"; + +function usage(): never { + console.error( + [ + "usage: proof-shim exec ", + " proof-shim materialize", + "", + "The mount directory is part of the workspace namespace: the same", + "database must always be mounted at the same absolute path.", + ].join("\n"), + ); + Deno.exit(2); +} + +main(function* () { + const [dbPath, mountDir, command, ...rest] = Deno.args; + if (dbPath === undefined || mountDir === undefined) { + usage(); + } + if (command !== "exec" && command !== "materialize") { + usage(); + } + + const started = performance.now(); + const wired = createFileBackedVfs(dbPath); + wired.vfs.mkdirSync(mountDir, { recursive: true }); + Deno.mkdirSync(mountDir, { recursive: true }); + + const shim = yield* until( + mountShim({ vfs: wired.vfs, mountPoint: mountDir }), + ); + const mountedMs = Math.round(performance.now() - started); + + let payload: Record = {}; + if (command === "exec") { + const shellCommand = rest.join(" "); + if (shellCommand === "") { + usage(); + } + yield* until(shim.flush()); + // cwd stays outside the mount: upstream computerd prefixes `cd` instead + // of passing cwd because fork+chdir into a mount served by the same + // process deadlocks under FUSE; the shim keeps the same shape. + const result = yield* exec("/bin/sh", { + arguments: ["-c", `cd ${mountDir} && (${shellCommand})`], + }).join(); + yield* until(shim.reconcileNow()); + payload = { + code: result.code, + stdout: result.stdout, + stderr: result.stderr, + }; + } + + yield* until(shim.unmount()); + wired.storage.close(); + console.log(JSON.stringify({ command, mountedMs, ...payload })); +}); diff --git a/spikes/349-dofs/host/types/computerd-shim.d.ts b/spikes/349-dofs/host/types/computerd-shim.d.ts new file mode 100644 index 00000000..10e82764 --- /dev/null +++ b/spikes/349-dofs/host/types/computerd-shim.d.ts @@ -0,0 +1,16 @@ +// Facade for the vendored shim's consumed surface; mirrors +// vendor/computerd-shim/dist/shim/shim.d.ts. + +import type { VirtualFileSystem } from "./platformatic-vfs.d.ts"; + +export interface ShimMount { + unmount(): Promise; + flush(): Promise; + reconcileNow(): Promise; +} + +export declare function mountShim(options: { + vfs: VirtualFileSystem; + mountPoint: string; + pollIntervalMs?: number; +}): Promise; diff --git a/spikes/349-dofs/host/types/dofs-rename.d.ts b/spikes/349-dofs/host/types/dofs-rename.d.ts new file mode 100644 index 00000000..248d6ec1 --- /dev/null +++ b/spikes/349-dofs/host/types/dofs-rename.d.ts @@ -0,0 +1,9 @@ +// Facade counterpart of vendor/dofs/dist/fs/rename.d.ts; see dofs.d.ts. + +import type { Database } from "./dofs.d.ts"; + +export declare function rename( + db: Database, + oldPath: string, + newPath: string, +): void; diff --git a/spikes/349-dofs/host/types/dofs.d.ts b/spikes/349-dofs/host/types/dofs.d.ts new file mode 100644 index 00000000..eee36a05 --- /dev/null +++ b/spikes/349-dofs/host/types/dofs.d.ts @@ -0,0 +1,72 @@ +// Typed facade for the surface this spike consumes from the vendored +// @cloudflare/dofs. Deno pairs .js imports with sibling .d.ts files for +// registry npm packages but not for file:-resolved ones, so the vendored +// package's own declarations cannot be used directly; this file mirrors +// vendor/dofs/dist/{types,storage,schema/index,fs/filesystem}.d.ts for +// exactly the members the host uses. + +export interface SQLCursorLike> { + toArray(): Row[]; +} + +export interface SQLStorageLike { + exec>( + query: string, + ...bindings: unknown[] + ): SQLCursorLike; +} + +export interface DurableObjectStorageLike { + sql: SQLStorageLike; + transaction?(closure: () => T | Promise): T | Promise; + transactionSync?(closure: () => T): T; +} + +export declare class Database { + constructor(storage: DurableObjectStorageLike); + all(query: string, ...bindings: unknown[]): Row[]; +} + +export declare function initializeSchema( + db: Database, + now: () => number, +): void; + +export declare const SCHEMA_VERSION: number; + +export interface SQLiteWorkspaceProviderOptions { + now?: () => number; + watchIntervalMs?: number; +} + +export declare class SQLiteWorkspaceProvider { + readonly db: Database; + constructor(db: Database, options?: SQLiteWorkspaceProviderOptions); +} + +export interface WorkspaceDirentResult { + name: string; +} + +export interface WorkspaceStatResult { + size: number; + mode: number; +} + +export interface WorkspaceFilesystemOptions { + now?: () => number; +} + +export declare class WorkspaceFilesystem { + constructor(db: Database, options?: WorkspaceFilesystemOptions); + readFile(path: string): Promise>; + readFile(path: string, encoding: "utf8"): Promise; + stat(path: string): Promise; + lstat(path: string): Promise; + readlink(path: string): Promise; + readdir(path: string): Promise; + writeFile(path: string, content: string | Uint8Array): Promise; + mkdir(path: string, options?: { recursive?: boolean }): Promise; + rm(path: string, options?: { recursive?: boolean }): Promise; + symlink(target: string, path: string): Promise; +} diff --git a/spikes/349-dofs/host/types/platformatic-vfs.d.ts b/spikes/349-dofs/host/types/platformatic-vfs.d.ts new file mode 100644 index 00000000..52a9fc45 --- /dev/null +++ b/spikes/349-dofs/host/types/platformatic-vfs.d.ts @@ -0,0 +1,22 @@ +// Typed facade for the @platformatic/vfs surface this spike consumes; the +// package ships CommonJS whose default-import shape differs by runtime, so +// the host normalizes at this boundary. + +export interface VirtualFileSystem { + provider: unknown; + mkdirSync(path: string, options?: { recursive?: boolean }): void; +} + +export declare class VirtualProvider { + private brand; +} + +declare const vfsModule: { + create( + provider: unknown, + options?: { moduleHooks?: boolean }, + ): VirtualFileSystem; + VirtualProvider: typeof VirtualProvider; +}; + +export default vfsModule; diff --git a/spikes/349-dofs/host/vfs-wiring.ts b/spikes/349-dofs/host/vfs-wiring.ts new file mode 100644 index 00000000..29885f96 --- /dev/null +++ b/spikes/349-dofs/host/vfs-wiring.ts @@ -0,0 +1,85 @@ +// Port of upstream packages/computerd/src/fuse/vfs.ts (v0.1.1) minus the +// RPC/sync-loop branches. Deviations from upstream, in full: +// 1. No upstream/SyncRPC support. +// 2. Storage is the file-backed adapter instead of the in-memory +// SQLiteTestStorage. +// 3. After create(), the wiring verifies the vfs kept our provider — +// @platformatic/vfs's create() silently falls back to a +// MemoryProvider when the instanceof check fails, which would make +// every test pass against the wrong store. + +// @ts-types="./types/dofs.d.ts" +import { + Database, + initializeSchema, + SQLiteWorkspaceProvider, +} from "@cloudflare/dofs"; +// @ts-types="./types/platformatic-vfs.d.ts" +import vfsModule from "@platformatic/vfs"; +import type { VirtualFileSystem } from "./types/platformatic-vfs.d.ts"; +import { FileSQLiteStorage } from "./file-storage.ts"; + +const { create, VirtualProvider } = vfsModule; + +let prototypePatched = false; +function ensureVirtualProviderPrototype(): void { + if (prototypePatched) { + return; + } + const proto = SQLiteWorkspaceProvider.prototype; + const parent = Object.getPrototypeOf(proto); + if (parent === VirtualProvider.prototype) { + prototypePatched = true; + return; + } + Object.setPrototypeOf(proto, VirtualProvider.prototype); + prototypePatched = true; +} + +const EXTRA_VFS_METHODS = [ + "linkSync", + "createFileSync", + "writeRangeSync", + "truncateFileSync", + "chmodSync", + "readRangeSync", + "openWriteBufferSync", + "openWriteBufferForCreateSync", + "releaseWriteBufferSync", +]; + +export interface WiredFileSystem { + vfs: VirtualFileSystem; + db: Database; + provider: SQLiteWorkspaceProvider; + storage: FileSQLiteStorage; +} + +export function createFileBackedVfs(dbPath: string): WiredFileSystem { + ensureVirtualProviderPrototype(); + const storage = new FileSQLiteStorage(dbPath); + const db = new Database(storage); + initializeSchema(db, Date.now); + + const provider = new SQLiteWorkspaceProvider(db); + const vfs = create(provider, { moduleHooks: false }); + if (vfs.provider !== provider) { + storage.close(); + throw new Error( + "@platformatic/vfs fell back to a MemoryProvider: the prototype splice did not take", + ); + } + const source: Record = Object(provider); + for (const name of EXTRA_VFS_METHODS) { + const fn = source[name]; + if (typeof fn !== "function") { + continue; + } + Object.defineProperty(vfs, name, { + value: (...args: unknown[]) => fn.apply(provider, args), + writable: true, + configurable: true, + }); + } + return { vfs, db, provider, storage }; +} diff --git a/spikes/349-dofs/package-lock.json b/spikes/349-dofs/package-lock.json new file mode 100644 index 00000000..cefabd07 --- /dev/null +++ b/spikes/349-dofs/package-lock.json @@ -0,0 +1,253 @@ +{ + "name": "spike-349-dofs", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "spike-349-dofs", + "dependencies": { + "@cloudflare/dofs": "file:./vendor/dofs", + "@effectionx/converge": "0.1.4", + "@effectionx/process": "0.8.1", + "@platformatic/vfs": "0.4.0", + "@xmd-spike/computerd-shim": "file:./vendor/computerd-shim", + "effection": "4.1.0" + } + }, + "node_modules/@cloudflare/dofs": { + "resolved": "vendor/dofs", + "link": true + }, + "node_modules/@cloudflare/workers-types": { + "version": "4.20260702.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260702.1.tgz", + "integrity": "sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==", + "dev": true, + "license": "MIT OR Apache-2.0" + }, + "node_modules/@effectionx/context-api": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@effectionx/context-api/-/context-api-0.6.0.tgz", + "integrity": "sha512-t004qvlkJDMB6EhHP1lOQ97PeIn90m7cv4+wsRPnx4YBem+pJzTL+Sm1KWbKMjMeFJz4oqllUWuBJZsCi+nuTw==", + "license": "MIT", + "dependencies": { + "@effectionx/middleware": "0.1.1" + }, + "peerDependencies": { + "effection": "^3 || ^4" + } + }, + "node_modules/@effectionx/converge": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@effectionx/converge/-/converge-0.1.4.tgz", + "integrity": "sha512-M11KU7jK3gOFGI4RT+Bf6ITMJntzzoGEO2KPx9r8uBoR+Rgwiyeuq+OeIHkuiHLUC+bB1g93KOj7aKR/G0F1/Q==", + "license": "MIT", + "dependencies": { + "@effectionx/timebox": "0.4.3" + }, + "peerDependencies": { + "effection": "^3 || ^4" + } + }, + "node_modules/@effectionx/middleware": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@effectionx/middleware/-/middleware-0.1.1.tgz", + "integrity": "sha512-ss/bZRkt/xzJNE59r8NR1+0K/xQcIyCm0y9n8FYC8jKdFn51SPe3m3t7EfPcK8zkdjCoTOU7k1UpIXRl26asYA==", + "license": "MIT" + }, + "node_modules/@effectionx/node": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@effectionx/node/-/node-0.2.4.tgz", + "integrity": "sha512-cPnp3fvfBKjGWekmBHdhZr5ScAr3Mg+x5IXpO8uKFe7AZ8EPAT9Di6skuB4kuGFJtRtS0Z1e5G4+2eJyapKhYA==", + "license": "MIT", + "peerDependencies": { + "effection": "^3 || ^4" + } + }, + "node_modules/@effectionx/process": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@effectionx/process/-/process-0.8.1.tgz", + "integrity": "sha512-xyXlFja0Ill80lQ3IYfksXtJkqVmWuUOogRn/qlHWCAGlZj+MGGF8gOFbyzk/3Kx4pj14riVGgF/cyT5XCzqDw==", + "license": "MIT", + "dependencies": { + "@effectionx/context-api": "0.6.0", + "@effectionx/node": "0.2.4", + "@effectionx/scope-eval": "0.1.3", + "cross-spawn": "^7", + "ctrlc-windows": "^2", + "shellwords-ts": "^3.0.1" + }, + "peerDependencies": { + "effection": "^3 || ^4" + } + }, + "node_modules/@effectionx/scope-eval": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@effectionx/scope-eval/-/scope-eval-0.1.3.tgz", + "integrity": "sha512-Acn45lb3H94WYhNVHXYtXOZYzjpBGDPPlsyW1Talb/vYjQzCzus5lkxxOlPyphzvi7d+7mGNXiIVt4JLSZmLnQ==", + "license": "MIT", + "peerDependencies": { + "effection": "^3 || ^4" + } + }, + "node_modules/@effectionx/timebox": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@effectionx/timebox/-/timebox-0.4.3.tgz", + "integrity": "sha512-cc7SLpL3svAYK8M5NS8kLQuL0lrZNoQb+Hi9NSaWOudzAW1HoewuDfUtfXLemPJnnLqLYhbghRhmpVqCm4Xg3Q==", + "license": "MIT", + "peerDependencies": { + "effection": "^3 || ^4" + } + }, + "node_modules/@platformatic/vfs": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@platformatic/vfs/-/vfs-0.4.0.tgz", + "integrity": "sha512-JwRxSIG63e/VaDSkYXkSBajySt5MJzxYIKuknYda28fhhmJMqyUGf5rbKrGW+vO9L83nIjQH3+YI12fp7EKICQ==", + "license": "MIT", + "engines": { + "node": ">= 22" + } + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@xmd-spike/computerd-shim": { + "resolved": "vendor/computerd-shim", + "link": true + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/ctrlc-windows": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ctrlc-windows/-/ctrlc-windows-2.2.0.tgz", + "integrity": "sha512-t9y568r+T8FUuBaqKK60YGFJdj3b3ktdJW9WXIT3CuBdQhAOYdSZu75jFUN0Ay4Yz5HHicVQqAYCwcnqhOn23g==", + "license": "MIT" + }, + "node_modules/effection": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/effection/-/effection-4.1.0.tgz", + "integrity": "sha512-/BJuaYhzDrvC/nrIW054z1itmCNjVNFtyv/J26xrS4pP2MLfm5TG8VFe5f4zl1oGcAoF1fq1GfDO+v0qx6URPw==", + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shellwords-ts": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shellwords-ts/-/shellwords-ts-3.0.1.tgz", + "integrity": "sha512-GabK4ApLMqHFRGlpgNqg8dmtHTnYHt0WUUJkIeMd3QaDrUUBEDXHSSNi3I0PzMimg8W+I0EN4TshQxsnHv1cwg==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "vendor/computerd-shim": { + "name": "@xmd-spike/computerd-shim", + "version": "0.0.0", + "devDependencies": { + "@types/node": "^24", + "typescript": "^6.0.3" + }, + "peerDependencies": { + "@platformatic/vfs": "^0.4.0" + } + }, + "vendor/dofs": { + "name": "@cloudflare/dofs", + "version": "0.0.0", + "devDependencies": { + "@cloudflare/workers-types": "^4.20260616.1", + "@types/node": "^24", + "typescript": "^6.0.3" + } + } + } +} diff --git a/spikes/349-dofs/package.json b/spikes/349-dofs/package.json new file mode 100644 index 00000000..2d9fc936 --- /dev/null +++ b/spikes/349-dofs/package.json @@ -0,0 +1,13 @@ +{ + "name": "spike-349-dofs", + "private": true, + "type": "module", + "dependencies": { + "@cloudflare/dofs": "file:./vendor/dofs", + "effection": "4.1.0", + "@effectionx/process": "0.8.1", + "@effectionx/converge": "0.1.4", + "@xmd-spike/computerd-shim": "file:./vendor/computerd-shim", + "@platformatic/vfs": "0.4.0" + } +} \ No newline at end of file diff --git a/spikes/349-dofs/tests/shim.test.ts b/spikes/349-dofs/tests/shim.test.ts new file mode 100644 index 00000000..6510e1e3 --- /dev/null +++ b/spikes/349-dofs/tests/shim.test.ts @@ -0,0 +1,88 @@ +import { type Operation, run } from "effection"; +import { exec } from "@effectionx/process"; + +const proofBinary = new URL("../dist/proof", import.meta.url).pathname; +const shimBinary = new URL("../dist/proof-shim", import.meta.url).pathname; + +interface Outcome { + code: number | undefined; + payload: Record; + stderr: string; +} + +function* invoke(binary: string, args: string[]): Operation { + const result = yield* exec(binary, { arguments: args }).join(); + let payload: Record = {}; + if (result.code === 0) { + const lastLine = result.stdout.trim().split("\n").at(-1) ?? ""; + const parsed: unknown = JSON.parse(lastLine); + if (typeof parsed === "object" && parsed !== null) { + payload = Object.fromEntries(Object.entries(parsed)); + } + } + return { code: result.code, payload, stderr: result.stderr }; +} + +function assertEquals(actual: unknown, expected: unknown, detail: string) { + const left = JSON.stringify(actual); + const right = JSON.stringify(expected); + if (left !== right) { + throw new Error(`${detail}: expected ${right}, got ${left}`); + } +} + +// The mount path is part of the workspace namespace, so each scenario pins +// one absolute mount directory and reuses it for the database's lifetime. +Deno.test("a native subprocess reads and mutates the workspace through the userspace shim", () => + run(function* () { + const root = Deno.makeTempDirSync({ prefix: "spike349-shim-" }); + const db = `${root}/ws.db`; + const mount = `${root}/mount`; + + yield* invoke(proofBinary, [db, "mkdir", mount]); + yield* invoke(proofBinary, [db, "write", `${mount}/api.txt`, "from-api"]); + + const execRun = yield* invoke(shimBinary, [ + db, + mount, + "exec", + "cat api.txt > copy.txt && printf sub > sub.txt", + ]); + assertEquals(execRun.payload.code, 0, "subprocess exit code"); + + const sub = yield* invoke(proofBinary, [db, "read", `${mount}/sub.txt`]); + assertEquals( + sub.payload.body, + "sub", + "a subprocess write lands in the SQLite workspace", + ); + const copy = yield* invoke(proofBinary, [db, "read", `${mount}/copy.txt`]); + assertEquals( + copy.payload.body, + "from-api", + "the subprocess read an API-written file through the mount", + ); + })); + +Deno.test("the shim rematerializes the persisted frontier into an emptied mount directory", () => + run(function* () { + const root = Deno.makeTempDirSync({ prefix: "spike349-shim-mat-" }); + const db = `${root}/ws.db`; + const mount = `${root}/mount`; + + yield* invoke(proofBinary, [db, "mkdir", mount]); + yield* invoke(proofBinary, [db, "write", `${mount}/keep.txt`, "durable"]); + + yield* invoke(shimBinary, [db, mount, "materialize"]); + const first = Deno.readTextFileSync(`${mount}/keep.txt`); + assertEquals(first, "durable", "boot materialization writes the file"); + + Deno.removeSync(mount, { recursive: true }); + yield* invoke(shimBinary, [db, mount, "materialize"]); + const again = Deno.readTextFileSync(`${mount}/keep.txt`); + assertEquals( + again, + "durable", + "an emptied mount directory is rebuilt from SQLite state", + ); + })); diff --git a/spikes/349-dofs/tests/spike.test.ts b/spikes/349-dofs/tests/spike.test.ts new file mode 100644 index 00000000..4097f0b0 --- /dev/null +++ b/spikes/349-dofs/tests/spike.test.ts @@ -0,0 +1,126 @@ +import { type Operation, run } from "effection"; +import { exec } from "@effectionx/process"; +import { DatabaseSync } from "node:sqlite"; + +const proofBinary = new URL("../dist/proof", import.meta.url).pathname; + +interface ProofOutcome { + code: number | undefined; + payload: Record; + stderr: string; +} + +function* proof( + dbPath: string, + op: string, + ...args: string[] +): Operation { + const result = yield* exec(proofBinary, { + arguments: [dbPath, op, ...args], + }).join(); + let payload: Record = {}; + if (result.code === 0) { + const lastLine = result.stdout.trim().split("\n").at(-1) ?? ""; + const parsed: unknown = JSON.parse(lastLine); + if (typeof parsed === "object" && parsed !== null) { + payload = Object.fromEntries(Object.entries(parsed)); + } + } + return { code: result.code, payload, stderr: result.stderr }; +} + +function assertEquals(actual: unknown, expected: unknown, detail: string) { + const left = JSON.stringify(actual); + const right = JSON.stringify(expected); + if (left !== right) { + throw new Error(`${detail}: expected ${right}, got ${left}`); + } +} + +Deno.test("filesystem frontier survives full process restarts, including create/delete/create", () => + run(function* () { + const dir = Deno.makeTempDirSync({ prefix: "spike349-" }); + const db = `${dir}/ws.db`; + + yield* proof(db, "mkdir", "/notes"); + yield* proof(db, "write", "/notes/a.md", "alpha"); + const alpha = yield* proof(db, "read", "/notes/a.md"); + assertEquals(alpha.payload.body, "alpha", "read after restart"); + + yield* proof(db, "write", "/notes/b.md", "beta"); + yield* proof(db, "rm", "/notes/a.md"); + const listing = yield* proof(db, "ls", "/notes"); + assertEquals( + listing.payload.entries, + ["b.md"], + "deletion survives into the next process", + ); + + yield* proof(db, "write", "/f.txt", "v1"); + yield* proof(db, "rm", "/f.txt"); + yield* proof(db, "write", "/f.txt", "v2"); + const recreated = yield* proof(db, "read", "/f.txt"); + assertEquals( + recreated.payload.body, + "v2", + "create/delete/create keeps the last content", + ); + + yield* proof(db, "rename", "/notes/b.md", "/notes/c.md"); + const renamed = yield* proof(db, "ls", "/notes"); + assertEquals(renamed.payload.entries, ["c.md"], "rename persists"); + + yield* proof(db, "symlink", "/notes/c.md", "/link"); + const linkTarget = yield* proof(db, "readlink", "/link"); + assertEquals(linkTarget.payload.target, "/notes/c.md", "readlink"); + const throughLink = yield* proof(db, "read", "/link"); + assertEquals( + throughLink.payload.body, + "beta", + "readFile follows symlinks across a restart", + ); + + const gone = yield* proof(db, "read", "/notes/a.md"); + if (gone.code === 0) { + throw new Error("reading a deleted file succeeded"); + } + })); + +Deno.test("separate database paths are separate workspaces", () => + run(function* () { + const dir = Deno.makeTempDirSync({ prefix: "spike349-iso-" }); + yield* proof(`${dir}/a.db`, "write", "/only-in-a.txt", "a"); + const other = yield* proof(`${dir}/b.db`, "ls", "/"); + assertEquals(other.payload.entries, [], "second database starts empty"); + })); + +Deno.test("clean close leaves a single-file artifact (WAL checkpointed and removed)", () => + run(function* () { + const dir = Deno.makeTempDirSync({ prefix: "spike349-wal-" }); + const db = `${dir}/ws.db`; + yield* proof(db, "write", "/x.txt", "wal-check"); + const files = Array.from(Deno.readDirSync(dir)).map((entry) => entry.name) + .sort(); + assertEquals(files, ["ws.db"], "no -wal/-shm files after close"); + })); + +Deno.test("a newer on-disk schema version is refused loudly, not recreated", () => + run(function* () { + const dir = Deno.makeTempDirSync({ prefix: "spike349-schema-" }); + const db = `${dir}/ws.db`; + yield* proof(db, "write", "/keep.txt", "content"); + + const raw = new DatabaseSync(db); + raw.exec("UPDATE vfs_meta SET v = '99' WHERE k = 'schema_version'"); + raw.close(); + + const refused = yield* proof(db, "read", "/keep.txt"); + if (refused.code === 0) { + throw new Error("opening a future-schema database succeeded"); + } + if (!refused.stderr.includes("schema version 99")) { + throw new Error( + `refusal does not name the schema version: ${refused.stderr.slice(-300)}`, + ); + } + })); diff --git a/spikes/349-dofs/vendor.ts b/spikes/349-dofs/vendor.ts new file mode 100644 index 00000000..42951069 --- /dev/null +++ b/spikes/349-dofs/vendor.ts @@ -0,0 +1,26 @@ +import { main } from "effection"; +import { exec } from "@effectionx/process"; + +const dofsDir = new URL("./vendor/dofs/", import.meta.url).pathname; +const shimDir = new URL("./vendor/computerd-shim/", import.meta.url).pathname; + +main(function* () { + yield* exec("npm", { + arguments: ["install", "--no-audit", "--no-fund"], + cwd: dofsDir, + }).expect(); + yield* exec("npx", { + arguments: ["tsc", "-p", "tsconfig.build.json"], + cwd: dofsDir, + }).expect(); + console.log("vendored dofs built to vendor/dofs/dist"); + yield* exec("npm", { + arguments: ["install", "--no-audit", "--no-fund"], + cwd: shimDir, + }).expect(); + yield* exec("npx", { + arguments: ["tsc", "-p", "tsconfig.json"], + cwd: shimDir, + }).expect(); + console.log("vendored shim built to vendor/computerd-shim/dist"); +}); diff --git a/spikes/349-dofs/vendor/computerd-shim/LICENSE b/spikes/349-dofs/vendor/computerd-shim/LICENSE new file mode 100644 index 00000000..631c4d3e --- /dev/null +++ b/spikes/349-dofs/vendor/computerd-shim/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2026 Cloudflare, Inc. + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/spikes/349-dofs/vendor/computerd-shim/PROVENANCE.md b/spikes/349-dofs/vendor/computerd-shim/PROVENANCE.md new file mode 100644 index 00000000..4c37ab2d --- /dev/null +++ b/spikes/349-dofs/vendor/computerd-shim/PROVENANCE.md @@ -0,0 +1,26 @@ +# Vendored: computerd userspace shim (subset) + +- Source: https://github.com/cloudflare/computer, `packages/computerd` +- Tag: `v0.1.1`, commit `63d363632e558f7e077794988d36ed75017c2a62` +- License: MIT (Cloudflare, Inc.) — see [LICENSE](LICENSE), copied from the + repository root. +- This is a deliberate subset, named `@xmd-spike/computerd-shim` rather than + `@cloudflare/computerd` because it is not the upstream package: it carries + only the userspace mount shim. +- Contents: + - `src/shim/shim.ts` — byte-identical to upstream + `packages/computerd/src/shim/shim.ts` + (sha1 `05e6a2acc0bdbd9970d7e1fea2c1e13d482b528d`). + - `src/fuse/vfs.ts` — a five-line type-only stub replacing upstream's + runtime module: shim.ts imports only the `NodeVirtualFileSystem` type + from it, which upstream aliases to `@platformatic/vfs`'s + `VirtualFileSystem`. The runtime wiring upstream keeps in that module + (prototype splice + method forwarding) lives in this spike's + `host/vfs-wiring.ts` instead, ported with its deviations documented + in-file. + - `package.json` / `tsconfig.json` — authored here (upstream builds the + whole computerd package CommonJS; this subset compiles standalone as + ESM with `tsc`). +- Upgrade procedure: re-copy `shim.ts` from the new upstream tag, verify the + type-only import of `../fuse/vfs.js` is still the only coupling, rebuild, + re-run `deno task spike:349:test`. diff --git a/spikes/349-dofs/vendor/computerd-shim/package-lock.json b/spikes/349-dofs/vendor/computerd-shim/package-lock.json new file mode 100644 index 00000000..1e26eb31 --- /dev/null +++ b/spikes/349-dofs/vendor/computerd-shim/package-lock.json @@ -0,0 +1,60 @@ +{ + "name": "@xmd-spike/computerd-shim", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@xmd-spike/computerd-shim", + "version": "0.0.0", + "devDependencies": { + "@types/node": "^24", + "typescript": "^6.0.3" + }, + "peerDependencies": { + "@platformatic/vfs": "^0.4.0" + } + }, + "node_modules/@platformatic/vfs": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@platformatic/vfs/-/vfs-0.4.0.tgz", + "integrity": "sha512-JwRxSIG63e/VaDSkYXkSBajySt5MJzxYIKuknYda28fhhmJMqyUGf5rbKrGW+vO9L83nIjQH3+YI12fp7EKICQ==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">= 22" + } + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/spikes/349-dofs/vendor/computerd-shim/package.json b/spikes/349-dofs/vendor/computerd-shim/package.json new file mode 100644 index 00000000..3d401e6c --- /dev/null +++ b/spikes/349-dofs/vendor/computerd-shim/package.json @@ -0,0 +1,21 @@ +{ + "name": "@xmd-spike/computerd-shim", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": { + "default": "./dist/shim/shim.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "devDependencies": { + "typescript": "^6.0.3", + "@types/node": "^24" + }, + "peerDependencies": { + "@platformatic/vfs": "^0.4.0" + } +} diff --git a/spikes/349-dofs/vendor/computerd-shim/src/fuse/vfs.ts b/spikes/349-dofs/vendor/computerd-shim/src/fuse/vfs.ts new file mode 100644 index 00000000..3a859796 --- /dev/null +++ b/spikes/349-dofs/vendor/computerd-shim/src/fuse/vfs.ts @@ -0,0 +1,5 @@ +// Type-only stub for the probe. Upstream vfs.ts line 7 is: +// export type NodeVirtualFileSystem = VirtualFileSystem; +// The runtime wiring (prototype splice + EXTRA_VFS_METHODS forwarding) +// is hand-ported to wiring.mjs; shim.ts only imports this type. +export type { VirtualFileSystem as NodeVirtualFileSystem } from "@platformatic/vfs"; diff --git a/spikes/349-dofs/vendor/computerd-shim/src/shim/shim.ts b/spikes/349-dofs/vendor/computerd-shim/src/shim/shim.ts new file mode 100644 index 00000000..65de9d1f --- /dev/null +++ b/spikes/349-dofs/vendor/computerd-shim/src/shim/shim.ts @@ -0,0 +1,578 @@ +// Userspace bidirectional sync between the @platformatic/vfs store +// and a real directory on the host filesystem. Used when FUSE is +// unavailable (no /dev/fuse, no macFUSE) and the user has opted in +// via FUSE_MOUNT=shim. Explicitly not production-grade: races between +// writers across the seam are resolved on the next reconcile tick, +// with VFS winning ties. +// +// Model: +// +// - A "shadow" snapshot records what we last successfully synced. +// Map. +// +// - VFS -> disk runs off vfs.watchAsync("/", { recursive: true }). +// On each event we read the VFS, write the host fs, and update +// the shadow to match what we just wrote. +// +// - Disk -> VFS runs off a periodic reconcile tick. We walk the +// host directory, diff it against the shadow, and push any new +// or changed entries into the VFS. Deletes are inferred from +// shadow keys that no longer exist on disk. +// +// Loop suppression falls out of the shadow: after writing in either +// direction the shadow matches both sides, so the next tick on the +// opposite side sees no diff and emits nothing. + +import { createHash } from "node:crypto"; +import { + mkdir as fsMkdir, + readdir as fsReaddir, + readFile as fsReadFile, + rm as fsRm, + stat as fsStat, + writeFile as fsWriteFile, +} from "node:fs/promises"; +import { dirname, join, posix } from "node:path"; +import type { NodeVirtualFileSystem } from "../fuse/vfs.js"; + +export interface ShimMount { + unmount(): Promise; + /** + * Block until the on-disk tree at `mountPoint` reflects the + * VFS's current state. Called by the SyncRPC `push` handler + * after applying a peer batch so a subsequent `shell.exec` is + * guaranteed to see the just-pushed files — the watcher-driven + * VFS→disk path is async, and reads from spawned processes go + * against the real fs. + * + * Walks the VFS tree and calls `syncVfsPathToDisk` for every + * entry. The shadow short-circuits files that already match, + * so flushing twice in a row is cheap. + */ + flush(): Promise; + /** + * Block until the VFS reflects the on-disk tree's current state. + * Called by the SyncRPC `fetchChanges` handler right before it + * computes the change set the puller will see, so a + * `Workspace.pull()` issued after `shell.exec` returns observes + * files the exec'd process wrote without waiting on the next + * periodic poll tick. + * + * Runs the same disk→VFS reconcile the polling loop runs, + * serialised through the same internal mutex so a request-time + * call can't race with the tick. Idempotent on a clean tree. + */ + reconcileNow(): Promise; +} + +export interface MountShimOptions { + vfs: NodeVirtualFileSystem; + mountPoint: string; + // Disk poll cadence. Default 250ms — fast enough for an editor + // save to surface in the VFS before the next exec command, slow + // enough that the recursive readdir stays in the noise. + pollIntervalMs?: number; +} + +interface ShadowEntry { + kind: "file" | "dir"; + size: number; + mtimeMs: number; + // sha1 of the file content the last time we synced. Used to + // detect "echo" events: when a disk→VFS write triggers the VFS + // watcher, the watcher reads the VFS and sees content that + // already matches the shadow. Skipping in that case prevents the + // shim from resurrecting a file the user just deleted from disk. + contentHash?: string; +} + +type Shadow = Map; + +const DEFAULT_POLL_MS = 250; + +export async function mountShim(options: MountShimOptions): Promise { + const { vfs, mountPoint } = options; + const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_MS; + + // VFS paths and host paths share the same absolute prefix: + // `${mountPoint}/foo.txt` lives at `${mountPoint}/foo.txt` in the + // VFS and at `${mountPoint}/foo.txt` on disk. The shim drives + // everything off the host path; toVfs / toHost are identities, + // kept as named helpers so the asymmetry shows up if it ever + // creeps back in. + await fsMkdir(mountPoint, { recursive: true }); + vfs.mkdirSync(mountPoint, { recursive: true }); + + // Reconcile mutex. Both the watch loop and the poll loop mutate + // disk + shadow; serialising them lets us keep the shadow as the + // single source of truth for "what we last synced" without + // racing on map updates. + let chain: Promise = Promise.resolve(); + const run = (fn: () => Promise): Promise => { + const next = chain.then(fn, fn); + chain = next.then( + () => {}, + () => {}, + ); + return next; + }; + + const shadow: Shadow = new Map(); + + // Initial materialisation: VFS is the source of truth at boot. + // Walk it depth-first, write everything to disk, populate shadow. + // If the mount point already has content from a previous run we + // leave it alone — the disk-poll on the first reconcile tick + // will treat any extra files as disk-side additions and push + // them into the VFS. That matches the "computerd is not the only + // writer" relaxation the shim explicitly takes on. + await run(async () => { + await materialiseVfsToDisk(vfs, mountPoint, shadow); + }); + + // The watcher is scoped to the mount point so any events outside + // it (e.g. unrelated VFS writes under sibling mounts) don't drive + // the shim. coalesceChanges emits filenames relative to the + // watched root, so we reconstruct the absolute VFS path before + // syncing. + + // VFS -> disk via the platformatic/dofs watcher. watchAsync + // returns an AsyncIterable; the iterator's return() method (called + // implicitly when we break out below, or explicitly in unmount) + // tears down the underlying interval. + let stopped = false; + // watchAsync lives on the provider, not the VFS facade — the dofs + // SQLiteWorkspaceProvider implements it via revision polling. + const watcher = vfs.provider.watchAsync(mountPoint, { recursive: true }) as AsyncIterable<{ + eventType: "rename" | "change"; + filename: string; + }> & { return?(): Promise }; + + const watchLoop = (async () => { + try { + for await (const event of watcher) { + if (stopped) break; + const vfsPath = joinMount(mountPoint, event.filename); + // watchAsync emits filename="" for the watched root on some + // operations; the mount point itself is not materialised + // beneath itself. + if (vfsPath === mountPoint) continue; + await run(() => syncVfsPathToDisk(vfs, mountPoint, vfsPath, shadow)); + } + } catch (error) { + if (!stopped) { + console.error("[shim] VFS watcher loop failed:", error); + } + } + })(); + + // Disk -> VFS via periodic reconcile. Walks the mount point, + // diffs against the shadow, applies changes to the VFS. + // Shared by the periodic poll below and the on-demand + // reconcileNow() hook; both go through `run` so they serialise + // against the VFS watcher loop. + const reconcile = (): Promise => run(() => reconcileDiskToVfs(vfs, mountPoint, shadow)); + const pollTimer = setInterval(() => { + if (stopped) return; + void reconcile().catch((error) => { + console.error("[shim] disk reconcile failed:", error); + }); + }, pollIntervalMs); + pollTimer.unref?.(); + + return { + async unmount(): Promise { + if (stopped) return; + stopped = true; + clearInterval(pollTimer); + // Best-effort: tell the AsyncIterable we're done so its + // backing interval clears. The watcher iterator may be parked + // inside `for await`; calling return() resolves that pending + // next() with done:true. + try { + await watcher.return?.(); + } catch { + // ignore + } + await watchLoop; + }, + async flush(): Promise { + if (stopped) return; + // Wait for anything the watcher loop has already queued, then + // walk the VFS once and reconcile every entry against disk. + // Both steps go through `run` so they serialise with the rest + // of the chain — a concurrent watcher event can't sneak in a + // write between our walk and the resolve of `flush()`. + await run(async () => { + await flushVfsToDisk(vfs, mountPoint, shadow); + }); + }, + async reconcileNow(): Promise { + if (stopped) return; + await reconcile(); + }, + }; +} + +// --- VFS -> disk ---------------------------------------------------------- + +async function materialiseVfsToDisk( + vfs: NodeVirtualFileSystem, + mountPoint: string, + shadow: Shadow, +): Promise { + const queue: string[] = [mountPoint]; + while (queue.length > 0) { + const current = queue.shift() as string; + let entries: string[]; + try { + entries = vfs.readdirSync(current) as string[]; + } catch { + continue; + } + for (const name of entries) { + const vfsPath = `${current}/${name}`; + const stat = safeVfsStat(vfs, vfsPath); + if (stat === undefined) continue; + const hostPath = toHostPath(mountPoint, vfsPath); + if (stat.isDirectory()) { + await fsMkdir(hostPath, { recursive: true }); + shadow.set(vfsPath, dirShadow(stat.mtimeMs)); + queue.push(vfsPath); + } else if (stat.isFile()) { + const bytes = Buffer.from(vfs.readFileSync(vfsPath) as Buffer); + await fsWriteFile(hostPath, bytes); + const after = await fsStat(hostPath); + shadow.set(vfsPath, { + kind: "file", + size: bytes.byteLength, + mtimeMs: after.mtimeMs, + contentHash: hash(bytes), + }); + } + } + } +} + +/** + * Walk the VFS tree and call `syncVfsPathToDisk` for every entry. + * Unlike `materialiseVfsToDisk` (which always writes), this honors + * the shadow so files that already match disk are no-ops. Used by + * `ShimMount.flush()` to settle a freshly-applied push batch. + */ +async function flushVfsToDisk( + vfs: NodeVirtualFileSystem, + mountPoint: string, + shadow: Shadow, +): Promise { + const queue: string[] = [mountPoint]; + while (queue.length > 0) { + const current = queue.shift() as string; + let entries: string[]; + try { + entries = vfs.readdirSync(current) as string[]; + } catch { + continue; + } + for (const name of entries) { + const vfsPath = `${current}/${name}`; + const stat = safeVfsStat(vfs, vfsPath); + if (stat === undefined) continue; + await syncVfsPathToDisk(vfs, mountPoint, vfsPath, shadow); + if (stat.isDirectory()) queue.push(vfsPath); + } + } +} + +async function syncVfsPathToDisk( + vfs: NodeVirtualFileSystem, + mountPoint: string, + vfsPath: string, + shadow: Shadow, +): Promise { + const stat = safeVfsStat(vfs, vfsPath); + const hostPath = toHostPath(mountPoint, vfsPath); + + if (stat === undefined) { + // VFS says it's gone. Remove from disk + shadow. Recursive rm + // covers the case where the VFS dropped an entire subtree in a + // single rev (rare, but cheap to handle). + await fsRm(hostPath, { recursive: true, force: true }); + deleteSubtreeFromShadow(shadow, vfsPath); + return; + } + + if (stat.isDirectory()) { + await fsMkdir(hostPath, { recursive: true }); + shadow.set(vfsPath, dirShadow(stat.mtimeMs)); + return; + } + + if (stat.isFile()) { + const bytes = Buffer.from(vfs.readFileSync(vfsPath) as Buffer); + const digest = hash(bytes); + // Echo guard: if the shadow already records this exact content, + // the watcher event we're servicing was triggered by our own + // disk→VFS write a moment ago. Don't touch disk — the user + // may have deleted or modified the file between then and now, + // and resurrecting it would lose their change. The disk-poll + // is responsible for propagating any such disk-side state. + const prev = shadow.get(vfsPath); + if (prev?.kind === "file" && prev.contentHash === digest) { + return; + } + // Content-equal short-circuit: if disk already matches, skip + // the write to avoid bumping mtime and confusing the next + // disk-poll tick. + const current = await readIfFile(hostPath); + if (current === undefined || !buffersEqual(current, bytes)) { + await fsMkdir(dirname(hostPath), { recursive: true }); + await fsWriteFile(hostPath, bytes); + } + const after = await fsStat(hostPath); + shadow.set(vfsPath, { + kind: "file", + size: bytes.byteLength, + mtimeMs: after.mtimeMs, + contentHash: digest, + }); + } +} + +// --- disk -> VFS ---------------------------------------------------------- + +async function reconcileDiskToVfs( + vfs: NodeVirtualFileSystem, + mountPoint: string, + shadow: Shadow, +): Promise { + const seen = new Set(); + await walkDisk(mountPoint, mountPoint, async (hostPath, vfsPath, stat) => { + seen.add(vfsPath); + const prev = shadow.get(vfsPath); + if (stat.isDirectory()) { + if (prev?.kind === "dir") return; + // Either new directory or replaces a file. Make sure the VFS + // has it. mkdir with recursive matches mkdir -p semantics — + // safe when the path already exists. + if (prev?.kind === "file") { + try { + vfs.unlinkSync(vfsPath); + } catch { + // ignore + } + } + try { + vfs.mkdirSync(vfsPath, { recursive: true }); + } catch { + // ignore — typically EEXIST after a concurrent create + } + shadow.set(vfsPath, dirShadow(stat.mtimeMs)); + return; + } + if (!stat.isFile()) return; + // File. Skip if the shadow says we already have this exact + // (size, mtime) — that's our "did anything change?" check. + if (prev?.kind === "file" && prev.size === stat.size && prev.mtimeMs === stat.mtimeMs) { + return; + } + let bytes: Buffer; + try { + bytes = await fsReadFile(hostPath); + } catch { + return; + } + // Content-equal short-circuit: if the VFS already holds the same + // bytes (e.g. an editor save that wrote identical content) + // refresh the shadow without re-writing the VFS, which would + // otherwise bump vfs_meta.rev and echo back over the watch loop. + const vfsBytes = safeVfsRead(vfs, vfsPath); + if (vfsBytes !== undefined && buffersEqual(vfsBytes, bytes)) { + shadow.set(vfsPath, { + kind: "file", + size: stat.size, + mtimeMs: stat.mtimeMs, + contentHash: hash(bytes), + }); + return; + } + try { + // Ensure the parent dir exists in the VFS — `walkDisk` walks + // breadth-first by default but we don't rely on parent + // ordering being friendly. + const parent = posix.dirname(vfsPath); + if (parent !== "/" && parent !== ".") { + try { + vfs.mkdirSync(parent, { recursive: true }); + } catch { + // ignore + } + } + vfs.writeFileSync(vfsPath, bytes); + } catch (error) { + console.error(`[shim] failed to write ${vfsPath} into VFS:`, error); + return; + } + shadow.set(vfsPath, { + kind: "file", + size: stat.size, + mtimeMs: stat.mtimeMs, + contentHash: hash(bytes), + }); + }); + + // Deletions: anything in the shadow that the walk didn't see is + // gone from disk. Drop it from the VFS too. Sort by depth so + // children come before parents — vfs.rmdirSync rejects non-empty + // directories. + const removed = [...shadow.keys()].filter((p) => !seen.has(p)); + removed.sort((a, b) => depth(b) - depth(a)); + for (const vfsPath of removed) { + const entry = shadow.get(vfsPath); + if (entry === undefined) continue; + try { + if (entry.kind === "dir") { + vfs.rmdirSync(vfsPath); + } else { + vfs.unlinkSync(vfsPath); + } + } catch { + // ignore — most often a race with a concurrent VFS write + // that already removed the entry, or a non-empty dir that + // will get cleaned up on the next pass. + } + shadow.delete(vfsPath); + } +} + +async function walkDisk( + root: string, + current: string, + visit: ( + hostPath: string, + vfsPath: string, + stat: { isDirectory(): boolean; isFile(): boolean; size: number; mtimeMs: number }, + ) => Promise, +): Promise { + let entries: string[]; + try { + entries = await fsReaddir(current); + } catch { + return; + } + for (const name of entries) { + const hostPath = join(current, name); + let stat: Awaited>; + try { + stat = await fsStat(hostPath); + } catch { + continue; + } + const vfsPath = toVfsPath(root, hostPath); + await visit(hostPath, vfsPath, stat); + if (stat.isDirectory()) { + await walkDisk(root, hostPath, visit); + } + } +} + +// --- helpers -------------------------------------------------------------- + +// VFS and host namespaces share the same absolute paths under +// `mountPoint`. The translators are identities on the prefix; they +// strip and re-attach `mountPoint` so the rest of the code can +// stay symmetrical without sprinkling string slicing throughout. +function toHostPath(mountPoint: string, vfsPath: string): string { + if (vfsPath === mountPoint) return mountPoint; + const prefix = mountPoint === "/" ? "/" : `${mountPoint}/`; + const rel = vfsPath.startsWith(prefix) + ? vfsPath.slice(prefix.length) + : vfsPath.replace(/^\/+/, ""); + return rel === "" ? mountPoint : join(mountPoint, rel); +} + +function toVfsPath(mountPoint: string, hostPath: string): string { + const rel = hostPath.slice(mountPoint.length).replace(/\\/g, "/"); + if (rel === "" || rel === "/") return mountPoint; + return rel.startsWith("/") ? `${mountPoint}${rel}` : `${mountPoint}/${rel}`; +} + +function joinMount(mountPoint: string, filename: string): string { + if (filename === "" || filename === "/") return mountPoint; + return filename.startsWith("/") ? `${mountPoint}${filename}` : `${mountPoint}/${filename}`; +} + +function dirShadow(mtimeMs: number): ShadowEntry { + return { kind: "dir", size: 0, mtimeMs }; +} + +function depth(path: string): number { + let count = 0; + for (let i = 0; i < path.length; i++) if (path.charCodeAt(i) === 47) count++; + return count; +} + +function deleteSubtreeFromShadow(shadow: Shadow, vfsPath: string): void { + const prefix = vfsPath === "/" ? "/" : `${vfsPath}/`; + for (const key of [...shadow.keys()]) { + if (key === vfsPath || key.startsWith(prefix)) shadow.delete(key); + } +} + +function safeVfsStat( + vfs: NodeVirtualFileSystem, + path: string, +): + | { + isFile(): boolean; + isDirectory(): boolean; + size: number; + mtimeMs: number; + } + | undefined { + try { + const s = vfs.statSync(path) as { + isFile(): boolean; + isDirectory(): boolean; + size: number; + mtime: Date; + }; + return { + isFile: () => s.isFile(), + isDirectory: () => s.isDirectory(), + size: s.size, + mtimeMs: s.mtime.getTime(), + }; + } catch { + return undefined; + } +} + +function safeVfsRead(vfs: NodeVirtualFileSystem, path: string): Buffer | undefined { + try { + return Buffer.from(vfs.readFileSync(path) as Buffer); + } catch { + return undefined; + } +} + +async function readIfFile(hostPath: string): Promise { + try { + return await fsReadFile(hostPath); + } catch { + return undefined; + } +} + +function buffersEqual(a: Buffer, b: Buffer): boolean { + if (a.byteLength !== b.byteLength) return false; + // For small files just compare directly; for large ones hashing + // avoids touching every byte twice (once for compare, once for + // write). Threshold is conservative. + if (a.byteLength < 64 * 1024) return a.equals(b); + return hash(a) === hash(b); +} + +function hash(buf: Buffer): string { + return createHash("sha1").update(buf).digest("hex"); +} diff --git a/spikes/349-dofs/vendor/computerd-shim/tsconfig.json b/spikes/349-dofs/vendor/computerd-shim/tsconfig.json new file mode 100644 index 00000000..d550a1a8 --- /dev/null +++ b/spikes/349-dofs/vendor/computerd-shim/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "strict": true, + "skipLibCheck": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/spikes/349-dofs/vendor/dofs/.gitignore b/spikes/349-dofs/vendor/dofs/.gitignore new file mode 100644 index 00000000..bcef19c8 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/.gitignore @@ -0,0 +1 @@ +tests/worker-configuration.d.ts diff --git a/spikes/349-dofs/vendor/dofs/LICENSE b/spikes/349-dofs/vendor/dofs/LICENSE new file mode 100644 index 00000000..631c4d3e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/LICENSE @@ -0,0 +1,21 @@ +MIT License Copyright (c) 2026 Cloudflare, Inc. + +Permission is hereby granted, free of +charge, to any person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice +(including the next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/spikes/349-dofs/vendor/dofs/PROVENANCE.md b/spikes/349-dofs/vendor/dofs/PROVENANCE.md new file mode 100644 index 00000000..76316937 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/PROVENANCE.md @@ -0,0 +1,22 @@ +# Vendored: @cloudflare/dofs + +- Source: https://github.com/cloudflare/computer, `packages/dofs` +- Tag: `v0.1.1`, commit `63d363632e558f7e077794988d36ed75017c2a62` +- License: MIT (Cloudflare, Inc.) — see [LICENSE](LICENSE), copied from the + repository root; the package directory carries no separate license file. +- Local modifications, in full: + - `package.json`: devDependencies trimmed to `typescript`, + `@cloudflare/workers-types`, `@types/node` (the upstream set fails + `npm install` on an ERESOLVE conflict between `wrangler@4.119` and + `@cloudflare/workers-types@^4`; the removed packages are test-only), and + the vitest/wrangler scripts dropped with them. + - `tsconfig.build.json`: `types` pinned to + `["@cloudflare/workers-types", "node"]` so the build resolves without + the removed dev dependencies. + - `package.json`: a `./fs/rename` exports entry added — upstream's + `index.ts` does not re-export `rename`, and the spike consumes the + package through its exports map. + - No source file under `src/` is modified. +- Upgrade procedure: re-copy `packages/dofs` from the new upstream tag, + re-apply the two manifest edits above, rebuild, re-run + `deno task spike:349:test`. diff --git a/spikes/349-dofs/vendor/dofs/README.md b/spikes/349-dofs/vendor/dofs/README.md new file mode 100644 index 00000000..8eacbf4a --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/README.md @@ -0,0 +1,61 @@ +# `@cloudflare/dofs` + +> [!IMPORTANT] +> **PREVIEW ONLY** This package is provided as a preview for feedback only. +> APIs are unstable and the design is subject to change. +> +> Suitable for experiments, exploration and prototypes. It is NOT suitable +> for production use at this time. +> +> The specification under [`docs/`](../../docs/README.md) is forward-looking — read it for +> intent, not as description of the code today. + +Durable Object SQLite-backed virtual filesystem for Cloudflare Computer. + +This package exposes a JavaScript module, not a CLI. It bundles three layers that can be used independently: + +- A `Database` wrapper around Durable Object SQL storage plus `initializeSchema` for the `vfs_*` tables. +- Filesystem primitives under `src/fs/*` (`mkdir`, `writeFile`, `readFile`, `rm`, `readdir`, `stat`, `lstat`, `chmod`, `find`, `ls`, `grep`, `symlink`, `readlink`, `gc`, `watch`) operating on a `Database`. +- `SQLiteWorkspaceProvider`, a `@platformatic/vfs` adapter that composes those primitives into a node-shaped filesystem (fd table, positional `readSync`/`writeSync`, `watchSync`, symlinks). This is what `computerd` mounts via FUSE. +- Sync protocol building blocks operating on the same `Database`: `applyChanges`, `stageBlob`, `materialiseChange`, `coalesceChanges`, `fetchChanges`, `fetchObjects`, `hasObjects`, `pushObjects`, `buildManifest`, `currentRev`, `compareChangeCursors`, `readWatermark`/`writeWatermark`, `assertAppliedPushCursor`, and `DEFAULT_IGNORE`/`isIgnored`. The wire wiring lives in `@cloudflare/computer-rpc`. + +Minimal DO-side usage — initialize the schema; the `Database` becomes the handle every other helper takes: + +```ts +import { Database, initializeSchema } from "@cloudflare/dofs"; + +export class WorkspaceDO extends DurableObject { + private readonly db: Database; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + this.db = new Database(ctx.storage); + initializeSchema(this.db, Date.now); + } +} +``` + +> The `src/fs/*` primitives (`mkdir`, `writeFile`, `readFile`, `rm`, `readdir`, `stat`, `find`, `ls`, `grep`, `symlink`, `readlink`, `gc`, `watch`) are not re-exported from the package root yet — they are consumed in-tree by `SQLiteWorkspaceProvider` and by the sync `applyChanges` path. On the node side, instantiate `SQLiteWorkspaceProvider` (the `@platformatic/vfs` adapter) for a familiar node:fs-shaped surface; this is what `@cloudflare/computerd` mounts via FUSE. A higher-level DO-side `Workspace` class with the `fs`/`shell`/`push`/`pull` surface described in [`../../docs/README.md`](../../docs/README.md) is still future work. + +## Implementation status + + +- `Database` wrapper around Durable Object SQL storage in place. +- Schema initialization for the documented `vfs_*` tables (FS and sync) implemented and split into `schema/core.ts` + `schema/sync.ts`. +- `incrementRev()` shared sequencer in place. FS writes stamp the returned value into `vfs_nodes.rev` and pass it to `sync/changes.ts` for tombstones. +- `SQLiteTestStorage` (backed by `node:sqlite`) available from `./testing` for unit tests against a real in-memory database; `RecordingStorage` available from the package root for workerd-safe schema assertions. +- All filesystem primitives listed above are implemented and unit-tested. +- `SQLiteWorkspaceProvider` (the `@platformatic/vfs` adapter) implemented and exported from the package entrypoint; consumed by `@cloudflare/computerd`. +- Buffered-write surface for the FUSE driver: `createFileSync`, + `writeRangeSync`, `truncateFileSync`, `readRangeSync`, `chmodSync`, + `openWriteBufferSync`, `openWriteBufferForCreateSync`, and + `releaseWriteBufferSync` on the provider. The driver opens a buffer + on FUSE create/open, mutates it through subsequent writes and + truncates, and commits chunks in one transaction at release time. + Reads against the same database see the buffered bytes immediately. +- Content-addressed blob cache: `readFile`, `readRangeSync`, + `provider.readFileSync`, and the partial-chunk read-modify-write + helper share a per-`Database` LRU keyed by `vfs_blob_bytes.hash`. + Repeated reads of dedup'd chunks (a file of zeroes, a re-used + package payload) skip SQLite after the first fetch. +- Sync protocol building blocks implemented and exported; the typed RPC surface on top of them lives in `@cloudflare/computer-rpc`. diff --git a/spikes/349-dofs/vendor/dofs/package-lock.json b/spikes/349-dofs/vendor/dofs/package-lock.json new file mode 100644 index 00000000..2b3bf5ee --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/package-lock.json @@ -0,0 +1,55 @@ +{ + "name": "@cloudflare/dofs", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@cloudflare/dofs", + "version": "0.0.0", + "devDependencies": { + "@cloudflare/workers-types": "^4.20260616.1", + "@types/node": "^24", + "typescript": "^6.0.3" + } + }, + "node_modules/@cloudflare/workers-types": { + "version": "4.20260702.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260702.1.tgz", + "integrity": "sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==", + "dev": true, + "license": "MIT OR Apache-2.0" + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/spikes/349-dofs/vendor/dofs/package.json b/spikes/349-dofs/vendor/dofs/package.json new file mode 100644 index 00000000..146ae123 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/package.json @@ -0,0 +1,33 @@ +{ + "name": "@cloudflare/dofs", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./testing": { + "types": "./dist/testing.d.ts", + "default": "./dist/testing.js" + }, + "./fs/rename": { + "types": "./dist/fs/rename.d.ts", + "default": "./dist/fs/rename.js" + } + }, + "types": "./dist/index.d.ts", + "scripts": { + "build": "tsc -p tsconfig.build.json", + "typecheck": "tsc -p tsconfig.build.json --noEmit", + "test": "vitest run", + "test:workers": "vitest run --config vitest.config.workers.ts", + "bench": "vitest run --config vitest.config.bench.ts" + }, + "devDependencies": { + "typescript": "^6.0.3", + "@cloudflare/workers-types": "^4.20260616.1", + "@types/node": "^24" + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/bench/counting-storage.ts b/spikes/349-dofs/vendor/dofs/src/bench/counting-storage.ts new file mode 100644 index 00000000..70797ae9 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/bench/counting-storage.ts @@ -0,0 +1,123 @@ +// Statement/row counter that wraps a real storage backend. +// +// The benchmark harness runs against a genuine Durable Object +// SqlStorage (see vitest.config.bench.ts). Wall-clock alone can't +// prove *why* an operation is slow, so this decorator sits between the +// `Database` wrapper and the real backend and records every +// `sql.exec` call: how many statements ran, split into reads +// (SELECT/WITH) and writes (INSERT/UPDATE/DELETE/REPLACE), plus a +// best-effort tally of rows touched. +// +// Statement counts are deterministic and backend-independent — they +// are the primary signal for the O(depth) resolution fingerprint +// (`resolveInode` = 1 + 2D statements) and for write-amplification +// analysis (added rows per mutation). Row counts are read opportunist- +// ically off the cursor (the DO backend exposes rowsRead/rowsWritten) +// and are reported as a secondary, best-effort figure. +// +// The decorator forwards transactionSync/transaction straight through +// to the real backend, so real SQLite transaction semantics are +// preserved; only `sql.exec` is instrumented. + +import type { DurableObjectStorageLike, SQLCursorLike, SQLStorageLike } from "../types.js"; + +export interface StatementCounts { + statements: number; + reads: number; + writes: number; + other: number; + rowsRead: number; + rowsWritten: number; +} + +function readNumber(source: unknown, key: string): number | undefined { + const value = (source as Record | null)?.[key]; + return typeof value === "number" ? value : undefined; +} + +export class CountingStorage implements DurableObjectStorageLike { + statements = 0; + reads = 0; + writes = 0; + other = 0; + rowsRead = 0; + rowsWritten = 0; + + readonly sql: SQLStorageLike; + readonly transactionSync?: (closure: () => T) => T; + readonly transaction?: (closure: () => T | Promise) => T | Promise; + + constructor(inner: DurableObjectStorageLike) { + this.sql = { + exec: >( + query: string, + ...bindings: unknown[] + ): SQLCursorLike => { + this.statements += 1; + this.classify(query); + const cursor = inner.sql.exec(query, ...bindings); + // Writes report rowsWritten eagerly after exec on the DO + // backend; run() never iterates the cursor, so capture it here. + const written = readNumber(cursor, "rowsWritten"); + if (written !== undefined) { + this.rowsWritten += written; + } + return { + toArray: (): Row[] => { + const rows = cursor.toArray(); + // rowsRead is only meaningful once the cursor is drained, + // which all() does exactly once. + this.rowsRead += readNumber(cursor, "rowsRead") ?? rows.length; + return rows; + }, + }; + }, + }; + + if (inner.transactionSync !== undefined) { + const delegate = inner.transactionSync.bind(inner); + this.transactionSync = (closure: () => T): T => delegate(closure); + } + if (inner.transaction !== undefined) { + const delegate = inner.transaction.bind(inner); + this.transaction = (closure: () => T | Promise): T | Promise => delegate(closure); + } + } + + private classify(query: string): void { + const head = query.trimStart().slice(0, 6).toLowerCase(); + if (head.startsWith("select") || head.startsWith("with")) { + this.reads += 1; + } else if ( + head.startsWith("insert") || + head.startsWith("update") || + head.startsWith("delete") || + head.startsWith("replac") + ) { + this.writes += 1; + } else { + // SAVEPOINT/RELEASE/PRAGMA/DDL etc. Not part of per-op data cost. + this.other += 1; + } + } + + reset(): void { + this.statements = 0; + this.reads = 0; + this.writes = 0; + this.other = 0; + this.rowsRead = 0; + this.rowsWritten = 0; + } + + snapshot(): StatementCounts { + return { + statements: this.statements, + reads: this.reads, + writes: this.writes, + other: this.other, + rowsRead: this.rowsRead, + rowsWritten: this.rowsWritten, + }; + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/bench/fs-ops.bench.ts b/spikes/349-dofs/vendor/dofs/src/bench/fs-ops.bench.ts new file mode 100644 index 00000000..38360b96 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/bench/fs-ops.bench.ts @@ -0,0 +1,640 @@ +// dofs micro-benchmark harness. +// +// Runs under @cloudflare/vitest-pool-workers (see +// vitest.config.bench.ts) so every operation drives a REAL Durable +// Object SqlStorage. It deliberately does NOT use the node +// SQLiteTestStorage fixture: that backend caches prepared statements +// and would understate the per-statement cost this harness exists to +// measure. +// +// Each scenario is reported two ways: +// * ns/op wall-clock, measured against the raw DO SqlStorage. +// * statement + row counts, measured against the same backend +// wrapped in CountingStorage. Statement counts are deterministic +// and are the primary signal — a resolve is one statement +// regardless of depth (fs.stat = 1, provider.statSync = 2), and the +// cold-vs-warm group isolates the CTE cold walk from the cache hit. +// +// Output is a set of tables plus a single-line JSON blob so before/ +// after deltas are easy to capture and diff. Run with: +// npm run bench --workspace @cloudflare/dofs +// (or: npx vitest run --config vitest.config.bench.ts, from the +// package dir). + +import { env, runInDurableObject } from "cloudflare:test"; +import { expect, it } from "vitest"; +import type { TestBindings } from "../../tests/worker.js"; +import { ls } from "../fs/ls.js"; +import { resolveInode } from "../fs/resolve.js"; +import { clearResolveCache } from "../fs/resolveCache.js"; +import { rm } from "../fs/rm.js"; +import { stat } from "../fs/stat.js"; +import { SQLiteWorkspaceProvider } from "../provider.js"; +import { initializeSchema } from "../schema/index.js"; +import { Database } from "../storage.js"; +import type { DurableObjectStorageLike } from "../types.js"; +import { CountingStorage } from "./counting-storage.js"; + +const NOW = (): number => 1000; + +interface RealStorage extends DurableObjectStorageLike { + readonly databaseSize?: number; +} + +function freshStub(): DurableObjectStub { + const ns = (env as unknown as TestBindings).TestStorage; + return ns.get(ns.newUniqueId()); +} + +// Raw real SqlStorage — used for clean wall-clock numbers with no +// counting overhead. +async function withRealDb( + fn: (db: Database, provider: SQLiteWorkspaceProvider, storage: RealStorage) => T, +): Promise { + const stub = freshStub(); + return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { + const storage = state.storage as unknown as DurableObjectStorageLike; + const db = new Database(storage); + initializeSchema(db, NOW); + const provider = new SQLiteWorkspaceProvider(db, { now: NOW }); + return fn(db, provider, (storage as { sql: RealStorage }).sql as unknown as RealStorage); + }); +} + +// Counting wrapper over the same real backend — used for deterministic +// statement/row counts. Schema init is excluded via reset(). +async function withCountingDb( + fn: (db: Database, provider: SQLiteWorkspaceProvider, counting: CountingStorage) => T, +): Promise { + const stub = freshStub(); + return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { + const counting = new CountingStorage(state.storage as unknown as DurableObjectStorageLike); + const db = new Database(counting); + initializeSchema(db, NOW); + counting.reset(); + const provider = new SQLiteWorkspaceProvider(db, { now: NOW }); + return fn(db, provider, counting); + }); +} + +type Build = (db: Database, provider: SQLiteWorkspaceProvider) => void; +type Op = (db: Database, provider: SQLiteWorkspaceProvider) => void; + +interface ReadResult { + name: string; + depth: number; + nsPerOp: number; + statements: number; + reads: number; + writes: number; +} + +interface MutationResult { + name: string; + items: number; + totalMs: number; + nsPerItem: number; + statements: number; + reads: number; + writes: number; + rowsWritten: number; +} + +// Repeatable read op: build the tree once, then time `op` over `iters`. +async function benchRead( + name: string, + depth: number, + build: Build, + op: Op, + iters: number, +): Promise { + const warmup = Math.min(200, iters); + const nsPerOp = await withRealDb((db, provider) => { + build(db, provider); + for (let i = 0; i < warmup; i++) { + op(db, provider); + } + const t0 = performance.now(); + for (let i = 0; i < iters; i++) { + op(db, provider); + } + const t1 = performance.now(); + return ((t1 - t0) * 1e6) / iters; + }); + const counts = await withCountingDb((db, provider, counting) => { + build(db, provider); + counting.reset(); + op(db, provider); + return counting.snapshot(); + }); + return { + name, + depth, + nsPerOp, + statements: counts.statements, + reads: counts.reads, + writes: counts.writes, + }; +} + +// One-shot mutation batch: build fresh state, run the whole batch once, +// report totals and per-item amortized cost. +async function benchMutation( + name: string, + build: Build, + batch: (db: Database, provider: SQLiteWorkspaceProvider) => number, +): Promise { + const timing = await withRealDb((db, provider) => { + build(db, provider); + const t0 = performance.now(); + const items = batch(db, provider); + const t1 = performance.now(); + return { totalMs: t1 - t0, items }; + }); + const counts = await withCountingDb((db, provider, counting) => { + build(db, provider); + counting.reset(); + batch(db, provider); + return counting.snapshot(); + }); + const items = Math.max(1, timing.items); + return { + name, + items: timing.items, + totalMs: timing.totalMs, + nsPerItem: (timing.totalMs * 1e6) / items, + statements: counts.statements, + reads: counts.reads, + writes: counts.writes, + rowsWritten: counts.rowsWritten, + }; +} + +// --- path helpers ------------------------------------------------- + +function chainOf(depth: number): { dir: string | null; file: string } { + const segs = Array.from({ length: depth }, (_, i) => `s${i + 1}`); + const file = `/${segs.join("/")}`; + const dir = depth > 1 ? `/${segs.slice(0, -1).join("/")}` : null; + return { dir, file }; +} + +function buildChainFile(provider: SQLiteWorkspaceProvider, depth: number, content = "x"): string { + const { dir, file } = chainOf(depth); + if (dir !== null) { + provider.mkdirSync(dir, { recursive: true }); + } + provider.writeFileSync(file, content); + return file; +} + +// --- formatting --------------------------------------------------- + +function ns(value: number): string { + if (value >= 1e6) { + return `${(value / 1e6).toFixed(3)}ms`; + } + if (value >= 1e3) { + return `${(value / 1e3).toFixed(2)}\u00b5s`; + } + return `${value.toFixed(0)}ns`; +} + +function pad(value: string | number, width: number): string { + return String(value).padStart(width); +} + +function padEnd(value: string | number, width: number): string { + return String(value).padEnd(width); +} + +// ------------------------------------------------------------------ + +it("dofs micro-benchmark (real DO SqlStorage)", async () => { + const lines: string[] = []; + const readResults: ReadResult[] = []; + const mutationResults: MutationResult[] = []; + + const depths = [1, 2, 4, 8, 16, 20]; + + // Group A — path-resolution depth sweep, both stat surfaces + a + // flat single-lookup baseline that should stay constant. + for (const depth of depths) { + readResults.push( + await benchRead( + "fs.stat", + depth, + (_db, provider) => { + buildChainFile(provider, depth); + }, + (db) => { + stat(db, chainOf(depth).file); + }, + 4000, + ), + ); + readResults.push( + await benchRead( + "provider.statSync", + depth, + (_db, provider) => { + buildChainFile(provider, depth); + }, + (_db, provider) => { + provider.statSync(chainOf(depth).file); + }, + 4000, + ), + ); + const holder = { inode: 0 }; + readResults.push( + await benchRead( + "flat-baseline(inode)", + depth, + (db, provider) => { + const file = buildChainFile(provider, depth); + holder.inode = resolveInode(db, file)?.inode ?? 0; + }, + (db) => { + db.one( + "SELECT inode, type, mode, mtime, size FROM vfs_nodes WHERE inode = ?", + holder.inode, + ); + }, + 4000, + ), + ); + } + + // Group B — exists, present vs missing leaf, at two depths. + for (const depth of [8, 16]) { + readResults.push( + await benchRead( + "exists(present)", + depth, + (_db, provider) => { + buildChainFile(provider, depth); + }, + (_db, provider) => { + provider.existsSync(chainOf(depth).file); + }, + 4000, + ), + ); + readResults.push( + await benchRead( + "exists(missing)", + depth, + (_db, provider) => { + const { dir } = chainOf(depth); + if (dir !== null) { + provider.mkdirSync(dir, { recursive: true }); + } + }, + (_db, provider) => { + provider.existsSync(chainOf(depth).file); + }, + 4000, + ), + ); + } + + // Group C — read paths at a representative depth. + { + const depth = 8; + const content = "x".repeat(4096); + readResults.push( + await benchRead( + "readFile(4KiB)", + depth, + (_db, provider) => { + buildChainFile(provider, depth, content); + }, + (_db, provider) => { + provider.readFileSync(chainOf(depth).file, "utf8"); + }, + 2000, + ), + ); + readResults.push( + await benchRead( + "readRange(1KiB)", + depth, + (_db, provider) => { + buildChainFile(provider, depth, content); + }, + (_db, provider) => { + provider.readRangeSync(chainOf(depth).file, 0, 1024); + }, + 2000, + ), + ); + } + + // Group D — directory listing: readdir (single dir) vs ls (resolves + // the prefix to its inode, then walks that subtree). + { + const width = 200; + const buildWide: Build = (_db, provider) => { + provider.mkdirSync("/wide", { recursive: true }); + for (let i = 0; i < width; i++) { + provider.writeFileSync(`/wide/f${i}.txt`, "x"); + } + }; + readResults.push( + await benchRead( + `readdir(${width})`, + 1, + buildWide, + (_db, provider) => { + provider.readdirSync("/wide"); + }, + 1000, + ), + ); + readResults.push( + await benchRead( + `ls(${width})`, + 1, + buildWide, + (db) => { + ls(db, "/wide"); + }, + 500, + ), + ); + } + + // Group E — recursive delete of a populated tree. + { + const files = 2000; + mutationResults.push( + await benchMutation( + `recursive-delete(${files})`, + (_db, provider) => { + provider.mkdirSync("/tree", { recursive: true }); + for (let i = 0; i < files; i++) { + provider.writeFileSync(`/tree/f${i}.txt`, "x"); + } + }, + (db) => { + rm(db, "/tree", { recursive: true, force: true }); + return files; + }, + ), + ); + } + + // Group F — write-heavy burst (agent edit session): create, then + // edit-in-place (overwrite), then delete N files at depth. + { + const depth = 8; + const count = 1000; + const { dir } = chainOf(depth); + const base = dir ?? ""; + const ensureDir: Build = (_db, provider) => { + if (dir !== null) { + provider.mkdirSync(dir, { recursive: true }); + } + }; + const createAll = (provider: SQLiteWorkspaceProvider, value: string): void => { + for (let i = 0; i < count; i++) { + provider.writeFileSync(`${base}/burst${i}.txt`, value); + } + }; + mutationResults.push( + await benchMutation(`write-burst:create(${count})`, ensureDir, (_db, provider) => { + createAll(provider, "x"); + return count; + }), + ); + mutationResults.push( + await benchMutation( + `write-burst:edit-in-place(${count})`, + (_db, provider) => { + ensureDir(_db, provider); + createAll(provider, "x"); + }, + (_db, provider) => { + createAll(provider, "yy"); + return count; + }, + ), + ); + mutationResults.push( + await benchMutation( + `write-burst:delete(${count})`, + (_db, provider) => { + ensureDir(_db, provider); + createAll(provider, "x"); + }, + (_db, provider) => { + for (let i = 0; i < count; i++) { + provider.unlinkSync(`${base}/burst${i}.txt`); + } + return count; + }, + ), + ); + } + + // Group G — rename: many single-file renames vs one subtree rename. + { + const count = 1000; + mutationResults.push( + await benchMutation( + `single-rename(${count})`, + (_db, provider) => { + provider.mkdirSync("/mv", { recursive: true }); + for (let i = 0; i < count; i++) { + provider.writeFileSync(`/mv/a${i}.txt`, "x"); + } + }, + (_db, provider) => { + for (let i = 0; i < count; i++) { + provider.renameSync(`/mv/a${i}.txt`, `/mv/b${i}.txt`); + } + return count; + }, + ), + ); + const descendants = 500; + mutationResults.push( + await benchMutation( + `subtree-rename(${descendants})`, + (_db, provider) => { + provider.mkdirSync("/sub/inner", { recursive: true }); + for (let i = 0; i < descendants; i++) { + provider.writeFileSync(`/sub/inner/f${i}.txt`, "x"); + } + }, + (_db, provider) => { + provider.renameSync("/sub", "/moved"); + return descendants; + }, + ), + ); + } + + // Group H — DB size / dedup guard: 100 identical 1 MiB files should + // dedup to ~1 MiB of blob bytes plus small metadata. + const dedup = await withRealDb((_db, provider, storage) => { + const oneMiB = "a".repeat(1024 * 1024); + provider.mkdirSync("/dup", { recursive: true }); + for (let i = 0; i < 100; i++) { + provider.writeFileSync(`/dup/f${i}.bin`, oneMiB); + } + return { bytes: storage.databaseSize ?? 0, files: 100, logicalMiB: 100 }; + }); + + // Group I — cold vs warm resolve: isolate the CTE cold walk from the + // cache hit. Cold clears the cache before every timed op (a fresh + // single-statement CTE that reads D rows internally); warm leaves it + // primed (a single readNode, O(1)). Same fs.stat, shallow and deep. + interface ColdWarmResult { + name: string; + depth: number; + coldNsPerOp: number; + warmNsPerOp: number; + } + const coldWarmResults: ColdWarmResult[] = []; + for (const depth of [4, 20]) { + const measured = await withRealDb((db, provider) => { + buildChainFile(provider, depth); + const file = chainOf(depth).file; + const iters = 4000; + const warmup = 200; + for (let i = 0; i < warmup; i++) { + clearResolveCache(db); + stat(db, file); + } + const cold0 = performance.now(); + for (let i = 0; i < iters; i++) { + clearResolveCache(db); + stat(db, file); + } + const cold1 = performance.now(); + for (let i = 0; i < warmup; i++) { + stat(db, file); + } + const warm0 = performance.now(); + for (let i = 0; i < iters; i++) { + stat(db, file); + } + const warm1 = performance.now(); + return { + cold: ((cold1 - cold0) * 1e6) / iters, + warm: ((warm1 - warm0) * 1e6) / iters, + }; + }); + coldWarmResults.push({ + name: "fs.stat", + depth, + coldNsPerOp: measured.cold, + warmNsPerOp: measured.warm, + }); + } + + // --- render report ------------------------------------------------ + + lines.push("=".repeat(96)); + lines.push( + "dofs micro-benchmark — backend: REAL Durable Object SqlStorage (vitest-pool-workers)", + ); + lines.push(`generated: ${new Date().toISOString()}`); + lines.push( + "note: statement counts are deterministic; ns/op is wall-clock under workerd. " + + "resolve = 1 statement (cold CTE or warm cache), depth-independent. " + + "Depth-sweep ns/op below is cache-warm; see COLD-VS-WARM for the CTE cold walk.", + ); + lines.push("=".repeat(96)); + + lines.push(""); + lines.push("READ / RESOLVE OPS"); + lines.push( + `${padEnd("operation", 22)}${pad("depth", 6)}${pad("ns/op", 12)}${pad("stmts", 8)}${pad("reads", 7)}${pad("writes", 8)}`, + ); + lines.push("-".repeat(63)); + for (const r of readResults) { + lines.push( + `${padEnd(r.name, 22)}${pad(r.depth, 6)}${pad(ns(r.nsPerOp), 12)}${pad(r.statements, 8)}${pad(r.reads, 7)}${pad(r.writes, 8)}`, + ); + } + + lines.push(""); + lines.push("MUTATION BATCHES (per-item amortized)"); + lines.push( + `${padEnd("operation", 30)}${pad("items", 7)}${pad("total", 11)}${pad("ns/item", 12)}${pad("stmts", 8)}${pad("writes", 8)}${pad("rowsWr", 8)}`, + ); + lines.push("-".repeat(84)); + for (const m of mutationResults) { + lines.push( + `${padEnd(m.name, 30)}${pad(m.items, 7)}${pad(`${m.totalMs.toFixed(1)}ms`, 11)}${pad(ns(m.nsPerItem), 12)}${pad(m.statements, 8)}${pad(m.writes, 8)}${pad(m.rowsWritten, 8)}`, + ); + } + + lines.push(""); + lines.push("COLD (CTE walk) VS WARM (cached) RESOLVE — fs.stat"); + lines.push( + `${padEnd("operation", 22)}${pad("depth", 6)}${pad("cold ns/op", 12)}${pad("warm ns/op", 12)}`, + ); + lines.push("-".repeat(52)); + for (const c of coldWarmResults) { + lines.push( + `${padEnd(c.name, 22)}${pad(c.depth, 6)}${pad(ns(c.coldNsPerOp), 12)}${pad(ns(c.warmNsPerOp), 12)}`, + ); + } + + lines.push(""); + lines.push("DB SIZE / DEDUP GUARD"); + lines.push( + `100 x 1 MiB identical files -> logical ${dedup.logicalMiB} MiB, on-disk ${(dedup.bytes / (1024 * 1024)).toFixed(2)} MiB (${dedup.bytes} bytes)`, + ); + + lines.push(""); + lines.push("JSON"); + lines.push( + JSON.stringify({ + backend: "durable-object-sqlstorage", + reads: readResults, + mutations: mutationResults, + coldWarm: coldWarmResults, + dedup, + }), + ); + lines.push("=".repeat(96)); + + console.log(`\n${lines.join("\n")}\n`); + + // Signature gate. Statement counts are deterministic, so the O(depth) + // fingerprint doubles as the harness's contract. Asserted AFTER the + // report is printed so the numbers stay visible even when the gate + // trips. A deliberate perf change is expected to update these, which + // is the point — silent drift becomes a failure. + // + // fs.stat = 1 — one recursive-CTE resolve. + // provider.statSync = 2 — one resolve plus linkCount. + // flat-baseline = 1 — a single indexed inode lookup. + // + // The CTE still reads O(depth) rows internally, but the statement + // count — what the DO bills and round-trips — is depth-independent, + // and a warm cache hit re-reads just the one node row. + const find = (name: string, depth: number): ReadResult => { + const row = readResults.find((r) => r.name === name && r.depth === depth); + if (row === undefined) { + throw new Error(`benchmark result missing: ${name} depth=${depth}`); + } + return row; + }; + for (const depth of depths) { + expect(find("fs.stat", depth).statements, `fs.stat depth=${depth}`).toBe(1); + expect(find("provider.statSync", depth).statements, `provider.statSync depth=${depth}`).toBe(2); + expect(find("flat-baseline(inode)", depth).statements, `flat-baseline depth=${depth}`).toBe(1); + } + // exists resolves in a single CTE statement too, whether the leaf is + // present or missing. + expect(find("exists(present)", 8).statements).toBe(1); + expect(find("exists(missing)", 8).statements).toBe(1); + // Dedup guarantee: 100 identical 1 MiB files must not balloon the DB. + expect(dedup.bytes).toBeLessThan(2 * 1024 * 1024); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/errors.ts b/spikes/349-dofs/vendor/dofs/src/errors.ts new file mode 100644 index 00000000..1c21257d --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/errors.ts @@ -0,0 +1,36 @@ +export type WorkspaceErrorCode = + | "ENOENT" + | "ENOTEMPTY" + | "ENOTDIR" + | "EISDIR" + | "EEXIST" + | "EINVAL" + | "EACCES" + | "EPERM" + | "EROFS" + | "ENOSYS" + | "EBADF" + | "ELOOP" + | "EUNKNOWN_HASH" + | "EIO"; + +export interface WorkspaceFsError extends Error { + code: WorkspaceErrorCode; + path?: string; +} + +export function createWorkspaceError( + code: WorkspaceErrorCode, + message: string, + path?: string, +): WorkspaceFsError { + const error = new Error(path === undefined ? message : `${message}: ${path}`) as WorkspaceFsError; + error.name = "WorkspaceFsError"; + error.code = code; + error.path = path; + return error; +} + +export function invalidPath(path: string, reason: string): WorkspaceFsError { + return createWorkspaceError("EINVAL", `Invalid path (${reason})`, path); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/blobCache.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/blobCache.test.ts new file mode 100644 index 00000000..9b2f5729 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/blobCache.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, vi } from "vitest"; +import { clearBlobCache, getBlobBytes } from "./blobCache.js"; +import { readRangeSync } from "./readFile.js"; +import { withDB } from "./with-db.js"; +import { CHUNK_SIZE, writeFileSync } from "./writeFile.js"; + +describe("blobCache", () => { + it("reuses bytes for the same hash across calls", async () => { + await withDB(async (db) => { + writeFileSync(db, "/seed.bin", new Uint8Array(CHUNK_SIZE).fill(7), {}, () => 1); + // Pull the chunk hash out of vfs_chunks so we can hit the + // cache helper directly without going through readFile. + const row = db.one<{ hash: Uint8Array }>( + "SELECT hash FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", + "seed.bin", + ); + expect(row).toBeDefined(); + const hash = row?.hash as Uint8Array; + + // First call populates the cache; the second returns the + // exact same Uint8Array reference rather than re-querying. + const first = getBlobBytes(db, hash); + expect(first).toBeInstanceOf(Uint8Array); + const second = getBlobBytes(db, hash); + expect(second).toBe(first); + }); + }); + + it("evicts the least-recently-used entry once 17 distinct hashes are cached", async () => { + await withDB(async (db) => { + // Stage 17 files with distinct content so each one gets a + // unique blob hash. The cache holds 16; the 17th fetch must + // evict the least-recently-used, which is the first hash. + const hashes: Uint8Array[] = []; + for (let i = 0; i < 17; i++) { + const path = `/distinct-${i}.bin`; + writeFileSync(db, path, new TextEncoder().encode(`payload-${i}`), {}, () => 1); + const row = db.one<{ hash: Uint8Array }>( + "SELECT hash FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", + `distinct-${i}.bin`, + ); + hashes.push(row?.hash as Uint8Array); + } + clearBlobCache(db); + + // First 16 fetches populate the cache. + for (let i = 0; i < 16; i++) getBlobBytes(db, hashes[i]); + // 17th fetch evicts the LRU (entry 0); spy to confirm the + // 18th fetch of hash 0 is a fresh SQL lookup, but hash 16 is + // cached and the 18th fetch of hash 1 (still the LRU) hits + // SQL too. + getBlobBytes(db, hashes[16]); + + const spy = vi.spyOn(db, "one"); + getBlobBytes(db, hashes[0]); // evicted, must hit SQL + getBlobBytes(db, hashes[16]); // hot, must NOT hit SQL + const lookups = spy.mock.calls.filter( + ([query]) => typeof query === "string" && query.includes("vfs_blob_bytes"), + ).length; + spy.mockRestore(); + + expect(lookups).toBe(1); + }); + }); + + it("moves a touched entry to most-recent so it survives eviction", async () => { + await withDB(async (db) => { + const hashes: Uint8Array[] = []; + for (let i = 0; i < 17; i++) { + const path = `/touched-${i}.bin`; + writeFileSync(db, path, new TextEncoder().encode(`touched-${i}`), {}, () => 1); + const row = db.one<{ hash: Uint8Array }>( + "SELECT hash FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", + `touched-${i}.bin`, + ); + hashes.push(row?.hash as Uint8Array); + } + clearBlobCache(db); + + for (let i = 0; i < 16; i++) getBlobBytes(db, hashes[i]); + // Touch hash 0 so it becomes most-recent. + getBlobBytes(db, hashes[0]); + // The 17th fetch should now evict hash 1, not hash 0. + getBlobBytes(db, hashes[16]); + + const spy = vi.spyOn(db, "one"); + getBlobBytes(db, hashes[0]); // touched, still cached + getBlobBytes(db, hashes[1]); // evicted, must hit SQL + const lookups = spy.mock.calls.filter( + ([query]) => typeof query === "string" && query.includes("vfs_blob_bytes"), + ).length; + spy.mockRestore(); + + expect(lookups).toBe(1); + }); + }); + + it("readRangeSync avoids repeating vfs_blob_bytes lookups on sequential reads", async () => { + await withDB(async (db) => { + // 4 MiB of repeated content → one dedup'd blob in the store. + // A sequential read in 128 KiB windows used to issue one + // SELECT bytes per window, even though every window came out + // of the same blob. + const payload = new Uint8Array(CHUNK_SIZE * 8).fill(3); + writeFileSync(db, "/big.bin", payload, {}, () => 1); + clearBlobCache(db); + + const spy = vi.spyOn(db, "one"); + const window = 128 * 1024; + for (let offset = 0; offset < payload.byteLength; offset += window) { + readRangeSync(db, "/big.bin", offset, window); + } + const blobLookups = spy.mock.calls.filter( + ([query]) => typeof query === "string" && query.includes("vfs_blob_bytes"), + ).length; + spy.mockRestore(); + + // 8 distinct chunks were written and they all share one + // hash; we should fetch the blob bytes at most once. + expect(blobLookups).toBeLessThanOrEqual(1); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/blobCache.ts b/spikes/349-dofs/vendor/dofs/src/fs/blobCache.ts new file mode 100644 index 00000000..b997ee82 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/blobCache.ts @@ -0,0 +1,87 @@ +// In-process LRU cache of vfs_blob_bytes payloads, keyed by hash. +// +// FUSE reads up to 128 KiB at a time (the kernel's default max_read); +// our chunk size is 512 KiB. A sequential read of a chunk-backed file +// re-fetches the same blob 4x by default. Worse, a 64 MiB file of +// repeated content (e.g. `dd if=/dev/zero`) deduplicates to a single +// blob in vfs_blobs, and we then re-fetch that one blob 512 times +// over the lifetime of one read pass. +// +// vfs_blob_bytes is content-addressed. The normal write path +// (upsertChunkBlob) uses ON CONFLICT DO NOTHING, so a correct +// (hash, bytes) pair is never overwritten and the cache stays valid +// for it. The one exception is repair: stageBlob (the sync receiver +// path) uses ON CONFLICT DO UPDATE SET bytes to replace an incomplete +// or size-mismatched payload left by an interrupted or corrupt write, +// and clears this cache afterward so a stale payload is never served +// after a repair. +// +// The cache is bounded (CHUNK_CACHE_MAX_ENTRIES) and per-Database so +// independent test databases don't pollute each other. Eviction is +// LRU; access moves an entry to the most-recent position. + +import type { Database } from "../storage.js"; + +// Number of distinct blob payloads kept in memory per Database. +// At 512 KiB per blob this caps the cache at ~8 MiB, large enough +// to hold a handful of hot chunks for sequential reads of large +// files without dominating process memory. +const CHUNK_CACHE_MAX_ENTRIES = 16; + +const caches = new WeakMap>(); + +function cacheFor(db: Database): Map { + let cache = caches.get(db); + if (cache === undefined) { + cache = new Map(); + caches.set(db, cache); + } + return cache; +} + +// Stringify a 32-byte hash so it can key a JS Map. Latin-1 +// preserves every byte exactly and avoids the allocation cost of +// hex encoding for what is a very hot path. +function hashKey(hash: Uint8Array): string { + let out = ""; + for (let i = 0; i < hash.byteLength; i++) { + out += String.fromCharCode(hash[i]); + } + return out; +} + +// Look up blob bytes by hash. Cache hit returns the cached +// Uint8Array directly (callers must not mutate it). Cache miss +// queries vfs_blob_bytes and stores the result. Returns undefined +// if the blob isn't in the store. +export function getBlobBytes(db: Database, hash: Uint8Array): Uint8Array | undefined { + const cache = cacheFor(db); + const key = hashKey(hash); + const cached = cache.get(key); + if (cached !== undefined) { + // Reinsert to move to the most-recent position. Map iteration + // order is insertion order, so this gives us LRU eviction for + // free without a separate doubly-linked list. + cache.delete(key); + cache.set(key, cached); + return cached; + } + const row = db.one<{ bytes: Uint8Array }>( + "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", + hash, + ); + if (row === undefined) return undefined; + cache.set(key, row.bytes); + while (cache.size > CHUNK_CACHE_MAX_ENTRIES) { + const first = cache.keys().next(); + if (first.done === true) break; + cache.delete(first.value); + } + return row.bytes; +} + +// Reset the cache for `db`. Tests use this to keep cache state from +// leaking between cases that share a Database constructor pattern. +export function clearBlobCache(db: Database): void { + caches.delete(db); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/chmod.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/chmod.test.ts new file mode 100644 index 00000000..68cd7d08 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/chmod.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; + +import { chmod } from "./chmod.js"; +import { mkdir } from "./mkdir.js"; +import { invalidateReadOnlyMountCache } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { symlink } from "./symlink.js"; +import { withDB } from "./with-db.js"; +import { writeFileSync } from "./writeFile.js"; + +const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); + +describe("chmod", () => { + it("updates the mode on a regular file", async () => { + await withDB((db) => { + writeFileSync(db, "/a.txt", utf8("hi"), {}, () => 1000); + chmod(db, "/a.txt", 0o600, () => 2000); + expect(resolveInode(db, "/a.txt")?.mode).toBe(0o600); + }); + }); + + it("updates the mode on a directory", async () => { + await withDB((db) => { + mkdir(db, "/d", { mode: 0o755 }, () => 1000); + chmod(db, "/d", 0o700, () => 2000); + expect(resolveInode(db, "/d")?.mode).toBe(0o700); + }); + }); + + it("masks the supplied mode to twelve bits", async () => { + // POSIX chmod truncates anything above 07777. Mirror that here + // so callers can hand us a Node-style stat.mode (which carries + // file-type bits in the upper byte) without corrupting the + // stored permissions. + await withDB((db) => { + writeFileSync(db, "/a", utf8("hi"), {}, () => 0); + chmod(db, "/a", 0o100644, () => 0); + expect(resolveInode(db, "/a")?.mode).toBe(0o644); + }); + }); + + it("bumps rev and stamps it onto the node", async () => { + await withDB((db) => { + writeFileSync(db, "/a", utf8("hi"), {}, () => 0); + const beforeRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); + chmod(db, "/a", 0o600, () => 0); + const afterRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); + expect(afterRev).toBe((beforeRev ?? 0) + 1); + const nodeRev = db.scalar( + "SELECT n.rev FROM vfs_nodes n JOIN vfs_dirents d ON d.child_inode = n.inode WHERE name = ?", + "a", + ); + expect(nodeRev).toBe(afterRev); + }); + }); + + it("updates mtime to now()", async () => { + await withDB((db) => { + writeFileSync(db, "/a", utf8("hi"), {}, () => 1000); + chmod(db, "/a", 0o600, () => 5000); + expect(resolveInode(db, "/a")?.mtime).toBe(5000); + }); + }); + + it("follows symlinks by default — POSIX chmod semantics", async () => { + // chmod("/link") changes the mode of the target, not the link. + // POSIX symlinks carry mode 0o777 and chmod against the link + // itself is a no-op on most kernels. We mirror that. + await withDB((db) => { + writeFileSync(db, "/target", utf8("hi"), {}, () => 0); + symlink(db, "/target", "/link", () => 0); + chmod(db, "/link", 0o600, () => 0); + expect(resolveInode(db, "/target")?.mode).toBe(0o600); + // The symlink node itself stays at 0o777. + expect(resolveInode(db, "/link", { followSymlinks: false })?.mode).toBe(0o777); + }); + }); + + it("rejects ENOENT for a missing path", async () => { + await withDB((db) => { + expect(() => chmod(db, "/missing", 0o600, () => 0)).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("rejects EROFS when the path overlaps a read-only mount root", async () => { + await withDB((db) => { + // Stage the read-only mount through the same SQL the indexer + // uses, then materialise a directory under it so chmod has + // something to land on. invalidateReadOnlyMountCache mirrors + // what the indexer does after writing _vfs_mounts. + db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES ('/mnt', 'r2', 'read-only')"); + invalidateReadOnlyMountCache(db); + // Directly stage a directory under the mount root via SQL + // since mkdir() would itself reject under the read-only + // guard. + db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('dir', 493, 0, 0)"); + const inode = db.scalar("SELECT last_insert_rowid()"); + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (1, 'mnt', ?)", + inode, + ); + expect(() => chmod(db, "/mnt", 0o755, () => 0)).toThrowError( + expect.objectContaining({ code: "EROFS" }), + ); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/chmod.ts b/spikes/349-dofs/vendor/dofs/src/fs/chmod.ts new file mode 100644 index 00000000..5c0800fb --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/chmod.ts @@ -0,0 +1,34 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; + +// Change the file mode bits of a path. Follows symlinks like POSIX +// chmod — the change lands on the target, not the link. Bumps rev +// and mtime so the sync protocol carries the change. +// +// The supplied mode is masked to 12 bits (the permission bits and +// the setuid / setgid / sticky bits). Callers that pass a Node-style +// stat.mode with file-type bits in the upper byte get only the +// permission half stored. +export function chmod(db: Database, path: string, mode: number, now: () => number): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + + db.transactionSync(() => { + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ? WHERE inode = ?", + mode & 0o7777, + now(), + rev, + node.inode, + ); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/filesystem.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/filesystem.test.ts new file mode 100644 index 00000000..3e2842d3 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/filesystem.test.ts @@ -0,0 +1,174 @@ +// Smoke tests for the WorkspaceFilesystem class wrapper. +// +// The class is a thin forward to the free fs/* functions, so per-op +// behaviour is already covered by neighbouring tests +// (stat.test.ts, readdir.test.ts, ...). What's tested here is the +// wrapper itself: methods land on the right free function, the +// (db, now) pair gets threaded through, and the documented shape +// at the class boundary matches the free functions. + +import { describe, expect, it } from "vitest"; + +import { initializeSchema } from "../schema/index.js"; +import { Database } from "../storage.js"; +import { SQLiteTestStorage } from "../testing.js"; +import { WorkspaceFilesystem } from "./filesystem.js"; + +async function withFs( + fn: (fs: WorkspaceFilesystem) => T | Promise, + now: () => number = () => 1234, +): Promise { + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + initializeSchema(db, now); + const fs = new WorkspaceFilesystem(db, { now }); + try { + return await fn(fs); + } finally { + storage.close(); + } +} + +describe("WorkspaceFilesystem", () => { + it("writeFile and readFile round-trip utf8", async () => { + await withFs(async (fs) => { + await fs.writeFile("/hello.txt", "hello world"); + expect(await fs.readFile("/hello.txt", "utf8")).toBe("hello world"); + }); + }); + + it("readFile returns a stream by default", async () => { + await withFs(async (fs) => { + await fs.writeFile("/bin", new Uint8Array([1, 2, 3, 4])); + const stream = await fs.readFile("/bin"); + expect(stream).toBeInstanceOf(ReadableStream); + const buf = new Uint8Array(await new Response(stream).arrayBuffer()); + expect(Array.from(buf)).toEqual([1, 2, 3, 4]); + }); + }); + + it("stat returns the documented shape for a file", async () => { + await withFs(async (fs) => { + await fs.writeFile("/a.txt", "ab"); + const s = await fs.stat("/a.txt"); + expect(s).toMatchObject({ + name: "a.txt", + size: 2, + isFile: true, + isDirectory: false, + }); + }); + }); + + it("stat throws ENOENT for a missing path", async () => { + await withFs(async (fs) => { + await expect(fs.stat("/missing")).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("mkdir creates a directory that stat recognises", async () => { + await withFs(async (fs) => { + await fs.mkdir("/d"); + const s = await fs.stat("/d"); + expect(s.isDirectory).toBe(true); + }); + }); + + it("readdir lists immediate children only", async () => { + await withFs(async (fs) => { + await fs.mkdir("/d"); + await fs.writeFile("/d/a.txt", "a"); + await fs.writeFile("/d/b.txt", "b"); + await fs.mkdir("/d/sub"); + await fs.writeFile("/d/sub/deep.txt", "deep"); + const names = (await fs.readdir("/d")).map((e) => e.name).sort(); + expect(names).toEqual(["a.txt", "b.txt", "sub"]); + }); + }); + + it("find walks subtrees and ls flattens paths", async () => { + await withFs(async (fs) => { + await fs.mkdir("/p"); + await fs.writeFile("/p/a.txt", "a"); + await fs.mkdir("/p/q"); + await fs.writeFile("/p/q/b.txt", "b"); + + const found = (await fs.find("/p")).map((e) => e.path).sort(); + expect(found).toContain("/p/a.txt"); + expect(found).toContain("/p/q/b.txt"); + + const flat = (await fs.ls("/p")).sort(); + expect(flat).toContain("/p/a.txt"); + expect(flat).toContain("/p/q/b.txt"); + }); + }); + + it("grep finds matching lines", async () => { + await withFs(async (fs) => { + await fs.writeFile("/notes.txt", "alpha\nbeta\ngamma\n"); + const hits = await fs.grep("beta", "/notes.txt"); + expect(hits).toHaveLength(1); + expect(hits[0]?.text).toBe("beta"); + }); + }); + + it("rm removes a file; rm with recursive removes a directory tree", async () => { + await withFs(async (fs) => { + await fs.writeFile("/x.txt", "x"); + await fs.rm("/x.txt"); + await expect(fs.stat("/x.txt")).rejects.toMatchObject({ code: "ENOENT" }); + + await fs.mkdir("/tree"); + await fs.writeFile("/tree/inner.txt", "i"); + await fs.rm("/tree", { recursive: true }); + await expect(fs.stat("/tree")).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("chmod updates the stored mode", async () => { + await withFs(async (fs) => { + await fs.writeFile("/a", "hi"); + await fs.chmod("/a", 0o600); + expect((await fs.stat("/a")).mode).toBe(0o600); + }); + }); + + it("symlink + readlink round-trip", async () => { + await withFs(async (fs) => { + await fs.writeFile("/target", "hi"); + await fs.symlink("/target", "/link"); + expect(await fs.readlink("/link")).toBe("/target"); + }); + }); + + it("stat follows symlinks; lstat reports the link itself", async () => { + await withFs(async (fs) => { + await fs.writeFile("/target", "hello"); + await fs.symlink("/target", "/link"); + const s = await fs.stat("/link"); + expect(s.isFile).toBe(true); + expect(s.isSymbolicLink).toBe(false); + const l = await fs.lstat("/link"); + expect(l.isSymbolicLink).toBe(true); + expect(l.isFile).toBe(false); + expect(l.size).toBe("/target".length); + }); + }); + + it("threads the injected clock through writeFile", async () => { + let t = 5000; + await withFs( + async (fs) => { + await fs.writeFile("/clock.txt", "c"); + const s = await fs.stat("/clock.txt"); + expect(s.mtime).toBe(5000); + + t = 9000; + await fs.writeFile("/clock.txt", "c2"); + const s2 = await fs.stat("/clock.txt"); + expect(s2.mtime).toBe(9000); + }, + () => t, + ); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/filesystem.ts b/spikes/349-dofs/vendor/dofs/src/fs/filesystem.ts new file mode 100644 index 00000000..82ea588c --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/filesystem.ts @@ -0,0 +1,128 @@ +// WorkspaceFilesystem — class wrapper that binds a Database and a +// clock to the free fs/* functions. +// +// Every method here is a thin forward to the matching free +// function. The class exists so callers (host-side Workspace, +// in-container tools, tests) get a single instance to thread +// through their code rather than passing (db, now) pairs into +// every call. +// +// Free functions remain exported for internal callers — the +// apply paths in sync/* operate on a Database directly, and the +// in-package tests skip the class wrapper when they only need a +// single op. + +import type { Database } from "../storage.js"; + +import { chmod } from "./chmod.js"; +import { find, type WorkspaceFoundEntry } from "./find.js"; +import { type GrepOptions, grep, type WorkspaceGrepMatch } from "./grep.js"; +import { ls } from "./ls.js"; +import { type MkdirOptions, mkdir } from "./mkdir.js"; +import { type ReaddirOptions, readdir, type WorkspaceDirentResult } from "./readdir.js"; +import { type ReadFileOptions, readFile } from "./readFile.js"; +import { readlink } from "./readlink.js"; +import { type RmOptions, rm } from "./rm.js"; +import { lstat, stat, type WorkspaceStatResult } from "./stat.js"; +import { symlink } from "./symlink.js"; +import { type WriteFileContent, type WriteFileOptions, writeFile } from "./writeFile.js"; + +export interface WorkspaceFilesystemOptions { + // Clock used for mtime / last_seen. Defaults to Date.now. + // Override for deterministic tests. + now?: () => number; +} + +export class WorkspaceFilesystem { + readonly db: Database; + readonly now: () => number; + + constructor(db: Database, options: WorkspaceFilesystemOptions = {}) { + this.db = db; + this.now = options.now ?? Date.now; + } + + // --- Reads ------------------------------------------------------- + + readFile(path: string): Promise>; + readFile(path: string, encoding: "utf8"): Promise; + readFile(path: string, options: ReadFileOptions): Promise>; + readFile( + path: string, + optionsOrEncoding?: "utf8" | ReadFileOptions, + ): Promise> { + // Forward through the free function's overload set. The + // individual overloads above let callers see the precise + // return type for each input shape. + // Cast through the union overload of the free function; + // the class's overloads above carry the precise return type + // for each input shape back to the caller. + return readFile(this.db, path, optionsOrEncoding as ReadFileOptions); + } + + async stat(path: string): Promise { + return stat(this.db, path); + } + + // POSIX lstat — like stat, but doesn't follow a trailing symlink. + // Use when the caller wants to inspect the link itself: readlink + // / unlink under a Node-style fs surface, or just-bash's adapter + // routing lstat through to the workspace. + async lstat(path: string): Promise { + return lstat(this.db, path); + } + + // Return the stored target of a symlink. EINVAL when path is + // not a symlink; ENOENT when path is missing. + async readlink(path: string): Promise { + return readlink(this.db, path); + } + + async readdir(path: string, options: ReaddirOptions = {}): Promise { + return readdir(this.db, path, options); + } + + async find(directory: string, pattern?: string): Promise { + return find(this.db, directory, pattern); + } + + async ls(prefix: string): Promise { + return ls(this.db, prefix); + } + + grep(pattern: string, path: string, options: GrepOptions = {}): Promise { + return grep(this.db, pattern, path, options); + } + + // --- Mutations --------------------------------------------------- + + writeFile( + path: string, + content: WriteFileContent, + options: WriteFileOptions = {}, + ): Promise { + return writeFile(this.db, path, content, options, this.now); + } + + async mkdir(path: string, options: MkdirOptions = {}): Promise { + mkdir(this.db, path, options, this.now); + } + + async rm(path: string, options: RmOptions = {}): Promise { + rm(this.db, path, options); + } + + // Change the permission bits on a path. Follows symlinks like + // POSIX chmod — the change lands on the target, not the link. + // The supplied mode is masked to twelve bits. + async chmod(path: string, mode: number): Promise { + chmod(this.db, path, mode, this.now); + } + + // Create a symbolic link at `path` pointing at `target`. The + // target is stored verbatim; it can be relative or absolute and + // is allowed to dangle. + async symlink(target: string, path: string): Promise { + symlink(this.db, target, path, this.now); + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/find.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/find.test.ts new file mode 100644 index 00000000..f2e8d064 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/find.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; + +import { find } from "./find.js"; +import { mkdir } from "./mkdir.js"; +import { withDB } from "./with-db.js"; +import { writeFile } from "./writeFile.js"; + +describe("find", () => { + it("returns nothing for an empty directory", async () => { + await withDB((db) => { + expect(find(db, "/")).toEqual([]); + }); + }); + + it("walks every entry without a pattern", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/x.ts", "", {}, () => 0); + await writeFile(db, "/a/b/y.md", "", {}, () => 0); + const entries = find(db, "/").sort((p, q) => p.path.localeCompare(q.path)); + expect(entries).toEqual([ + { path: "/a", type: "dir" }, + { path: "/a/b", type: "dir" }, + { path: "/a/b/y.md", type: "file" }, + { path: "/a/x.ts", type: "file" }, + ]); + }); + }); + + it("treats an empty pattern like no pattern and returns every entry", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/x.ts", "", {}, () => 0); + await writeFile(db, "/a/b/y.md", "", {}, () => 0); + const entries = find(db, "/", "").sort((p, q) => p.path.localeCompare(q.path)); + expect(entries).toEqual([ + { path: "/a", type: "dir" }, + { path: "/a/b", type: "dir" }, + { path: "/a/b/y.md", type: "file" }, + { path: "/a/x.ts", type: "file" }, + ]); + }); + }); + + it("matches a single-level glob *.ts within the directory only", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/x.ts", "", {}, () => 0); + await writeFile(db, "/a/b/y.ts", "", {}, () => 0); + const paths = find(db, "/a", "*.ts") + .map((e) => e.path) + .sort(); + expect(paths).toEqual(["/a/x.ts"]); + }); + }); + + it("matches ** recursively", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b/c", { recursive: true }, () => 0); + await writeFile(db, "/a/x.md", "", {}, () => 0); + await writeFile(db, "/a/b/y.md", "", {}, () => 0); + await writeFile(db, "/a/b/c/z.md", "", {}, () => 0); + const paths = find(db, "/a", "**/*.md") + .map((e) => e.path) + .sort(); + expect(paths).toEqual(["/a/b/c/z.md", "/a/b/y.md", "/a/x.md"]); + }); + }); + + it("walks from a nested directory", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b/c", { recursive: true }, () => 0); + await writeFile(db, "/a/b/y.ts", "", {}, () => 0); + await writeFile(db, "/a/b/c/z.ts", "", {}, () => 0); + const paths = find(db, "/a/b", "**/*.ts") + .map((e) => e.path) + .sort(); + expect(paths).toEqual(["/a/b/c/z.ts", "/a/b/y.ts"]); + }); + }); + + it("does not match files outside the start directory even with **", async () => { + await withDB(async (db) => { + mkdir(db, "/a", {}, () => 0); + mkdir(db, "/b", {}, () => 0); + await writeFile(db, "/a/x.ts", "", {}, () => 0); + await writeFile(db, "/b/x.ts", "", {}, () => 0); + const paths = find(db, "/a", "**/*.ts").map((e) => e.path); + expect(paths).toEqual(["/a/x.ts"]); + }); + }); + + it("throws ENOENT when the directory is missing", async () => { + await withDB((db) => { + expect(() => find(db, "/missing")).toThrowError(expect.objectContaining({ code: "ENOENT" })); + }); + }); + + it("throws ENOTDIR when called on a file", async () => { + await withDB(async (db) => { + await writeFile(db, "/file.txt", "x", {}, () => 0); + expect(() => find(db, "/file.txt")).toThrowError( + expect.objectContaining({ code: "ENOTDIR" }), + ); + }); + }); + + it("escapes regex metacharacters in literal segments of a pattern", async () => { + await withDB(async (db) => { + mkdir(db, "/a", {}, () => 0); + await writeFile(db, "/a/file.ts", "", {}, () => 0); + // The dot in `*.ts` is a regex metacharacter; make sure we don't match + // any other single character against it. + await writeFile(db, "/a/fileXts", "", {}, () => 0); + const paths = find(db, "/a", "*.ts").map((e) => e.path); + expect(paths).toEqual(["/a/file.ts"]); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/find.ts b/spikes/349-dofs/vendor/dofs/src/fs/find.ts new file mode 100644 index 00000000..67e1f747 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/find.ts @@ -0,0 +1,102 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; + +export interface WorkspaceFoundEntry { + path: string; + type: "file" | "dir"; +} + +interface ChildRow { + name: string; + child_inode: number; + type: "file" | "dir"; +} + +export function find(db: Database, directory: string, pattern?: string): WorkspaceFoundEntry[] { + const { path: canonical } = canonicalizePath(directory); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); + } + + const out: WorkspaceFoundEntry[] = []; + // An empty pattern is equivalent to no pattern: walk and return + // everything rather than compiling it into `^$`, which would match + // only empty relative paths and yield no results. + const regex = pattern ? compileGlob(pattern) : undefined; + + walk(db, node.inode, canonical, out); + + if (regex === undefined) { + return out; + } + // Glob matches against the path relative to the start directory. + const prefix = canonical === "/" ? "/" : `${canonical}/`; + return out.filter((entry) => { + if (!entry.path.startsWith(prefix)) return false; + const rel = entry.path.slice(prefix.length); + return regex.test(rel); + }); +} + +function walk(db: Database, parentInode: number, parentPath: string, out: WorkspaceFoundEntry[]) { + const children = db.all( + `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name`, + parentInode, + ); + for (const child of children) { + const childPath = parentPath === "/" ? `/${child.name}` : `${parentPath}/${child.name}`; + out.push({ path: childPath, type: child.type }); + if (child.type === "dir") { + walk(db, child.child_inode, childPath, out); + } + } +} + +// Compile a simple glob into a regex. Supported: +// * matches any run of characters except '/' +// ** matches any run of characters including '/' +// Anything else is a literal. Regex metacharacters in literals are +// escaped so '.' in '*.ts' doesn't match an arbitrary character. +function compileGlob(pattern: string): RegExp { + let re = ""; + let i = 0; + while (i < pattern.length) { + const ch = pattern[i]; + if (ch === "*") { + if (pattern[i + 1] === "*") { + // '**/' matches zero or more path segments. Without the slash, '**' + // matches any run including slashes. + if (pattern[i + 2] === "/") { + re += "(?:.*/)?"; + i += 3; + } else { + re += ".*"; + i += 2; + } + } else { + re += "[^/]*"; + i += 1; + } + continue; + } + if (REGEX_METACHARS.has(ch)) { + re += `\\${ch}`; + } else { + re += ch; + } + i += 1; + } + return new RegExp(`^${re}$`); +} + +const REGEX_METACHARS = new Set([".", "+", "?", "^", "$", "(", ")", "[", "]", "{", "}", "|", "\\"]); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/gc.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/gc.test.ts new file mode 100644 index 00000000..64cce055 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/gc.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; + +import type { Database } from "../storage.js"; +import { gc } from "./gc.js"; +import { rm } from "./rm.js"; +import { withDB } from "./with-db.js"; +import { writeFile } from "./writeFile.js"; + +function blobCount(db: Database): number { + return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; +} + +function blobBytesCount(db: Database): number { + return db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes") ?? 0; +} + +describe("gc", () => { + it("returns { blobsFreed: 0, manifestsFreed: 0 } on an empty FS", async () => { + await withDB((db) => { + expect(gc(db, { now: () => 1_000_000 })).toEqual({ blobsFreed: 0, manifestsFreed: 0 }); + }); + }); + + it("does not free blobs that are still referenced", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "live content", {}, () => 1000); + expect(blobCount(db)).toBe(1); + expect(gc(db, { now: () => 999_999_999 })).toEqual({ + blobsFreed: 0, + manifestsFreed: 0, + }); + expect(blobCount(db)).toBe(1); + }); + }); + + it("frees orphan blobs left behind by overwrite", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "first", {}, () => 1000); + await writeFile(db, "/x.txt", "second", {}, () => 1000); + // Both blobs exist; the 'first' content is orphaned. + expect(blobCount(db)).toBe(2); + + const result = gc(db, { now: () => 2000, safetyWindowMs: 0 }); + expect(result.blobsFreed).toBe(1); + expect(blobCount(db)).toBe(1); + // The remaining blob is the one referenced by the current chunk. + const referenced = db.scalar( + "SELECT COUNT(*) FROM vfs_blobs b WHERE EXISTS (SELECT 1 FROM vfs_chunks c WHERE c.hash = b.hash)", + ); + expect(referenced).toBe(1); + }); + }); + + it("cascades the delete to vfs_blob_bytes", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "first", {}, () => 1000); + await writeFile(db, "/x.txt", "second", {}, () => 1000); + expect(blobBytesCount(db)).toBe(2); + gc(db, { now: () => 2000, safetyWindowMs: 0 }); + expect(blobBytesCount(db)).toBe(1); + }); + }); + + it("frees orphan blobs left behind by rm", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "unique-content", {}, () => 1000); + expect(blobCount(db)).toBe(1); + rm(db, "/a.txt", {}); + expect(blobCount(db)).toBe(1); // rm doesn't sweep blobs + const result = gc(db, { now: () => 2000, safetyWindowMs: 0 }); + expect(result.blobsFreed).toBe(1); + expect(blobCount(db)).toBe(0); + }); + }); + + it("respects the safety window", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "first", {}, () => 1000); + await writeFile(db, "/x.txt", "second", {}, () => 1000); + // Inside the safety window the orphan stays. + expect(gc(db, { now: () => 1500, safetyWindowMs: 1_000 })).toEqual({ + blobsFreed: 0, + manifestsFreed: 0, + }); + expect(blobCount(db)).toBe(2); + // Outside the window it gets swept. + expect(gc(db, { now: () => 5000, safetyWindowMs: 1_000 })).toEqual({ + blobsFreed: 1, + manifestsFreed: 1, + }); + expect(blobCount(db)).toBe(1); + }); + }); + + it("uses a conservative default safety window when none is provided", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "first", {}, () => 1000); + await writeFile(db, "/x.txt", "second", {}, () => 1000); + // The default is conservative enough that a small elapsed time + // does not sweep anything. + expect(gc(db, { now: () => 1500 })).toEqual({ blobsFreed: 0, manifestsFreed: 0 }); + expect(blobCount(db)).toBe(2); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/gc.ts b/spikes/349-dofs/vendor/dofs/src/fs/gc.ts new file mode 100644 index 00000000..4229fee9 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/gc.ts @@ -0,0 +1,55 @@ +import type { Database } from "../storage.js"; + +export interface GcOptions { + // Wall-clock at the moment GC runs. Defaults to Date.now so callers + // can pin it from tests. + now?: () => number; + // Blobs whose last_seen is younger than now() - safetyWindowMs are + // never swept. The default is generous (1 hour) so a misconfigured + // GC pass cannot wipe blobs the application is actively writing. + safetyWindowMs?: number; +} + +export interface GcResult { + blobsFreed: number; + manifestsFreed: number; +} + +const DEFAULT_SAFETY_WINDOW_MS = 60 * 60 * 1000; // 1 hour + +export function gc(db: Database, options: GcOptions = {}): GcResult { + const now = (options.now ?? Date.now)(); + const safety = options.safetyWindowMs ?? DEFAULT_SAFETY_WINDOW_MS; + const cutoff = now - safety; + + return db.transactionSync(() => { + // Sweep orphan blobs: no row in vfs_chunks references the hash and + // last_seen is older than the safety cutoff. vfs_blob_bytes + // cascades on delete via the foreign key, so the bytes row goes + // with its parent. + db.run( + `DELETE FROM vfs_blobs + WHERE last_seen < ? + AND NOT EXISTS (SELECT 1 FROM vfs_chunks c WHERE c.hash = vfs_blobs.hash)`, + cutoff, + ); + const blobsFreed = db.scalar("SELECT changes()") ?? 0; + + // Manifests share the blob safety window: a writer might have + // inserted a manifest row but not yet linked it from vfs_nodes. + // Inside the same transactionSync block this can't happen, but + // keep the window as defence in depth for sync-layer code that + // might stage manifests before linking nodes. + db.run( + `DELETE FROM vfs_manifests + WHERE last_seen < ? + AND NOT EXISTS ( + SELECT 1 FROM vfs_nodes n WHERE n.manifest_hash = vfs_manifests.hash + )`, + cutoff, + ); + const manifestsFreed = db.scalar("SELECT changes()") ?? 0; + + return { blobsFreed, manifestsFreed }; + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/grep.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/grep.test.ts new file mode 100644 index 00000000..aef8fbcf --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/grep.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; + +import { grep } from "./grep.js"; +import { mkdir } from "./mkdir.js"; +import { withDB } from "./with-db.js"; +import { CHUNK_SIZE, writeFile } from "./writeFile.js"; + +describe("grep", () => { + it("returns no matches for an empty file", async () => { + await withDB(async (db) => { + await writeFile(db, "/empty", "", {}, () => 0); + expect(await grep(db, "TODO", "/empty")).toEqual([]); + }); + }); + + it("finds a single match in a single file", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "line one\nTODO fix me\nline three\n", {}, () => 0); + expect(await grep(db, "TODO", "/a.txt")).toEqual([ + { path: "/a.txt", line: 2, text: "TODO fix me" }, + ]); + }); + }); + + it("returns multiple matches in one file", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "a TODO here\nplain\nb TODO there\n", {}, () => 0); + expect(await grep(db, "TODO", "/a.txt")).toEqual([ + { path: "/a.txt", line: 1, text: "a TODO here" }, + { path: "/a.txt", line: 3, text: "b TODO there" }, + ]); + }); + }); + + it("walks a directory recursively", async () => { + await withDB(async (db) => { + mkdir(db, "/d/sub", { recursive: true }, () => 0); + await writeFile(db, "/d/a.txt", "TODO root\n", {}, () => 0); + await writeFile(db, "/d/sub/b.txt", "TODO nested\n", {}, () => 0); + await writeFile(db, "/d/sub/c.txt", "no match\n", {}, () => 0); + const matches = await grep(db, "TODO", "/d"); + expect(matches.map((m) => m.path).sort()).toEqual(["/d/a.txt", "/d/sub/b.txt"]); + }); + }); + + it("respects ignoreCase", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "todo\nTODO\nTodo\n", {}, () => 0); + expect((await grep(db, "TODO", "/a.txt", { ignoreCase: true })).length).toBe(3); + expect((await grep(db, "TODO", "/a.txt")).length).toBe(1); + }); + }); + + it("matches across a chunk boundary", async () => { + await withDB(async (db) => { + // Lay out a file whose line straddles the 512KiB chunk boundary. + const pad = "a".repeat(CHUNK_SIZE - 5); + const content = `${pad}TODO straddle\nafter\n`; + await writeFile(db, "/big.txt", content, {}, () => 0); + const matches = await grep(db, "TODO", "/big.txt"); + expect(matches).toHaveLength(1); + expect(matches[0].path).toBe("/big.txt"); + expect(matches[0].text.endsWith("TODO straddle")).toBe(true); + }); + }); + + it("returns lines 1-indexed", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "first\nsecond\nthird\n", {}, () => 0); + expect(await grep(db, "first", "/a.txt")).toEqual([ + { path: "/a.txt", line: 1, text: "first" }, + ]); + expect(await grep(db, "third", "/a.txt")).toEqual([ + { path: "/a.txt", line: 3, text: "third" }, + ]); + }); + }); + + it("rejects ENOENT when the path is missing", async () => { + await withDB(async (db) => { + await expect(grep(db, "x", "/missing")).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/grep.ts b/spikes/349-dofs/vendor/dofs/src/fs/grep.ts new file mode 100644 index 00000000..fd28471c --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/grep.ts @@ -0,0 +1,107 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { find } from "./find.js"; +import { readFile } from "./readFile.js"; +import { resolveInode } from "./resolve.js"; + +export interface WorkspaceGrepMatch { + path: string; + line: number; + text: string; +} + +export interface GrepOptions { + ignoreCase?: boolean; +} + +export async function grep( + db: Database, + pattern: string, + path: string, + options: GrepOptions = {}, +): Promise { + const { path: canonical } = canonicalizePath(path); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + + const filePaths = + node.type === "file" + ? [canonical] + : find(db, canonical) + .filter((entry) => entry.type === "file") + .map((entry) => entry.path); + + const matches: WorkspaceGrepMatch[] = []; + for (const filePath of filePaths) { + await scanFile(db, filePath, pattern, options, matches); + } + return matches; +} + +// Stream the file in chunks so very large files don't load fully into +// memory. Carry a partial-line tail between chunks (everything after +// the last '\n') so a line that straddles a chunk boundary still +// matches as one line. Line numbers are 1-indexed. +async function scanFile( + db: Database, + path: string, + pattern: string, + options: GrepOptions, + out: WorkspaceGrepMatch[], +): Promise { + const stream = await readFile(db, path); + const reader = stream.getReader(); + const decoder = new TextDecoder("utf-8", { fatal: false }); + const needle = options.ignoreCase ? pattern.toUpperCase() : pattern; + + let tail = ""; + let lineNo = 1; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + const text = tail + decoder.decode(value, { stream: true }); + const newlineIdx = text.lastIndexOf("\n"); + const ready = newlineIdx === -1 ? "" : text.slice(0, newlineIdx); + tail = newlineIdx === -1 ? text : text.slice(newlineIdx + 1); + if (ready.length > 0) { + lineNo = scanLines(ready, lineNo, needle, options.ignoreCase === true, path, out); + } + } + // Drain the decoder and scan whatever's left (final line without a + // trailing newline). + tail += decoder.decode(); + if (tail.length > 0) { + scanLines(tail, lineNo, needle, options.ignoreCase === true, path, out); + } +} + +// Walk `block` line-by-line, push matches into `out`, return the next +// 1-indexed line number to use for the following block. +function scanLines( + block: string, + startLine: number, + needle: string, + ignoreCase: boolean, + path: string, + out: WorkspaceGrepMatch[], +): number { + let line = startLine; + let cursor = 0; + while (cursor <= block.length) { + const next = block.indexOf("\n", cursor); + const end = next === -1 ? block.length : next; + const text = block.slice(cursor, end); + const haystack = ignoreCase ? text.toUpperCase() : text; + if (haystack.includes(needle)) { + out.push({ path, line, text }); + } + line += 1; + if (next === -1) break; + cursor = next + 1; + } + return line; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/link.ts b/spikes/349-dofs/vendor/dofs/src/fs/link.ts new file mode 100644 index 00000000..c5e21d17 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/link.ts @@ -0,0 +1,84 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact } from "./resolveCache.js"; + +function resolveParent(db: Database, parts: string[], canonical: string): number { + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + parts[i], + ); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one<{ inode: number; type: "file" | "dir" | "symlink" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + child.child_inode, + ); + if (next === undefined) { + throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); + } + if (next.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = next.inode; + } + return parentInode; +} + +export function link(db: Database, existingPath: string, newPath: string): void { + const { parts, path: canonicalNew } = canonicalizePath(newPath); + if (parts.length === 0) { + throw createWorkspaceError("EEXIST", "cannot link onto root", canonicalNew); + } + + assertNotReadOnly(db, canonicalNew); + + db.transactionSync(() => { + const source = resolveInode(db, existingPath); + if (source === null) { + throw createWorkspaceError("ENOENT", `no such file: ${existingPath}`, existingPath); + } + if (source.type !== "file") { + throw createWorkspaceError( + "EPERM", + `cannot hardlink non-file: ${existingPath}`, + existingPath, + ); + } + + const parentInode = resolveParent(db, parts, canonicalNew); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonicalNew}`, canonicalNew); + } + + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + leafName, + source.inode, + ); + const rev = incrementRev(db); + db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); + // A new hardlink name for an existing file: a leaf with no + // descendants, so drop just the (possibly negative) entry for it. + invalidateResolveExact(db, canonicalNew); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/ls.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/ls.test.ts new file mode 100644 index 00000000..108c94ee --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/ls.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; + +import { ls } from "./ls.js"; +import { mkdir } from "./mkdir.js"; +import { symlink } from "./symlink.js"; +import { withDB } from "./with-db.js"; +import { writeFile } from "./writeFile.js"; + +describe("ls", () => { + it("returns an empty array when nothing matches", async () => { + await withDB((db) => { + expect(ls(db, "/")).toEqual([]); + }); + }); + + it("lists every file under root sorted by path", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/b/x.ts", "", {}, () => 0); + await writeFile(db, "/a/y.ts", "", {}, () => 0); + await writeFile(db, "/z.ts", "", {}, () => 0); + expect(ls(db, "/")).toEqual(["/a/b/x.ts", "/a/y.ts", "/z.ts"]); + }); + }); + + it("returns only files, not directory entries", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/b/x", "", {}, () => 0); + expect(ls(db, "/")).toEqual(["/a/b/x"]); + }); + }); + + it("matches an exact subtree by prefix", async () => { + await withDB(async (db) => { + mkdir(db, "/wsp", {}, () => 0); + mkdir(db, "/workspace", {}, () => 0); + await writeFile(db, "/wsp/a", "", {}, () => 0); + await writeFile(db, "/workspace/b", "", {}, () => 0); + await writeFile(db, "/workspace/c", "", {}, () => 0); + expect(ls(db, "/workspace")).toEqual(["/workspace/b", "/workspace/c"]); + }); + }); + + it("returns just the file path when the prefix is a file", async () => { + await withDB(async (db) => { + mkdir(db, "/a", {}, () => 0); + await writeFile(db, "/a/x.ts", "hi", {}, () => 0); + expect(ls(db, "/a/x.ts")).toEqual(["/a/x.ts"]); + }); + }); + + it("returns an empty array for a missing prefix", async () => { + await withDB((db) => { + expect(ls(db, "/no/such/prefix")).toEqual([]); + }); + }); + + it("lists a nested subdirectory without scanning sibling subtrees", async () => { + await withDB(async (db) => { + mkdir(db, "/a/deep", { recursive: true }, () => 0); + mkdir(db, "/b", { recursive: true }, () => 0); + await writeFile(db, "/a/deep/x.ts", "", {}, () => 0); + await writeFile(db, "/a/deep/y.ts", "", {}, () => 0); + await writeFile(db, "/a/top.ts", "", {}, () => 0); + await writeFile(db, "/b/other.ts", "", {}, () => 0); + expect(ls(db, "/a/deep")).toEqual(["/a/deep/x.ts", "/a/deep/y.ts"]); + }); + }); + + it("does not follow a symlink prefix", async () => { + await withDB(async (db) => { + mkdir(db, "/real", {}, () => 0); + await writeFile(db, "/real/f.ts", "", {}, () => 0); + symlink(db, "/real", "/link", () => 0); + // A symlink is a leaf with no dirents, so listing at or through + // it yields nothing; the real directory still lists normally. + expect(ls(db, "/link")).toEqual([]); + expect(ls(db, "/link/f.ts")).toEqual([]); + expect(ls(db, "/real")).toEqual(["/real/f.ts"]); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/ls.ts b/spikes/349-dofs/vendor/dofs/src/fs/ls.ts new file mode 100644 index 00000000..6a5c2387 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/ls.ts @@ -0,0 +1,59 @@ +import { canonicalizePath } from "../path.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; + +interface PathRow { + path: string; +} + +// Recursive CTE that materializes the file paths under one listing +// root. Files only (no directory entries) because that's the +// documented "flat list of file paths" semantics. +// +// The walk is seeded at the listing root's inode: each row is +// (inode, path, type), built by concatenating dirent names with '/' +// separators onto the seed path. Scoping the seed to the requested +// directory keeps the walk O(subtree) instead of O(whole tree). +const LS_QUERY = ` + WITH RECURSIVE walk(inode, path, type) AS ( + SELECT inode, ?, type FROM vfs_nodes WHERE inode = ? + UNION ALL + SELECT n.inode, w.path || '/' || d.name, n.type + FROM walk w + JOIN vfs_dirents d ON d.parent_inode = w.inode + JOIN vfs_nodes n ON n.inode = d.child_inode + ) + SELECT path FROM walk + WHERE type = 'file' + ORDER BY path +`; + +// Walk dirents from the root to `parts` without following symlinks, so +// the seed matches the CTE's structural view: a symlink component has +// no dirents and thus lists nothing, and a missing or non-directory +// component resolves to null (an empty listing). Returns the root +// inode for an empty path. +function resolvePrefixInode(db: Database, parts: string[]): number | null { + let inode = ROOT_INODE; + for (const name of parts) { + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + inode, + name, + ); + if (child === undefined) return null; + inode = child.child_inode; + } + return inode; +} + +export function ls(db: Database, prefix: string): string[] { + const { parts, path: canonical } = canonicalizePath(prefix); + const inode = resolvePrefixInode(db, parts); + if (inode === null) return []; + // Root contributes the empty string so its children start with '/'; + // a non-root prefix seeds its own path so descendants read as + // absolute paths. + const seedPath = canonical === "/" ? "" : canonical; + return db.all(LS_QUERY, seedPath, inode).map((row) => row.path); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mkdir.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/mkdir.test.ts new file mode 100644 index 00000000..e1fe0221 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/mkdir.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; + +import { ROOT_INODE } from "../schema/index.js"; +import { mkdir } from "./mkdir.js"; +import { resolveInode } from "./resolve.js"; +import { withDB } from "./with-db.js"; + +describe("mkdir", () => { + it("creates a top-level directory with the default mode", async () => { + await withDB((db) => { + mkdir(db, "/a", {}, () => 2000); + const resolved = resolveInode(db, "/a"); + expect(resolved).toMatchObject({ type: "dir", mode: 0o755, mtime: 2000 }); + }); + }); + + it("honors the supplied mode", async () => { + await withDB((db) => { + mkdir(db, "/locked", { mode: 0o700 }, () => 0); + expect(resolveInode(db, "/locked")?.mode).toBe(0o700); + }); + }); + + it("bumps rev and stamps it onto the new node", async () => { + await withDB((db) => { + const beforeRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); + mkdir(db, "/a", {}, () => 0); + const afterRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); + expect(afterRev).toBe((beforeRev ?? 0) + 1); + + const nodeRev = db.scalar( + "SELECT n.rev FROM vfs_nodes n JOIN vfs_dirents d ON d.child_inode = n.inode WHERE d.parent_inode = ? AND d.name = ?", + ROOT_INODE, + "a", + ); + expect(nodeRev).toBe(afterRev); + }); + }); + + it("rejects when the parent directory is missing", async () => { + await withDB((db) => { + expect(() => mkdir(db, "/no/such/parent", {}, () => 0)).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("rejects when the parent path segment is a file (ENOTDIR)", async () => { + await withDB((db) => { + db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); + const inode = db.scalar("SELECT last_insert_rowid()"); + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + ROOT_INODE, + "a", + inode, + ); + expect(() => mkdir(db, "/a/b", {}, () => 0)).toThrowError( + expect.objectContaining({ code: "ENOTDIR" }), + ); + }); + }); + + it("rejects EEXIST when the path already exists as a directory without recursive", async () => { + await withDB((db) => { + mkdir(db, "/a", {}, () => 0); + expect(() => mkdir(db, "/a", {}, () => 0)).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + }); + }); + + it("rejects EEXIST when the path already exists as a file", async () => { + await withDB((db) => { + db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); + const inode = db.scalar("SELECT last_insert_rowid()"); + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + ROOT_INODE, + "a", + inode, + ); + expect(() => mkdir(db, "/a", {}, () => 0)).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + }); + }); + + it("recursive: creates missing ancestors", async () => { + await withDB((db) => { + mkdir(db, "/x/y/z", { recursive: true }, () => 1234); + expect(resolveInode(db, "/x")?.type).toBe("dir"); + expect(resolveInode(db, "/x/y")?.type).toBe("dir"); + expect(resolveInode(db, "/x/y/z")?.type).toBe("dir"); + }); + }); + + it("recursive: is idempotent when the target already exists as a dir", async () => { + await withDB((db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + expect(() => mkdir(db, "/a/b", { recursive: true }, () => 0)).not.toThrow(); + }); + }); + + it("recursive: still rejects EEXIST when the target exists as a file", async () => { + await withDB((db) => { + mkdir(db, "/a", {}, () => 0); + db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); + const inode = db.scalar("SELECT last_insert_rowid()"); + const aInode = resolveInode(db, "/a")?.inode; + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + aInode, + "b", + inode, + ); + expect(() => mkdir(db, "/a/b", { recursive: true }, () => 0)).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + }); + }); + + it("rejects EEXIST when creating root", async () => { + await withDB((db) => { + expect(() => mkdir(db, "/", {}, () => 0)).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + }); + }); + + it("accepts recursive: false for node:fs/promises parity", async () => { + await withDB((db) => { + // boolean false should be accepted by the type and behave as default. + mkdir(db, "/dir", { recursive: false }, () => 0); + expect(() => mkdir(db, "/dir", { recursive: false }, () => 0)).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mkdir.ts b/spikes/349-dofs/vendor/dofs/src/fs/mkdir.ts new file mode 100644 index 00000000..4d664c61 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/mkdir.ts @@ -0,0 +1,130 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveExact } from "./resolveCache.js"; + +export interface MkdirOptions { + recursive?: boolean; + mode?: number; +} + +interface ResolvedSegment { + inode: number; + type: "file" | "dir"; +} + +// Look up a child by name under a parent directory. Returns undefined +// when there's no dirent. The caller decides whether that's an error. +function lookupChild(db: Database, parentInode: number, name: string): ResolvedSegment | undefined { + const row = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + name, + ); + if (row === undefined) { + return undefined; + } + const node = db.one<{ inode: number; type: "file" | "dir" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + row.child_inode, + ); + if (node === undefined) { + return undefined; + } + return node; +} + +// Create one directory entry under `parentInode`, returning the new +// inode. The caller has already verified the name is not taken. +function createDir( + db: Database, + parentInode: number, + name: string, + mode: number, + mtime: number, + rev: number, +): number { + // RETURNING folds the rowid read into the INSERT. + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('dir', ?, ?, ?) RETURNING inode", + mode, + mtime, + rev, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + const inode = row.inode; + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + name, + inode, + ); + return inode; +} + +export function mkdir(db: Database, path: string, options: MkdirOptions, now: () => number): void { + const { parts, path: canonical } = canonicalizePath(path); + const recursive = options.recursive === true; + const mode = (options.mode ?? 0o755) & 0o7777; + + if (parts.length === 0) { + // Root always exists post-initializeSchema; mkdir("/") is EEXIST + // even with recursive (matches Node fs.mkdir's "EEXIST on root" + // behaviour for non-recursive; for recursive Node returns + // undefined, but our docs treat mkdir("/") as nonsensical). + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + assertNotReadOnly(db, canonical); + + db.transactionSync(() => { + const rev = incrementRev(db); + const mtime = now(); + + let parentInode = ROOT_INODE; + // Walk all but the final segment. Each must already exist as a + // directory; if `recursive`, we create missing ones. + for (let i = 0; i < parts.length - 1; i++) { + const name = parts[i]; + const existing = lookupChild(db, parentInode, name); + if (existing === undefined) { + if (!recursive) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + parentInode = createDir(db, parentInode, name, 0o755, mtime, rev); + // A newly created directory is empty, so a cached negative for + // its own path is the only stale entry possible; drop it exact. + invalidateResolveExact(db, `/${parts.slice(0, i + 1).join("/")}`); + continue; + } + if (existing.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = existing.inode; + } + + // Final segment. + const leafName = parts[parts.length - 1]; + const existing = lookupChild(db, parentInode, leafName); + if (existing !== undefined) { + // EEXIST is correct for both "already a directory" and + // "already a file" per docs/04. Recursive only swallows the + // already-a-directory case. + if (recursive && existing.type === "dir") { + return; + } + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + + createDir(db, parentInode, leafName, mode, mtime, rev); + invalidateResolveExact(db, canonical); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.test.ts new file mode 100644 index 00000000..cc729a48 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.test.ts @@ -0,0 +1,318 @@ +import { describe, expect, it } from "vitest"; + +import type { Database } from "../storage.js"; +import { mkdir } from "./mkdir.js"; +import { + assertNotReadOnly, + getReadOnlyMountRoots, + invalidateReadOnlyMountCache, +} from "./mount-guard.js"; +import { rename } from "./rename.js"; +import { resolveInode } from "./resolve.js"; +import { rm } from "./rm.js"; +import { symlink } from "./symlink.js"; +import { withDB } from "./with-db.js"; +import { writeFile, writeFileSync } from "./writeFile.js"; + +// Stage a read-only mount the way the workspace-side indexer +// eventually will: a row in `_vfs_mounts` plus an actual subtree +// stamped with `mount_root`. Tests that want the cache to pick this +// up should invalidate it after staging. +function stageMount(db: Database, root: string, mode: "read-only" | "read-write"): void { + db.run( + "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, ?)", + root, + "test", + mode, + ); + invalidateReadOnlyMountCache(db); +} + +// Create a stub directory hierarchy at the mount root, stamped with +// `mount_root`. The guard only consults `_vfs_mounts`, so for the +// rm tests we materialise enough of the subtree for the walk to +// find something to delete. +async function materialiseRootDir(db: Database, root: string, now: () => number): Promise { + mkdir(db, root, { recursive: true }, now); + // Stamp the inode so a later "drop the workspace.mount_root + // column" sweep would notice if anything else relies on it. + db.run( + "UPDATE vfs_nodes SET mount_root = ? WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ? AND parent_inode = 1)", + root, + root.slice(1), + ); +} + +describe("mount-guard helpers", () => { + it("caches read-only roots per database and reloads after invalidation", async () => { + await withDB(async (db) => { + // Cold cache: empty list, no rows. + expect(getReadOnlyMountRoots(db)).toEqual([]); + + // Stage a row without invalidating; cache stays empty. + db.run( + "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-only')", + "/workspace/r2", + "r2", + ); + expect(getReadOnlyMountRoots(db)).toEqual([]); + + // After invalidation the next call re-reads. + invalidateReadOnlyMountCache(db); + expect(getReadOnlyMountRoots(db)).toEqual(["/workspace/r2"]); + + // A read-write row stays out of the read-only set. + db.run( + "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-write')", + "/workspace/scratch", + "r2", + ); + invalidateReadOnlyMountCache(db); + expect(getReadOnlyMountRoots(db)).toEqual(["/workspace/r2"]); + }); + }); + + it("assertNotReadOnly is a no-op when no read-only mounts are registered", async () => { + await withDB(async (db) => { + expect(() => assertNotReadOnly(db, "/anywhere")).not.toThrow(); + }); + }); + + it("assertNotReadOnly throws EROFS for paths under, at, or above a read-only root", async () => { + await withDB(async (db) => { + stageMount(db, "/workspace/r2", "read-only"); + + // Direct paths inside. + expect(() => assertNotReadOnly(db, "/workspace/r2/hello.txt")).toThrow(/EROFS|read-only/); + // Path equal to the mount root. + expect(() => assertNotReadOnly(db, "/workspace/r2")).toThrow(/EROFS|read-only/); + // Ancestor of the mount root (the rm-the-whole-workspace + // shape). + expect(() => assertNotReadOnly(db, "/workspace")).toThrow(/EROFS|read-only/); + + // Paths outside the mount are fine. + expect(() => assertNotReadOnly(db, "/workspace/r2-sibling")).not.toThrow(); + expect(() => assertNotReadOnly(db, "/scratch/elsewhere")).not.toThrow(); + }); + }); + + it("read-write mounts do not register as read-only", async () => { + await withDB(async (db) => { + stageMount(db, "/workspace/rw", "read-write"); + expect(getReadOnlyMountRoots(db)).toEqual([]); + expect(() => assertNotReadOnly(db, "/workspace/rw/file")).not.toThrow(); + }); + }); +}); + +describe("writeFile under a read-only mount", () => { + it("rejects a streaming write under the mount root with EROFS", async () => { + await withDB(async (db) => { + // Materialise the directory before flipping the mount to + // read-only so the guard doesn't block our own setup. + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageMount(db, "/workspace/r2", "read-only"); + + const source = new ReadableStream({ + start(c) { + c.enqueue(new TextEncoder().encode("blocked")); + c.close(); + }, + }); + await expect( + writeFile(db, "/workspace/r2/hello.txt", source, {}, () => 0), + ).rejects.toMatchObject({ code: "EROFS" }); + + // The reject happens before we stage blobs, so no orphan + // rows land. + const blobs = db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; + expect(blobs).toBe(0); + }); + }); + + it("rejects writeFileSync under the mount root with EROFS", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageMount(db, "/workspace/r2", "read-only"); + + expect(() => + writeFileSync(db, "/workspace/r2/hello.txt", new Uint8Array([1, 2, 3]), {}, () => 0), + ).toThrow(/EROFS|read-only/); + }); + }); + + it("allows writes under a read-write mount", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/rw", { recursive: true }, () => 0); + stageMount(db, "/workspace/rw", "read-write"); + + // No throw; the bytes land in vfs_nodes. + writeFileSync(db, "/workspace/rw/ok.txt", new TextEncoder().encode("hi"), {}, () => 0); + const inode = db.scalar( + "SELECT inode FROM vfs_nodes WHERE manifest_hash IS NOT NULL", + ); + expect(inode).toBeDefined(); + }); + }); +}); + +describe("mkdir under a read-only mount", () => { + it("rejects mkdir under the mount root with EROFS", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageMount(db, "/workspace/r2", "read-only"); + + expect(() => mkdir(db, "/workspace/r2/sub", { recursive: true }, () => 0)).toThrow( + /EROFS|read-only/, + ); + }); + }); + + it("rejects mkdir of a read-only mount root that doesn't exist yet", async () => { + await withDB(async (db) => { + stageMount(db, "/workspace/r2", "read-only"); + expect(() => mkdir(db, "/workspace/r2", { recursive: true }, () => 0)).toThrow( + /EROFS|read-only/, + ); + }); + }); +}); + +describe("rm under a read-only mount", () => { + it("rejects rm of a path inside the mount", async () => { + await withDB(async (db) => { + // Stage the row, materialise the subtree before stamping + // read-only so writeFile can land a file. + stageMount(db, "/workspace/r2", "read-write"); + await materialiseRootDir(db, "/workspace/r2", () => 0); + writeFileSync(db, "/workspace/r2/hello.txt", new Uint8Array([1]), {}, () => 0); + + // Flip to read-only. + db.run("UPDATE _vfs_mounts SET mode = 'read-only' WHERE root = ?", "/workspace/r2"); + invalidateReadOnlyMountCache(db); + + expect(() => rm(db, "/workspace/r2/hello.txt", {})).toThrow(/EROFS|read-only/); + }); + }); + + it("rejects rm of the mount root itself", async () => { + await withDB(async (db) => { + await materialiseRootDir(db, "/workspace/r2", () => 0); + stageMount(db, "/workspace/r2", "read-only"); + + expect(() => rm(db, "/workspace/r2", { recursive: true, force: true })).toThrow( + /EROFS|read-only/, + ); + }); + }); + + it("rejects rm of an ancestor whose subtree contains a read-only mount", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageMount(db, "/workspace/r2", "read-only"); + + // The ancestor path /workspace overlaps the read-only root + // via the symmetric check; rm with recursive/force must + // reject before deleting anything. + expect(() => rm(db, "/workspace", { recursive: true, force: true })).toThrow( + /EROFS|read-only/, + ); + + // The mount root inode is still present. + const remaining = db.scalar("SELECT COUNT(*) FROM vfs_dirents WHERE name = ?", "r2"); + expect(remaining).toBeGreaterThan(0); + }); + }); + + it("allows rm of a path outside any mount", async () => { + await withDB(async (db) => { + mkdir(db, "/scratch", { recursive: true }, () => 0); + writeFileSync(db, "/scratch/file.txt", new Uint8Array([1]), {}, () => 0); + stageMount(db, "/workspace/r2", "read-only"); + + expect(() => rm(db, "/scratch/file.txt", {})).not.toThrow(); + }); + }); + + it("allows rm under a read-write mount", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/rw", { recursive: true }, () => 0); + writeFileSync(db, "/workspace/rw/hi.txt", new Uint8Array([1]), {}, () => 0); + stageMount(db, "/workspace/rw", "read-write"); + + expect(() => rm(db, "/workspace/rw/hi.txt", {})).not.toThrow(); + }); + }); + + it("rejects rm through a symlinked parent that resolves into a read-only mount", async () => { + await withDB((db) => { + mkdir(db, "/mnt", { recursive: true }, () => 0); + writeFileSync(db, "/mnt/file.txt", new Uint8Array([1]), {}, () => 0); + symlink(db, "/mnt", "/link", () => 0); + stageMount(db, "/mnt", "read-only"); + const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + expect(() => rm(db, "/link/file.txt", {})).toThrowError( + expect.objectContaining({ code: "EROFS" }), + ); + + expect(resolveInode(db, "/mnt/file.txt")).not.toBeNull(); + expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); + }); + }); + + it("rejects recursive rm of a directory inside a read-only mount via a symlinked parent", async () => { + await withDB((db) => { + mkdir(db, "/mnt/dir", { recursive: true }, () => 0); + writeFileSync(db, "/mnt/dir/file.txt", new Uint8Array([1]), {}, () => 0); + symlink(db, "/mnt", "/link", () => 0); + stageMount(db, "/mnt", "read-only"); + const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + expect(() => rm(db, "/link/dir", { recursive: true })).toThrowError( + expect.objectContaining({ code: "EROFS" }), + ); + + expect(resolveInode(db, "/mnt/dir/file.txt")).not.toBeNull(); + expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); + }); + }); +}); + +describe("rename under a read-only mount", () => { + it("rejects rename from a symlinked parent that resolves into a read-only mount", async () => { + await withDB((db) => { + mkdir(db, "/mnt", { recursive: true }, () => 0); + writeFileSync(db, "/mnt/file.txt", new Uint8Array([1]), {}, () => 0); + symlink(db, "/mnt", "/link", () => 0); + stageMount(db, "/mnt", "read-only"); + const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + expect(() => rename(db, "/link/file.txt", "/moved.txt")).toThrowError( + expect.objectContaining({ code: "EROFS" }), + ); + + expect(resolveInode(db, "/mnt/file.txt")).not.toBeNull(); + expect(resolveInode(db, "/moved.txt")).toBeNull(); + expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); + }); + }); + + it("rejects rename to a symlinked parent that resolves into a read-only mount", async () => { + await withDB((db) => { + mkdir(db, "/mnt", { recursive: true }, () => 0); + writeFileSync(db, "/src.txt", new Uint8Array([1]), {}, () => 0); + symlink(db, "/mnt", "/link", () => 0); + stageMount(db, "/mnt", "read-only"); + const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + expect(() => rename(db, "/src.txt", "/link/file.txt")).toThrowError( + expect.objectContaining({ code: "EROFS" }), + ); + + expect(resolveInode(db, "/src.txt")).not.toBeNull(); + expect(resolveInode(db, "/mnt/file.txt")).toBeNull(); + expect(db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'")).toBe(before); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.ts b/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.ts new file mode 100644 index 00000000..6a2aadd3 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/mount-guard.ts @@ -0,0 +1,82 @@ +// Read-only mount guard. +// +// Every dofs mutating entry point (writeFile, mkdir, rm, and the +// apply path in sync/apply.ts) consults this module to reject +// writes that fall under a registered read-only mount root. The +// guard lives at the data layer so container-side writes that +// arrive via pullOnce -> applyChanges are caught too — the +// workspace-side surface wrapper alone cannot see them. +// +// The set of read-only roots is small (one row per registered +// mount per workspace, typically <10) and changes only at indexer +// write time. Cache it per Database in a WeakMap so repeat lookups +// don't hit SQLite. The mount indexer in @cloudflare/computer +// invalidates the cache via `invalidateReadOnlyMountCache(db)` after +// it writes _vfs_mounts. + +import { createWorkspaceError } from "../errors.js"; +import type { Database } from "../storage.js"; + +// undefined sentinel = "not loaded yet"; an empty array means +// "loaded, no read-only mounts registered". The two are not the +// same: the empty case must skip the SQL lookup on every check. +const cache = new WeakMap(); + +// Public so the workspace-side indexer can drop the cache after it +// writes a new _vfs_mounts row. Tests also call it when they stage +// a mount fixture by direct SQL. +export function invalidateReadOnlyMountCache(db: Database): void { + cache.delete(db); +} + +function loadReadOnlyRoots(db: Database): readonly string[] { + const rows = db.all<{ root: string }>("SELECT root FROM _vfs_mounts WHERE mode = 'read-only'"); + const roots = rows.map((r) => r.root); + cache.set(db, roots); + return roots; +} + +export function getReadOnlyMountRoots(db: Database): readonly string[] { + const cached = cache.get(db); + if (cached !== undefined) return cached; + return loadReadOnlyRoots(db); +} + +// Symmetric overlap check between a candidate write path and a +// mount root. Either: +// - `path` is at or below `root` (a direct write or rm under the +// mount root), OR +// - `root` is below `path` (an ancestor rm that would recurse +// through the mount). +// Both shapes must be blocked so a read-only mount survives both +// vectors. Mirrors the predicate that lived in +// GuardedWorkspaceFilesystem before the data-layer move. +function overlapsRoot(path: string, root: string): boolean { + return path === root || path.startsWith(`${root}/`) || root.startsWith(`${path}/`); +} + +// Throws EROFS when the path overlaps any read-only mount root. +// Callers should invoke this before any DB mutation. The error +// shape matches the existing createWorkspaceError contract so +// surface callers see a normal WorkspaceFsError. +export function assertNotReadOnly(db: Database, path: string): void { + const roots = getReadOnlyMountRoots(db); + if (roots.length === 0) return; + for (const root of roots) { + if (overlapsRoot(path, root)) { + throw createWorkspaceError("EROFS", `read-only mount at ${root}: cannot modify`, path); + } + } +} + +// Variant for callers that already know the path is canonicalised +// and want to reject a single descendant during a recursive walk +// (rm's walkPostOrder). Returns the matching root or undefined; the +// caller decides whether to throw, log, or skip. +export function readOnlyRootFor(db: Database, path: string): string | undefined { + const roots = getReadOnlyMountRoots(db); + for (const root of roots) { + if (overlapsRoot(path, root)) return root; + } + return undefined; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readFile.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/readFile.test.ts new file mode 100644 index 00000000..c8d80c56 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/readFile.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; + +import { mkdir } from "./mkdir.js"; +import { readFile } from "./readFile.js"; +import { withDB } from "./with-db.js"; +import { CHUNK_SIZE, writeFile } from "./writeFile.js"; + +async function drain(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const parts: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value !== undefined) { + parts.push(value); + total += value.byteLength; + } + } + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; +} + +describe("readFile", () => { + it("returns a ReadableStream by default", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "hello workspace", {}, () => 0); + const stream = await readFile(db, "/a.txt"); + expect(stream).toBeInstanceOf(ReadableStream); + expect(new TextDecoder().decode(await drain(stream))).toBe("hello workspace"); + }); + }); + + it("returns a string when encoding is 'utf8'", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "hello", {}, () => 0); + expect(await readFile(db, "/a.txt", "utf8")).toBe("hello"); + }); + }); + + it("accepts the object-form encoding option", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "hello", {}, () => 0); + expect(await readFile(db, "/a.txt", { encoding: "utf8" })).toBe("hello"); + }); + }); + + it("streams a multi-chunk file in chunk-sized pieces", async () => { + await withDB(async (db) => { + const bytes = new Uint8Array(CHUNK_SIZE + 100); + bytes.fill(0x41); + for (let i = CHUNK_SIZE; i < bytes.byteLength; i++) bytes[i] = 0x42; + await writeFile(db, "/big", bytes, {}, () => 0); + + const stream = await readFile(db, "/big"); + const reader = stream.getReader(); + const first = await reader.read(); + expect(first.done).toBe(false); + expect(first.value?.byteLength).toBe(CHUNK_SIZE); + expect(first.value?.[0]).toBe(0x41); + const second = await reader.read(); + expect(second.done).toBe(false); + expect(second.value?.byteLength).toBe(100); + expect(second.value?.[0]).toBe(0x42); + const end = await reader.read(); + expect(end.done).toBe(true); + }); + }); + + it("returns an empty stream for an empty file", async () => { + await withDB(async (db) => { + await writeFile(db, "/empty", "", {}, () => 0); + const stream = await readFile(db, "/empty"); + const bytes = await drain(stream); + expect(bytes.byteLength).toBe(0); + expect(await readFile(db, "/empty", "utf8")).toBe(""); + }); + }); + + it("does not modify vfs_blobs.last_seen when chunks are read", async () => { + await withDB(async (db) => { + const bytes = new Uint8Array(CHUNK_SIZE + 1); + bytes.fill(0x61); + await writeFile(db, "/x.txt", bytes, {}, () => 100); + expect(db.scalar("SELECT MIN(last_seen) FROM vfs_blobs")).toBe(100); + + // String form reads every chunk and must leave last_seen alone. + await readFile(db, "/x.txt", "utf8"); + expect(db.scalar("SELECT MIN(last_seen) FROM vfs_blobs")).toBe(100); + + // Stream form: drain it so every chunk is pulled, then confirm + // no restamp happened in the pull callback. + const stream = await readFile(db, "/x.txt"); + const reader = stream.getReader(); + while (true) { + const { done } = await reader.read(); + if (done) break; + } + expect(db.scalar("SELECT MIN(last_seen) FROM vfs_blobs")).toBe(100); + }); + }); + + it("rejects ENOENT when the path does not exist", async () => { + await withDB(async (db) => { + await expect(readFile(db, "/missing")).rejects.toMatchObject({ code: "ENOENT" }); + await expect(readFile(db, "/missing", "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); + + it("rejects EISDIR when the path is a directory", async () => { + await withDB(async (db) => { + mkdir(db, "/d", {}, () => 0); + await expect(readFile(db, "/d")).rejects.toMatchObject({ code: "EISDIR" }); + await expect(readFile(db, "/d", "utf8")).rejects.toMatchObject({ code: "EISDIR" }); + }); + }); + + it("rejects ENOENT when an intermediate segment is missing", async () => { + await withDB(async (db) => { + await expect(readFile(db, "/no/such/file")).rejects.toMatchObject({ code: "ENOENT" }); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readFile.ts b/spikes/349-dofs/vendor/dofs/src/fs/readFile.ts new file mode 100644 index 00000000..7f89de40 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/readFile.ts @@ -0,0 +1,194 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { getBlobBytes } from "./blobCache.js"; +import { resolveInode } from "./resolve.js"; +import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; +import { CHUNK_SIZE } from "./writeFile.js"; + +export interface ReadFileOptions { + encoding?: "utf8"; +} + +interface ChunkRow { + hash: Uint8Array; + size: number; +} + +// Overloads match docs/04_filesystem_interface.md exactly. +export function readFile(db: Database, path: string): Promise>; +export function readFile(db: Database, path: string, encoding: "utf8"): Promise; +export function readFile( + db: Database, + path: string, + options: ReadFileOptions, +): Promise>; +export async function readFile( + db: Database, + path: string, + optionsOrEncoding?: "utf8" | ReadFileOptions, +): Promise> { + const wantString = + optionsOrEncoding === "utf8" || + (typeof optionsOrEncoding === "object" && optionsOrEncoding?.encoding === "utf8"); + + // Pending-create files surface through the path-keyed buffer. + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + const snapshot = new Uint8Array(pending.size); + snapshot.set(pending.buf.subarray(0, pending.size)); + if (wantString) return new TextDecoder().decode(snapshot); + return new ReadableStream({ + start(controller) { + controller.enqueue(snapshot); + controller.close(); + }, + }); + } + + // Resolve up front so we surface ENOENT/EISDIR before doing any + // streaming work. + const node = resolveInode(db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + + // While a write buffer is open for this inode it is the source of + // truth. Skip the chunk store and serve the buffered bytes. + const buffered = getWriteBuffer(db, node.inode); + if (buffered?.dirty) { + const snapshot = new Uint8Array(buffered.size); + snapshot.set(buffered.buf.subarray(0, buffered.size)); + if (wantString) return new TextDecoder().decode(snapshot); + return new ReadableStream({ + start(controller) { + controller.enqueue(snapshot); + controller.close(); + }, + }); + } + + const chunks = db.all( + "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node.inode, + ); + + if (wantString) { + // Fast path — concatenate everything and decode once. Matches the + // node:fs/promises.readFile semantics for an encoding argument: + // memory cost = whole file. + const totalSize = chunks.reduce((acc, c) => acc + c.size, 0); + const out = new Uint8Array(totalSize); + let offset = 0; + for (const chunk of chunks) { + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); + } + out.set(bytes, offset); + offset += bytes.byteLength; + } + return new TextDecoder().decode(out); + } + + // Stream form. We enqueue one Uint8Array per chunk, lazily pulled. + // Reads resolve bytes by hash and never restamp last_seen: a chunk + // being read is already linked to a node, so gc's orphan gate keeps + // it. last_seen only guards blobs staged but not yet linked. + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i >= chunks.length) { + controller.close(); + return; + } + const chunk = chunks[i++]; + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + controller.error(createWorkspaceError("EIO", `missing blob bytes for ${path}`, path)); + return; + } + controller.enqueue(bytes); + }, + }); +} + +// Positional read primitive. Walks only the chunk rows that overlap +// [offset, offset+length), so the FUSE driver can serve a kernel +// read without materializing the whole file. +export function readRangeSync( + db: Database, + path: string, + offset: number, + length: number, +): Uint8Array { + if (!Number.isInteger(offset) || offset < 0) { + throw createWorkspaceError("EINVAL", `invalid read offset: ${offset}`, path); + } + if (!Number.isInteger(length) || length < 0) { + throw createWorkspaceError("EINVAL", `invalid read length: ${length}`, path); + } + // Pending-create files have no inode yet. Serve reads from the + // path-keyed buffer until release commits the row. + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + if (length === 0) return new Uint8Array(); + if (offset >= pending.size) return new Uint8Array(); + const end = Math.min(offset + length, pending.size); + return pending.buf.subarray(offset, end); + } + const node = resolveInode(db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + if (length === 0) return new Uint8Array(); + + // If a write buffer is open for this inode, it is the source of + // truth: pending writes have not yet committed to vfs_chunks. + // Reading from SQLite here would return stale bytes. + const buffered = getWriteBuffer(db, node.inode); + if (buffered?.dirty) { + if (offset >= buffered.size) return new Uint8Array(); + const end = Math.min(offset + length, buffered.size); + return buffered.buf.subarray(offset, end); + } + + // node.size is the cached value resolveInode just loaded. + const totalSize = node.size; + if (offset >= totalSize) return new Uint8Array(); + const end = Math.min(offset + length, totalSize); + const firstIdx = Math.floor(offset / CHUNK_SIZE); + const lastIdx = Math.floor((end - 1) / CHUNK_SIZE); + // Pull every overlapping chunk in one indexed range scan. Missing + // indices (a sparse file) simply don't come back, so the assembly + // below compacts around the gaps exactly as a per-index walk would. + const chunks = db.all<{ idx: number; hash: Uint8Array }>( + "SELECT idx, hash FROM vfs_chunks WHERE inode = ? AND idx BETWEEN ? AND ? ORDER BY idx", + node.inode, + firstIdx, + lastIdx, + ); + const out = new Uint8Array(end - offset); + let written = 0; + for (const { idx, hash } of chunks) { + const start = idx * CHUNK_SIZE; + const bytes = getBlobBytes(db, hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); + } + const srcStart = Math.max(0, offset - start); + const srcEnd = Math.min(bytes.byteLength, end - start); + if (srcEnd <= srcStart) continue; + out.set(bytes.subarray(srcStart, srcEnd), written); + written += srcEnd - srcStart; + } + return written === out.byteLength ? out : out.subarray(0, written); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readRange.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/readRange.test.ts new file mode 100644 index 00000000..626b5ab9 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/readRange.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import { readRangeSync } from "./readFile.js"; +import { resolveInode } from "./resolve.js"; +import { withDB } from "./with-db.js"; +import { CHUNK_SIZE, writeFileSync } from "./writeFile.js"; + +describe("readRangeSync", () => { + it("reads small chunk-backed files at non-zero offset", async () => { + await withDB((db) => { + writeFileSync(db, "/inline.txt", new TextEncoder().encode("hello world"), {}, () => 1); + + const slice = readRangeSync(db, "/inline.txt", 6, 5); + expect(new TextDecoder().decode(slice)).toBe("world"); + }); + }); + + it("clamps the read at end of file", async () => { + await withDB((db) => { + writeFileSync(db, "/inline.txt", new TextEncoder().encode("abc"), {}, () => 1); + + expect(readRangeSync(db, "/inline.txt", 0, 100).byteLength).toBe(3); + expect(readRangeSync(db, "/inline.txt", 2, 100).byteLength).toBe(1); + expect(readRangeSync(db, "/inline.txt", 3, 100).byteLength).toBe(0); + }); + }); + + it("reads a single chunk window without materializing other chunks", async () => { + await withDB((db) => { + const original = new Uint8Array(CHUNK_SIZE * 3); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + original.fill(3, CHUNK_SIZE * 2); + writeFileSync(db, "/large.bin", original, {}, () => 1); + + const slice = readRangeSync(db, "/large.bin", CHUNK_SIZE + 10, 5); + expect(Array.from(slice)).toEqual([2, 2, 2, 2, 2]); + }); + }); + + it("reads across a chunk boundary", async () => { + await withDB((db) => { + const original = new Uint8Array(CHUNK_SIZE + 100); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE); + writeFileSync(db, "/large.bin", original, {}, () => 1); + + const slice = readRangeSync(db, "/large.bin", CHUNK_SIZE - 2, 4); + expect(Array.from(slice)).toEqual([1, 1, 2, 2]); + }); + }); + + it("returns an empty view past the end of a chunk-backed file", async () => { + await withDB((db) => { + const original = new Uint8Array(CHUNK_SIZE + 1); + original.fill(7); + writeFileSync(db, "/large.bin", original, {}, () => 1); + + expect(readRangeSync(db, "/large.bin", CHUNK_SIZE + 1, 10).byteLength).toBe(0); + }); + }); + + it("assembles a read spanning multiple chunks with partial ends", async () => { + await withDB((db) => { + const original = new Uint8Array(CHUNK_SIZE * 3); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + original.fill(3, CHUNK_SIZE * 2); + writeFileSync(db, "/large.bin", original, {}, () => 1); + + // Start inside chunk 0 and end inside chunk 2, so the range query + // returns all three rows and they must assemble in idx order with + // correct partial-chunk trimming. + const start = CHUNK_SIZE - 3; + const len = CHUNK_SIZE + 6; + const slice = readRangeSync(db, "/large.bin", start, len); + expect(slice.byteLength).toBe(len); + expect(equalBytes(slice, original.subarray(start, start + len))).toBe(true); + }); + }); + + it("reads an entire multi-chunk file byte-for-byte", async () => { + await withDB((db) => { + const original = new Uint8Array(CHUNK_SIZE * 2 + 50); + for (let i = 0; i < original.byteLength; i++) original[i] = i % 251; + writeFileSync(db, "/large.bin", original, {}, () => 1); + + const slice = readRangeSync(db, "/large.bin", 0, original.byteLength); + expect(equalBytes(slice, original)).toBe(true); + }); + }); + + it("compacts around a missing chunk row rather than zero-filling", async () => { + await withDB((db) => { + const original = new Uint8Array(CHUNK_SIZE * 3); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + original.fill(3, CHUNK_SIZE * 2); + writeFileSync(db, "/large.bin", original, {}, () => 1); + const node = resolveInode(db, "/large.bin"); + // Drop the middle chunk row (node.size still reports three + // chunks). The read elides the gap and returns the present + // chunks concatenated, trimmed to what was actually read. + db.run("DELETE FROM vfs_chunks WHERE inode = ? AND idx = 1", node?.inode ?? 0); + + const slice = readRangeSync(db, "/large.bin", 0, CHUNK_SIZE * 3); + expect(slice.byteLength).toBe(CHUNK_SIZE * 2); + expect(slice[0]).toBe(1); + expect(slice[CHUNK_SIZE - 1]).toBe(1); + expect(slice[CHUNK_SIZE]).toBe(3); + expect(slice[CHUNK_SIZE * 2 - 1]).toBe(3); + }); + }); + + it("throws EIO when a referenced chunk's blob bytes are gone", async () => { + await withDB((db) => { + writeFileSync(db, "/inline.txt", new TextEncoder().encode("hello"), {}, () => 1); + // vfs_chunks still references the hash, but the bytes are gone + // (cascade from vfs_blobs) — a read must surface EIO. + db.run("DELETE FROM vfs_blobs"); + + expect(() => readRangeSync(db, "/inline.txt", 0, 5)).toThrowError( + expect.objectContaining({ code: "EIO" }), + ); + }); + }); +}); + +function equalBytes(a: Uint8Array, b: Uint8Array): boolean { + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) { + if (a[i] !== b[i]) return false; + } + return true; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readdir.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/readdir.test.ts new file mode 100644 index 00000000..77b69ea6 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/readdir.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; + +import { mkdir } from "./mkdir.js"; +import { readdir } from "./readdir.js"; +import { withDB } from "./with-db.js"; +import { writeFile } from "./writeFile.js"; + +describe("readdir", () => { + it("returns an empty array for an empty directory", async () => { + await withDB((db) => { + expect(readdir(db, "/")).toEqual([]); + }); + }); + + it("lists files and directories with dirent shape", async () => { + await withDB(async (db) => { + mkdir(db, "/sub", {}, () => 0); + await writeFile(db, "/file.txt", "x", {}, () => 0); + + const entries = readdir(db, "/"); + expect(entries).toHaveLength(2); + expect(entries).toContainEqual({ + name: "file.txt", + parentPath: "/", + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }); + expect(entries).toContainEqual({ + name: "sub", + parentPath: "/", + isFile: false, + isDirectory: true, + isSymbolicLink: false, + }); + }); + }); + + it("sorts entries by name", async () => { + await withDB(async (db) => { + await writeFile(db, "/b", "", {}, () => 0); + await writeFile(db, "/a", "", {}, () => 0); + await writeFile(db, "/c", "", {}, () => 0); + expect(readdir(db, "/").map((e) => e.name)).toEqual(["a", "b", "c"]); + }); + }); + + it("limits committed entries before materializing the result", async () => { + await withDB(async (db) => { + for (const name of ["a", "b", "c"]) await writeFile(db, `/${name}`, "", {}, () => 0); + expect(readdir(db, "/", { limit: 2 }).map((entry) => entry.name)).toEqual(["a", "b"]); + }); + }); + + it("uses the canonical parent path for nested directories", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/b/leaf.txt", "x", {}, () => 0); + + const entries = readdir(db, "/a/b"); + expect(entries).toEqual([ + { + name: "leaf.txt", + parentPath: "/a/b", + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }, + ]); + }); + }); + + it("canonicalizes the parentPath even when called with a non-canonical input", async () => { + await withDB(async (db) => { + mkdir(db, "/a", {}, () => 0); + await writeFile(db, "/a/x", "", {}, () => 0); + const entries = readdir(db, "/a//."); + expect(entries[0]).toMatchObject({ parentPath: "/a" }); + }); + }); + + it("throws ENOENT for a missing path", async () => { + await withDB((db) => { + expect(() => readdir(db, "/missing")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("throws ENOENT when an intermediate segment is missing", async () => { + await withDB((db) => { + expect(() => readdir(db, "/no/such/path")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("includes symlink entries with isSymbolicLink set", async () => { + // resolveInode + readdir originally only filtered file and dir + // rows; symlinks were invisible. The dirent shape now carries + // an explicit isSymbolicLink flag so just-bash and other + // adapters can branch on the type without a follow-up lstat. + const { symlink } = await import("./symlink.js"); + await withDB(async (db) => { + await writeFile(db, "/target", "x", {}, () => 0); + symlink(db, "/target", "/link", () => 0); + const entries = readdir(db, "/"); + const link = entries.find((e) => e.name === "link"); + expect(link).toMatchObject({ + name: "link", + isFile: false, + isDirectory: false, + isSymbolicLink: true, + }); + }); + }); + + it("throws ENOTDIR when called on a file", async () => { + await withDB(async (db) => { + await writeFile(db, "/file.txt", "x", {}, () => 0); + expect(() => readdir(db, "/file.txt")).toThrowError( + expect.objectContaining({ code: "ENOTDIR" }), + ); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readdir.ts b/spikes/349-dofs/vendor/dofs/src/fs/readdir.ts new file mode 100644 index 00000000..dddfa8d1 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/readdir.ts @@ -0,0 +1,84 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; +import { listPendingByParent } from "./writeBuffer.js"; + +export interface WorkspaceDirentResult { + name: string; + parentPath: string; + isFile: boolean; + isDirectory: boolean; + isSymbolicLink: boolean; +} + +interface DirentRow { + name: string; + type: "file" | "dir" | "symlink"; +} + +export interface ReaddirOptions { + /** Maximum committed entries to materialize. Pending entries may extend the result. */ + limit?: number; +} + +export function readdir( + db: Database, + path: string, + options: ReaddirOptions = {}, +): WorkspaceDirentResult[] { + const { path: canonical } = canonicalizePath(path); + const node = resolveInode(db, canonical); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + if (node.type !== "dir") { + throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); + } + + const limit = options.limit; + if (limit !== undefined && (!Number.isSafeInteger(limit) || limit < 0)) { + throw new TypeError("readdir limit must be a non-negative safe integer"); + } + const rows = db.all( + `SELECT d.name AS name, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name + ${limit === undefined ? "" : "LIMIT ?"}`, + ...(limit === undefined ? [node.inode] : [node.inode, limit]), + ); + + const entries = rows.map((row) => ({ + name: row.name, + parentPath: canonical, + isFile: row.type === "file", + isDirectory: row.type === "dir", + isSymbolicLink: row.type === "symlink", + })); + + // Merge in pending-create buffers parented under this directory so + // a `readdir` between FUSE create and release still surfaces the + // file. Skip any whose name already appears in the SQL rows (in + // case a concurrent commit just landed it). + const pending = listPendingByParent(db, node.inode); + if (pending.length > 0) { + const seen = new Set(entries.map((e) => e.name)); + for (const entry of pending) { + if (entry.pending === undefined) continue; + const { leafName } = entry.pending; + if (seen.has(leafName)) continue; + entries.push({ + name: leafName, + parentPath: canonical, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }); + } + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + } + + return entries; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/readlink.ts b/spikes/349-dofs/vendor/dofs/src/fs/readlink.ts new file mode 100644 index 00000000..8b9b5048 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/readlink.ts @@ -0,0 +1,17 @@ +import { createWorkspaceError } from "../errors.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; + +// Return the stored target of a symlink. Does not follow the link. +// Mirrors POSIX semantics: ENOENT for a missing path, EINVAL when +// the path resolves to something that isn't a symlink. +export function readlink(db: Database, path: string): string { + const node = resolveInode(db, path, { followSymlinks: false }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + if (node.type !== "symlink" || node.linkTarget === undefined) { + throw createWorkspaceError("EINVAL", `not a symlink: ${path}`, path); + } + return node.linkTarget; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/rename.ts b/spikes/349-dofs/vendor/dofs/src/fs/rename.ts new file mode 100644 index 00000000..c2692b1e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/rename.ts @@ -0,0 +1,230 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import type { Database } from "../storage.js"; +import { recordDelete } from "../sync/changes.js"; +import { pathOf } from "../sync/paths.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; +import { unlinkDirent } from "./unlink.js"; + +type NodeType = "file" | "dir" | "symlink"; + +export function rename(db: Database, oldPath: string, newPath: string): void { + const { path: oldCanonical } = canonicalizePath(oldPath); + const { parts: newParts, path: newCanonical } = canonicalizePath(newPath); + + if (oldCanonical === "/") { + throw createWorkspaceError("EINVAL", "cannot rename root", oldCanonical); + } + if (newParts.length === 0) { + throw createWorkspaceError("EINVAL", "cannot rename onto root", newCanonical); + } + + assertNotReadOnly(db, oldCanonical); + assertNotReadOnly(db, newCanonical); + + db.transactionSync(() => { + const source = resolveInode(db, oldCanonical, { followSymlinks: false }); + if (source === null) { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + + // Resolve the source's real parent dirent. The parent path is + // resolved with symlinks followed so a request through a symlinked + // directory lands on the real container; the inode is then + // identified by (parent_inode, name) rather than by child_inode so + // a hardlinked source touches only the requested name. + const { parts: oldParts } = canonicalizePath(oldCanonical); + const oldName = oldParts[oldParts.length - 1]; + const oldParentPath = oldParts.length === 1 ? "/" : `/${oldParts.slice(0, -1).join("/")}`; + const oldParent = resolveInode(db, oldParentPath); + if (oldParent === null || oldParent.type !== "dir") { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + const oldParentReal = pathOf(db, oldParent.inode); + if (oldParentReal === null) { + throw createWorkspaceError("ENOENT", `no such path: ${oldCanonical}`, oldCanonical); + } + const oldRealPath = oldParentReal === "/" ? `/${oldName}` : `${oldParentReal}/${oldName}`; + assertNotReadOnly(db, oldRealPath); + + if (oldCanonical === newCanonical) return; + + const newName = newParts[newParts.length - 1]; + const newParentPath = newParts.length === 1 ? "/" : `/${newParts.slice(0, -1).join("/")}`; + const newParent = resolveInode(db, newParentPath); + if (newParent === null || newParent.type !== "dir") { + throw createWorkspaceError( + "ENOENT", + `parent directory missing: ${newCanonical}`, + newCanonical, + ); + } + const newParentReal = pathOf(db, newParent.inode); + if (newParentReal === null) { + throw createWorkspaceError( + "ENOENT", + `parent directory missing: ${newCanonical}`, + newCanonical, + ); + } + const newRealPath = newParentReal === "/" ? `/${newName}` : `${newParentReal}/${newName}`; + assertNotReadOnly(db, newRealPath); + + // A rename whose source and destination resolve to the very same + // dirent (same real parent and name, e.g. through a symlinked path) + // is a true no-op: leave the tree and the change stream untouched. + // This is distinct from renaming one hardlink onto another, where + // the names differ and the source link must still be removed. + if (oldParent.inode === newParent.inode && oldName === newName) return; + + const existing = db.one<{ child_inode: number; type: "file" | "dir" | "symlink" }>( + `SELECT d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? AND d.name = ?`, + newParent.inode, + newName, + ); + + // Authoritative directory self-move guard. It tests the *resolved* + // destination parent inode against the source subtree, so it catches + // a symlinked destination that lands inside the source and allows one + // that resolves outside it. A textual prefix test on the unresolved + // path could do neither and is intentionally absent. + if ( + source.type === "dir" && + renamedSubtreeContains(db, source.inode, oldRealPath, newParent.inode) + ) { + throw createWorkspaceError( + "EINVAL", + `cannot rename a directory into itself: ${oldRealPath}`, + newCanonical, + ); + } + + if (existing !== undefined) { + assertCompatibleOverwrite(source.type, existing.type, newCanonical); + if (existing.type === "dir") { + const childCount = db.scalar( + "SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", + existing.child_inode, + ); + if ((childCount ?? 0) > 0) { + throw createWorkspaceError("ENOTEMPTY", `not empty: ${newCanonical}`, newCanonical); + } + } + // Displace only the destination name. The displaced inode may + // carry other hardlinks (or be the source inode itself), so reap + // its chunks and node row only once the final link disappears. + // Order matters: displace before unlinking the source so a + // hardlink-onto-hardlink rename never momentarily drops to zero + // links and reaps the inode it is about to re-point. + unlinkDirent(db, newParent.inode, newName, existing.child_inode, existing.type); + } + + // Unlink only the source name; a hardlinked source keeps its other + // names alive. + db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", oldParent.inode, oldName); + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + newParent.inode, + newName, + source.inode, + ); + + const rev = incrementRev(db); + // Rename is represented on the wire as old-path tombstones plus + // live entries for the moved inode subtree, so stamp only that + // subtree with the shared rev. Parent directory mtimes are left + // unchanged on purpose; this diverges from POSIX rename(2), but + // avoids treating the old and new parents as content changes. A + // directory move stamps and tombstones its whole subtree in two + // set-based statements; a file or symlink touches one inode and + // one path. + if (source.type === "dir") { + stampRenamedSubtree(db, source.inode, oldRealPath, rev); + } else { + db.run("UPDATE vfs_nodes SET rev = ? WHERE inode = ?", rev, source.inode); + recordDelete(db, rev, oldRealPath); + } + + // Drop cached resolutions for both endpoints. A directory move + // changes every descendant's path, so both sides need a subtree + // drop; a file/symlink move only touches the two leaf paths. The + // destination drop also covers any entry displaced by an overwrite. + if (source.type === "dir") { + invalidateResolveSubtree(db, oldRealPath); + invalidateResolveSubtree(db, newRealPath); + } else { + invalidateResolveExact(db, oldRealPath); + invalidateResolveExact(db, newRealPath); + } + }); +} + +function assertCompatibleOverwrite( + sourceType: NodeType, + existingType: NodeType, + path: string, +): void { + if (sourceType === "dir" && existingType === "dir") return; + if (existingType === "dir") { + throw createWorkspaceError("EISDIR", `cannot overwrite directory: ${path}`, path); + } + if (sourceType === "dir") { + throw createWorkspaceError("ENOTDIR", `cannot overwrite non-directory: ${path}`, path); + } +} + +// Recursive walk of a directory subtree seeded at an inode and its +// path. Descends through directory dirents only, so files and symlinks +// are leaves and each hardlink name yields its own row (matching the +// per-component collection it replaces). Bound as a reusable WITH +// clause whose two placeholders are the seed inode and path; callers +// append their own projection. +const SUBTREE_CTE = `WITH RECURSIVE subtree(inode, type, path) AS ( + SELECT ?, 'dir', ? + UNION ALL + SELECT n.inode, n.type, + CASE WHEN s.path = '/' THEN '/' || d.name ELSE s.path || '/' || d.name END + FROM subtree s + JOIN vfs_dirents d ON d.parent_inode = s.inode + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE s.type = 'dir' +)`; + +function renamedSubtreeContains( + db: Database, + rootInode: number, + rootPath: string, + targetInode: number, +): boolean { + const hit = db.one<{ hit: number }>( + `${SUBTREE_CTE} SELECT 1 AS hit FROM subtree WHERE inode = ? LIMIT 1`, + rootInode, + rootPath, + targetInode, + ); + return hit !== undefined; +} + +// Stamp the shared rev on every inode in the moved subtree and record +// an old-path tombstone for each entry, in two set-based statements +// over the same walk. +function stampRenamedSubtree(db: Database, rootInode: number, rootPath: string, rev: number): void { + db.run( + `${SUBTREE_CTE} UPDATE vfs_nodes SET rev = ? WHERE inode IN (SELECT inode FROM subtree)`, + rootInode, + rootPath, + rev, + ); + db.run( + `${SUBTREE_CTE} INSERT INTO vfs_changes (rev, path, op) SELECT ?, path, 'delete' FROM subtree ORDER BY path`, + rootInode, + rootPath, + rev, + ); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolve.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolve.test.ts new file mode 100644 index 00000000..c8a939e0 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/resolve.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; + +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; +import { withDB } from "./with-db.js"; + +// Convenience: insert a node row and a dirent under a given parent. +// Returns the new inode. type defaults to 'file' so directory tests are +// explicit. +function addNode( + db: Database, + parentInode: number, + name: string, + options: { type?: "file" | "dir"; mode?: number; mtime?: number } = {}, +): number { + const type = options.type ?? "file"; + const mode = options.mode ?? (type === "dir" ? 0o755 : 0o644); + const mtime = options.mtime ?? 0; + db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES (?, ?, ?, 0)", type, mode, mtime); + const inode = db.scalar("SELECT last_insert_rowid()"); + if (inode === undefined) { + throw new Error("failed to allocate inode"); + } + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + name, + inode, + ); + return inode; +} + +describe("resolveInode", () => { + it("resolves the root", async () => { + await withDB( + (db) => { + expect(resolveInode(db, "/")).toEqual({ + inode: ROOT_INODE, + type: "dir", + mode: 0o755, + mtime: 0, + size: 0, + }); + }, + { now: () => 0 }, + ); + }); + + it("resolves a top-level file", async () => { + await withDB((db) => { + const inode = addNode(db, ROOT_INODE, "hello.txt", { type: "file", mode: 0o644, mtime: 99 }); + expect(resolveInode(db, "/hello.txt")).toEqual({ + inode, + type: "file", + mode: 0o644, + mtime: 99, + size: 0, + }); + }); + }); + + it("resolves a nested directory", async () => { + await withDB((db) => { + const dir = addNode(db, ROOT_INODE, "a", { type: "dir" }); + const sub = addNode(db, dir, "b", { type: "dir" }); + const leaf = addNode(db, sub, "c.txt", { type: "file", mtime: 7 }); + expect(resolveInode(db, "/a/b/c.txt")).toEqual({ + inode: leaf, + type: "file", + mode: 0o644, + mtime: 7, + size: 0, + }); + }); + }); + + it("returns null when the final segment is missing", async () => { + await withDB((db) => { + addNode(db, ROOT_INODE, "a", { type: "dir" }); + expect(resolveInode(db, "/a/missing")).toBeNull(); + }); + }); + + it("returns null when an intermediate segment is missing", async () => { + await withDB((db) => { + expect(resolveInode(db, "/no/such/path")).toBeNull(); + }); + }); + + it("returns null when an intermediate segment is a file (not a dir)", async () => { + await withDB((db) => { + addNode(db, ROOT_INODE, "a", { type: "file" }); + expect(resolveInode(db, "/a/b")).toBeNull(); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolve.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolve.ts new file mode 100644 index 00000000..defaa66c --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/resolve.ts @@ -0,0 +1,264 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { lookupResolveCache, storeResolveCache } from "./resolveCache.js"; + +export interface ResolvedInode { + inode: number; + type: "file" | "dir" | "symlink"; + mode: number; + mtime: number; + // Cached file size from vfs_nodes.size. Always 0 for directories + // and symlinks; for files this matches SUM(vfs_chunks.size) for + // the inode. Stat callers consume it directly instead of doing a + // separate aggregate query. + size: number; + // Populated only when type === "symlink". Higher layers (readlink, + // lstat) consume this; resolveInode follows it transparently unless + // the caller asks otherwise. + linkTarget?: string; +} + +export interface ResolveOptions { + // Default true. Pass false to land on a symlink itself — the + // lstat / readlink code paths rely on this. Loops are still + // detected when following. + followSymlinks?: boolean; +} + +interface NodeRow { + inode: number; + type: "file" | "dir" | "symlink"; + mode: number; + mtime: number; + size: number; + link_target: string | null; +} + +interface ChildRow { + child_inode: number; +} + +// Cap the total number of symlinks resolved across a single +// resolveInode() call. Matches Linux's default SYMLOOP_MAX of 40. +const MAX_SYMLINK_FOLLOWS = 40; + +// Walk vfs_dirents from ROOT_INODE down to `path`. Returns null when +// any segment is missing, when an intermediate segment is a file +// (which a real filesystem would surface as ENOTDIR — callers map +// the `null` to the appropriate POSIX code), or when a final-segment +// symlink dangles. Throws ELOOP when a cycle is detected. +// +// `path` is canonicalized internally so callers can pass user input +// directly. Pre-canonicalized paths are also accepted and incur the +// same trivial re-canonicalization cost. +export function resolveInode( + db: Database, + path: string, + options: ResolveOptions = {}, +): ResolvedInode | null { + const followFinal = options.followSymlinks !== false; + const { parts, path: canonical } = canonicalizePath(path); + + // Cache + single-statement CTE serve only cache-eligible reads: + // follow-symlinks resolutions outside a transaction. Everything else + // uses the per-component loop: + // * followSymlinks:false (lstat / readlink / the provider's + // pre-mutation captures) — not cached, and the loop is cheaper + // for these shallow one-shot resolves than the recursive CTE. + // * inside a transaction (every mutation path) — resolves are + // shallow and hot, the CTE competes with the mutation's own + // statements for the plan cache (recompiling it is far dearer + // than the loop), and the cache must not be populated + // mid-transaction anyway (rollback safety). + // Mutations still invalidate the cache; that is independent of this. + if (!followFinal || db.inTransaction) { + return resolveParts(db, parts, followFinal, 0); + } + + // Repeat reads of the same path are served from the per-Database + // cache. Only the path -> inode mapping is cached; re-read the node + // row so mode/size/mtime/type are always current. A stale mapping + // (inode reaped without invalidation) reads back null and falls + // through to a full resolve that re-populates the cache. + const hit = lookupResolveCache(db, canonical); + if (hit !== undefined) { + if (hit.kind === "negative") { + return null; + } + const node = readNode(db, hit.inode); + if (node !== null) { + return toResolved(node); + } + } + + // One recursive-CTE statement resolves the common symlink-free + // case. Any symlink on the path falls back to the per-component loop, + // which follows links and enforces ELOOP; those resolutions are not + // cached (a followed path is an alias whose invalidation can't be + // reasoned about structurally). + const cte = resolveViaCte(db, parts); + if (cte.kind === "symlink") { + return resolveParts(db, parts, followFinal, 0); + } + storeResolveCache(db, canonical, cte.node === null ? null : cte.node.inode); + return cte.node; +} + +interface CteRow { + level: number; + inode: number; + type: "file" | "dir" | "symlink"; + mode: number; + mtime: number; + size: number; + link_target: string | null; +} + +type CteResolution = + // Walk completed with no symlink on the path: `node` is the resolved + // final node, or null when a segment was missing or an intermediate + // was not a directory (both map to null, exactly like the loop). + | { kind: "resolved"; node: ResolvedInode | null } + // A symlink was encountered anywhere on the path (intermediate or + // final). The CTE can't follow links, so the caller must fall back to + // the loop for byte-identical follow / ELOOP / dangling behaviour. + | { kind: "symlink" }; + +// Single-statement path walk. Binds the canonical path segments as a +// JSON array and walks vfs_dirents -> vfs_nodes from ROOT_INODE, one +// level per segment. Descends only through directories (WHERE +// w.type = 'dir'), so a file intermediate stalls the walk (ENOTDIR) +// and a missing dirent produces no row (ENOENT) — both surface as a +// missing level-D row, matching the loop's null. Every node the walk +// touches is returned so the caller can detect any symlink and fall +// back. +function resolveViaCte(db: Database, parts: string[]): CteResolution { + const rows = db.all( + `WITH RECURSIVE + segs(level, name) AS ( + SELECT key, value FROM json_each(?) + ), + walk(level, inode, type, mode, mtime, size, link_target) AS ( + SELECT 0, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target + FROM vfs_nodes n + WHERE n.inode = ? + UNION ALL + SELECT w.level + 1, n.inode, n.type, n.mode, n.mtime, n.size, n.link_target + FROM walk w + JOIN segs s ON s.level = w.level + JOIN vfs_dirents d ON d.parent_inode = w.inode AND d.name = s.name + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE w.type = 'dir' + ) + SELECT level, inode, type, mode, mtime, size, link_target + FROM walk + ORDER BY level`, + JSON.stringify(parts), + ROOT_INODE, + ); + + const depth = parts.length; + let target: CteRow | undefined; + for (const row of rows) { + // Any symlink on the walk (root is level 0 and always a dir) means + // the loop must take over to follow it. + if (row.level >= 1 && row.type === "symlink") { + return { kind: "symlink" }; + } + if (row.level === depth) { + target = row; + } + } + return { + kind: "resolved", + node: target === undefined ? null : toResolved(target), + }; +} + +function toResolved(node: NodeRow): ResolvedInode { + return { + inode: node.inode, + type: node.type, + mode: node.mode, + mtime: node.mtime, + size: node.size, + linkTarget: node.link_target ?? undefined, + }; +} + +function resolveParts( + db: Database, + parts: string[], + followFinal: boolean, + follows: number, +): ResolvedInode | null { + const root = readNode(db, ROOT_INODE); + if (root === null) { + return null; + } + + let current: NodeRow = root; + for (let i = 0; i < parts.length; i++) { + const isFinal = i === parts.length - 1; + if (current.type !== "dir") { + return null; + } + const child = db.one( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + current.inode, + parts[i], + ); + if (child === undefined) { + return null; + } + const next = readNode(db, child.child_inode); + if (next === null) { + return null; + } + // Intermediate symlinks always get followed; final-segment symlinks + // are only followed when the caller wants. A dangling intermediate + // is the same as a missing intermediate (return null). + if (next.type === "symlink" && (!isFinal || followFinal)) { + follows += 1; + if (follows > MAX_SYMLINK_FOLLOWS) { + throw createWorkspaceError("ELOOP", "too many symlinks resolving path"); + } + const target = next.link_target ?? ""; + const resolved = resolveParts(db, canonicalizePath(target).parts, true, follows); + if (resolved === null) { + return null; + } + // Replace the current dirent-resolved node with the followed + // result, then keep walking remaining segments (if any). + current = { + inode: resolved.inode, + type: resolved.type, + mode: resolved.mode, + mtime: resolved.mtime, + size: resolved.size, + link_target: resolved.linkTarget ?? null, + }; + continue; + } + current = next; + } + + return { + inode: current.inode, + type: current.type, + mode: current.mode, + mtime: current.mtime, + size: current.size, + linkTarget: current.link_target ?? undefined, + }; +} + +function readNode(db: Database, inode: number): NodeRow | null { + const row = db.one( + "SELECT inode, type, mode, mtime, size, link_target FROM vfs_nodes WHERE inode = ?", + inode, + ); + return row ?? null; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.test.ts new file mode 100644 index 00000000..b3e61b52 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.test.ts @@ -0,0 +1,248 @@ +import { describe, expect, it } from "vitest"; +import { applyChangesSync } from "../sync/apply.js"; +import { link } from "./link.js"; +import { mkdir } from "./mkdir.js"; +import { rename } from "./rename.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact } from "./resolveCache.js"; +import { rm } from "./rm.js"; +import { symlink } from "./symlink.js"; +import { withDB } from "./with-db.js"; +import { writeFileSync } from "./writeFile.js"; + +const NOW = (): number => 1000; + +function bytesOf(text: string): Uint8Array { + return new TextEncoder().encode(text); +} + +// These prove the path->inode cache never serves a stale result across +// every mutation shape, and that the recursive-CTE resolve's symlink +// fallback matches the per-component loop. They run under both backends +// (node:sqlite and real DO SqlStorage) via withDB, so cache correctness +// is exercised on the shipping storage. +describe("resolve cache + CTE resolve", () => { + it("drops a negative entry the instant the path is created", async () => { + await withDB(async (db) => { + mkdir(db, "/d", { recursive: true }, NOW); + // Prime a negative entry. + expect(resolveInode(db, "/d/f")).toBeNull(); + writeFileSync(db, "/d/f", bytesOf("x"), {}, NOW); + // Must resolve to the new inode, not a stale ENOENT. + expect(resolveInode(db, "/d/f")?.type).toBe("file"); + }); + }); + + it("invalidates a positive entry on unlink, rename, and rmdir", async () => { + await withDB(async (db) => { + mkdir(db, "/p", { recursive: true }, NOW); + + writeFileSync(db, "/p/a", bytesOf("a"), {}, NOW); + expect(resolveInode(db, "/p/a")?.inode).toBeGreaterThan(0); // prime + rm(db, "/p/a", {}); + expect(resolveInode(db, "/p/a")).toBeNull(); + + writeFileSync(db, "/p/b", bytesOf("b"), {}, NOW); + expect(resolveInode(db, "/p/b")).not.toBeNull(); // prime + rename(db, "/p/b", "/p/c"); + expect(resolveInode(db, "/p/b")).toBeNull(); + expect(resolveInode(db, "/p/c")).not.toBeNull(); + + mkdir(db, "/p/sub", { recursive: true }, NOW); + expect(resolveInode(db, "/p/sub")).not.toBeNull(); // prime + rm(db, "/p/sub", {}); + expect(resolveInode(db, "/p/sub")).toBeNull(); + }); + }); + + it("invalidates every descendant path on a directory rename", async () => { + await withDB(async (db) => { + mkdir(db, "/src/inner", { recursive: true }, NOW); + writeFileSync(db, "/src/inner/deep.txt", bytesOf("d"), {}, NOW); + // Prime positives for the whole chain. + const deepInode = resolveInode(db, "/src/inner/deep.txt")?.inode; + expect(resolveInode(db, "/src/inner")).not.toBeNull(); + expect(deepInode).toBeGreaterThan(0); + + rename(db, "/src", "/dst"); + + // Old descendant paths must be gone, not stale positives. + expect(resolveInode(db, "/src/inner/deep.txt")).toBeNull(); + expect(resolveInode(db, "/src/inner")).toBeNull(); + expect(resolveInode(db, "/src")).toBeNull(); + // New paths resolve; the moved file keeps its inode. + expect(resolveInode(db, "/dst/inner/deep.txt")?.inode).toBe(deepInode); + }); + }); + + it("resolves a hardlink's second name to the shared inode", async () => { + await withDB(async (db) => { + mkdir(db, "/h", { recursive: true }, NOW); + writeFileSync(db, "/h/a", bytesOf("shared"), {}, NOW); + const inode = resolveInode(db, "/h/a")?.inode; + // Prime a negative for the not-yet-existing link path. + expect(resolveInode(db, "/h/b")).toBeNull(); + link(db, "/h/a", "/h/b"); + expect(resolveInode(db, "/h/b")?.inode).toBe(inode); + }); + }); + + it("reflects a sync-applied change through cached negative and positive paths", async () => { + await withDB(async (db) => { + mkdir(db, "/s", { recursive: true }, NOW); + writeFileSync(db, "/s/existing", bytesOf("e"), {}, NOW); + // Prime: negative for a dir we will create, positive for a file we + // will delete. + expect(resolveInode(db, "/s/newdir")).toBeNull(); + expect(resolveInode(db, "/s/existing")).not.toBeNull(); + + // applyChangesSync funnels through mkdir (create) and rm (delete), + // both of which invalidate the cache. + applyChangesSync( + db, + [ + { kind: "dir", rev: 1, path: "/s/newdir", mode: 0o755, mtime: 1000 }, + { kind: "delete", rev: 2, path: "/s/existing" }, + ], + new Map(), + ); + + expect(resolveInode(db, "/s/newdir")?.type).toBe("dir"); + expect(resolveInode(db, "/s/existing")).toBeNull(); + + // Apply a symlink over a POPULATED directory: exercises apply's own + // conflict-cleanup branch (removeReplaceableFinalEntry -> + // removeInodeTreeAtPath subtree invalidation) plus symlink-create + // invalidation, with a primed descendant positive. + mkdir(db, "/s/dir/child", { recursive: true }, NOW); + expect(resolveInode(db, "/s/dir/child")).not.toBeNull(); + expect(resolveInode(db, "/s/dir")?.type).toBe("dir"); + applyChangesSync( + db, + [ + { + kind: "symlink", + rev: 3, + path: "/s/dir", + target: "/elsewhere", + mode: 0o777, + mtime: 1000, + }, + ], + new Map(), + ); + // /s/dir is now a symlink; the former descendant no longer resolves. + expect(resolveInode(db, "/s/dir", { followSymlinks: false })?.type).toBe("symlink"); + expect(resolveInode(db, "/s/dir/child")).toBeNull(); + }); + }); + + it("leaves no stale entry after a rolled-back write", async () => { + await withDB(async (db) => { + mkdir(db, "/r", { recursive: true }, NOW); + const rDir = resolveInode(db, "/r")?.inode ?? 0; + writeFileSync(db, "/r/keep", bytesOf("k"), {}, NOW); + const keepInode = resolveInode(db, "/r/keep")?.inode; // prime positive + expect(keepInode).toBeGreaterThan(0); + + // A mutation is (structural write + cache invalidation) inside one + // transaction. This drives that shape with raw statements at a + // single transaction level — the DO backend forbids the nested + // savepoints an outer db.transactionSync around an fs op would use, + // and the cache's rollback-safety is backend-independent anyway. + + // Rolled-back delete: the invalidation drops the entry mid-txn; the + // rollback restores the dirent; the recompute finds it alive again + // (no stale ENOENT). + expect(() => + db.transactionSync(() => { + db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", rDir, "keep"); + invalidateResolveExact(db, "/r/keep"); + throw new Error("boom-delete"); + }), + ).toThrow("boom-delete"); + expect(resolveInode(db, "/r/keep")?.inode).toBe(keepInode); + + // Rolled-back create + inode reuse: population is gated inside a + // transaction, so the doomed inode is never cached for /r/new. A + // broken gate would cache it and, after AUTOINCREMENT reuses the + // number on the next committed create, alias /r/new to /r/other. + expect(resolveInode(db, "/r/new")).toBeNull(); // prime negative + expect(() => + db.transactionSync(() => { + db.run("INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', 420, 0, 0)"); + const inode = db.scalar("SELECT last_insert_rowid() AS v") ?? 0; + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + rDir, + "new", + inode, + ); + invalidateResolveExact(db, "/r/new"); + // Reading inside the txn must not populate the cache (the gate). + expect(resolveInode(db, "/r/new")).not.toBeNull(); + throw new Error("boom-create"); + }), + ).toThrow("boom-create"); + writeFileSync(db, "/r/other", bytesOf("o"), {}, NOW); + expect(resolveInode(db, "/r/new")).toBeNull(); + expect(resolveInode(db, "/r/other")).not.toBeNull(); + }); + }); + + it("falls back to the loop for symlinks, resolving identically", async () => { + await withDB(async (db) => { + mkdir(db, "/target/sub", { recursive: true }, NOW); + writeFileSync(db, "/target/sub/file.txt", bytesOf("hello"), {}, NOW); + const realInode = resolveInode(db, "/target/sub/file.txt")?.inode; + + // Intermediate symlink: /link -> /target. + symlink(db, "/target", "/link", NOW); + // Following through the link reaches the real file inode. + expect(resolveInode(db, "/link/sub/file.txt")?.inode).toBe(realInode); + // Following the link itself lands on the directory it points at. + expect(resolveInode(db, "/link")?.type).toBe("dir"); + // lstat (no follow) lands on the symlink node itself. + const withoutFollow = resolveInode(db, "/link", { followSymlinks: false }); + expect(withoutFollow?.type).toBe("symlink"); + expect(withoutFollow?.linkTarget).toBe("/target"); + + // Dangling symlink: ENOENT when followed, symlink node when not. + symlink(db, "/nope", "/dangling", NOW); + expect(resolveInode(db, "/dangling")).toBeNull(); + expect(resolveInode(db, "/dangling", { followSymlinks: false })?.type).toBe("symlink"); + }); + }); + + it("drops a negative primed beneath a path that a new symlink makes resolvable", async () => { + await withDB(async (db) => { + mkdir(db, "/target", { recursive: true }, NOW); + writeFileSync(db, "/target/x", bytesOf("x"), {}, NOW); + const realInode = resolveInode(db, "/target/x")?.inode; + // Prime a negative for a path beneath where the link will land + // (no symlink on the path yet, so the negative is cached). + expect(resolveInode(db, "/link/x")).toBeNull(); + + // Creating the symlink makes "/link/x" resolvable through it; the + // subtree invalidation must drop the stale negative beneath it. + symlink(db, "/target", "/link", NOW); + expect(resolveInode(db, "/link/x")?.inode).toBe(realInode); + }); + }); + + it("does not cache a negative when a symlink on the path forces the loop", async () => { + await withDB(async (db) => { + mkdir(db, "/target", { recursive: true }, NOW); + symlink(db, "/target", "/link", NOW); + // Resolve through the link before the leaf exists: the CTE bails + // to the loop and must cache nothing for the aliased path. + expect(resolveInode(db, "/link/x")).toBeNull(); + + // Create the real leaf. This invalidates "/target/x", not the + // "/link/x" alias — so a negative wrongly cached by the bail would + // linger and this read would still see null. + writeFileSync(db, "/target/x", bytesOf("x"), {}, NOW); + expect(resolveInode(db, "/link/x")?.type).toBe("file"); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.ts b/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.ts new file mode 100644 index 00000000..0b40f054 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/resolveCache.ts @@ -0,0 +1,129 @@ +// Per-Database path -> inode resolution cache. +// +// Maps a canonical absolute path (as produced by +// canonicalizePath().path) to the inode that resolveInode(path, +// { followSymlinks: true }) lands on, or a NEGATIVE marker when the +// path does not resolve. It turns repeat stat/exists/read of the same +// path into an O(1) lookup instead of an O(depth) walk. +// +// Deliberately narrow, for correctness: +// +// * Only the path -> inode MAPPING is cached. resolveInode always +// re-reads the node row on a hit, so content changes (chmod, +// size/mtime, type) are never served stale — only structural +// mutations that move a dirent can invalidate an entry. +// +// * Only symlink-free resolutions are cached. Following a symlink +// makes the cached path an alias of the target whose invalidation +// can't be reasoned about from the path alone, so resolveInode +// stores nothing when a symlink was traversed. +// +// * Population is gated on Database.inTransaction: entries are only +// written outside a transaction, so a rolled-back mutation can +// never leave a positive/negative entry reflecting uncommitted +// state. Mutations invalidate (drop) freely — dropping is safe +// under rollback because the worst case is a recompute. +// +// The cache is per-Database (WeakMap) and bounded (LRU by Map +// insertion order, same discipline as blobCache). + +import type { Database } from "../storage.js"; + +// Sentinel value for "this path resolves to nothing" (ENOENT/ENOTDIR). +const NEGATIVE = -1; + +// Upper bound on cached paths per Database. Entries are tiny (a string +// key and a number), so this caps memory at a few MB while covering +// the working set of a busy tree. +const MAX_ENTRIES = 8192; + +// Keyed by the Database instance, so correctness assumes exactly one +// Database wraps each SqlStorage. Two Databases over the same storage +// would hold independent caches and could serve each other stale +// results; the DO owns a single Database, which upholds this. +const caches = new WeakMap>(); + +function cacheFor(db: Database): Map { + let cache = caches.get(db); + if (cache === undefined) { + cache = new Map(); + caches.set(db, cache); + } + return cache; +} + +export type ResolveCacheHit = { kind: "inode"; inode: number } | { kind: "negative" }; + +// Look up a canonical path. Returns undefined on a miss, a positive +// inode hit, or a negative (known-absent) hit. Bumps LRU recency. +export function lookupResolveCache( + db: Database, + canonicalPath: string, +): ResolveCacheHit | undefined { + const cache = cacheFor(db); + const value = cache.get(canonicalPath); + if (value === undefined) { + return undefined; + } + // Move to most-recent position for LRU eviction. + cache.delete(canonicalPath); + cache.set(canonicalPath, value); + return value === NEGATIVE ? { kind: "negative" } : { kind: "inode", inode: value }; +} + +// Cache a resolution. `inode === null` records a negative entry. No-op +// while a transaction is active so the cache never reflects +// uncommitted state (rollback safety). +export function storeResolveCache(db: Database, canonicalPath: string, inode: number | null): void { + if (db.inTransaction) { + return; + } + const cache = cacheFor(db); + cache.set(canonicalPath, inode === null ? NEGATIVE : inode); + while (cache.size > MAX_ENTRIES) { + const oldest = cache.keys().next(); + if (oldest.done === true) { + break; + } + cache.delete(oldest.value); + } +} + +// Drop the entry for exactly `canonicalPath`. Use after a mutation +// that changes a single leaf's existence without affecting anything +// beneath it: creating/removing a file, symlink, hardlink, or an +// empty directory. O(1). +export function invalidateResolveExact(db: Database, canonicalPath: string): void { + const cache = caches.get(db); + cache?.delete(canonicalPath); +} + +// Drop `canonicalPath` and every entry beneath it (keys prefixed +// `canonicalPath + "/"`). Use when a mutation changes a whole subtree's +// resolution: a recursive delete, any directory rename (every +// descendant's path changes), a structural subtree replacement, or a +// symlink create (paths *through* the new link become resolvable, so +// stale negatives beneath it must go). Root ("/") clears everything. +export function invalidateResolveSubtree(db: Database, canonicalPath: string): void { + const cache = caches.get(db); + if (cache === undefined || cache.size === 0) { + return; + } + if (canonicalPath === "/") { + cache.clear(); + return; + } + cache.delete(canonicalPath); + const prefix = `${canonicalPath}/`; + for (const key of cache.keys()) { + if (key.startsWith(prefix)) { + cache.delete(key); + } + } +} + +// Drop the entire cache for a Database. Used by tests and available as +// a blunt reset. +export function clearResolveCache(db: Database): void { + caches.get(db)?.clear(); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/rm.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/rm.test.ts new file mode 100644 index 00000000..5acd002d --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/rm.test.ts @@ -0,0 +1,256 @@ +import { describe, expect, it } from "vitest"; + +import type { Database } from "../storage.js"; +import { mkdir } from "./mkdir.js"; +import { readdir } from "./readdir.js"; +import { readFile } from "./readFile.js"; +import { resolveInode } from "./resolve.js"; +import { rm } from "./rm.js"; +import { symlink } from "./symlink.js"; +import { withDB } from "./with-db.js"; +import { writeFile } from "./writeFile.js"; + +interface ChangeRow { + rev: number; + path: string; + op: string; +} + +function listChanges(db: Database): ChangeRow[] { + return db.all("SELECT rev, path, op FROM vfs_changes ORDER BY rev"); +} + +function countBlobs(db: Database): number { + return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; +} + +describe("rm", () => { + it("removes a single file", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "hi", {}, () => 0); + rm(db, "/a.txt", {}); + expect(resolveInode(db, "/a.txt")).toBeNull(); + expect(readdir(db, "/")).toEqual([]); + }); + }); + + it("records a tombstone for the removed path", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "hi", {}, () => 0); + rm(db, "/a.txt", {}); + expect(listChanges(db)).toEqual([expect.objectContaining({ path: "/a.txt", op: "delete" })]); + }); + }); + + it("records tombstones at the resolved path through intermediate symlinks", async () => { + await withDB(async (db) => { + mkdir(db, "/real", {}, () => 0); + await writeFile(db, "/real/file.txt", "content", {}, () => 0); + symlink(db, "/real", "/link", () => 0); + + rm(db, "/link/file.txt", {}); + + expect(listChanges(db)).toContainEqual( + expect.objectContaining({ path: "/real/file.txt", op: "delete" }), + ); + expect(listChanges(db)).not.toContainEqual( + expect.objectContaining({ path: "/link/file.txt", op: "delete" }), + ); + }); + }); + + it("bumps rev once per call", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "hi", {}, () => 0); + const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + rm(db, "/a.txt", {}); + const after = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + expect(after).toBe(before + 1); + }); + }); + + it("leaves orphan blob rows alive for gc()", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "unique-content", {}, () => 0); + const before = countBlobs(db); + expect(before).toBe(1); + rm(db, "/a.txt", {}); + expect(countBlobs(db)).toBe(1); + }); + }); + + it("removes a symlink itself rather than its target", async () => { + await withDB(async (db) => { + await writeFile(db, "/target.txt", "still here", {}, () => 0); + symlink(db, "/target.txt", "/link.txt", () => 0); + + rm(db, "/link.txt", {}); + + expect(resolveInode(db, "/link.txt", { followSymlinks: false })).toBeNull(); + expect(resolveInode(db, "/target.txt")).not.toBeNull(); + }); + }); + + it("recursive rm does not follow symlinks out of the removed tree", async () => { + await withDB(async (db) => { + await writeFile(db, "/outside.txt", "still here", {}, () => 0); + mkdir(db, "/d", {}, () => 0); + symlink(db, "/outside.txt", "/d/link.txt", () => 0); + + rm(db, "/d", { recursive: true }); + + expect(resolveInode(db, "/d", { followSymlinks: false })).toBeNull(); + expect(resolveInode(db, "/outside.txt")).not.toBeNull(); + }); + }); + + it("removes a dangling symlink", async () => { + await withDB((db) => { + symlink(db, "/missing", "/dangling", () => 0); + + rm(db, "/dangling", {}); + + expect(resolveInode(db, "/dangling", { followSymlinks: false })).toBeNull(); + }); + }); + + it("rejects ENOENT for a missing path", async () => { + await withDB((db) => { + expect(() => rm(db, "/missing", {})).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("force swallows ENOENT", async () => { + await withDB((db) => { + expect(() => rm(db, "/missing", { force: true })).not.toThrow(); + expect(listChanges(db)).toEqual([]); + }); + }); + + it("rejects EPERM on root", async () => { + await withDB((db) => { + expect(() => rm(db, "/", {})).toThrowError(expect.objectContaining({ code: "EPERM" })); + expect(() => rm(db, "/", { recursive: true })).toThrowError( + expect.objectContaining({ code: "EPERM" }), + ); + expect(() => rm(db, "/", { recursive: true, force: true })).toThrowError( + expect.objectContaining({ code: "EPERM" }), + ); + }); + }); + + it("removes an empty directory without recursive", async () => { + await withDB((db) => { + mkdir(db, "/d", {}, () => 0); + rm(db, "/d", {}); + expect(resolveInode(db, "/d")).toBeNull(); + }); + }); + + it("rejects ENOTEMPTY on a non-empty directory without recursive", async () => { + await withDB(async (db) => { + mkdir(db, "/d", {}, () => 0); + await writeFile(db, "/d/a", "x", {}, () => 0); + expect(() => rm(db, "/d", {})).toThrowError(expect.objectContaining({ code: "ENOTEMPTY" })); + }); + }); + + it("recursive removes a directory tree", async () => { + await withDB(async (db) => { + mkdir(db, "/d/e/f", { recursive: true }, () => 0); + await writeFile(db, "/d/a", "x", {}, () => 0); + await writeFile(db, "/d/e/b", "y", {}, () => 0); + await writeFile(db, "/d/e/f/c", "z", {}, () => 0); + rm(db, "/d", { recursive: true }); + expect(resolveInode(db, "/d")).toBeNull(); + expect(resolveInode(db, "/d/a")).toBeNull(); + expect(resolveInode(db, "/d/e/f/c")).toBeNull(); + expect(readdir(db, "/")).toEqual([]); + }); + }); + + it("recursive removes a symlink to a directory without deleting its target", async () => { + await withDB(async (db) => { + mkdir(db, "/target/sub", { recursive: true }, () => 0); + await writeFile(db, "/target/sub/file.txt", "content", {}, () => 0); + symlink(db, "/target", "/link", () => 0); + + rm(db, "/link", { recursive: true }); + + expect(resolveInode(db, "/link", { followSymlinks: false })).toBeNull(); + expect(await readFile(db, "/target/sub/file.txt", "utf8")).toBe("content"); + }); + }); + + it("recursive records one tombstone per removed path", async () => { + await withDB(async (db) => { + mkdir(db, "/d", {}, () => 0); + await writeFile(db, "/d/a", "x", {}, () => 0); + await writeFile(db, "/d/b", "y", {}, () => 0); + rm(db, "/d", { recursive: true }); + const paths = listChanges(db) + .map((r) => r.path) + .sort(); + expect(paths).toEqual(["/d", "/d/a", "/d/b"]); + }); + }); + + it("recursive records resolved subtree tombstones through intermediate symlinks", async () => { + await withDB(async (db) => { + mkdir(db, "/real/dir", { recursive: true }, () => 0); + await writeFile(db, "/real/dir/a", "x", {}, () => 0); + await writeFile(db, "/real/dir/b", "y", {}, () => 0); + symlink(db, "/real", "/link", () => 0); + + rm(db, "/link/dir", { recursive: true }); + + const paths = listChanges(db) + .map((r) => r.path) + .sort(); + expect(paths).toEqual(expect.arrayContaining(["/real/dir", "/real/dir/a", "/real/dir/b"])); + expect(paths).not.toEqual( + expect.arrayContaining(["/link/dir", "/link/dir/a", "/link/dir/b"]), + ); + }); + }); + + it("recursive still bumps rev only once for the whole tree", async () => { + await withDB(async (db) => { + mkdir(db, "/d", {}, () => 0); + await writeFile(db, "/d/a", "x", {}, () => 0); + await writeFile(db, "/d/b", "y", {}, () => 0); + const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + rm(db, "/d", { recursive: true }); + const after = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + expect(after).toBe(before + 1); + }); + }); + + it("recursive cleans up chunk rows for removed files", async () => { + await withDB(async (db) => { + mkdir(db, "/d", {}, () => 0); + await writeFile(db, "/d/a", "first", {}, () => 0); + await writeFile(db, "/d/b", "second", {}, () => 0); + rm(db, "/d", { recursive: true }); + const chunkRows = db.scalar("SELECT COUNT(*) FROM vfs_chunks") ?? 0; + expect(chunkRows).toBe(0); + }); + }); + + it("force is idempotent on missing intermediate segments", async () => { + await withDB((db) => { + expect(() => rm(db, "/no/such/path", { force: true })).not.toThrow(); + }); + }); + + it("accepts recursive: false / force: false for node:fs/promises parity", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "hello", {}, () => 0); + // boolean false should be accepted by the type and behave as default. + rm(db, "/a.txt", { recursive: false, force: false }); + expect(resolveInode(db, "/a.txt")).toBeNull(); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/rm.ts b/spikes/349-dofs/vendor/dofs/src/fs/rm.ts new file mode 100644 index 00000000..cf82fb8e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/rm.ts @@ -0,0 +1,181 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import type { Database } from "../storage.js"; +import { recordDelete } from "../sync/changes.js"; +import { pathOf } from "../sync/paths.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { resolveInode } from "./resolve.js"; +import { invalidateResolveExact, invalidateResolveSubtree } from "./resolveCache.js"; +import { unlinkDirent } from "./unlink.js"; + +export interface RmOptions { + recursive?: boolean; + force?: boolean; +} + +interface DirChild { + name: string; + child_inode: number; + type: "file" | "dir" | "symlink"; +} + +// Walk a directory subtree post-order so we delete leaves before +// parents. Yields each node together with the parent inode and name +// the walk already knows, so the caller can unlink the dirent by +// (parent, name) without re-resolving the parent from root. The caller +// appends one tombstone per yielded path and clears vfs_chunks for +// file inodes. +function* walkPostOrder( + db: Database, + rootInode: number, + rootPath: string, + rootParentInode: number, + rootName: string, +): Generator<{ + path: string; + inode: number; + type: "file" | "dir" | "symlink"; + parentInode: number; + name: string; +}> { + // Stack-based DFS to avoid recursion limits on deep trees. + type Frame = { + inode: number; + path: string; + type: "file" | "dir" | "symlink"; + parentInode: number; + name: string; + expanded: boolean; + }; + const stack: Frame[] = [ + { + inode: rootInode, + path: rootPath, + type: "dir", + parentInode: rootParentInode, + name: rootName, + expanded: false, + }, + ]; + + while (stack.length > 0) { + const top = stack[stack.length - 1]; + if (top.type !== "dir" || top.expanded) { + stack.pop(); + yield { + path: top.path, + inode: top.inode, + type: top.type, + parentInode: top.parentInode, + name: top.name, + }; + continue; + } + top.expanded = true; + const children = db.all( + `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ? + ORDER BY d.name`, + top.inode, + ); + for (const child of children) { + const childPath = top.path === "/" ? `/${child.name}` : `${top.path}/${child.name}`; + stack.push({ + inode: child.child_inode, + path: childPath, + type: child.type, + parentInode: top.inode, + name: child.name, + expanded: false, + }); + } + } +} + +export function rm(db: Database, path: string, options: RmOptions): void { + const { parts, path: canonical } = canonicalizePath(path); + + if (parts.length === 0) { + // The workspace root is structural; refuse to delete it even with + // recursive+force. Matches the doc's example. + throw createWorkspaceError("EPERM", `cannot remove the root directory`, canonical); + } + + // assertNotReadOnly uses the symmetric overlap predicate, so a + // recursive rm of an ancestor whose subtree contains a read-only + // mount root is caught here without walking the tree. + assertNotReadOnly(db, canonical); + + const force = options.force === true; + const recursive = options.recursive === true; + + db.transactionSync(() => { + const node = resolveInode(db, canonical, { followSymlinks: false }); + if (node === null) { + if (force) return; + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + + if (node.type === "dir" && !recursive) { + const childCount = db.scalar( + "SELECT COUNT(*) FROM vfs_dirents WHERE parent_inode = ?", + node.inode, + ); + if ((childCount ?? 0) > 0) { + throw createWorkspaceError("ENOTEMPTY", `directory not empty: ${canonical}`, canonical); + } + } + + // Resolve the entry's real path from its parent rather than from + // the inode: a hardlinked file has several names, and pathOf would + // pick an arbitrary one. Following symlinks on the parent lets a + // request through a symlinked directory land on the real container + // while still removing exactly the requested name. + const name = parts[parts.length - 1]; + const parentPath = parts.length === 1 ? "/" : `/${parts.slice(0, -1).join("/")}`; + const parent = resolveInode(db, parentPath); + if (parent === null || parent.type !== "dir") { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const parentReal = pathOf(db, parent.inode); + if (parentReal === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + const realPath = parentReal === "/" ? `/${name}` : `${parentReal}/${name}`; + assertNotReadOnly(db, realPath); + + const rev = incrementRev(db); + + if (node.type !== "dir" || !recursive) { + // Single entry removal — file, symlink, or empty directory. A + // file inode may have multiple dirents (hardlinks), so remove + // only the requested name and reap chunks/node after the final + // link disappears. `parent` is already resolved above, so unlink + // by (parent, name) directly rather than re-resolving. The + // tombstone is recorded at the resolved real path so sync sees + // the move-aware location. + unlinkDirent(db, parent.inode, name, node.inode, node.type); + recordDelete(db, rev, realPath); + // A single removed entry is a file, symlink, or empty directory: + // no cached descendants to worry about, so drop it exact. + invalidateResolveExact(db, realPath); + return; + } + + // Recursive directory removal. Walk leaves first so each delete + // sees an empty parent by the time we get to it. File entries may + // be hardlinked outside this subtree, so delete by path rather + // than by child inode. The walk carries each node's parent inode + // and name, so unlinkDirent needs no per-node re-resolve from root. + for (const entry of walkPostOrder(db, node.inode, realPath, parent.inode, name)) { + unlinkDirent(db, entry.parentInode, entry.name, entry.inode, entry.type); + recordDelete(db, rev, entry.path); + } + // The whole subtree under realPath is gone; one subtree drop covers + // every descendant's cached resolution. + invalidateResolveSubtree(db, realPath); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/stat.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/stat.test.ts new file mode 100644 index 00000000..8878bfd9 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/stat.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; + +import { mkdir } from "./mkdir.js"; +import { lstat, stat } from "./stat.js"; +import { symlink } from "./symlink.js"; +import { withDB } from "./with-db.js"; +import { writeFileSync } from "./writeFile.js"; + +const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s); + +describe("stat", () => { + it("reports a regular file", async () => { + await withDB((db) => { + writeFileSync(db, "/a.txt", utf8("hello"), { mode: 0o644 }, () => 1234); + const s = stat(db, "/a.txt"); + expect(s).toMatchObject({ + name: "a.txt", + mode: 0o644, + size: 5, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + mtime: 1234, + }); + }); + }); + + it("reports a directory", async () => { + await withDB((db) => { + mkdir(db, "/d", { mode: 0o700 }, () => 0); + const s = stat(db, "/d"); + expect(s).toMatchObject({ + name: "d", + mode: 0o700, + size: 0, + isFile: false, + isDirectory: true, + isSymbolicLink: false, + }); + }); + }); + + it("follows symlinks", async () => { + // stat() on a symlink reports the target. The link itself is + // observable only via lstat(). + await withDB((db) => { + writeFileSync(db, "/target", utf8("hello"), { mode: 0o600 }, () => 0); + symlink(db, "/target", "/link", () => 0); + const s = stat(db, "/link"); + expect(s.isFile).toBe(true); + expect(s.isSymbolicLink).toBe(false); + expect(s.mode).toBe(0o600); + expect(s.size).toBe(5); + }); + }); + + it("throws ENOENT for a missing path", async () => { + await withDB((db) => { + expect(() => stat(db, "/missing")).toThrowError(expect.objectContaining({ code: "ENOENT" })); + }); + }); +}); + +describe("lstat", () => { + it("reports a symlink without following it", async () => { + // POSIX lstat: size is the byte length of the stored target, + // mode is the symlink node's own mode (always 0o777 today). + await withDB((db) => { + writeFileSync(db, "/target", utf8("hello world"), {}, () => 0); + symlink(db, "/target", "/link", () => 0); + const s = lstat(db, "/link"); + expect(s.isSymbolicLink).toBe(true); + expect(s.isFile).toBe(false); + expect(s.isDirectory).toBe(false); + expect(s.size).toBe("/target".length); + expect(s.mode).toBe(0o777); + }); + }); + + it("matches stat for non-symlink nodes", async () => { + await withDB((db) => { + writeFileSync(db, "/a.txt", utf8("hi"), {}, () => 0); + const s = stat(db, "/a.txt"); + const l = lstat(db, "/a.txt"); + expect(l).toEqual(s); + }); + }); + + it("throws ENOENT for a missing path", async () => { + await withDB((db) => { + expect(() => lstat(db, "/missing")).toThrowError(expect.objectContaining({ code: "ENOENT" })); + }); + }); + + it("returns the dangling symlink itself when the target is missing", async () => { + await withDB((db) => { + symlink(db, "/nowhere", "/dangling", () => 0); + const s = lstat(db, "/dangling"); + expect(s.isSymbolicLink).toBe(true); + expect(s.size).toBe("/nowhere".length); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/stat.ts b/spikes/349-dofs/vendor/dofs/src/fs/stat.ts new file mode 100644 index 00000000..4ce2734f --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/stat.ts @@ -0,0 +1,86 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { resolveInode } from "./resolve.js"; +import { getPendingWriteBufferByPath, getWriteBuffer } from "./writeBuffer.js"; + +export interface WorkspaceStatResult { + name: string; + // Inode of the resolved node, or 0 for a pending-create file that + // has no inode yet. Exposed so provider stat surfaces can read the + // inode from the same resolve instead of walking the path twice. + inode: number; + mode: number; + mtime: number; + size: number; + isFile: boolean; + isDirectory: boolean; + // True when the result describes a symlink itself rather than + // its target. Only lstat() can produce a true value here; stat() + // follows links and reports the final node. + isSymbolicLink: boolean; +} + +export function stat(db: Database, path: string): WorkspaceStatResult { + return statShared(db, path, true); +} + +// Like stat, but does not follow a trailing symlink. Mirrors POSIX +// lstat: the returned size for a symlink is the byte length of the +// stored target, and mode is the symlink node's own mode. +export function lstat(db: Database, path: string): WorkspaceStatResult { + return statShared(db, path, false); +} + +function statShared(db: Database, path: string, followFinal: boolean): WorkspaceStatResult { + const { name, path: canonical } = canonicalizePath(path); + // Pending-create files have no inode yet; serve the buffer state + // so callers between create and release see the file as it stands. + // Pending creates never apply to symlinks, so this is safe to run + // even on the lstat path — a hit here always corresponds to a + // file mid-open. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined && pending.pending !== undefined) { + return { + name, + // A pending create has no inode until releaseWriteBufferSync + // commits it; report 0, which yields nlink 1 in the provider. + inode: 0, + mode: pending.mode & 0o7777, + mtime: pending.pending.mtime, + size: pending.size, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + }; + } + const node = resolveInode(db, path, { followSymlinks: followFinal }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + + const isDirectory = node.type === "dir"; + const isFile = node.type === "file"; + const isSymbolicLink = node.type === "symlink"; + let size = 0; + if (isFile) { + // Prefer the in-memory buffer when an open file has unflushed + // writes; otherwise read the cached size off vfs_nodes that + // resolveInode just loaded for us, no extra SQL. + const buffered = getWriteBuffer(db, node.inode); + size = buffered?.dirty ? buffered.size : node.size; + } else if (isSymbolicLink) { + size = (node.linkTarget ?? "").length; + } + + return { + name, + inode: node.inode, + mode: node.mode, + mtime: node.mtime, + size, + isFile, + isDirectory, + isSymbolicLink, + }; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/symlink.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/symlink.test.ts new file mode 100644 index 00000000..f4317a3e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/symlink.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from "vitest"; + +import { mkdir } from "./mkdir.js"; +import { invalidateReadOnlyMountCache } from "./mount-guard.js"; +import { readlink } from "./readlink.js"; +import { resolveInode } from "./resolve.js"; +import { symlink } from "./symlink.js"; +import { withDB } from "./with-db.js"; +import { writeFile } from "./writeFile.js"; + +describe("symlink", () => { + it("creates a symlink node with the requested target", async () => { + await withDB((db) => { + symlink(db, "/target", "/link", () => 5000); + expect(readlink(db, "/link")).toBe("/target"); + }); + }); + + it("rejects EEXIST when the path already exists", async () => { + await withDB(async (db) => { + await writeFile(db, "/a", "x", {}, () => 0); + expect(() => symlink(db, "/wherever", "/a", () => 0)).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + }); + }); + + it("rejects ENOENT when the parent directory is missing", async () => { + await withDB((db) => { + expect(() => symlink(db, "/t", "/no/such/parent/link", () => 0)).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("bumps rev and records mtime", async () => { + await withDB((db) => { + const before = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + symlink(db, "/t", "/link", () => 4242); + const after = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + expect(after).toBe(before + 1); + const row = db.one<{ mtime: number; rev: number; link_target: string }>( + "SELECT n.mtime, n.rev, n.link_target FROM vfs_nodes n JOIN vfs_dirents d ON d.child_inode = n.inode WHERE d.name = ?", + "link", + ); + expect(row?.mtime).toBe(4242); + expect(row?.rev).toBe(after); + expect(row?.link_target).toBe("/t"); + }); + }); + + it("rejects EROFS when the link path overlaps a read-only mount root", async () => { + // Same guard that writeFile and mkdir consult — a symlink that + // lands inside a read-only mount is a write that the indexer + // must reject before the node table sees it. + await withDB((db) => { + db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES ('/mnt', 'r2', 'read-only')"); + invalidateReadOnlyMountCache(db); + expect(() => symlink(db, "/elsewhere", "/mnt/link", () => 0)).toThrowError( + expect.objectContaining({ code: "EROFS" }), + ); + }); + }); + + it("creates a symlink inside a nested directory", async () => { + await withDB((db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + symlink(db, "/t", "/a/b/link", () => 0); + expect(readlink(db, "/a/b/link")).toBe("/t"); + }); + }); +}); + +describe("readlink", () => { + it("returns the stored target", async () => { + await withDB((db) => { + symlink(db, "/some/target", "/link", () => 0); + expect(readlink(db, "/link")).toBe("/some/target"); + }); + }); + + it("throws ENOENT when the path does not exist", async () => { + await withDB((db) => { + expect(() => readlink(db, "/missing")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("throws EINVAL when the path is not a symlink", async () => { + await withDB(async (db) => { + await writeFile(db, "/file", "x", {}, () => 0); + expect(() => readlink(db, "/file")).toThrowError(expect.objectContaining({ code: "EINVAL" })); + }); + }); +}); + +describe("resolveInode + symlinks", () => { + it("resolveInode follows symlinks by default", async () => { + await withDB(async (db) => { + await writeFile(db, "/target", "content", {}, () => 0); + symlink(db, "/target", "/link", () => 0); + const node = resolveInode(db, "/link"); + // The target is a file, so following lands on the file node. + expect(node?.type).toBe("file"); + }); + }); + + it("resolveInode with followSymlinks=false returns the link itself", async () => { + await withDB(async (db) => { + await writeFile(db, "/target", "content", {}, () => 0); + symlink(db, "/target", "/link", () => 0); + const node = resolveInode(db, "/link", { followSymlinks: false }); + expect(node?.type).toBe("symlink"); + }); + }); + + it("follows a chain of symlinks", async () => { + await withDB(async (db) => { + await writeFile(db, "/target", "content", {}, () => 0); + symlink(db, "/target", "/a", () => 0); + symlink(db, "/a", "/b", () => 0); + symlink(db, "/b", "/c", () => 0); + expect(resolveInode(db, "/c")?.type).toBe("file"); + }); + }); + + it("returns null on a dangling symlink when following", async () => { + await withDB((db) => { + symlink(db, "/no/such/target", "/dangling", () => 0); + expect(resolveInode(db, "/dangling")).toBeNull(); + }); + }); + + it("returns the symlink node on a dangling symlink with followSymlinks=false", async () => { + await withDB((db) => { + symlink(db, "/no/such/target", "/dangling", () => 0); + const node = resolveInode(db, "/dangling", { followSymlinks: false }); + expect(node?.type).toBe("symlink"); + }); + }); + + it("throws ELOOP on a cycle", async () => { + await withDB((db) => { + symlink(db, "/b", "/a", () => 0); + symlink(db, "/a", "/b", () => 0); + expect(() => resolveInode(db, "/a")).toThrowError(expect.objectContaining({ code: "ELOOP" })); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/symlink.ts b/spikes/349-dofs/vendor/dofs/src/fs/symlink.ts new file mode 100644 index 00000000..cbd6f2e9 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/symlink.ts @@ -0,0 +1,81 @@ +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveSubtree } from "./resolveCache.js"; + +// Create a symlink node. The target is stored as-is — it can be a +// relative or absolute path, dangling or live. resolveInode follows +// it transparently when callers walk through this entry. +export function symlink(db: Database, target: string, path: string, now: () => number): void { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EEXIST", "cannot symlink onto root", canonical); + } + assertNotReadOnly(db, canonical); + + db.transactionSync(() => { + // Walk to the parent dirent. Intermediate segments must be real + // directories; we don't auto-create them. + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + parts[i], + ); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one<{ inode: number; type: "file" | "dir" | "symlink" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + child.child_inode, + ); + if (next === undefined || next.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = next.inode; + } + + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + + const rev = incrementRev(db); + const mtime = now(); + // RETURNING folds the rowid read into the INSERT. + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, link_target) VALUES ('symlink', ?, ?, ?, ?) RETURNING inode", + 0o777, + mtime, + rev, + target, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + const inode = row.inode; + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + leafName, + inode, + ); + // Subtree, not exact: paths *through* the new link (e.g. /s/x when + // /s -> a populated dir) now resolve, so any cached negative + // beneath the link must be dropped. + invalidateResolveSubtree(db, canonical); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/unlink.ts b/spikes/349-dofs/vendor/dofs/src/fs/unlink.ts new file mode 100644 index 00000000..7a7d33c2 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/unlink.ts @@ -0,0 +1,34 @@ +import type { Database } from "../storage.js"; + +type NodeType = "file" | "dir" | "symlink"; + +// Remove a single (parent, name) dirent and reap the child inode's +// node and chunk rows only once its last link disappears. A file inode +// can carry several hardlink names, so the node and its chunks survive +// until the final dirent is gone. Returns true when the inode was +// reaped, false when other links keep it alive. +// +// Callers own rev bumps and tombstones; this helper touches only +// vfs_dirents, vfs_chunks, and vfs_nodes. It is the single place the +// refcount-gated reap is implemented — rm, rename, and the sync apply +// path all funnel through here so the invariant lives once. +export function unlinkDirent( + db: Database, + parentInode: number, + name: string, + childInode: number, + type: NodeType, +): boolean { + db.run("DELETE FROM vfs_dirents WHERE parent_inode = ? AND name = ?", parentInode, name); + const remaining = db.scalar( + "SELECT COUNT(*) FROM vfs_dirents WHERE child_inode = ?", + childInode, + ); + if ((remaining ?? 0) > 0) return false; + + if (type === "file") { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", childInode); + } + db.run("DELETE FROM vfs_nodes WHERE inode = ?", childInode); + return true; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/watch.ts b/spikes/349-dofs/vendor/dofs/src/fs/watch.ts new file mode 100644 index 00000000..6c087e6f --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/watch.ts @@ -0,0 +1,163 @@ +// Directory watcher backed by polling vfs_meta.rev. +// +// On each tick, coalesceChanges yields every path touched since +// the watcher last looked. We filter by the watched directory and +// recursive flag, then emit one 'change' event per path with +// fs.watch-compatible (eventType, filename) arguments. +// +// The polling cadence is the provider's watchIntervalMs (default +// 100 ms). That's already what node's fs.watch uses internally on +// platforms without inotify, and it's slow enough that the SQL +// scan stays in the noise even with many watchers active. +// +// Coverage: there is no watch.test.ts here because watch is only +// reachable through SQLiteWorkspaceProvider.watch / +// watchAsyncIterable; the test surface is provider.watch.test.ts, +// which exercises both the EventEmitter and the AsyncIterable +// adapters end-to-end. + +import { EventEmitter } from "node:events"; + +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; +import { coalesceChanges } from "../sync/coalesce.js"; +import { currentRev } from "../sync/watermarks.js"; + +export interface WatchOptions { + // Recurse into subdirectories. When false (default) the watcher + // only fires for direct children of `path`. + recursive?: boolean; + // AbortSignal that closes the watcher when triggered. + signal?: AbortSignal; + // Override the poll interval. Default comes from the provider. + interval?: number; +} + +export interface WatchEvent { + eventType: "rename" | "change"; + filename: string; +} + +export interface WatchHandle extends EventEmitter { + close(): void; +} + +export function createWatcher( + db: Database, + path: string, + options: WatchOptions, + defaultInterval: number, +): WatchHandle { + const { path: canonical } = canonicalizePath(path); + const prefix = canonical === "/" ? "/" : `${canonical}/`; + const recursive = options.recursive === true; + const interval = options.interval ?? defaultInterval; + + const emitter = new EventEmitter() as WatchHandle; + let cursor = currentRev(db); + let closed = false; + + const tick = async () => { + if (closed) return; + try { + const seen = new Set(); + for await (const entry of coalesceChanges(db, cursor)) { + // Filter to entries inside the watched scope. + if (!isInScope(entry.path, canonical, prefix, recursive)) continue; + if (seen.has(entry.path)) continue; + seen.add(entry.path); + const filename = relativeName(entry.path, canonical); + const eventType: "rename" | "change" = entry.kind === "delete" ? "rename" : "change"; + emitter.emit("change", eventType, filename); + } + cursor = currentRev(db); + } catch (error) { + emitter.emit("error", error); + } + }; + + const handle = setInterval(() => void tick(), interval); + handle.unref?.(); + + emitter.close = () => { + if (closed) return; + closed = true; + clearInterval(handle); + emitter.emit("close"); + }; + + if (options.signal !== undefined) { + if (options.signal.aborted) { + emitter.close(); + } else { + options.signal.addEventListener("abort", () => emitter.close(), { + once: true, + }); + } + } + + return emitter; +} + +function isInScope( + entryPath: string, + watchedPath: string, + prefix: string, + recursive: boolean, +): boolean { + if (entryPath === watchedPath) return true; + if (!entryPath.startsWith(prefix)) return false; + if (recursive) return true; + // Non-recursive: only direct children. No extra '/' in the + // remainder past the prefix. + const remainder = entryPath.slice(prefix.length); + return !remainder.includes("/"); +} + +function relativeName(entryPath: string, watchedPath: string): string { + if (entryPath === watchedPath) return ""; + const prefix = watchedPath === "/" ? "/" : `${watchedPath}/`; + return entryPath.startsWith(prefix) ? entryPath.slice(prefix.length) : entryPath; +} + +// Adapter from EventEmitter-based watcher to AsyncIterable for +// for-await consumers. Mirrors @platformatic/vfs's VFSWatchAsyncIterable. +export function createWatchAsyncIterable(watcher: WatchHandle): AsyncIterable & { + return(): Promise<{ value: undefined; done: true }>; +} { + const pending: WatchEvent[] = []; + const waiters: ((result: IteratorResult) => void)[] = []; + let done = false; + + watcher.on("change", (eventType: "rename" | "change", filename: string) => { + const event: WatchEvent = { eventType, filename }; + const next = waiters.shift(); + if (next) next({ value: event, done: false }); + else pending.push(event); + }); + watcher.on("close", () => { + done = true; + while (waiters.length > 0) { + const next = waiters.shift(); + if (next) next({ value: undefined as never, done: true }); + } + }); + + return { + [Symbol.asyncIterator]() { + return this as unknown as AsyncIterator; + }, + next(): Promise> { + const buffered = pending.shift(); + if (buffered) return Promise.resolve({ value: buffered, done: false }); + if (done) return Promise.resolve({ value: undefined as never, done: true }); + return new Promise((resolve) => waiters.push(resolve)); + }, + async return() { + watcher.close(); + return { value: undefined, done: true as const }; + }, + } as unknown as AsyncIterable & { + return(): Promise<{ value: undefined; done: true }>; + }; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/with-db.ts b/spikes/349-dofs/vendor/dofs/src/fs/with-db.ts new file mode 100644 index 00000000..38f82ff7 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/with-db.ts @@ -0,0 +1,55 @@ +// Node-backed implementation. Selected by the default vitest config. +// The workers config aliases this module to ./with-db.workers.ts so the +// same test source runs against a real Durable Object. + +import { initializeSchema } from "../schema/index.js"; +import { Database } from "../storage.js"; +import { SQLiteTestStorage } from "../testing.js"; + +export interface WithDBOptions { + now?: () => number; +} + +export async function withDB( + fn: (db: Database) => T | Promise, + options: WithDBOptions = {}, +): Promise { + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + initializeSchema(db, options.now ?? (() => 1000)); + try { + return await fn(db); + } finally { + storage.close(); + } +} + +// Two independent DBs split across a snapshot/apply pair. The shape +// matches the workerd backend, which can't hold two DOs alive at +// once because of cross-DO I/O isolation — the snapshot pass +// captures everything the apply pass needs as plain serializable +// values. +// +// Under node we run both callbacks in the same process with two +// SQLiteTestStorage instances. Either shape would work here; we +// match the workerd API so test code stays uniform. +export async function withTwoDBs( + snapshot: (a: Database) => S | Promise, + apply: (b: Database, snapshot: S) => T | Promise, + options: WithDBOptions = {}, +): Promise { + const storageA = new SQLiteTestStorage(); + const storageB = new SQLiteTestStorage(); + const a = new Database(storageA); + const b = new Database(storageB); + const now = options.now ?? (() => 1000); + initializeSchema(a, now); + initializeSchema(b, now); + try { + const captured = await snapshot(a); + return await apply(b, captured); + } finally { + storageA.close(); + storageB.close(); + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/with-db.workers.ts b/spikes/349-dofs/vendor/dofs/src/fs/with-db.workers.ts new file mode 100644 index 00000000..addfaf7a --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/with-db.workers.ts @@ -0,0 +1,68 @@ +// Workers-backed implementation. Vitest config aliases ./with-db.js to +// this file when running under @cloudflare/vitest-pool-workers, so every +// test that calls withDB(fn) ends up driving a real Durable Object's +// SQLite storage instead of a node:sqlite in-memory DB. + +import { env, runInDurableObject } from "cloudflare:test"; +import type { TestBindings } from "../../tests/worker.js"; +import { initializeSchema } from "../schema/index.js"; +import { Database } from "../storage.js"; +import type { DurableObjectStorageLike } from "../types.js"; + +export interface WithDBOptions { + now?: () => number; +} + +// Each call gets a fresh DO instance so tests don't bleed into each +// other. newUniqueId() gives a name that never collides between runs. +function freshStub() { + const id = (env as unknown as TestBindings).TestStorage.newUniqueId(); + return (env as unknown as TestBindings).TestStorage.get(id); +} + +export async function withDB( + fn: (db: Database) => T | Promise, + options: WithDBOptions = {}, +): Promise { + const stub = freshStub(); + return runInDurableObject(stub, async (_instance: unknown, state: DurableObjectState) => { + const db = new Database(state.storage as unknown as DurableObjectStorageLike); + initializeSchema(db, options.now ?? (() => 1000)); + return await fn(db); + }); +} + +// Two independent DOs. Each newUniqueId() gives a fresh DO with its +// own SqlStorage. We can't nest runInDurableObject calls because +// workerd hard-isolates I/O objects between DOs in the same isolate +// (the Database wrapper captures the SqlStorage as such an object). +// Snapshot A's outputs to plain serializable values inside its DO +// context, then apply them in B's. The caller drives the two passes +// via a snapshot and apply callback pair. +// +// This is the workerd shape; the node-side withTwoDBs runs both +// callbacks in the same process with two real SQLiteTestStorage +// instances. Tests that need the simpler signature should branch on +// the node-only path. +export async function withTwoDBs( + snapshot: (a: Database) => S | Promise, + apply: (b: Database, snapshot: S) => T | Promise, + options: WithDBOptions = {}, +): Promise { + const stubA = freshStub(); + const stubB = freshStub(); + const now = options.now ?? (() => 1000); + const captured = await runInDurableObject( + stubA, + async (_a: unknown, stateA: DurableObjectState) => { + const a = new Database(stateA.storage as unknown as DurableObjectStorageLike); + initializeSchema(a, now); + return await snapshot(a); + }, + ); + return runInDurableObject(stubB, async (_b: unknown, stateB: DurableObjectState) => { + const b = new Database(stateB.storage as unknown as DurableObjectStorageLike); + initializeSchema(b, now); + return await apply(b, captured); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.test.ts new file mode 100644 index 00000000..891f5e3e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it } from "vitest"; + +import type { Database } from "../storage.js"; +import { readRangeSync } from "./readFile.js"; +import { resolveInode } from "./resolve.js"; +import { stat } from "./stat.js"; +import { withDB } from "./with-db.js"; +import { + CHUNK_SIZE, + createFileSync, + openWriteBufferForCreateSync, + openWriteBufferSync, + releaseWriteBufferSync, + truncateFileSync, + writeRangeSync, +} from "./writeFile.js"; + +function bytesOf(text: string): Uint8Array { + return new TextEncoder().encode(text); +} + +function blobCount(db: Database): number { + return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; +} + +function orphanBlobCount(db: Database): number { + return ( + db.scalar( + "SELECT COUNT(*) FROM vfs_blobs b WHERE NOT EXISTS (SELECT 1 FROM vfs_chunks c WHERE c.hash = b.hash)", + ) ?? 0 + ); +} + +function chunkCount(db: Database, path: string): number { + const node = resolveInode(db, path); + if (node === null) throw new Error(`missing node: ${path}`); + return db.scalar("SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", node.inode) ?? 0; +} + +describe("buffered write lifecycle", () => { + it("buffers many small writes and stages blobs only on release", async () => { + await withDB(async (db) => { + createFileSync(db, "/buffered.bin", {}, () => 1000); + openWriteBufferSync(db, "/buffered.bin"); + + const big = new Uint8Array(CHUNK_SIZE); + big.fill(7); + // Write the same chunk-sized payload eight times at offset 0. + // Pre-buffer, each write would stage one orphan blob per + // intermediate state. + for (let i = 0; i < 8; i++) { + writeRangeSync(db, "/buffered.bin", big, 0, {}, () => 1001 + i); + } + // While the buffer is open we keep no chunk or blob rows yet. + expect(blobCount(db)).toBe(0); + + releaseWriteBufferSync(db, "/buffered.bin", () => 1100); + + // After release: exactly one blob, no orphans, content matches. + expect(blobCount(db)).toBe(1); + expect(orphanBlobCount(db)).toBe(0); + const final = readRangeSync(db, "/buffered.bin", 0, CHUNK_SIZE); + expect(final.byteLength).toBe(CHUNK_SIZE); + expect(final[0]).toBe(7); + }); + }); + + it("serves buffered reads before release", async () => { + await withDB(async (db) => { + createFileSync(db, "/buffered.txt", {}, () => 1000); + openWriteBufferSync(db, "/buffered.txt"); + + writeRangeSync(db, "/buffered.txt", bytesOf("hello"), 0, {}, () => 1001); + // Reading through the same db sees the buffered bytes even + // though no chunk row has been written. + expect(new TextDecoder().decode(readRangeSync(db, "/buffered.txt", 0, 5))).toBe("hello"); + expect(chunkCount(db, "/buffered.txt")).toBe(0); + + releaseWriteBufferSync(db, "/buffered.txt", () => 1100); + expect(new TextDecoder().decode(readRangeSync(db, "/buffered.txt", 0, 5))).toBe("hello"); + }); + }); + + it("truncate updates the buffer instead of rewriting chunks", async () => { + await withDB(async (db) => { + createFileSync(db, "/trunc.bin", {}, () => 1000); + openWriteBufferSync(db, "/trunc.bin"); + + const payload = new Uint8Array(CHUNK_SIZE * 2); + payload.fill(1); + writeRangeSync(db, "/trunc.bin", payload, 0, {}, () => 1001); + truncateFileSync(db, "/trunc.bin", CHUNK_SIZE - 100, () => 1002); + expect(chunkCount(db, "/trunc.bin")).toBe(0); + + releaseWriteBufferSync(db, "/trunc.bin", () => 1100); + expect(chunkCount(db, "/trunc.bin")).toBe(1); + const final = readRangeSync(db, "/trunc.bin", 0, CHUNK_SIZE); + expect(final.byteLength).toBe(CHUNK_SIZE - 100); + }); + }); + + it("hardlinks share the same buffered bytes by inode", async () => { + await withDB(async (db) => { + createFileSync(db, "/a.txt", {}, () => 1000); + // Both paths point at the same inode. Open under /a.txt, then + // write under /b.txt: the buffer is keyed by inode so the write + // lands in the same cache entry. + const { link } = await import("./link.js"); + link(db, "/a.txt", "/b.txt"); + openWriteBufferSync(db, "/a.txt"); + + // Multiple intermediate writes through both paths. Pre-buffer + // each one would have staged its own blob and orphaned the + // previous state; the buffer keeps them in memory until release. + writeRangeSync(db, "/b.txt", bytesOf("step-1"), 0, {}, () => 1001); + writeRangeSync(db, "/a.txt", bytesOf("step-2"), 0, {}, () => 1002); + writeRangeSync(db, "/b.txt", bytesOf("shared"), 0, {}, () => 1003); + expect(new TextDecoder().decode(readRangeSync(db, "/a.txt", 0, 6))).toBe("shared"); + expect(blobCount(db)).toBe(0); + + releaseWriteBufferSync(db, "/a.txt", () => 1100); + expect(new TextDecoder().decode(readRangeSync(db, "/b.txt", 0, 6))).toBe("shared"); + // Exactly one blob for the final state, regardless of how many + // intermediate writes the open window saw. + expect(blobCount(db)).toBe(1); + }); + }); + + it("commits a chunked file with one blob per chunk on release", async () => { + await withDB(async (db) => { + createFileSync(db, "/big.bin", {}, () => 1000); + openWriteBufferSync(db, "/big.bin"); + + // Three chunks of distinct content. Pre-buffer, each write + // would create the chunk row eagerly and a partial-tail write + // would create an orphan blob for the previous tail size. + const payload = new Uint8Array(CHUNK_SIZE * 3); + payload.fill(1, 0, CHUNK_SIZE); + payload.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + payload.fill(3, CHUNK_SIZE * 2); + writeRangeSync(db, "/big.bin", payload, 0, {}, () => 1001); + + releaseWriteBufferSync(db, "/big.bin", () => 1100); + expect(chunkCount(db, "/big.bin")).toBe(3); + expect(orphanBlobCount(db)).toBe(0); + }); + }); +}); + +describe("deferred-create lifecycle", () => { + it("holds the file in memory until release commits one transaction", async () => { + await withDB(async (db) => { + openWriteBufferForCreateSync(db, "/pending.txt", { mode: 0o600 }, () => 1000); + + // No SQL row exists yet but the path is reachable via the + // path-keyed pending cache: stat, read, and write all see it. + expect(stat(db, "/pending.txt").size).toBe(0); + writeRangeSync(db, "/pending.txt", bytesOf("hello"), 0, {}, () => 1001); + expect(stat(db, "/pending.txt").size).toBe(5); + expect(new TextDecoder().decode(readRangeSync(db, "/pending.txt", 0, 5))).toBe("hello"); + expect(resolveInode(db, "/pending.txt")).toBeNull(); + expect(blobCount(db)).toBe(0); + + releaseWriteBufferSync(db, "/pending.txt", () => 1100); + + // Release committed the INSERT, dirent, and chunk rows in one + // transaction; the path now resolves to a real inode and the + // stat sees the persisted size. + const node = resolveInode(db, "/pending.txt"); + expect(node?.type).toBe("file"); + expect(node?.mode).toBe(0o600); + expect(stat(db, "/pending.txt").size).toBe(5); + expect(blobCount(db)).toBe(1); + expect(orphanBlobCount(db)).toBe(0); + }); + }); + + it("rejects a second openWriteBufferForCreateSync against the same path", async () => { + await withDB(async (db) => { + openWriteBufferForCreateSync(db, "/dupe.txt", {}, () => 1000); + expect(() => openWriteBufferForCreateSync(db, "/dupe.txt", {}, () => 1001)).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + }); + }); + + it("surfaces pending files in readdir before release", async () => { + await withDB(async (db) => { + const { readdir } = await import("./readdir.js"); + const { mkdir } = await import("./mkdir.js"); + mkdir(db, "/d", {}, () => 1000); + openWriteBufferForCreateSync(db, "/d/pending.txt", {}, () => 1001); + + const names = readdir(db, "/d").map((entry) => entry.name); + expect(names).toEqual(["pending.txt"]); + + releaseWriteBufferSync(db, "/d/pending.txt", () => 1100); + const after = readdir(db, "/d").map((entry) => entry.name); + expect(after).toEqual(["pending.txt"]); + }); + }); + + it("defers commit until the matching release count is reached", async () => { + await withDB(async (db) => { + createFileSync(db, "/multi.txt", {}, () => 1000); + writeRangeSync(db, "/multi.txt", bytesOf("seed"), 0, {}, () => 1001); + // Establish the on-disk shape we'll observe against. + const seedBlobs = blobCount(db); + const seedChunks = chunkCount(db, "/multi.txt"); + + // Two opens of the same path share a single inode-keyed entry. + openWriteBufferSync(db, "/multi.txt"); + openWriteBufferSync(db, "/multi.txt"); + writeRangeSync(db, "/multi.txt", bytesOf("DIRTY"), 0, {}, () => 1002); + + // First release decrements but does not commit — chunk/blob + // shape unchanged. + releaseWriteBufferSync(db, "/multi.txt", () => 1003); + expect(blobCount(db)).toBe(seedBlobs); + expect(chunkCount(db, "/multi.txt")).toBe(seedChunks); + + // Second release commits. Reading back through the chunk + // store sees the dirty bytes. + releaseWriteBufferSync(db, "/multi.txt", () => 1004); + const final = readRangeSync(db, "/multi.txt", 0, 5); + expect(new TextDecoder().decode(final)).toBe("DIRTY"); + + // A fresh open after release starts a clean buffer; an + // immediate release without writes is a no-op. + openWriteBufferSync(db, "/multi.txt"); + releaseWriteBufferSync(db, "/multi.txt", () => 1005); + // No corruption: file content still matches. + const stable = readRangeSync(db, "/multi.txt", 0, 5); + expect(new TextDecoder().decode(stable)).toBe("DIRTY"); + }); + }); + + it("flushPendingByPath promotes the buffer without consuming the open count", async () => { + const { flushPendingByPath } = await import("./writeFile.js"); + await withDB(async (db) => { + openWriteBufferForCreateSync(db, "/promote.txt", {}, () => 1000); + writeRangeSync(db, "/promote.txt", bytesOf("before"), 0, {}, () => 1001); + expect(resolveInode(db, "/promote.txt")).toBeNull(); + + // Promote the pending entry without releasing it. + const committed = flushPendingByPath(db, "/promote.txt", () => 1002); + expect(committed).toBe(true); + + // The path now resolves; the open buffer survived the + // promotion under the real inode key. + const node = resolveInode(db, "/promote.txt"); + expect(node?.type).toBe("file"); + expect(chunkCount(db, "/promote.txt")).toBeGreaterThan(0); + + // Further writes route through the inode-keyed cache. The + // matching release commits the final bytes over the + // promoted state. + writeRangeSync(db, "/promote.txt", bytesOf("after-x"), 0, {}, () => 1003); + releaseWriteBufferSync(db, "/promote.txt", () => 1004); + const final = readRangeSync(db, "/promote.txt", 0, 7); + expect(new TextDecoder().decode(final)).toBe("after-x"); + }); + }); + + it("flushPendingByPath returns false when no pending buffer is open", async () => { + const { flushPendingByPath } = await import("./writeFile.js"); + await withDB(async (db) => { + createFileSync(db, "/already.txt", {}, () => 1000); + expect(flushPendingByPath(db, "/already.txt", () => 1001)).toBe(false); + expect(flushPendingByPath(db, "/missing.txt", () => 1002)).toBe(false); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.ts new file mode 100644 index 00000000..72ff047a --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/writeBuffer.ts @@ -0,0 +1,138 @@ +// In-process write buffer cache. +// +// Holds per-inode mutable byte buffers between an explicit open and +// release. While a buffer is open, all reads and writes for that +// inode go through the buffer rather than the SQLite blob/chunk +// store. Release commits the bytes to chunks once per file +// and evicts the entry, so per-syscall writes no longer accumulate +// orphan blob rows in the store. +// +// The cache is keyed by Database so a fresh database (a test, a +// rebooted DO incarnation) starts with an empty cache. + +import type { Database } from "../storage.js"; + +export interface WriteBufferEntry { + // Growable backing store. byteLength is capacity; logical length + // lives in `size`. + buf: Uint8Array; + // Logical end-of-file in `buf`. + size: number; + // True once writeRange/truncate mutates the buffer. A non-dirty + // buffer is one that the caller opened but never wrote to; release + // is a no-op in that case so we do not touch the existing chunks. + dirty: boolean; + // Open handle count. Each FUSE open/create increments this; each + // release decrements. The buffer commits and evicts when the count + // reaches zero. + openCount: number; + // Mode the caller wants persisted on release. Defaults to the + // inode's existing mode at open time when the caller has none. + mode: number; + // Pending-create state. When set, no inode row exists yet; release + // will INSERT the node + dirent + chunks in one transaction. The + // synthetic inode id used to key this entry in the cache is stored + // here so release can find and remove the entry without scanning + // the cache. + pending?: { + parentInode: number; + leafName: string; + canonicalPath: string; + pendingInode: number; + mtime: number; + }; +} + +interface DatabaseCache { + byInode: Map; + byPendingPath: Map; + nextPendingInode: number; +} + +const caches = new WeakMap(); + +function cacheFor(db: Database): DatabaseCache { + let cache = caches.get(db); + if (cache === undefined) { + cache = { byInode: new Map(), byPendingPath: new Map(), nextPendingInode: -1 }; + caches.set(db, cache); + } + return cache; +} + +export function getWriteBuffer(db: Database, inode: number): WriteBufferEntry | undefined { + return caches.get(db)?.byInode.get(inode); +} + +export function getPendingWriteBufferByPath( + db: Database, + canonicalPath: string, +): WriteBufferEntry | undefined { + return caches.get(db)?.byPendingPath.get(canonicalPath); +} + +// List pending-create buffers whose parent dirent matches `parentInode`. +// Used by readdir so freshly-created-but-not-yet-released files show +// up in directory listings between open and release. +export function listPendingByParent(db: Database, parentInode: number): WriteBufferEntry[] { + const cache = caches.get(db); + if (cache === undefined) return []; + const out: WriteBufferEntry[] = []; + for (const entry of cache.byPendingPath.values()) { + if (entry.pending?.parentInode === parentInode) out.push(entry); + } + return out; +} + +export function setWriteBuffer(db: Database, inode: number, entry: WriteBufferEntry): void { + const cache = cacheFor(db); + cache.byInode.set(inode, entry); + if (entry.pending !== undefined) { + cache.byPendingPath.set(entry.pending.canonicalPath, entry); + } +} + +export function deleteWriteBuffer(db: Database, inode: number): void { + const cache = caches.get(db); + if (cache === undefined) return; + const entry = cache.byInode.get(inode); + if (entry?.pending !== undefined) { + cache.byPendingPath.delete(entry.pending.canonicalPath); + } + cache.byInode.delete(inode); +} + +// Allocate a synthetic negative inode id for a pending file. The +// real id is assigned by SQLite when release INSERTs the node row; +// the synthetic value just lets the buffer cache key entries +// before that point. +export function allocatePendingInode(db: Database): number { + const cache = cacheFor(db); + const next = cache.nextPendingInode; + cache.nextPendingInode -= 1; + return next; +} + +// Re-key a pending entry to the real inode assigned by SQLite at +// commit time, dropping the pending-path index. +export function promotePendingToInode(db: Database, pendingInode: number, realInode: number): void { + const cache = caches.get(db); + if (cache === undefined) return; + const entry = cache.byInode.get(pendingInode); + if (entry === undefined) return; + if (entry.pending !== undefined) { + cache.byPendingPath.delete(entry.pending.canonicalPath); + entry.pending = undefined; + } + cache.byInode.delete(pendingInode); + cache.byInode.set(realInode, entry); +} + +export function ensureCapacity(entry: WriteBufferEntry, needed: number): void { + if (entry.buf.byteLength >= needed) return; + let cap = Math.max(entry.buf.byteLength * 2, 64 * 1024); + while (cap < needed) cap *= 2; + const next = new Uint8Array(cap); + next.set(entry.buf.subarray(0, entry.size), 0); + entry.buf = next; +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeFile.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeFile.test.ts new file mode 100644 index 00000000..d8df6b74 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/writeFile.test.ts @@ -0,0 +1,395 @@ +import { describe, expect, it } from "vitest"; + +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { mkdir } from "./mkdir.js"; +import { resolveInode } from "./resolve.js"; +import { withDB } from "./with-db.js"; +import { CHUNK_SIZE, writeFile, writeFileRangesSync, writeFileSync } from "./writeFile.js"; + +// Reassemble a file's bytes by stitching its chunk rows together. +// A deliberately minimal helper so writeFile tests can stand alone +// without depending on readFile. +function readBack(db: Database, path: string): Uint8Array { + const node = resolveInode(db, path); + if (node === null) throw new Error(`no such path: ${path}`); + if (node.type !== "file") throw new Error(`not a file: ${path}`); + const chunks = db.all<{ hash: Uint8Array; size: number }>( + "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node.inode, + ); + const parts: Uint8Array[] = []; + let total = 0; + for (const chunk of chunks) { + const row = db.one<{ bytes: Uint8Array }>( + "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", + chunk.hash, + ); + if (row === undefined) throw new Error("missing blob bytes"); + parts.push(row.bytes); + total += row.bytes.byteLength; + } + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; +} + +function chunkRows(db: Database, path: string): Array<{ hash: Uint8Array; size: number }> { + const node = resolveInode(db, path); + if (node === null) throw new Error(`no such path: ${path}`); + return db.all<{ hash: Uint8Array; size: number }>( + "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node.inode, + ); +} + +function countBlobs(db: Database): number { + return db.scalar("SELECT COUNT(*) FROM vfs_blobs") ?? 0; +} + +function streamOf(...chunks: Uint8Array[]): ReadableStream { + let i = 0; + return new ReadableStream({ + pull(controller) { + if (i < chunks.length) { + controller.enqueue(chunks[i++]); + } else { + controller.close(); + } + }, + }); +} + +describe("writeFile", () => { + it("writes a small string and stores one chunk", async () => { + await withDB(async (db) => { + await writeFile(db, "/hello.txt", "hello fuse", {}, () => 1234); + + const bytes = readBack(db, "/hello.txt"); + expect(new TextDecoder().decode(bytes)).toBe("hello fuse"); + + const chunkCount = db.scalar( + "SELECT COUNT(*) FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?)", + ROOT_INODE, + "hello.txt", + ); + expect(chunkCount).toBe(1); + }); + }); + + it("writeFileSync stores small strings as a single chunk row", async () => { + await withDB(async (db) => { + writeFileSync(db, "/hello.txt", new TextEncoder().encode("hello fuse"), {}, () => 1234); + + const bytes = readBack(db, "/hello.txt"); + expect(new TextDecoder().decode(bytes)).toBe("hello fuse"); + + const chunkCount = db.scalar( + "SELECT COUNT(*) FROM vfs_chunks WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?)", + ROOT_INODE, + "hello.txt", + ); + expect(chunkCount).toBe(1); + }); + }); + + it("atomically rejects an exclusive write when the target exists", async () => { + await withDB(async (db) => { + await writeFile(db, "/exclusive.txt", "first", {}, () => 1); + await expect( + writeFile(db, "/exclusive.txt", "second", { exclusive: true }, () => 2), + ).rejects.toMatchObject({ code: "EEXIST" }); + expect(new TextDecoder().decode(readBack(db, "/exclusive.txt"))).toBe("first"); + }); + }); + + it("rejects an exclusive streaming write before reading its source", async () => { + await withDB(async (db) => { + await writeFile(db, "/exclusive-stream.txt", "first", {}, () => 1); + let pulls = 0; + const source = new ReadableStream( + { + pull(controller) { + pulls += 1; + controller.enqueue(new TextEncoder().encode("second")); + }, + }, + { highWaterMark: 0 }, + ); + await expect( + writeFile(db, "/exclusive-stream.txt", source, { exclusive: true }, () => 2), + ).rejects.toMatchObject({ code: "EEXIST" }); + expect(pulls).toBe(0); + }); + }); + + it("accepts a Uint8Array", async () => { + await withDB(async (db) => { + const data = new Uint8Array([1, 2, 3, 4, 5]); + await writeFile(db, "/data.bin", data, {}, () => 0); + expect(Array.from(readBack(db, "/data.bin"))).toEqual([1, 2, 3, 4, 5]); + }); + }); + + it("accepts a ReadableStream and joins its chunks", async () => { + await withDB(async (db) => { + await writeFile( + db, + "/streamed.txt", + streamOf(new TextEncoder().encode("hello "), new TextEncoder().encode("stream")), + {}, + () => 0, + ); + expect(new TextDecoder().decode(readBack(db, "/streamed.txt"))).toBe("hello stream"); + }); + }); + + it("writes an empty file (zero chunks, zero size)", async () => { + await withDB(async (db) => { + await writeFile(db, "/empty", "", {}, () => 0); + const node = resolveInode(db, "/empty"); + expect(node?.type).toBe("file"); + const chunks = db.scalar( + "SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", + node?.inode, + ); + expect(chunks).toBe(0); + }); + }); + + it("splits content larger than CHUNK_SIZE across multiple chunks", async () => { + await withDB(async (db) => { + const oneChunk = new Uint8Array(CHUNK_SIZE); + oneChunk.fill(0x41); + const trailing = new Uint8Array(100); + trailing.fill(0x42); + const combined = new Uint8Array(CHUNK_SIZE + 100); + combined.set(oneChunk, 0); + combined.set(trailing, CHUNK_SIZE); + await writeFile(db, "/big", combined, {}, () => 0); + + const node = resolveInode(db, "/big"); + const chunkCount = db.scalar( + "SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", + node?.inode, + ); + expect(chunkCount).toBe(2); + + const sizes = db + .all<{ idx: number; size: number }>( + "SELECT idx, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node?.inode, + ) + .map((r) => r.size); + expect(sizes).toEqual([CHUNK_SIZE, 100]); + + const round = readBack(db, "/big"); + expect(round.byteLength).toBe(CHUNK_SIZE + 100); + expect(round[0]).toBe(0x41); + expect(round[CHUNK_SIZE]).toBe(0x42); + }); + }); + + it("range writes reuse unchanged chunks", async () => { + await withDB(async (db) => { + const first = new Uint8Array(CHUNK_SIZE); + first.fill(0x41); + const second = new Uint8Array(CHUNK_SIZE); + second.fill(0x42); + const third = new Uint8Array(CHUNK_SIZE); + third.fill(0x43); + const original = new Uint8Array(3 * CHUNK_SIZE); + original.set(first, 0); + original.set(second, CHUNK_SIZE); + original.set(third, 2 * CHUNK_SIZE); + await writeFile(db, "/big", original, {}, () => 100); + const beforeChunks = chunkRows(db, "/big"); + + let stagedBlobs = 0; + const run = db.run.bind(db); + db.run = (query: string, ...bindings: unknown[]) => { + if (query.startsWith("INSERT INTO vfs_blobs")) stagedBlobs += 1; + return run(query, ...bindings); + }; + + const next = new Uint8Array(original); + const changedOffset = CHUNK_SIZE + 123; + next[changedOffset] = 0x99; + writeFileRangesSync( + db, + "/big", + next, + [{ start: changedOffset, end: changedOffset + 1 }], + {}, + () => 200, + ); + + expect(stagedBlobs).toBe(1); + expect(Array.from(readBack(db, "/big"))).toEqual(Array.from(next)); + const afterChunks = chunkRows(db, "/big"); + expect(afterChunks).toHaveLength(3); + expect(afterChunks[0].hash).toEqual(beforeChunks[0].hash); + expect(afterChunks[1].hash).not.toEqual(beforeChunks[1].hash); + expect(afterChunks[2].hash).toEqual(beforeChunks[2].hash); + }); + }); + + it("dedups identical content across two paths into one blob row", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "shared", {}, () => 0); + await writeFile(db, "/b.txt", "shared", {}, () => 0); + expect(countBlobs(db)).toBe(1); + }); + }); + + it("overwriting reuses the blob when content is unchanged", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "same", {}, () => 0); + const before = countBlobs(db); + await writeFile(db, "/x.txt", "same", {}, () => 0); + expect(countBlobs(db)).toBe(before); + }); + }); + + it("overwriting replaces chunk rows; old content blob remains for GC", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "first", {}, () => 0); + await writeFile(db, "/x.txt", "second-version", {}, () => 0); + expect(new TextDecoder().decode(readBack(db, "/x.txt"))).toBe("second-version"); + expect(countBlobs(db)).toBe(2); + }); + }); + + it("rejects ENOENT when the parent directory is missing", async () => { + await withDB(async (db) => { + await expect(writeFile(db, "/no/such/dir/file.txt", "hi", {}, () => 0)).rejects.toMatchObject( + { + code: "ENOENT", + }, + ); + }); + }); + + it("rejects EISDIR when the path resolves to a directory", async () => { + await withDB(async (db) => { + mkdir(db, "/d", {}, () => 0); + await expect(writeFile(db, "/d", "x", {}, () => 0)).rejects.toMatchObject({ + code: "EISDIR", + }); + }); + }); + + it("honors mode and bumps rev on first write", async () => { + await withDB(async (db) => { + const beforeRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); + await writeFile(db, "/run.sh", "#!/bin/sh\n", { mode: 0o755 }, () => 4242); + const node = resolveInode(db, "/run.sh"); + expect(node?.mode).toBe(0o755); + expect(node?.mtime).toBe(4242); + const afterRev = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'"); + expect(afterRev).toBe((beforeRev ?? 0) + 1); + const nodeRev = db.scalar("SELECT rev FROM vfs_nodes WHERE inode = ?", node?.inode); + expect(nodeRev).toBe(afterRev); + }); + }); + + it("updates mtime and rev on overwrite", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "v1", {}, () => 100); + const v1 = resolveInode(db, "/x.txt"); + const v1Rev = db.scalar("SELECT rev FROM vfs_nodes WHERE inode = ?", v1?.inode); + + await writeFile(db, "/x.txt", "v2", {}, () => 200); + const v2 = resolveInode(db, "/x.txt"); + expect(v2?.inode).toBe(v1?.inode); + expect(v2?.mtime).toBe(200); + const v2Rev = db.scalar("SELECT rev FROM vfs_nodes WHERE inode = ?", v2?.inode); + expect((v2Rev ?? 0) > (v1Rev ?? 0)).toBe(true); + }); + }); + + it("writes into a nested directory", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/b/c.txt", "nested", {}, () => 0); + expect(new TextDecoder().decode(readBack(db, "/a/b/c.txt"))).toBe("nested"); + }); + }); + + it("stages blobs incrementally as the stream produces them", async () => { + await withDB(async (db) => { + // Stream 3 CHUNK_SIZE-aligned source chunks. After the first + // is pulled, the receiver should have already staged it — + // we don't want to wait for the whole stream to drain. + const filler = new Uint8Array(CHUNK_SIZE); + filler.fill(0x41); + const filler2 = new Uint8Array(CHUNK_SIZE); + filler2.fill(0x42); + const filler3 = new Uint8Array(CHUNK_SIZE); + filler3.fill(0x43); + + let pulled = 0; + let blobsAfterFirstPull: number | undefined; + const stream = new ReadableStream({ + async pull(controller) { + if (pulled === 0) { + controller.enqueue(filler); + } else if (pulled === 1) { + // Snapshot blob count after the writer has consumed + // the first source chunk but before we hand it the + // second. With streaming this is ≥ 1; with buffering + // it stays at 0 until end-of-stream. + blobsAfterFirstPull = countBlobs(db); + controller.enqueue(filler2); + } else if (pulled === 2) { + controller.enqueue(filler3); + } else { + controller.close(); + } + pulled++; + }, + }); + + await writeFile(db, "/big.bin", stream, {}, () => 0); + expect(blobsAfterFirstPull).toBeGreaterThanOrEqual(1); + expect(countBlobs(db)).toBe(3); + const back = readBack(db, "/big.bin"); + expect(back.byteLength).toBe(3 * CHUNK_SIZE); + expect(back[0]).toBe(0x41); + expect(back[CHUNK_SIZE]).toBe(0x42); + expect(back[2 * CHUNK_SIZE]).toBe(0x43); + }); + }); + + it("chunks correctly when source ReadableStream chunks don't align to CHUNK_SIZE", async () => { + await withDB(async (db) => { + // Source emits oddly-sized parts: 100 bytes, then CHUNK_SIZE, + // then 50 bytes. Total = CHUNK_SIZE + 150 → 2 chunks. + const a = new Uint8Array(100); + a.fill(0x31); + const b = new Uint8Array(CHUNK_SIZE); + b.fill(0x32); + const c = new Uint8Array(50); + c.fill(0x33); + await writeFile(db, "/oddly.bin", streamOf(a, b, c), {}, () => 0); + const back = readBack(db, "/oddly.bin"); + expect(back.byteLength).toBe(CHUNK_SIZE + 150); + expect(back[0]).toBe(0x31); + expect(back[99]).toBe(0x31); + expect(back[100]).toBe(0x32); + expect(back[CHUNK_SIZE + 99]).toBe(0x32); + expect(back[CHUNK_SIZE + 100]).toBe(0x33); + // 2 chunks (first 512KiB, then 150-byte trailing). + const node = resolveInode(db, "/oddly.bin"); + const chunkCount = db.scalar( + "SELECT COUNT(*) FROM vfs_chunks WHERE inode = ?", + node?.inode, + ); + expect(chunkCount).toBe(2); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeFile.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeFile.ts new file mode 100644 index 00000000..48f6fc53 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/writeFile.ts @@ -0,0 +1,1070 @@ +import { createHash } from "node:crypto"; +import { createWorkspaceError } from "../errors.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; +import { stageBlob } from "../sync/blobs.js"; +import { buildManifest } from "../sync/manifests.js"; +import { getBlobBytes } from "./blobCache.js"; +import { assertNotReadOnly } from "./mount-guard.js"; +import { invalidateResolveExact } from "./resolveCache.js"; +import { + allocatePendingInode, + deleteWriteBuffer, + ensureCapacity as ensureBufferCapacity, + getPendingWriteBufferByPath, + getWriteBuffer, + promotePendingToInode, + setWriteBuffer, + type WriteBufferEntry, +} from "./writeBuffer.js"; + +// Fixed chunk size. Exported so tests can size inputs precisely +// without hard-coding the magic number twice. +export const CHUNK_SIZE = 512 * 1024; + +export type WriteFileContent = string | Uint8Array | ReadableStream; + +export interface WriteFileOptions { + mode?: number; + /** Fail with EEXIST when the target already exists. */ + exclusive?: boolean; +} + +export interface WriteFileRange { + start: number; + end: number; +} + +// Resolve directory-only paths (the parent of the target file). The +// final segment is handled by the caller. Returns the parent inode or +// throws ENOENT/ENOTDIR. +function resolveParent(db: Database, parts: string[], canonical: string): number { + let parentInode = ROOT_INODE; + for (let i = 0; i < parts.length - 1; i++) { + const name = parts[i]; + const child = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + name, + ); + if (child === undefined) { + throw createWorkspaceError("ENOENT", `parent directory missing: ${canonical}`, canonical); + } + const next = db.one<{ inode: number; type: "file" | "dir" }>( + "SELECT inode, type FROM vfs_nodes WHERE inode = ?", + child.child_inode, + ); + if (next === undefined) { + throw createWorkspaceError("ENOENT", `dangling dirent: ${canonical}`, canonical); + } + if (next.type !== "dir") { + throw createWorkspaceError( + "ENOTDIR", + `parent path segment is not a directory: ${canonical}`, + canonical, + ); + } + parentInode = next.inode; + } + return parentInode; +} + +async function materialize(content: string | Uint8Array): Promise { + if (typeof content === "string") { + return new TextEncoder().encode(content); + } + return content; +} + +// sha256 with a synchronous code path so writeFile can be called both +// from async drivers (the FS API) and from sync drivers (the +// VirtualProvider). node:crypto is available natively on Node and +// polyfilled by workerd. +function sha256(bytes: Uint8Array): Uint8Array { + const hash = createHash("sha256"); + hash.update(bytes); + return new Uint8Array(hash.digest()); +} + +interface PreparedChunk { + hash: Uint8Array; + bytes: Uint8Array; + size: number; +} + +interface ChunkRef { + hash: Uint8Array; + size: number; +} + +export function chunksOf(bytes: Uint8Array): PreparedChunk[] { + const chunks: PreparedChunk[] = []; + for (let offset = 0; offset < bytes.byteLength; offset += CHUNK_SIZE) { + const end = Math.min(offset + CHUNK_SIZE, bytes.byteLength); + // subarray (not slice) avoids an extra copy; sha256() takes its own + // copy when needed. + const slice = bytes.subarray(offset, end); + const hash = sha256(slice); + chunks.push({ hash, bytes: slice, size: slice.byteLength }); + } + return chunks; +} + +export async function writeFile( + db: Database, + path: string, + content: WriteFileContent, + options: WriteFileOptions, + now: () => number, +): Promise { + if (content instanceof ReadableStream) { + await writeFileStreaming(db, path, content, options, now); + return; + } + const bytes = await materialize(content); + writeFileSync(db, path, bytes, options, now); +} + +// Streaming write path. Reads the source one source-chunk at a time, +// re-windows into fixed CHUNK_SIZE pieces, hashes each window, and +// stages it into vfs_blobs / vfs_blob_bytes as it goes. The final +// inode / dirent / vfs_chunks / manifest writes happen in a single +// short transaction once the source is drained, against a list of +// {hash, size} entries that's O(file_size / CHUNK_SIZE) bytes — not +// O(file_size). +// +// Failure mid-stream leaves blob rows behind; gc() reaps orphans on +// its next pass since no node references them. +async function writeFileStreaming( + db: Database, + path: string, + source: ReadableStream, + options: WriteFileOptions, + now: () => number, +): Promise { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + // Reject before we stage any blob bytes so known failures do not grow + // orphan blob rows that gc() then has to reap. + assertNotReadOnly(db, canonical); + if (options.exclusive) { + const parentInode = resolveParent(db, parts, canonical); + const existing = db.one( + "SELECT 1 FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + parts[parts.length - 1], + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + } + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + + const chunkRefs: Array<{ hash: Uint8Array; size: number }> = []; + // Carry-over buffer: bytes left over from the previous source chunk + // that didn't fill a CHUNK_SIZE window. + let carry: Uint8Array | undefined; + + const flush = (chunk: Uint8Array): void => { + const hash = sha256(chunk); + stageBlob(db, hash, chunk, mtime); + chunkRefs.push({ hash, size: chunk.byteLength }); + }; + + const reader = source.getReader(); + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined || value.byteLength === 0) continue; + let input = value; + if (carry !== undefined) { + // Splice carry-over onto the front of this source chunk so + // we can re-window cleanly. + const merged = new Uint8Array(carry.byteLength + input.byteLength); + merged.set(carry, 0); + merged.set(input, carry.byteLength); + input = merged; + carry = undefined; + } + let offset = 0; + while (input.byteLength - offset >= CHUNK_SIZE) { + // Copy the window so the staged blob doesn't alias a + // larger backing buffer. + const window = input.slice(offset, offset + CHUNK_SIZE); + flush(window); + offset += CHUNK_SIZE; + } + if (offset < input.byteLength) { + carry = input.slice(offset); + } + } + } finally { + reader.releaseLock(); + } + if (carry !== undefined && carry.byteLength > 0) { + flush(carry); + } + + // Wire up the inode against the staged blobs in one short + // transaction. From this point on the SQL is the same shape as the + // synchronous path — only the chunk-bytes step is skipped because + // stageBlob already landed them above. + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + let inode: number; + if (existing !== undefined) { + if (options.exclusive) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const node = db.one<{ type: "file" | "dir" }>( + "SELECT type FROM vfs_nodes WHERE inode = ?", + existing.child_inode, + ); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + } else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + for (let idx = 0; idx < chunkRefs.length; idx++) { + const ref = chunkRefs[idx]; + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + ref.hash, + ref.size, + ); + } + const manifestHash = buildManifest(db, chunkRefs, mtime); + const rev = incrementRev(db); + let totalSize = 0; + for (const ref of chunkRefs) totalSize += ref.size; + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", + mode, + mtime, + rev, + totalSize, + manifestHash, + inode, + ); + }); +} + +// Allocate a fresh file inode row with the supplied mode and mtime, +// using SQLite's RETURNING so the new rowid comes back in the same +// statement instead of through a follow-up SELECT last_insert_rowid(). +// Link a freshly created file inode into its parent directory and drop +// any cached negative resolution for the new path. The single choke +// point for every new-file dirent, so the resolve cache stays correct +// on create without touching the overwrite path (which reuses the +// existing inode and dirent, so its resolution is unchanged). A new +// file is a leaf with no descendants, so exact invalidation suffices. +function insertFileDirent( + db: Database, + parentInode: number, + leafName: string, + childInode: number, + canonicalPath: string, +): void { + db.run( + "INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES (?, ?, ?)", + parentInode, + leafName, + childInode, + ); + invalidateResolveExact(db, canonicalPath); +} + +function insertFileNode(db: Database, mode: number, mtime: number): number { + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev) VALUES ('file', ?, ?, 0) RETURNING inode", + mode, + mtime, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + return row.inode; +} + +function upsertChunkBlob(db: Database, chunk: PreparedChunk, lastSeen: number): void { + db.run( + "INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", + chunk.hash, + chunk.size, + lastSeen, + ); + db.run( + "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO NOTHING", + chunk.hash, + chunk.bytes, + ); +} + +function replaceChunkRows( + db: Database, + inode: number, + chunks: ChunkRef[], + manifestTime: number, +): Uint8Array { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + for (let idx = 0; idx < chunks.length; idx++) { + const chunk = chunks[idx]; + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + return buildManifest(db, chunks, manifestTime); +} + +function rangesOverlap(start: number, end: number, ranges: WriteFileRange[]): boolean { + for (const range of ranges) { + if (range.start < end && start < range.end) return true; + } + return false; +} + +function normalizeRanges(ranges: WriteFileRange[], size: number): WriteFileRange[] { + const normalized = ranges + .map((range) => ({ + start: Math.max(0, Math.min(size, Math.floor(range.start))), + end: Math.max(0, Math.min(size, Math.ceil(range.end))), + })) + .filter((range) => range.start < range.end) + .sort((a, b) => a.start - b.start); + + const merged: WriteFileRange[] = []; + for (const range of normalized) { + const previous = merged.at(-1); + if (previous === undefined || previous.end < range.start) { + merged.push({ ...range }); + } else { + previous.end = Math.max(previous.end, range.end); + } + } + return merged; +} + +function existingChunkRefs(db: Database, inode: number): ChunkRef[] { + return db.all("SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", inode); +} + +function fileSizeForInode(db: Database, inode: number): number { + return db.scalar("SELECT size FROM vfs_nodes WHERE inode = ?", inode) ?? 0; +} + +function readChunkBytes(db: Database, inode: number, idx: number): Uint8Array { + const chunk = db.one<{ hash: Uint8Array }>( + "SELECT hash FROM vfs_chunks WHERE inode = ? AND idx = ?", + inode, + idx, + ); + if (chunk === undefined) return new Uint8Array(); + const bytes = getBlobBytes(db, chunk.hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", "missing blob bytes"); + } + return bytes; +} + +function resolveFileInode(db: Database, path: string): { inode: number; mode: number } { + const { path: canonical } = canonicalizePath(path); + const node = db.one<{ inode: number; type: "file" | "dir"; mode: number }>( + `SELECT n.inode AS inode, n.type AS type, n.mode AS mode + FROM vfs_nodes n + WHERE n.inode = ( + SELECT child_inode + FROM vfs_dirents + WHERE parent_inode = ? AND name = ? + )`, + ...parentAndNameForResolvedPath(db, path), + ); + if (node === undefined) { + throw createWorkspaceError("ENOENT", `no such file: ${canonical}`, canonical); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + return { inode: node.inode, mode: node.mode }; +} + +function parentAndNameForResolvedPath(db: Database, path: string): [number, string] { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + return [resolveParent(db, parts, canonical), parts[parts.length - 1]]; +} + +// Update an inode's chunk-backed representation in place. Iterates over +// the full chunk grid but only touches `vfs_chunks` rows whose contents +// or size actually changed, so untouched chunk rows keep their +// rowids and the surrounding rows do not churn. The manifest is +// invalidated rather than recomputed; sync rebuilds it lazily. +function applyChunkedInodeUpdate( + db: Database, + inode: number, + size: number, + mode: number, + mtime: number, + isTouched: (idx: number, start: number, end: number) => boolean, + buildChunkBytes: (idx: number, start: number, end: number, existing: Uint8Array) => Uint8Array, +): void { + const oldChunks = existingChunkRefs(db, inode); + const chunkCount = Math.ceil(size / CHUNK_SIZE); + const oldChunkCount = oldChunks.length; + + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, size); + const intendedSize = end - start; + const old = oldChunks[idx]; + const touched = isTouched(idx, start, end); + // Stable chunk: existed before with the same logical size and the + // caller did not flag it as touched. Skip without issuing SQL so + // its rowid stays put. + if (old !== undefined && old.size === intendedSize && !touched) continue; + + const existingBytes = old !== undefined ? readChunkBytes(db, inode, idx) : new Uint8Array(); + const chunkBytes = buildChunkBytes(idx, start, end, existingBytes); + if (chunkBytes.byteLength !== intendedSize) { + throw createWorkspaceError("EIO", "chunk builder returned wrong size"); + } + const chunk = { hash: sha256(chunkBytes), bytes: chunkBytes, size: chunkBytes.byteLength }; + upsertChunkBlob(db, chunk, mtime); + db.run( + "INSERT OR REPLACE INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + + // Drop any old chunks past the new end of file (shrink case). + if (oldChunkCount > chunkCount) { + db.run("DELETE FROM vfs_chunks WHERE inode = ? AND idx >= ?", inode, chunkCount); + } + + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = NULL WHERE inode = ?", + mode, + mtime, + rev, + size, + inode, + ); +} + +export function createFileSync( + db: Database, + path: string, + options: WriteFileOptions, + now: () => number, +): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + + db.transactionSync(() => { + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const rev = incrementRev(db); + // INSERT with RETURNING folds the last_insert_rowid lookup into + // the same statement, and computing rev up front lets us write + // the node row with its final stamp in one shot. + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, manifest_hash) VALUES ('file', ?, ?, ?, NULL) RETURNING inode", + mode, + mtime, + rev, + ); + if (row === undefined) throw createWorkspaceError("EIO", "failed to allocate inode"); + insertFileDirent(db, parentInode, leafName, row.inode, canonical); + }); +} + +// Open a write buffer for an existing file. Subsequent writes, +// truncates, and reads against the same Database operate on the +// buffer instead of the SQLite chunk/blob store. Release commits +// the bytes back to chunks. +export function openWriteBufferSync(db: Database, path: string): void { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + pending.openCount += 1; + return; + } + const { inode, mode } = resolveFileInode(db, path); + const existing = getWriteBuffer(db, inode); + if (existing !== undefined) { + existing.openCount += 1; + return; + } + setWriteBuffer(db, inode, { + buf: new Uint8Array(0), + size: 0, + dirty: false, + openCount: 1, + mode, + }); +} + +// Create a new file lazily: stash a pending-create write buffer +// keyed by path, without touching SQL until release. createFileSync +// + openWriteBufferSync + writes + releaseWriteBufferSync would +// otherwise spend two transactions per file (one INSERT round and +// one chunk-commit round); this collapses them into a single +// INSERT-and-chunks transaction at release time. +// +// Throws EEXIST if a path already resolves to a live node or to +// another pending buffer. +export function openWriteBufferForCreateSync( + db: Database, + path: string, + options: WriteFileOptions, + now: () => number, +): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (getPendingWriteBufferByPath(db, canonical) !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const [parentInode, leafName] = parentAndNameForResolvedPath(db, path); + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (existing !== undefined) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + const pendingInode = allocatePendingInode(db); + setWriteBuffer(db, pendingInode, { + buf: new Uint8Array(0), + size: 0, + dirty: true, + openCount: 1, + mode, + pending: { parentInode, leafName, canonicalPath: canonical, pendingInode, mtime }, + }); +} + +// Release one open of an inode's write buffer. When the open count +// reaches zero, commit the buffered bytes to chunk rows and drop +// the entry. The committed mode is the buffer's mode at release +// time so an intermediate chmod survives. Pending-create entries +// emit their INSERT + dirent + chunks in the same transaction. +export function releaseWriteBufferSync(db: Database, path: string, now: () => number): void { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + releasePendingBuffer(db, pending, now); + return; + } + const node = resolveFileInode(db, path); + const entry = getWriteBuffer(db, node.inode); + if (entry === undefined) return; + entry.openCount -= 1; + if (entry.openCount > 0) return; + + if (!entry.dirty) { + deleteWriteBuffer(db, node.inode); + return; + } + + const mtime = now(); + const mode = entry.mode & 0o7777; + const buffered = entry.buf.subarray(0, entry.size); + + db.transactionSync(() => { + if (entry.size === 0) { + // An empty file owns no chunk rows; clear any old ones the + // buffer would otherwise have replaced and bump metadata. + db.run("DELETE FROM vfs_chunks WHERE inode = ?", node.inode); + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", + mode, + mtime, + rev, + node.inode, + ); + return; + } + applyChunkedInodeUpdate( + db, + node.inode, + entry.size, + mode, + mtime, + (_idx, start, end) => start < entry.size && end > 0, + (_idx, start, end) => buffered.subarray(start, Math.min(end, entry.size)), + ); + }); + + deleteWriteBuffer(db, node.inode); +} + +// Commit a pending-create buffer to SQLite. Returns the real inode +// allocated by the INSERT, or throws. Promotes the cache entry's key +// from the synthetic pending id to the real inode so subsequent +// reads/writes through the inode-keyed cache still see the same +// buffer. Caller owns the lifecycle of the now-promoted entry. +function commitPendingBuffer(db: Database, entry: WriteBufferEntry, now: () => number): number { + if (entry.pending === undefined) { + throw createWorkspaceError("EIO", "commitPendingBuffer called on non-pending entry"); + } + const { parentInode, leafName, canonicalPath, pendingInode } = entry.pending; + const mtime = now(); + const mode = entry.mode & 0o7777; + const buffered = entry.buf.subarray(0, entry.size); + + let realInode = 0; + try { + db.transactionSync(() => { + // Re-check at commit time: a non-buffered writeFile or another + // out-of-band path could have landed between open and release. + const collision = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + if (collision !== undefined) { + throw createWorkspaceError( + "EEXIST", + `path exists at commit time: ${canonicalPath}`, + canonicalPath, + ); + } + const rev = incrementRev(db); + const row = db.one<{ inode: number }>( + "INSERT INTO vfs_nodes (type, mode, mtime, rev, size, manifest_hash) VALUES ('file', ?, ?, ?, ?, NULL) RETURNING inode", + mode, + mtime, + rev, + entry.size, + ); + if (row === undefined) { + throw createWorkspaceError("EIO", "failed to allocate inode"); + } + insertFileDirent(db, parentInode, leafName, row.inode, canonicalPath); + if (entry.size > 0) { + const inode = row.inode; + const chunkCount = Math.ceil(entry.size / CHUNK_SIZE); + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, entry.size); + const chunkBytes = buffered.subarray(start, end); + const chunk = { + hash: sha256(chunkBytes), + bytes: chunkBytes, + size: chunkBytes.byteLength, + }; + upsertChunkBlob(db, chunk, mtime); + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + } + realInode = row.inode; + }); + } catch (error) { + // Transaction rolled back; drop the buffer so the next caller + // starts clean. + deleteWriteBuffer(db, pendingInode); + throw error; + } + promotePendingToInode(db, pendingInode, realInode); + return realInode; +} + +/** + * @internal + * Bridges a pending-create write buffer into the SQL world ahead of a + * dirent-mutating provider operation (link, rename, unlink). Leaves + * the open count untouched so a still-open handle keeps writing into + * the now-promoted buffer. Returns true when a pending buffer was + * committed. External callers should never invoke this directly. + */ +export function flushPendingByPath(db: Database, path: string, now: () => number): boolean { + const { path: canonical } = canonicalizePath(path); + const entry = getPendingWriteBufferByPath(db, canonical); + if (entry === undefined || entry.pending === undefined) return false; + commitPendingBuffer(db, entry, now); + return true; +} + +function releasePendingBuffer(db: Database, entry: WriteBufferEntry, now: () => number): void { + if (entry.pending === undefined) return; + entry.openCount -= 1; + if (entry.openCount > 0) return; + + const inode = commitPendingBuffer(db, entry, now); + // File is closed; drop the now-promoted entry. A subsequent open + // hits the SQL path and gets a fresh buffer if needed. + deleteWriteBuffer(db, inode); +} + +// Hydrate a freshly-opened buffer with the inode's current bytes +// the first time we mutate it. Avoids paying the read cost when the +// caller opens a file just to truncate or overwrite it. +function hydrateBufferIfNeeded(db: Database, inode: number, entry: WriteBufferEntry): void { + if (entry.dirty) return; + const existingSize = fileSizeForInode(db, inode); + if (existingSize === 0) { + entry.dirty = true; + return; + } + ensureBufferCapacity(entry, existingSize); + let copied = 0; + for (let idx = 0; copied < existingSize; idx++) { + const chunk = readChunkBytes(db, inode, idx); + if (chunk.byteLength === 0) break; + entry.buf.set(chunk, copied); + copied += chunk.byteLength; + } + entry.size = existingSize; + entry.dirty = true; +} + +export function writeRangeSync( + db: Database, + path: string, + bytes: Uint8Array, + offset: number, + options: WriteFileOptions, + now: () => number, +): number { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (!Number.isInteger(offset) || offset < 0) { + throw createWorkspaceError("EINVAL", `invalid write offset: ${offset}`, canonical); + } + if (bytes.byteLength === 0) return 0; + const mtime = now(); + + // Pending-create files don't have an inode yet; route the write + // straight into the path-keyed buffer. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + const writeEnd = offset + bytes.byteLength; + ensureBufferCapacity(pending, writeEnd); + if (offset > pending.size) { + pending.buf.fill(0, pending.size, offset); + } + pending.buf.set(bytes, offset); + if (writeEnd > pending.size) pending.size = writeEnd; + pending.mode = (options.mode ?? pending.mode) & 0o7777; + pending.dirty = true; + return bytes.byteLength; + } + + const { inode, mode: existingMode } = resolveFileInode(db, path); + const mode = (options.mode ?? existingMode) & 0o7777; + const buffered = getWriteBuffer(db, inode); + + // Buffered path: mutate the in-memory bytes and defer storage + // writes until release. Reads through the same Database see the + // buffer's current bytes via readRangeSync's buffer check. + if (buffered !== undefined) { + hydrateBufferIfNeeded(db, inode, buffered); + const writeEnd = offset + bytes.byteLength; + ensureBufferCapacity(buffered, writeEnd); + if (offset > buffered.size) { + buffered.buf.fill(0, buffered.size, offset); + } + buffered.buf.set(bytes, offset); + if (writeEnd > buffered.size) buffered.size = writeEnd; + buffered.mode = mode; + buffered.dirty = true; + return bytes.byteLength; + } + + db.transactionSync(() => { + const oldSize = fileSizeForInode(db, inode); + const writeEnd = offset + bytes.byteLength; + const nextSize = Math.max(oldSize, writeEnd); + + applyChunkedInodeUpdate( + db, + inode, + nextSize, + mode, + mtime, + (_idx, start, end) => offset < end && start < writeEnd, + (_idx, start, end, existing) => { + const chunkBytes = new Uint8Array(end - start); + chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); + if (offset < end && start < writeEnd) { + const copyStart = Math.max(start, offset); + const copyEnd = Math.min(end, writeEnd); + chunkBytes.set(bytes.subarray(copyStart - offset, copyEnd - offset), copyStart - start); + } + return chunkBytes; + }, + ); + }); + + return bytes.byteLength; +} + +export function truncateFileSync( + db: Database, + path: string, + size: number, + now: () => number, +): void { + const { path: canonical } = canonicalizePath(path); + assertNotReadOnly(db, canonical); + if (!Number.isInteger(size) || size < 0) { + throw createWorkspaceError("EINVAL", `invalid truncate size: ${size}`, canonical); + } + const mtime = now(); + + // Pending-create files truncate in-place on the path-keyed buffer. + const pending = getPendingWriteBufferByPath(db, canonical); + if (pending !== undefined) { + if (size > pending.size) { + ensureBufferCapacity(pending, size); + pending.buf.fill(0, pending.size, size); + } + pending.size = size; + pending.dirty = true; + return; + } + + const { inode, mode } = resolveFileInode(db, path); + const buffered = getWriteBuffer(db, inode); + + if (buffered !== undefined) { + hydrateBufferIfNeeded(db, inode, buffered); + if (size > buffered.size) { + ensureBufferCapacity(buffered, size); + buffered.buf.fill(0, buffered.size, size); + } + buffered.size = size; + buffered.dirty = true; + return; + } + + db.transactionSync(() => { + const oldSize = fileSizeForInode(db, inode); + if (oldSize === size) return; + + if (size === 0) { + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = 0, manifest_hash = NULL WHERE inode = ?", + mode, + mtime, + rev, + inode, + ); + return; + } + + applyChunkedInodeUpdate( + db, + inode, + size, + mode, + mtime, + () => false, + (_idx, start, end, existing) => { + const chunkBytes = new Uint8Array(end - start); + chunkBytes.set(existing.subarray(0, Math.min(existing.byteLength, chunkBytes.byteLength))); + return chunkBytes; + }, + ); + }); +} + +// Synchronous entry point used by the VirtualProvider. Identical SQL +// to the async path; differs only in that the bytes have already been +// materialized. +export function writeFileSync( + db: Database, + path: string, + bytes: Uint8Array, + options: WriteFileOptions, + now: () => number, +): void { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + assertNotReadOnly(db, canonical); + const mode = (options.mode ?? 0o644) & 0o7777; + const mtime = now(); + + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + + let inode: number; + if (existing !== undefined) { + if (options.exclusive) { + throw createWorkspaceError("EEXIST", `path exists: ${canonical}`, canonical); + } + const node = db.one<{ type: "file" | "dir" }>( + "SELECT type FROM vfs_nodes WHERE inode = ?", + existing.child_inode, + ); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + // Replace the existing representation. Orphaned blobs (if any) + // are cleaned up by a later gc() pass. + db.run("DELETE FROM vfs_chunks WHERE inode = ?", inode); + } else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + + const rev = incrementRev(db); + const chunks = chunksOf(bytes); + // Upsert blobs and write the new chunk list. + for (let idx = 0; idx < chunks.length; idx++) { + const chunk = chunks[idx]; + upsertChunkBlob(db, chunk, mtime); + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + inode, + idx, + chunk.hash, + chunk.size, + ); + } + + const manifestHash = buildManifest(db, chunks, mtime); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", + mode, + mtime, + rev, + bytes.byteLength, + manifestHash, + inode, + ); + }); +} + +export function writeFileRangesSync( + db: Database, + path: string, + bytes: Uint8Array, + dirtyRanges: WriteFileRange[], + options: WriteFileOptions, + now: () => number, +): void { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw createWorkspaceError("EISDIR", "cannot write to the root directory", canonical); + } + assertNotReadOnly(db, canonical); + const mode = (options.mode ?? 0o644) & 0o7777; + const ranges = normalizeRanges(dirtyRanges, bytes.byteLength); + const mtime = now(); + db.transactionSync(() => { + const parentInode = resolveParent(db, parts, canonical); + const leafName = parts[parts.length - 1]; + const existing = db.one<{ child_inode: number }>( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parentInode, + leafName, + ); + + let inode: number; + let oldChunks: ChunkRef[] = []; + if (existing !== undefined) { + const node = db.one<{ type: "file" | "dir" }>( + "SELECT type FROM vfs_nodes WHERE inode = ?", + existing.child_inode, + ); + if (node?.type === "dir") { + throw createWorkspaceError("EISDIR", `path is a directory: ${canonical}`, canonical); + } + inode = existing.child_inode; + oldChunks = existingChunkRefs(db, inode); + } else { + inode = insertFileNode(db, mode, mtime); + insertFileDirent(db, parentInode, leafName, inode, canonical); + } + + const rev = incrementRev(db); + const nextChunks: ChunkRef[] = []; + const chunkCount = Math.ceil(bytes.byteLength / CHUNK_SIZE); + for (let idx = 0; idx < chunkCount; idx++) { + const start = idx * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, bytes.byteLength); + const size = end - start; + const oldChunk = oldChunks[idx]; + if (oldChunk !== undefined && oldChunk.size === size && !rangesOverlap(start, end, ranges)) { + nextChunks.push(oldChunk); + continue; + } + const chunk = { + hash: sha256(bytes.subarray(start, end)), + bytes: bytes.subarray(start, end), + size, + }; + upsertChunkBlob(db, chunk, mtime); + nextChunks.push({ hash: chunk.hash, size: chunk.size }); + } + + const manifestHash = replaceChunkRows(db, inode, nextChunks, mtime); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ?, size = ?, manifest_hash = ? WHERE inode = ?", + mode, + mtime, + rev, + bytes.byteLength, + manifestHash, + inode, + ); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/fs/writeRange.test.ts b/spikes/349-dofs/vendor/dofs/src/fs/writeRange.test.ts new file mode 100644 index 00000000..46d08548 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/fs/writeRange.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from "vitest"; + +import type { Database } from "../storage.js"; +import { link } from "./link.js"; +import { readFile } from "./readFile.js"; +import { resolveInode } from "./resolve.js"; +import { withDB } from "./with-db.js"; +import { + CHUNK_SIZE, + createFileSync, + truncateFileSync, + writeFileSync, + writeRangeSync, +} from "./writeFile.js"; + +function bytesOf(text: string): Uint8Array { + return new TextEncoder().encode(text); +} + +async function readBytes(db: Database, path: string): Promise { + const stream = await readFile(db, path); + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value === undefined) continue; + chunks.push(value); + total += value.byteLength; + } + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; +} + +function manifestHash(db: Database, path: string): Uint8Array | null { + const node = resolveInode(db, path); + if (node === null) throw new Error(`missing node: ${path}`); + return ( + db.one<{ manifest_hash: Uint8Array | null }>( + "SELECT manifest_hash FROM vfs_nodes WHERE inode = ?", + node.inode, + )?.manifest_hash ?? null + ); +} + +function chunkRows( + db: Database, + path: string, +): Array<{ idx: number; hash: Uint8Array; size: number }> { + const node = resolveInode(db, path); + if (node === null) throw new Error(`missing node: ${path}`); + return db.all<{ idx: number; hash: Uint8Array; size: number }>( + "SELECT idx, hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node.inode, + ); +} + +describe("direct range writes", () => { + it("creates an empty file with no chunk rows", async () => { + await withDB(async (db) => { + createFileSync(db, "/empty.txt", { mode: 0o600 }, () => 1000); + + const node = resolveInode(db, "/empty.txt"); + expect(node?.type).toBe("file"); + expect(node?.mode).toBe(0o600); + expect(chunkRows(db, "/empty.txt")).toEqual([]); + }); + }); + + it("writes small ranges and stores them as a single chunk", async () => { + await withDB(async (db) => { + createFileSync(db, "/small.txt", {}, () => 1000); + + expect(writeRangeSync(db, "/small.txt", bytesOf("hello"), 0, {}, () => 1001)).toBe(5); + expect(writeRangeSync(db, "/small.txt", bytesOf("y"), 4, {}, () => 1002)).toBe(1); + + expect(new TextDecoder().decode(await readBytes(db, "/small.txt"))).toBe("helly"); + expect(chunkRows(db, "/small.txt")).toHaveLength(1); + }); + }); + + it("zero-fills sparse writes", async () => { + await withDB(async (db) => { + createFileSync(db, "/sparse.txt", {}, () => 1000); + + writeRangeSync(db, "/sparse.txt", bytesOf("x"), 3, {}, () => 1001); + + expect(Array.from(await readBytes(db, "/sparse.txt"))).toEqual([0, 0, 0, 120]); + }); + }); + + it("updates only affected chunk hashes for chunk-backed files", async () => { + await withDB(async (db) => { + const original = new Uint8Array(CHUNK_SIZE * 3); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + original.fill(3, CHUNK_SIZE * 2, CHUNK_SIZE * 3); + writeFileSync(db, "/large.bin", original, {}, () => 1000); + const before = chunkRows(db, "/large.bin"); + + writeRangeSync(db, "/large.bin", new Uint8Array([9, 9, 9]), CHUNK_SIZE + 10, {}, () => 1001); + const after = chunkRows(db, "/large.bin"); + + expect(after).toHaveLength(3); + expect(Buffer.from(after[0].hash).equals(Buffer.from(before[0].hash))).toBe(true); + expect(Buffer.from(after[1].hash).equals(Buffer.from(before[1].hash))).toBe(false); + expect(Buffer.from(after[2].hash).equals(Buffer.from(before[2].hash))).toBe(true); + const bytes = await readBytes(db, "/large.bin"); + expect(bytes[CHUNK_SIZE + 9]).toBe(2); + expect(Array.from(bytes.subarray(CHUNK_SIZE + 10, CHUNK_SIZE + 13))).toEqual([9, 9, 9]); + expect(bytes[CHUNK_SIZE + 13]).toBe(2); + }); + }); + + it("writes through hardlinks by shared inode", async () => { + await withDB(async (db) => { + createFileSync(db, "/a.txt", {}, () => 1000); + link(db, "/a.txt", "/b.txt"); + + writeRangeSync(db, "/b.txt", bytesOf("shared"), 0, {}, () => 1001); + + expect(new TextDecoder().decode(await readBytes(db, "/a.txt"))).toBe("shared"); + expect(resolveInode(db, "/a.txt")?.inode).toBe(resolveInode(db, "/b.txt")?.inode); + }); + }); + + it("skips rewriting untouched chunks on a small range write", async () => { + await withDB(async (db) => { + const original = new Uint8Array(CHUNK_SIZE * 3); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + original.fill(3, CHUNK_SIZE * 2, CHUNK_SIZE * 3); + writeFileSync(db, "/large.bin", original, {}, () => 1000); + + // Touch only the middle chunk, at a later mtime. + writeRangeSync(db, "/large.bin", new Uint8Array([7]), CHUNK_SIZE + 10, {}, () => 1001); + + // applyChunkedInodeUpdate skips SQL entirely for unchanged chunks, + // so their blobs are never re-upserted and keep the original + // last_seen; only the rewritten middle chunk's blob is stamped with + // the new mtime. (vfs_chunks is WITHOUT ROWID, so there is no rowid + // to watch. last_seen is bumped only by upsertChunkBlob, so an + // unchanged last_seen proves the chunk row was skipped.) + const rows = chunkRows(db, "/large.bin"); + expect(rows).toHaveLength(3); + const lastSeen = (hash: Uint8Array): number | undefined => + db.one<{ last_seen: number }>("SELECT last_seen FROM vfs_blobs WHERE hash = ?", hash) + ?.last_seen; + expect(lastSeen(rows[0].hash)).toBe(1000); + expect(lastSeen(rows[1].hash)).toBe(1001); + expect(lastSeen(rows[2].hash)).toBe(1000); + }); + }); + + it("invalidates the manifest hash after a direct range write", async () => { + await withDB(async (db) => { + const original = new Uint8Array(CHUNK_SIZE * 2); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE); + writeFileSync(db, "/large.bin", original, {}, () => 1000); + expect(manifestHash(db, "/large.bin")).not.toBe(null); + + writeRangeSync(db, "/large.bin", new Uint8Array([5]), 10, {}, () => 1001); + expect(manifestHash(db, "/large.bin")).toBe(null); + }); + }); + + it("truncates chunk-backed files without rewriting untouched chunks", async () => { + await withDB(async (db) => { + const original = new Uint8Array(CHUNK_SIZE * 2 + 100); + original.fill(1, 0, CHUNK_SIZE); + original.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + original.fill(3, CHUNK_SIZE * 2); + writeFileSync(db, "/truncate.bin", original, {}, () => 1000); + const before = chunkRows(db, "/truncate.bin"); + + truncateFileSync(db, "/truncate.bin", CHUNK_SIZE + 50, () => 1001); + const after = chunkRows(db, "/truncate.bin"); + + expect(after).toHaveLength(2); + expect(after[1].size).toBe(50); + expect(Buffer.from(after[0].hash).equals(Buffer.from(before[0].hash))).toBe(true); + expect(Buffer.from(after[1].hash).equals(Buffer.from(before[1].hash))).toBe(false); + expect((await readBytes(db, "/truncate.bin")).byteLength).toBe(CHUNK_SIZE + 50); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/index.ts b/spikes/349-dofs/vendor/dofs/src/index.ts new file mode 100644 index 00000000..76ef05a5 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/index.ts @@ -0,0 +1,68 @@ +export type { WorkspaceErrorCode, WorkspaceFsError } from "./errors.js"; +export { createWorkspaceError } from "./errors.js"; +export { chmod } from "./fs/chmod.js"; +export { + WorkspaceFilesystem, + type WorkspaceFilesystemOptions, +} from "./fs/filesystem.js"; +export type { WorkspaceFoundEntry } from "./fs/find.js"; +export type { GrepOptions, WorkspaceGrepMatch } from "./fs/grep.js"; +export { link } from "./fs/link.js"; +export type { MkdirOptions } from "./fs/mkdir.js"; +// Read-only mount enforcement. The workspace-side indexer writes +// _vfs_mounts; the helpers here let it invalidate the in-Database +// cache after a write, and let dofs callers (and tests) inspect or +// assert against the registered roots without re-implementing the +// overlap check. +export { + assertNotReadOnly, + getReadOnlyMountRoots, + invalidateReadOnlyMountCache, + readOnlyRootFor, +} from "./fs/mount-guard.js"; +export type { ReaddirOptions, WorkspaceDirentResult } from "./fs/readdir.js"; +export type { ReadFileOptions } from "./fs/readFile.js"; +export { readlink } from "./fs/readlink.js"; +export type { RmOptions } from "./fs/rm.js"; +export { lstat, stat, type WorkspaceStatResult } from "./fs/stat.js"; +export { symlink } from "./fs/symlink.js"; +export type { WriteFileContent, WriteFileOptions } from "./fs/writeFile.js"; +export type { SQLiteWorkspaceProviderOptions } from "./provider.js"; +export { SQLiteWorkspaceProvider } from "./provider.js"; +export { initializeSchema, ROOT_INODE, SCHEMA_VERSION } from "./schema/index.js"; +export { Database } from "./storage.js"; +export type { ApplyOptions, ApplyResult, SkippedEntry } from "./sync/apply.js"; +// Sync protocol building blocks. The wire wiring lives in +// @cloudflare/computer-rpc; these are the helpers that wiring binds +// to a Database. +export { applyChanges, applyChangesSync } from "./sync/apply.js"; +export { stageBlob } from "./sync/blobs.js"; +export type { ChangeEntry } from "./sync/changes.js"; +export { materialiseChange } from "./sync/changes.js"; +export type { CoalesceOptions } from "./sync/coalesce.js"; +export { coalesceChanges } from "./sync/coalesce.js"; +export { fetchChanges, fetchObjects, hasObjects } from "./sync/fetch.js"; +export { DEFAULT_IGNORE, isIgnored } from "./sync/ignore.js"; +export { assertAppliedPushCursor } from "./sync/invariant.js"; +export type { ManifestChunk } from "./sync/manifests.js"; +export { buildManifest, MANIFEST_VERSION } from "./sync/manifests.js"; +export { pushObjects } from "./sync/push.js"; +export type { ChangeCursor, WatermarkKey } from "./sync/watermarks.js"; +export { + compareChangeCursors, + currentRev, + readFetchCursor, + readWatermark, + writeFetchCursor, + writeWatermark, +} from "./sync/watermarks.js"; +export type { ExecutedStatement } from "./testing-recording.js"; +// RecordingStorage is workerd-safe (pure JS). SQLiteTestStorage +// wraps node:sqlite and must be imported from +// '@cloudflare/dofs/testing' under node-only call sites. +export { RecordingStorage } from "./testing-recording.js"; +export type { + DurableObjectStorageLike, + SQLCursorLike, + SQLStorageLike, +} from "./types.js"; diff --git a/spikes/349-dofs/vendor/dofs/src/path.test.ts b/spikes/349-dofs/vendor/dofs/src/path.test.ts new file mode 100644 index 00000000..6d513baa --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/path.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; + +import { canonicalizePath } from "./path.js"; + +describe("canonicalizePath", () => { + it("requires an absolute path", () => { + expect(() => canonicalizePath("workspace/file.txt")).toThrowError(/must be absolute/); + }); + + it("normalizes duplicate slashes and dot segments", () => { + expect(canonicalizePath("/workspace//src/./index.ts")).toEqual({ + path: "/workspace/src/index.ts", + parts: ["workspace", "src", "index.ts"], + name: "index.ts", + parentPath: "/workspace/src", + }); + }); + + it("canonicalizes parent segments without escaping root", () => { + expect(canonicalizePath("/workspace/src/../README.md").path).toBe("/workspace/README.md"); + expect(() => canonicalizePath("/..")).toThrowError(/escapes root/); + }); + + it("represents root explicitly", () => { + expect(canonicalizePath("/")).toEqual({ + path: "/", + parts: [], + name: "", + parentPath: undefined, + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/path.ts b/spikes/349-dofs/vendor/dofs/src/path.ts new file mode 100644 index 00000000..777a39a0 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/path.ts @@ -0,0 +1,52 @@ +import { invalidPath } from "./errors.js"; + +export interface CanonicalPath { + path: string; + parts: string[]; + name: string; + parentPath: string | undefined; +} + +export function canonicalizePath(path: string): CanonicalPath { + if (path.length === 0) { + throw invalidPath(path, "empty"); + } + + if (!path.startsWith("/")) { + throw invalidPath(path, "must be absolute"); + } + + if (path.includes("\0")) { + throw invalidPath(path, "contains NUL byte"); + } + + const parts: string[] = []; + for (const part of path.split("/")) { + if (part === "" || part === ".") { + continue; + } + + if (part === "..") { + if (parts.length === 0) { + throw invalidPath(path, "escapes root"); + } + parts.pop(); + continue; + } + + parts.push(part); + } + + const canonical = parts.length === 0 ? "/" : `/${parts.join("/")}`; + const name = parts.length === 0 ? "" : parts[parts.length - 1]; + const parentParts = parts.slice(0, -1); + const parentPath = + parts.length === 0 ? undefined : parentParts.length === 0 ? "/" : `/${parentParts.join("/")}`; + + return { + path: canonical, + parts, + name, + parentPath, + }; +} diff --git a/spikes/349-dofs/vendor/dofs/src/provider.fd.test.ts b/spikes/349-dofs/vendor/dofs/src/provider.fd.test.ts new file mode 100644 index 00000000..e7b0f5b8 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/provider.fd.test.ts @@ -0,0 +1,376 @@ +// File descriptor / positional I/O tests for SQLiteWorkspaceProvider. +// Separate file so the read/write coverage can grow without bloating +// the scaffold test file. + +import { describe, expect, it } from "vitest"; + +import { resolveInode } from "./fs/resolve.js"; +import { withDB } from "./fs/with-db.js"; +import { SQLiteWorkspaceProvider } from "./provider.js"; + +async function withProvider(fn: (p: SQLiteWorkspaceProvider) => T | Promise): Promise { + return withDB((db) => fn(new SQLiteWorkspaceProvider(db, { now: () => 1000 }))); +} + +// 512 KiB to match writeFile's CHUNK_SIZE so tests can deliberately +// straddle chunk boundaries. +const CHUNK_SIZE = 512 * 1024; + +function chunkHashes(p: SQLiteWorkspaceProvider, path: string): Buffer[] { + const node = resolveInode(p.db, path); + if (node === null) throw new Error(`missing node: ${path}`); + return p.db + .all<{ hash: Uint8Array }>( + "SELECT hash FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node.inode, + ) + .map((row) => Buffer.from(row.hash)); +} + +describe("SQLiteWorkspaceProvider — file descriptors", () => { + it("openSync allocates a positive integer", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello"); + const fd = p.openSync("/a", "r"); + expect(typeof fd).toBe("number"); + expect((fd as number) > 0).toBe(true); + p.closeSync(fd as number); + }); + }); + + it("openSync('r') on a missing file throws ENOENT", async () => { + await withProvider((p) => { + expect(() => p.openSync("/missing", "r")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("openSync('w') creates a missing file as empty", async () => { + await withProvider((p) => { + const fd = p.openSync("/new", "w"); + expect(p.existsSync("/new")).toBe(true); + expect(p.statSync("/new").size).toBe(0); + p.closeSync(fd as number); + }); + }); + + it("openSync('w') truncates an existing file", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "before"); + const fd = p.openSync("/a", "w"); + expect(p.statSync("/a").size).toBe(0); + p.closeSync(fd as number); + }); + }); + + it("openSync('a') opens for append without truncating", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello"); + const fd = p.openSync("/a", "a"); + expect(p.statSync("/a").size).toBe(5); + p.closeSync(fd as number); + }); + }); + + it("closeSync on an unknown fd throws EBADF", async () => { + await withProvider((p) => { + expect(() => p.closeSync(9999)).toThrowError(expect.objectContaining({ code: "EBADF" })); + }); + }); + + it("fstatSync mirrors statSync for the fd's path", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello", { mode: 0o644 }); + const fd = p.openSync("/a", "r") as number; + const s = p.fstatSync(fd); + expect(s.size).toBe(5); + expect(s.isFile()).toBe(true); + p.closeSync(fd); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — readSync", () => { + it("reads from the fd's position when position is null", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello workspace"); + const fd = p.openSync("/a", "r") as number; + const buf = Buffer.alloc(5); + const n = p.readSync(fd, buf, 0, 5, null); + expect(n).toBe(5); + expect(buf.toString()).toBe("hello"); + // Position advanced; next read continues. + const n2 = p.readSync(fd, buf, 0, 5, null); + expect(n2).toBe(5); + expect(buf.toString()).toBe(" work"); + p.closeSync(fd); + }); + }); + + it("reads at an explicit position without moving the fd", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello workspace"); + const fd = p.openSync("/a", "r") as number; + const buf = Buffer.alloc(5); + const n = p.readSync(fd, buf, 0, 5, 6); + expect(n).toBe(5); + expect(buf.toString()).toBe("works"); + // Fd position unchanged: still at 0. + p.readSync(fd, buf, 0, 5, null); + expect(buf.toString()).toBe("hello"); + p.closeSync(fd); + }); + }); + + it("returns 0 when reading past EOF", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "tiny"); + const fd = p.openSync("/a", "r") as number; + const buf = Buffer.alloc(10); + expect(p.readSync(fd, buf, 0, 10, 100)).toBe(0); + p.closeSync(fd); + }); + }); + + it("respects the buffer offset", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "abcde"); + const fd = p.openSync("/a", "r") as number; + const buf = Buffer.alloc(10).fill(0x2e); // '.' + const n = p.readSync(fd, buf, 3, 5, 0); + expect(n).toBe(5); + expect(buf.toString()).toBe("...abcde.."); + p.closeSync(fd); + }); + }); + + it("reads across a chunk boundary", async () => { + await withProvider((p) => { + // Build a file that crosses one 512KiB boundary. + const bytes = new Uint8Array(CHUNK_SIZE + 100); + bytes.fill(0x41); + for (let i = CHUNK_SIZE; i < bytes.byteLength; i++) bytes[i] = 0x42; + p.writeFileSync("/big", Buffer.from(bytes)); + const fd = p.openSync("/big", "r") as number; + const buf = Buffer.alloc(200); + // Straddle the boundary: read 200 bytes starting 100 bytes before it. + const n = p.readSync(fd, buf, 0, 200, CHUNK_SIZE - 100); + expect(n).toBe(200); + // First 100 bytes are 'A' (pre-boundary), remaining 100 are 'B'. + for (let i = 0; i < 100; i++) expect(buf[i]).toBe(0x41); + for (let i = 100; i < 200; i++) expect(buf[i]).toBe(0x42); + p.closeSync(fd); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — direct range methods", () => { + it("exposes direct create, write range, and truncate methods", async () => { + await withProvider((p) => { + p.createFileSync("/direct.txt", { mode: 0o600 }); + expect(p.statSync("/direct.txt").mode & 0o777).toBe(0o600); + + expect(p.writeRangeSync("/direct.txt", Buffer.from("abcdef"), 0)).toBe(6); + expect(p.writeRangeSync("/direct.txt", Buffer.from("Z"), 3)).toBe(1); + expect(p.readFileSync("/direct.txt", "utf8")).toBe("abcZef"); + + p.truncateFileSync("/direct.txt", 4); + expect(p.readFileSync("/direct.txt", "utf8")).toBe("abcZ"); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — writeSync", () => { + it("writes at position 0 and updates content", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello"); + const fd = p.openSync("/a", "r+") as number; + const n = p.writeSync(fd, Buffer.from("HELLO"), 0, 5, 0); + expect(n).toBe(5); + p.closeSync(fd); + expect(p.readFileSync("/a", "utf8")).toBe("HELLO"); + }); + }); + + it("writes at a non-zero offset", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello world"); + const fd = p.openSync("/a", "r+") as number; + p.writeSync(fd, Buffer.from("WORLD"), 0, 5, 6); + p.closeSync(fd); + expect(p.readFileSync("/a", "utf8")).toBe("hello WORLD"); + }); + }); + + it("extends the file when writing past EOF", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hi"); + const fd = p.openSync("/a", "r+") as number; + p.writeSync(fd, Buffer.from("bye"), 0, 3, 2); + p.closeSync(fd); + expect(p.readFileSync("/a", "utf8")).toBe("hibye"); + }); + }); + + it("zero-fills the gap when writing past current size", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "ab"); + const fd = p.openSync("/a", "r+") as number; + p.writeSync(fd, Buffer.from("z"), 0, 1, 5); + p.closeSync(fd); + const stat = p.statSync("/a"); + expect(stat.size).toBe(6); + const out = p.readFileSync("/a") as Buffer; + expect(out[0]).toBe(0x61); // 'a' + expect(out[1]).toBe(0x62); // 'b' + expect(out[2]).toBe(0); + expect(out[3]).toBe(0); + expect(out[4]).toBe(0); + expect(out[5]).toBe(0x7a); // 'z' + }); + }); + + it("advances the fd position when position is null", async () => { + await withProvider((p) => { + p.writeFileSync("/a", ""); + const fd = p.openSync("/a", "w") as number; + p.writeSync(fd, Buffer.from("abc"), 0, 3, null); + p.writeSync(fd, Buffer.from("def"), 0, 3, null); + p.closeSync(fd); + expect(p.readFileSync("/a", "utf8")).toBe("abcdef"); + }); + }); + + it("writes across a chunk boundary, splicing in the affected chunks only", async () => { + await withProvider((p) => { + const before = new Uint8Array(CHUNK_SIZE + 100); + before.fill(0x41); + for (let i = CHUNK_SIZE; i < before.byteLength; i++) before[i] = 0x42; + p.writeFileSync("/big", Buffer.from(before)); + + const fd = p.openSync("/big", "r+") as number; + // Overwrite 200 bytes that straddle the boundary with 'Z'. + const stamp = Buffer.alloc(200, 0x5a); + p.writeSync(fd, stamp, 0, 200, CHUNK_SIZE - 100); + p.closeSync(fd); + + const out = p.readFileSync("/big") as Buffer; + expect(out.byteLength).toBe(CHUNK_SIZE + 100); + expect(out[0]).toBe(0x41); + expect(out[CHUNK_SIZE - 101]).toBe(0x41); + expect(out[CHUNK_SIZE - 100]).toBe(0x5a); + expect(out[CHUNK_SIZE + 99]).toBe(0x5a); + // Anything past the overwrite is whatever remained of 'B'. + // (The original had only 100 B-bytes total, all of which we overwrote.) + }); + }); + + it("reuses untouched chunk rows for positional writes", async () => { + await withProvider((p) => { + const before = new Uint8Array(CHUNK_SIZE * 3); + before.fill(1, 0, CHUNK_SIZE); + before.fill(2, CHUNK_SIZE, CHUNK_SIZE * 2); + before.fill(3, CHUNK_SIZE * 2); + p.writeFileSync("/big", Buffer.from(before)); + const oldHashes = chunkHashes(p, "/big"); + + const fd = p.openSync("/big", "r+") as number; + p.writeSync(fd, Buffer.from([9, 9, 9]), 0, 3, CHUNK_SIZE + 10); + p.closeSync(fd); + const newHashes = chunkHashes(p, "/big"); + + expect(newHashes[0].equals(oldHashes[0])).toBe(true); + expect(newHashes[1].equals(oldHashes[1])).toBe(false); + expect(newHashes[2].equals(oldHashes[2])).toBe(true); + }); + }); + + it("openSync('a') starts the fd at EOF", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello"); + const fd = p.openSync("/a", "a") as number; + p.writeSync(fd, Buffer.from(" world"), 0, 6, null); + p.closeSync(fd); + expect(p.readFileSync("/a", "utf8")).toBe("hello world"); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — truncateSync / ftruncateSync", () => { + it("truncateSync shrinks a file", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello world"); + p.truncateSync("/a", 5); + expect(p.readFileSync("/a", "utf8")).toBe("hello"); + }); + }); + + it("truncateSync grows a file with zero fill", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "abc"); + p.truncateSync("/a", 6); + const out = p.readFileSync("/a") as Buffer; + expect(out.byteLength).toBe(6); + expect(out[0]).toBe(0x61); + expect(out[3]).toBe(0); + }); + }); + + it("truncateSync to 0 leaves an empty file", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello"); + p.truncateSync("/a", 0); + expect(p.statSync("/a").size).toBe(0); + }); + }); + + it("truncateSync at the same size is a no-op", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello"); + p.truncateSync("/a", 5); + expect(p.readFileSync("/a", "utf8")).toBe("hello"); + }); + }); + + it("truncateSync shrinks across a chunk boundary", async () => { + await withProvider((p) => { + const bytes = new Uint8Array(CHUNK_SIZE + 100); + bytes.fill(0x41); + p.writeFileSync("/big", Buffer.from(bytes)); + p.truncateSync("/big", CHUNK_SIZE - 10); + expect(p.statSync("/big").size).toBe(CHUNK_SIZE - 10); + }); + }); + + it("truncateSync grows across a chunk boundary", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "ab"); + p.truncateSync("/a", CHUNK_SIZE + 100); + expect(p.statSync("/a").size).toBe(CHUNK_SIZE + 100); + const out = p.readFileSync("/a") as Buffer; + expect(out[0]).toBe(0x61); + expect(out[1]).toBe(0x62); + expect(out[2]).toBe(0); + expect(out[CHUNK_SIZE + 99]).toBe(0); + }); + }); + + it("truncateSync on a missing file throws ENOENT", async () => { + await withProvider((p) => { + expect(() => p.truncateSync("/missing", 0)).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("ftruncateSync mirrors truncateSync through an fd", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "hello world"); + const fd = p.openSync("/a", "r+") as number; + p.ftruncateSync(fd, 5); + p.closeSync(fd); + expect(p.readFileSync("/a", "utf8")).toBe("hello"); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/provider.test.ts b/spikes/349-dofs/vendor/dofs/src/provider.test.ts new file mode 100644 index 00000000..769086d7 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/provider.test.ts @@ -0,0 +1,854 @@ +import { describe, expect, it } from "vitest"; +import { withDB } from "./fs/with-db.js"; +import { SQLiteWorkspaceProvider } from "./provider.js"; +import type { Database } from "./storage.js"; +import type { ChangeEntry } from "./sync/changes.js"; +import { coalesceChanges } from "./sync/coalesce.js"; + +// Each provider test gets a fresh DB via withDB, which the workers +// runner aliases to a DO-backed implementation. The provider holds +// no I/O resources of its own, so it's safe to construct inside the +// withDB callback and let the storage handle teardown. +async function withProvider(fn: (p: SQLiteWorkspaceProvider) => T | Promise): Promise { + return withDB((db) => fn(new SQLiteWorkspaceProvider(db, { now: () => 1000 }))); +} + +async function withProviderAndDB( + fn: (p: SQLiteWorkspaceProvider, db: Database) => T | Promise, +): Promise { + return withDB((db) => fn(new SQLiteWorkspaceProvider(db, { now: () => 1000 }), db)); +} + +async function drainChanges(db: Database, afterRev: number): Promise { + const out: ChangeEntry[] = []; + for await (const entry of coalesceChanges(db, afterRev)) out.push(entry); + return out; +} + +function kindPath(entries: ChangeEntry[]): Array<[ChangeEntry["kind"], string]> { + return entries.map((entry) => [entry.kind, entry.path]).sort((a, b) => a[1].localeCompare(b[1])); +} + +describe("SQLiteWorkspaceProvider — capability flags", () => { + it("reports the supported feature set", async () => { + await withProvider((p) => { + expect(p.readonly).toBe(false); + expect(p.supportsSymlinks).toBe(true); + expect(p.supportsWatch).toBe(true); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — implemented methods", () => { + it("mkdirSync creates a directory", async () => { + await withProvider((p) => { + p.mkdirSync("/a", { mode: 0o755 }); + expect(p.existsSync("/a")).toBe(true); + }); + }); + + it("statSync returns a VirtualStats-shaped object", async () => { + await withProvider((p) => { + p.mkdirSync("/a", {}); + const s = p.statSync("/a"); + expect(s.isDirectory()).toBe(true); + expect(s.isFile()).toBe(false); + expect(s.isSymbolicLink()).toBe(false); + // 0o40755 — S_IFDIR or permissions. Linux FUSE rejects a + // stat without the file-type bits, so we always set them. + expect(s.mode).toBe(0o40755); + expect(typeof s.ino).toBe("number"); + expect(typeof s.mtimeMs).toBe("number"); + expect(s.mtime).toBeInstanceOf(Date); + }); + }); + + it("lstatSync returns the same shape as statSync today (no symlinks yet)", async () => { + await withProvider((p) => { + p.mkdirSync("/a", {}); + expect(p.lstatSync("/a").isDirectory()).toBe(true); + }); + }); + + it("readdirSync returns names by default and dirent objects with withFileTypes", async () => { + await withProvider((p) => { + p.mkdirSync("/a", {}); + p.mkdirSync("/b", {}); + expect(p.readdirSync("/")).toEqual(["a", "b"]); + const dirents = p.readdirSync("/", { withFileTypes: true }); + expect(Array.isArray(dirents)).toBe(true); + expect((dirents as Array<{ name: string; isDirectory(): boolean }>)[0].isDirectory()).toBe( + true, + ); + }); + }); + + it("unlinkSync removes a file", async () => { + await withProvider((p) => { + p.writeFileSync("/a.txt", "hi"); + p.unlinkSync("/a.txt"); + expect(p.existsSync("/a.txt")).toBe(false); + }); + }); + + it("linkSync creates a second path to the same file inode", async () => { + await withProvider((p) => { + p.writeFileSync("/a.txt", "hi"); + p.linkSync("/a.txt", "/b.txt"); + + const a = p.statSync("/a.txt"); + const b = p.statSync("/b.txt"); + expect(a.ino).toBe(b.ino); + expect(a.nlink).toBe(2); + expect(b.nlink).toBe(2); + expect(p.readFileSync("/b.txt", "utf8")).toBe("hi"); + }); + }); + + it("writes through one hardlink are visible through the other", async () => { + await withProvider((p) => { + p.writeFileSync("/a.txt", "hi"); + p.linkSync("/a.txt", "/b.txt"); + p.writeFileSync("/b.txt", "bye"); + + expect(p.readFileSync("/a.txt", "utf8")).toBe("bye"); + expect(p.statSync("/a.txt").nlink).toBe(2); + expect(p.statSync("/b.txt").nlink).toBe(2); + }); + }); + + it("unlinkSync removes one hardlink without deleting the inode", async () => { + await withProvider((p) => { + p.writeFileSync("/a.txt", "hi"); + p.linkSync("/a.txt", "/b.txt"); + p.unlinkSync("/a.txt"); + + expect(p.existsSync("/a.txt")).toBe(false); + expect(p.readFileSync("/b.txt", "utf8")).toBe("hi"); + expect(p.statSync("/b.txt").nlink).toBe(1); + }); + }); + + it("renameSync from one hardlink onto another removes only the source name", async () => { + await withProvider((p) => { + p.writeFileSync("/a.txt", "hi"); + p.linkSync("/a.txt", "/b.txt"); + p.renameSync("/a.txt", "/b.txt"); + + expect(p.existsSync("/a.txt")).toBe(false); + expect(p.readFileSync("/b.txt", "utf8")).toBe("hi"); + expect(p.statSync("/b.txt").nlink).toBe(1); + }); + }); + + it("linkSync rejects invalid links", async () => { + await withProvider((p) => { + p.writeFileSync("/a.txt", "hi"); + p.mkdirSync("/dir", {}); + + expect(() => p.linkSync("/missing", "/missing-link")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + expect(() => p.linkSync("/a.txt", "/a.txt")).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + expect(() => p.linkSync("/dir", "/dir-link")).toThrowError( + expect.objectContaining({ code: "EPERM" }), + ); + expect(() => p.linkSync("/a.txt", "/missing-parent/b.txt")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("unlinkSync removes a symlink without deleting its target", async () => { + await withProvider((p) => { + p.writeFileSync("/target", "content"); + p.symlinkSync("/target", "/link"); + + p.unlinkSync("/link"); + + expect(p.existsSync("/link")).toBe(false); + expect(p.readFileSync("/target", "utf8")).toBe("content"); + }); + }); + + it("rmdirSync removes an empty directory", async () => { + await withProvider((p) => { + p.mkdirSync("/a", {}); + p.rmdirSync("/a"); + expect(p.existsSync("/a")).toBe(false); + }); + }); + + it("renameSync moves an entry", async () => { + await withProvider((p) => { + p.writeFileSync("/a", "x"); + p.renameSync("/a", "/b"); + expect(p.existsSync("/a")).toBe(false); + expect(p.existsSync("/b")).toBe(true); + }); + }); + + it("writeFileSync + readFileSync round-trip a string", async () => { + await withProvider((p) => { + p.writeFileSync("/a.txt", "hello workspace"); + expect(p.readFileSync("/a.txt", "utf8")).toBe("hello workspace"); + }); + }); + + it("writeFileSync + readFileSync round-trip bytes", async () => { + await withProvider((p) => { + p.writeFileSync("/a.bin", Buffer.from([1, 2, 3])); + const back = p.readFileSync("/a.bin"); + expect(back).toBeInstanceOf(Buffer); + expect(Array.from(back as Buffer)).toEqual([1, 2, 3]); + }); + }); + + it("existsSync returns true / false correctly", async () => { + await withProvider((p) => { + expect(p.existsSync("/missing")).toBe(false); + p.mkdirSync("/d", {}); + expect(p.existsSync("/d")).toBe(true); + }); + }); + + it("realpathSync returns the canonical path", async () => { + await withProvider((p) => { + p.mkdirSync("/a", {}); + expect(p.realpathSync("/a/./../a")).toBe("/a"); + }); + }); + + it("accessSync resolves for existing paths and throws ENOENT for missing", async () => { + await withProvider((p) => { + p.mkdirSync("/a", {}); + expect(() => p.accessSync("/a")).not.toThrow(); + expect(() => p.accessSync("/missing")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — renameSync overwrite matrix", () => { + it("records rename as an old-path delete and new-path live entry", async () => { + await withProviderAndDB(async (p, db) => { + p.writeFileSync("/src", "new"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/src", "/dst"); + + const entries = await drainChanges(db, cursor); + expect(kindPath(entries)).toEqual([ + ["file", "/dst"], + ["delete", "/src"], + ]); + }); + }); + + it("records directory rename for the whole moved subtree", async () => { + await withProviderAndDB(async (p, db) => { + p.mkdirSync("/src", {}); + p.writeFileSync("/src/a.txt", "a"); + p.mkdirSync("/src/sub", {}); + p.writeFileSync("/src/sub/b.txt", "b"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/src", "/dst"); + + const entries = await drainChanges(db, cursor); + expect(kindPath(entries)).toEqual([ + ["dir", "/dst"], + ["file", "/dst/a.txt"], + ["dir", "/dst/sub"], + ["file", "/dst/sub/b.txt"], + ["delete", "/src"], + ["delete", "/src/a.txt"], + ["delete", "/src/sub"], + ["delete", "/src/sub/b.txt"], + ]); + }); + }); + + it("records rename tombstones at the resolved old file path", async () => { + await withProviderAndDB(async (p, db) => { + p.mkdirSync("/real", {}); + p.writeFileSync("/real/file.txt", "x"); + p.symlinkSync("/real", "/link"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/link/file.txt", "/dst.txt"); + + const entries = await drainChanges(db, cursor); + expect(kindPath(entries)).toEqual([ + ["file", "/dst.txt"], + ["delete", "/real/file.txt"], + ]); + }); + }); + + it("records directory rename tombstones at the resolved old subtree paths", async () => { + await withProviderAndDB(async (p, db) => { + p.mkdirSync("/real", {}); + p.mkdirSync("/real/dir", {}); + p.writeFileSync("/real/dir/a.txt", "a"); + p.mkdirSync("/real/dir/sub", {}); + p.writeFileSync("/real/dir/sub/b.txt", "b"); + p.symlinkSync("/real", "/link"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/link/dir", "/dst"); + + const entries = await drainChanges(db, cursor); + expect(kindPath(entries)).toEqual([ + ["dir", "/dst"], + ["file", "/dst/a.txt"], + ["dir", "/dst/sub"], + ["file", "/dst/sub/b.txt"], + ["delete", "/real/dir"], + ["delete", "/real/dir/a.txt"], + ["delete", "/real/dir/sub"], + ["delete", "/real/dir/sub/b.txt"], + ]); + }); + }); + + it("subtree rename writes one tombstone per edge and one rev stamp per inode", async () => { + await withProviderAndDB(async (p, db) => { + // A nested subtree with a hardlink inside it: the file inode is + // reachable by two names, so both edges must be tombstoned while + // the single inode is stamped once. + p.mkdirSync("/src", {}); + p.writeFileSync("/src/a.txt", "a"); + p.mkdirSync("/src/sub", {}); + p.writeFileSync("/src/sub/b.txt", "b"); + p.linkSync("/src/a.txt", "/src/sub/a2.txt"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/src", "/dst"); + + // Every edge of the moved subtree, tombstoned at its old path and + // sharing the rename's rev. + const tombstones = db.all<{ rev: number; path: string; op: string }>( + "SELECT rev, path, op FROM vfs_changes WHERE rev > ? ORDER BY path", + cursor, + ); + const rev = tombstones[0]?.rev; + expect(tombstones).toEqual([ + { rev, path: "/src", op: "delete" }, + { rev, path: "/src/a.txt", op: "delete" }, + { rev, path: "/src/sub", op: "delete" }, + { rev, path: "/src/sub/a2.txt", op: "delete" }, + { rev, path: "/src/sub/b.txt", op: "delete" }, + ]); + + // The shared rev lands on exactly the four subtree inodes (the + // hardlinked file counted once) and on nothing else. + const stamped = db + .all<{ inode: number }>("SELECT inode FROM vfs_nodes WHERE rev = ? ORDER BY inode", rev) + .map((r) => r.inode); + const expected = [ + p.statSync("/dst").ino, + p.statSync("/dst/a.txt").ino, + p.statSync("/dst/sub").ino, + p.statSync("/dst/sub/b.txt").ino, + ].sort((a, b) => a - b); + expect(stamped).toEqual(expected); + }); + }); + + it("same-inode rename through a symlinked file path is a no-op", async () => { + await withProviderAndDB(async (p, db) => { + p.mkdirSync("/real", {}); + p.writeFileSync("/real/file.txt", "x"); + p.symlinkSync("/real", "/link"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/real/file.txt", "/link/file.txt"); + + expect(p.readFileSync("/real/file.txt", "utf8")).toBe("x"); + expect(p.readFileSync("/link/file.txt", "utf8")).toBe("x"); + expect(await drainChanges(db, cursor)).toEqual([]); + }); + }); + + it("same-inode rename through a symlinked directory path is a no-op", async () => { + await withProviderAndDB(async (p, db) => { + p.mkdirSync("/real", {}); + p.mkdirSync("/real/dir", {}); + p.writeFileSync("/real/dir/a.txt", "a"); + p.symlinkSync("/real", "/link"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/real/dir", "/link/dir"); + + expect(p.readFileSync("/real/dir/a.txt", "utf8")).toBe("a"); + expect(p.readFileSync("/link/dir/a.txt", "utf8")).toBe("a"); + expect(await drainChanges(db, cursor)).toEqual([]); + }); + }); + + it("file → existing file overwrites atomically", async () => { + await withProvider((p) => { + p.writeFileSync("/src", "new"); + p.writeFileSync("/dst", "old"); + p.renameSync("/src", "/dst"); + expect(p.existsSync("/src")).toBe(false); + expect(p.readFileSync("/dst", "utf8")).toBe("new"); + }); + }); + + it("dir → existing non-empty dir throws ENOTEMPTY", async () => { + await withProvider((p) => { + p.mkdirSync("/src", {}); + p.mkdirSync("/dst", {}); + p.writeFileSync("/dst/inside", "x"); + expect(() => p.renameSync("/src", "/dst")).toThrowError( + expect.objectContaining({ code: "ENOTEMPTY" }), + ); + // Both directories survive the failed rename. + expect(p.existsSync("/src")).toBe(true); + expect(p.existsSync("/dst/inside")).toBe(true); + }); + }); + + it("dir → existing empty dir succeeds", async () => { + await withProvider((p) => { + p.mkdirSync("/src", {}); + p.writeFileSync("/src/inside", "x"); + p.mkdirSync("/dst", {}); + p.renameSync("/src", "/dst"); + expect(p.existsSync("/src")).toBe(false); + expect(p.readFileSync("/dst/inside", "utf8")).toBe("x"); + }); + }); + + it.each([ + { + name: "file → existing symlink", + setup(p: SQLiteWorkspaceProvider) { + p.writeFileSync("/target", "x"); + p.writeFileSync("/src", "new"); + p.symlinkSync("/target", "/dst"); + }, + assertRenamed(p: SQLiteWorkspaceProvider) { + expect(p.existsSync("/src")).toBe(false); + expect(p.lstatSync("/dst").isFile()).toBe(true); + expect(p.readFileSync("/dst", "utf8")).toBe("new"); + expect(p.readFileSync("/target", "utf8")).toBe("x"); + }, + }, + { + name: "symlink → existing file", + setup(p: SQLiteWorkspaceProvider) { + p.writeFileSync("/target", "x"); + p.symlinkSync("/target", "/src"); + p.writeFileSync("/dst", "old"); + }, + assertRenamed(p: SQLiteWorkspaceProvider) { + expect(p.existsSync("/src")).toBe(false); + expect(p.lstatSync("/dst").isSymbolicLink()).toBe(true); + expect(p.readlinkSync("/dst")).toBe("/target"); + }, + }, + { + name: "symlink → existing symlink", + setup(p: SQLiteWorkspaceProvider) { + p.writeFileSync("/target", "x"); + p.writeFileSync("/other", "y"); + p.symlinkSync("/target", "/src"); + p.symlinkSync("/other", "/dst"); + }, + assertRenamed(p: SQLiteWorkspaceProvider) { + expect(p.existsSync("/src")).toBe(false); + expect(p.lstatSync("/dst").isSymbolicLink()).toBe(true); + expect(p.readlinkSync("/dst")).toBe("/target"); + }, + }, + ])("$name overwrites atomically", async ({ setup, assertRenamed }) => { + await withProvider((p) => { + setup(p); + p.renameSync("/src", "/dst"); + assertRenamed(p); + }); + }); + + it.each([ + { + name: "file → existing empty dir", + code: "EISDIR", + setup(p: SQLiteWorkspaceProvider) { + p.writeFileSync("/src", "new"); + p.mkdirSync("/dst", {}); + }, + assertUnchanged(p: SQLiteWorkspaceProvider) { + expect(p.readFileSync("/src", "utf8")).toBe("new"); + expect(p.statSync("/dst").isDirectory()).toBe(true); + }, + }, + { + name: "symlink → existing empty dir", + code: "EISDIR", + setup(p: SQLiteWorkspaceProvider) { + p.writeFileSync("/target", "x"); + p.symlinkSync("/target", "/src"); + p.mkdirSync("/dst", {}); + }, + assertUnchanged(p: SQLiteWorkspaceProvider) { + expect(p.readlinkSync("/src")).toBe("/target"); + expect(p.statSync("/dst").isDirectory()).toBe(true); + }, + }, + { + name: "dir → existing file", + code: "ENOTDIR", + setup(p: SQLiteWorkspaceProvider) { + p.mkdirSync("/src", {}); + p.writeFileSync("/dst", "old"); + }, + assertUnchanged(p: SQLiteWorkspaceProvider) { + expect(p.statSync("/src").isDirectory()).toBe(true); + expect(p.readFileSync("/dst", "utf8")).toBe("old"); + }, + }, + { + name: "dir → existing symlink", + code: "ENOTDIR", + setup(p: SQLiteWorkspaceProvider) { + p.writeFileSync("/target", "x"); + p.mkdirSync("/src", {}); + p.symlinkSync("/target", "/dst"); + }, + assertUnchanged(p: SQLiteWorkspaceProvider) { + expect(p.statSync("/src").isDirectory()).toBe(true); + expect(p.readlinkSync("/dst")).toBe("/target"); + }, + }, + ])("$name rejects without recording sync changes", async ({ code, setup, assertUnchanged }) => { + await withProviderAndDB(async (p, db) => { + setup(p); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + expect(() => p.renameSync("/src", "/dst")).toThrowError(expect.objectContaining({ code })); + + assertUnchanged(p); + expect(await drainChanges(db, cursor)).toEqual([]); + }); + }); + + it("source missing throws ENOENT", async () => { + await withProvider((p) => { + expect(() => p.renameSync("/missing", "/dst")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("same-path rename validates the source before no-op", async () => { + await withProvider((p) => { + expect(() => p.renameSync("/missing", "/missing")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("same-path rename of an existing file is a no-op", async () => { + await withProviderAndDB(async (p, db) => { + p.writeFileSync("/src", "x"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/src", "/src"); + + expect(p.readFileSync("/src", "utf8")).toBe("x"); + expect(await drainChanges(db, cursor)).toEqual([]); + }); + }); + + it("rename of a file into itself as a parent does not report directory self-move", async () => { + await withProvider((p) => { + p.writeFileSync("/file", "x"); + expect(() => p.renameSync("/file", "/file/child")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("rename onto root throws EINVAL", async () => { + await withProvider((p) => { + p.writeFileSync("/src", "x"); + expect(() => p.renameSync("/src", "/")).toThrowError( + expect.objectContaining({ code: "EINVAL" }), + ); + }); + }); + + it("rename into a missing parent throws ENOENT", async () => { + await withProvider((p) => { + p.writeFileSync("/src", "x"); + expect(() => p.renameSync("/src", "/no-such-dir/dst")).toThrowError( + expect.objectContaining({ code: "ENOENT" }), + ); + }); + }); + + it("rename of a symlink moves the link itself", async () => { + await withProvider((p) => { + p.writeFileSync("/target", "x"); + p.symlinkSync("/target", "/link"); + p.renameSync("/link", "/moved"); + expect(p.existsSync("/target")).toBe(true); + expect(p.readlinkSync("/moved")).toBe("/target"); + expect(p.existsSync("/link")).toBe(false); + }); + }); + + it("rename of a directory into its own subtree throws EINVAL", async () => { + await withProvider((p) => { + p.mkdirSync("/src", {}); + p.mkdirSync("/src/sub", {}); + expect(() => p.renameSync("/src", "/src/sub/dst")).toThrowError( + expect.objectContaining({ code: "EINVAL" }), + ); + }); + }); + + it("rename of a directory through a symlink into its own subtree throws EINVAL", async () => { + await withProviderAndDB(async (p, db) => { + p.mkdirSync("/src", {}); + p.mkdirSync("/src/sub", {}); + p.symlinkSync("/src/sub", "/link"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + expect(() => p.renameSync("/src", "/link/dst")).toThrowError( + expect.objectContaining({ code: "EINVAL" }), + ); + + expect(p.statSync("/src/sub").isDirectory()).toBe(true); + expect(p.readlinkSync("/link")).toBe("/src/sub"); + expect(await drainChanges(db, cursor)).toEqual([]); + }); + }); + + it("rename of a directory through a symlink out of its subtree succeeds", async () => { + await withProviderAndDB(async (p, db) => { + p.mkdirSync("/src", {}); + p.mkdirSync("/other", {}); + // /src/link points outside the source subtree, so the destination + // /src/link/dst resolves to /other/dst even though the literal path + // is lexically under /src. The self-move guard is inode-based and + // must allow this move rather than reject it on a textual prefix. + p.symlinkSync("/other", "/src/link"); + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + + p.renameSync("/src", "/src/link/dst"); + + expect(p.statSync("/other/dst").isDirectory()).toBe(true); + expect(p.existsSync("/src")).toBe(false); + expect(await drainChanges(db, cursor)).not.toEqual([]); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — unimplemented surface (stubs)", () => { + it.each([ + ["appendFileSync", (p: SQLiteWorkspaceProvider) => p.appendFileSync("/x", "y")], + ["copyFileSync", (p: SQLiteWorkspaceProvider) => p.copyFileSync("/x", "/y")], + ["internalModuleStat", (p: SQLiteWorkspaceProvider) => p.internalModuleStat("/x")], + + ["watchFile", (p: SQLiteWorkspaceProvider) => p.watchFile("/x")], + ])("%s throws ENOSYS", async (_name, call) => { + await withProvider((p) => { + expect(() => call(p)).toThrowError(expect.objectContaining({ code: "ENOSYS" })); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — pending-create flush on rename/link/unlink", () => { + it("linkSync commits a pending-create source before adding the second dirent", async () => { + await withProvider((p) => { + p.openWriteBufferForCreateSync("/src.txt", { mode: 0o644 }); + p.writeRangeSync("/src.txt", Buffer.from("linked"), 0); + p.linkSync("/src.txt", "/dst.txt"); + + expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("linked"); + expect(p.statSync("/src.txt").ino).toBe(p.statSync("/dst.txt").ino); + expect(p.statSync("/src.txt").nlink).toBe(2); + + p.releaseWriteBufferSync("/src.txt"); + expect((p.readFileSync("/src.txt") as Buffer).toString()).toBe("linked"); + }); + }); + + it("renameSync commits a pending-create source before moving the dirent", async () => { + await withProvider((p) => { + p.openWriteBufferForCreateSync("/from.txt", { mode: 0o644 }); + p.writeRangeSync("/from.txt", Buffer.from("moved"), 0); + p.renameSync("/from.txt", "/to.txt"); + + expect((p.readFileSync("/to.txt") as Buffer).toString()).toBe("moved"); + expect(p.existsSync("/from.txt")).toBe(false); + }); + }); + + it("unlinkSync commits then removes a pending-create file", async () => { + await withProvider((p) => { + p.openWriteBufferForCreateSync("/gone.txt", { mode: 0o644 }); + p.writeRangeSync("/gone.txt", Buffer.from("bye"), 0); + p.unlinkSync("/gone.txt"); + expect(p.existsSync("/gone.txt")).toBe(false); + }); + }); + + it("linkSync flushes a pending-create destination before colliding", async () => { + // Pending /dst is committed before link's existence check + // runs, so the user sees a normal EEXIST against a real inode + // rather than silently losing the pending bytes when the later + // release would have tripped its own EEXIST against the link's + // dirent. + await withProvider((p) => { + p.writeFileSync("/src.txt", "src bytes"); + p.openWriteBufferForCreateSync("/dst.txt", { mode: 0o644 }); + p.writeRangeSync("/dst.txt", Buffer.from("pending dst"), 0); + + expect(() => p.linkSync("/src.txt", "/dst.txt")).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + + // /dst.txt now exists with the previously-pending bytes; the + // release-after-collision finds the inode it expects and is a + // no-op rather than a data loss. + expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("pending dst"); + expect(() => p.releaseWriteBufferSync("/dst.txt")).not.toThrow(); + expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("pending dst"); + }); + }); + + it("renameSync overwrite evicts the displaced destination's buffer", async () => { + // Open a buffer over an existing /dst, mutate it, then overwrite + // /dst via rename. The buffer's inode is gone from SQL after + // rename; release must not commit chunks against the dead row + // and must not leave a dangling cache entry. + await withProvider((p) => { + p.writeFileSync("/src.txt", "src bytes"); + p.writeFileSync("/dst.txt", "dst bytes"); + const dstInodeBefore = p.statSync("/dst.txt").ino; + p.openWriteBufferSync("/dst.txt"); + p.writeRangeSync("/dst.txt", Buffer.from("dirty"), 0); + + p.renameSync("/src.txt", "/dst.txt"); + + // The path now resolves to the renamed source's inode, not + // the displaced one. Release is a no-op on the now-gone + // displaced inode; the renamed file's bytes are unchanged. + expect(p.statSync("/dst.txt").ino).not.toBe(dstInodeBefore); + expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("src bytes"); + expect(() => p.releaseWriteBufferSync("/dst.txt")).not.toThrow(); + expect((p.readFileSync("/dst.txt") as Buffer).toString()).toBe("src bytes"); + }); + }); + + it("unlinkSync drops the inode-keyed buffer when the last link disappears", async () => { + const { getWriteBuffer } = await import("./fs/writeBuffer.js"); + await withProvider((p) => { + p.writeFileSync("/a.txt", "hello"); + const inode = p.statSync("/a.txt").ino; + p.openWriteBufferSync("/a.txt"); + p.writeRangeSync("/a.txt", Buffer.from("WORLD"), 0); + // Buffer is staged in the inode-keyed cache. + expect(getWriteBuffer(p.db, inode)).toBeDefined(); + p.unlinkSync("/a.txt"); + // unlink removed the last link, so the inode row is gone and + // the buffer must not be cached against the dead inode. + expect(p.existsSync("/a.txt")).toBe(false); + expect(getWriteBuffer(p.db, inode)).toBeUndefined(); + }); + }); + + it("unlinkSync keeps the buffer alive when a hardlink remains", async () => { + const { getWriteBuffer } = await import("./fs/writeBuffer.js"); + await withProvider((p) => { + p.writeFileSync("/a.txt", "shared"); + p.linkSync("/a.txt", "/b.txt"); + const inode = p.statSync("/a.txt").ino; + p.openWriteBufferSync("/a.txt"); + p.writeRangeSync("/a.txt", Buffer.from("UPDATED"), 0); + p.unlinkSync("/a.txt"); + // /b.txt still references the inode; the buffer survives in + // the inode-keyed cache and a release through the remaining + // name commits the staged bytes. + expect(p.existsSync("/b.txt")).toBe(true); + expect(getWriteBuffer(p.db, inode)).toBeDefined(); + p.releaseWriteBufferSync("/b.txt"); + expect((p.readFileSync("/b.txt") as Buffer).toString()).toBe("UPDATED"); + }); + }); +}); + +describe("SQLiteWorkspaceProvider — cached vfs_nodes.size", () => { + function readSize(p: SQLiteWorkspaceProvider, name: string): number | undefined { + return p.db.one<{ size: number }>( + "SELECT size FROM vfs_nodes WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE name = ?)", + name, + )?.size; + } + + it("writeFileSync stamps size on first write and on overwrite", async () => { + await withProvider((p) => { + p.writeFileSync("/sized.bin", Buffer.alloc(123, 0x41)); + expect(p.statSync("/sized.bin").size).toBe(123); + expect(readSize(p, "sized.bin")).toBe(123); + + p.writeFileSync("/sized.bin", Buffer.alloc(7, 0x42)); + expect(p.statSync("/sized.bin").size).toBe(7); + expect(readSize(p, "sized.bin")).toBe(7); + }); + }); + + it("writeRangeSync extends the cached size on growth", async () => { + await withProvider((p) => { + p.createFileSync("/range.bin", { mode: 0o644 }); + p.writeRangeSync("/range.bin", Buffer.from("hello"), 0); + expect(readSize(p, "range.bin")).toBe(5); + p.writeRangeSync("/range.bin", Buffer.from("!!"), 10); + expect(p.statSync("/range.bin").size).toBe(12); + expect(readSize(p, "range.bin")).toBe(12); + }); + }); + + it("truncateFileSync updates the cached size on grow and shrink", async () => { + await withProvider((p) => { + p.writeFileSync("/trunc.bin", Buffer.alloc(100, 0x55)); + expect(readSize(p, "trunc.bin")).toBe(100); + + p.truncateFileSync("/trunc.bin", 250); + expect(p.statSync("/trunc.bin").size).toBe(250); + expect(readSize(p, "trunc.bin")).toBe(250); + + p.truncateFileSync("/trunc.bin", 0); + expect(p.statSync("/trunc.bin").size).toBe(0); + expect(readSize(p, "trunc.bin")).toBe(0); + }); + }); + + it("buffered release stamps the cached size of the committed bytes", async () => { + await withProvider((p) => { + p.openWriteBufferForCreateSync("/buffered.bin", { mode: 0o644 }); + p.writeRangeSync("/buffered.bin", Buffer.from("buffered-write"), 0); + expect(readSize(p, "buffered.bin")).toBeUndefined(); + p.releaseWriteBufferSync("/buffered.bin"); + expect(p.statSync("/buffered.bin").size).toBe(14); + expect(readSize(p, "buffered.bin")).toBe(14); + }); + }); + + it("async writeFile stamps size from the buffered bytes", async () => { + await withProvider(async (p) => { + const payload = Buffer.from("async payload"); + await p.writeFile("/streamed.bin", payload); + expect(p.statSync("/streamed.bin").size).toBe(payload.byteLength); + expect(readSize(p, "streamed.bin")).toBe(payload.byteLength); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/provider.ts b/spikes/349-dofs/vendor/dofs/src/provider.ts new file mode 100644 index 00000000..5adcafc8 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/provider.ts @@ -0,0 +1,936 @@ +// SQLiteWorkspaceProvider — a @platformatic/vfs VirtualProvider backed +// by the dofs SQLite store. +// +// Every method on VirtualProvider is declared. Methods we already have +// synchronous building blocks for delegate to the existing fs/ helpers; +// the rest throw ENOSYS so the gaps are visible at the call site. +// Subsequent commits fill in the stubs (file descriptors, positional +// I/O, truncate, symlinks, watch). + +import { createWorkspaceError } from "./errors.js"; +import { getBlobBytes } from "./fs/blobCache.js"; +import { link as linkImpl } from "./fs/link.js"; +import type { MkdirOptions } from "./fs/mkdir.js"; +import { mkdir as mkdirImpl } from "./fs/mkdir.js"; +import { readdir as readdirImpl } from "./fs/readdir.js"; +import { readRangeSync as readRangeSyncImpl } from "./fs/readFile.js"; +import { readlink as readlinkImpl } from "./fs/readlink.js"; +import { rename as renameImpl } from "./fs/rename.js"; +import { resolveInode } from "./fs/resolve.js"; +import { rm as rmImpl } from "./fs/rm.js"; +import { stat as statImpl } from "./fs/stat.js"; +import { symlink as symlinkImpl } from "./fs/symlink.js"; +import { + createWatchAsyncIterable, + createWatcher, + type WatchEvent, + type WatchHandle, + type WatchOptions, +} from "./fs/watch.js"; +import { + deleteWriteBuffer, + getPendingWriteBufferByPath, + getWriteBuffer, +} from "./fs/writeBuffer.js"; +import { + createFileSync as createFileSyncImpl, + flushPendingByPath, + openWriteBufferForCreateSync as openWriteBufferForCreateSyncImpl, + openWriteBufferSync as openWriteBufferSyncImpl, + releaseWriteBufferSync as releaseWriteBufferSyncImpl, + truncateFileSync as truncateFileSyncImpl, + type WriteFileRange, + writeFileRangesSync as writeFileRangesSyncImpl, + writeFileSync as writeFileSyncImpl, + writeRangeSync as writeRangeSyncImpl, +} from "./fs/writeFile.js"; +import { canonicalizePath } from "./path.js"; +import { incrementRev } from "./rev.js"; +import type { Database } from "./storage.js"; + +export interface SQLiteWorkspaceProviderOptions { + // Wall-clock source. Defaults to Date.now so production callers + // don't need to thread one through; tests pin it. + now?: () => number; + // Poll interval for watch() in milliseconds. Defaults to 100 ms + // to match node's fs.watch on most filesystems; tests can lower + // it to keep durations short. + watchIntervalMs?: number; +} + +interface VirtualStatsLike { + dev: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + blksize: number; + ino: number; + size: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; + isFile(): boolean; + isDirectory(): boolean; + isSymbolicLink(): boolean; + isBlockDevice(): boolean; + isCharacterDevice(): boolean; + isFIFO(): boolean; + isSocket(): boolean; +} + +interface VirtualDirentLike { + name: string; + parentPath: string; + path: string; + isFile(): boolean; + isDirectory(): boolean; + isSymbolicLink(): boolean; + isBlockDevice(): boolean; + isCharacterDevice(): boolean; + isFIFO(): boolean; + isSocket(): boolean; +} + +interface FdState { + path: string; + position: number; + readable: boolean; + writable: boolean; + // append mode pins every writeSync to current EOF rather than + // honouring an explicit position argument. + append: boolean; +} + +export class SQLiteWorkspaceProvider { + readonly db: Database; + readonly now: () => number; + + // Capability flags consulted by @platformatic/vfs callers. + readonly readonly = false; + readonly supportsSymlinks = true; + readonly supportsWatch = true; + + // Fd table. Start at 3 — 0/1/2 are reserved by convention even + // though we don't expose them — so consumers that pass them around + // can't accidentally collide with stdio mental models. + #fds = new Map(); + #nextFd = 3; + + readonly watchIntervalMs: number; + + constructor(db: Database, options: SQLiteWorkspaceProviderOptions = {}) { + this.db = db; + this.now = options.now ?? Date.now; + this.watchIntervalMs = options.watchIntervalMs ?? 100; + } + + // -- Essential primitives ------------------------------------------ + + open(path: string, flags?: string, mode?: number): Promise { + return Promise.resolve(this.openSync(path, flags, mode)); + } + + openSync(path: string, flags: string = "r", _mode?: number): number { + const { read, write, truncate, append, create, exclusive } = parseFlags(flags); + const existing = resolveInode(this.db, path); + + if (existing === null) { + if (!create) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + writeFileSyncImpl(this.db, path, new Uint8Array(), {}, this.now); + } else { + if (existing.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + if (exclusive) { + throw createWorkspaceError("EEXIST", `path exists: ${path}`, path); + } + if (truncate) { + writeFileSyncImpl(this.db, path, new Uint8Array(), {}, this.now); + } + } + + const stat = statImpl(this.db, path); + const fd = this.#nextFd++; + this.#fds.set(fd, { + path, + position: append ? stat.size : 0, + readable: read, + writable: write, + append, + }); + return fd; + } + + stat(path: string, options?: { bigint?: boolean }): Promise { + return Promise.resolve(this.statSync(path, options)); + } + + statSync(path: string, _options?: { bigint?: boolean }): VirtualStatsLike { + // statImpl resolves the path once (following symlinks) and returns + // the inode, so nlink comes from the same walk. A pending-create + // file reports inode 0, which yields nlink 1. + const s = statImpl(this.db, path); + return wrapStats({ + mode: s.mode, + size: s.size, + mtimeMs: s.mtime, + ino: s.inode, + isFile: s.isFile, + isDirectory: s.isDirectory, + isSymbolicLink: false, + nlink: linkCount(this.db, s.inode), + }); + } + + lstat(path: string, options?: { bigint?: boolean }): Promise { + return Promise.resolve(this.lstatSync(path, options)); + } + + lstatSync(path: string, _options?: { bigint?: boolean }): VirtualStatsLike { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(this.db, canonical); + if (pending !== undefined && pending.pending !== undefined) { + return wrapStats({ + mode: pending.mode & 0o7777, + size: pending.size, + mtimeMs: pending.pending.mtime, + ino: 0, + isFile: true, + isDirectory: false, + isSymbolicLink: false, + nlink: 1, + }); + } + const node = resolveInode(this.db, path, { followSymlinks: false }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + const isSymlink = node.type === "symlink"; + const size = isSymlink + ? (node.linkTarget ?? "").length + : node.type === "file" + ? fileSize(this.db, node.inode) + : 0; + return wrapStats({ + mode: node.mode, + size, + mtimeMs: node.mtime, + ino: node.inode, + isFile: node.type === "file", + isDirectory: node.type === "dir", + isSymbolicLink: isSymlink, + nlink: linkCount(this.db, node.inode), + }); + } + + readdir( + path: string, + options?: { withFileTypes?: boolean }, + ): Promise { + return Promise.resolve(this.readdirSync(path, options)); + } + + readdirSync(path: string, options?: { withFileTypes?: boolean }): string[] | VirtualDirentLike[] { + const entries = readdirImpl(this.db, path); + if (options?.withFileTypes === true) { + return entries.map((entry) => wrapDirent(entry)); + } + return entries.map((entry) => entry.name); + } + + mkdir(path: string, options?: MkdirOptions): Promise { + return Promise.resolve(this.mkdirSync(path, options)); + } + + mkdirSync(path: string, options?: MkdirOptions): string | undefined { + mkdirImpl(this.db, path, options ?? {}, this.now); + return undefined; + } + + rmdir(path: string): Promise { + this.rmdirSync(path); + return Promise.resolve(); + } + + rmdirSync(path: string): void { + rmImpl(this.db, path, {}); + } + + unlink(path: string): Promise { + this.unlinkSync(path); + return Promise.resolve(); + } + + unlinkSync(path: string): void { + // If a buffered create is still pending for this path, commit + // it first so rm sees a real inode to unlink (and so the + // resulting GC sees the orphaned blob, matching the non-buffered + // shape). The buffer's open handles continue to address bytes + // through the inode-keyed cache. + flushPendingByPath(this.db, path, this.now); + // Capture the target inode before rm runs so we can evict its + // write-buffer cache entry if rm removed the last link. Without + // this, a release-after-unlink leaves the buffer dangling on a + // dead inode and the eventual commit silently affects no rows. + const target = resolveInode(this.db, path, { followSymlinks: false }); + rmImpl(this.db, path, {}); + if (target !== null) { + const stillAlive = this.db.scalar( + "SELECT inode FROM vfs_nodes WHERE inode = ?", + target.inode, + ); + if (stillAlive === undefined) { + deleteWriteBuffer(this.db, target.inode); + } + } + } + + link(existingPath: string, newPath: string): Promise { + this.linkSync(existingPath, newPath); + return Promise.resolve(); + } + + linkSync(existingPath: string, newPath: string): void { + // Commit a still-pending source before adding the second dirent, + // otherwise link has nothing real to point at. Also commit a + // still-pending destination: link's existence check looks at + // dirents, so a pending buffer at newPath wouldn't trip it, and + // the eventual release on that pending buffer would re-check the + // dirent in commitPendingBuffer, throw EEXIST, drop the entry, + // and silently lose the user's bytes. + flushPendingByPath(this.db, existingPath, this.now); + flushPendingByPath(this.db, newPath, this.now); + linkImpl(this.db, existingPath, newPath); + } + + rename(oldPath: string, newPath: string): Promise { + this.renameSync(oldPath, newPath); + return Promise.resolve(); + } + + renameSync(oldPath: string, newPath: string): void { + // Commit any still-pending creates at either end before the rename + // touches dirents: the source needs a real inode to move, and a + // pending buffer at the destination would otherwise slip past + // rename's dirent-based existence check and lose bytes on release. + flushPendingByPath(this.db, oldPath, this.now); + flushPendingByPath(this.db, newPath, this.now); + // Capture the destination inode before the rename so we can evict + // its write-buffer cache entry if the rename displaced and reaped + // it. Without this, a release on an open destination would commit + // chunks against a dead inode (0-row UPDATE, silent data loss). + const displaced = resolveInode(this.db, newPath, { followSymlinks: false }); + renameImpl(this.db, oldPath, newPath); + if (displaced !== null) { + const stillAlive = this.db.scalar( + "SELECT inode FROM vfs_nodes WHERE inode = ?", + displaced.inode, + ); + if (stillAlive === undefined) { + deleteWriteBuffer(this.db, displaced.inode); + } + } + } + + // -- Default implementations --------------------------------------- + + readFile( + path: string, + options?: BufferEncoding | { encoding?: BufferEncoding | null } | null, + ): Promise { + return Promise.resolve(this.readFileSync(path, options)); + } + + readFileSync( + path: string, + options?: BufferEncoding | { encoding?: BufferEncoding | null } | null, + ): Buffer | string { + const encoding = typeof options === "string" ? options : options?.encoding; + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(this.db, canonical); + if (pending !== undefined) { + const snapshot = Buffer.alloc(pending.size); + snapshot.set(pending.buf.subarray(0, pending.size)); + return encoding ? snapshot.toString(encoding) : snapshot; + } + const node = resolveInode(this.db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such file: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + // While a buffer is open for this inode it owns the latest + // bytes; serve from it instead of the chunk store. + const buffered = getWriteBuffer(this.db, node.inode); + if (buffered?.dirty) { + const snapshot = Buffer.alloc(buffered.size); + snapshot.set(buffered.buf.subarray(0, buffered.size)); + return encoding ? snapshot.toString(encoding) : snapshot; + } + const chunks = this.db.all<{ hash: Uint8Array; size: number }>( + "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + node.inode, + ); + let total = 0; + for (const c of chunks) total += c.size; + const out = Buffer.alloc(total); + let offset = 0; + for (const chunk of chunks) { + const bytes = getBlobBytes(this.db, chunk.hash); + if (bytes === undefined) { + throw createWorkspaceError("EIO", `missing blob bytes for ${path}`, path); + } + out.set(bytes, offset); + offset += bytes.byteLength; + } + return encoding ? out.toString(encoding) : out; + } + + writeFile( + path: string, + data: string | Buffer, + options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, + ): Promise { + this.writeFileSync(path, data, options); + return Promise.resolve(); + } + + writeFileSync( + path: string, + data: string | Buffer, + options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, + ): void { + const mode = typeof options === "string" ? undefined : options?.mode; + const bytes = + typeof data === "string" + ? new TextEncoder().encode(data) + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + writeFileSyncImpl(this.db, path, bytes, { mode }, this.now); + } + + writeFileRangesSync( + path: string, + data: string | Buffer, + ranges: WriteFileRange[], + options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, + ): void { + const mode = typeof options === "string" ? undefined : options?.mode; + const bytes = + typeof data === "string" + ? new TextEncoder().encode(data) + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + writeFileRangesSyncImpl(this.db, path, bytes, ranges, { mode }, this.now); + } + + createFileSync(path: string, options?: { mode?: number }): void { + createFileSyncImpl(this.db, path, { mode: options?.mode }, this.now); + } + + writeRangeSync( + path: string, + data: string | Buffer | Uint8Array, + offset: number, + options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, + ): number { + const mode = typeof options === "string" ? undefined : options?.mode; + const bytes = + typeof data === "string" + ? new TextEncoder().encode(data) + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + return writeRangeSyncImpl(this.db, path, bytes, offset, { mode }, this.now); + } + + truncateFileSync(path: string, len: number): void { + truncateFileSyncImpl(this.db, path, len, this.now); + } + + openWriteBufferSync(path: string): void { + openWriteBufferSyncImpl(this.db, path); + } + + openWriteBufferForCreateSync(path: string, options?: { mode?: number }): void { + openWriteBufferForCreateSyncImpl(this.db, path, { mode: options?.mode }, this.now); + } + + releaseWriteBufferSync(path: string): void { + releaseWriteBufferSyncImpl(this.db, path, this.now); + } + + chmodSync(path: string, mode: number): void { + const { path: canonical } = canonicalizePath(path); + const pending = getPendingWriteBufferByPath(this.db, canonical); + if (pending !== undefined) { + // Pending-create files don't have a row yet; stash the mode on + // the buffer so the eventual INSERT picks it up. + pending.mode = mode & 0o7777; + return; + } + const node = resolveInode(this.db, path, { followSymlinks: false }); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + const rev = incrementRev(this.db); + this.db.run( + "UPDATE vfs_nodes SET mode = ?, rev = ? WHERE inode = ?", + mode & 0o7777, + rev, + node.inode, + ); + } + + appendFile( + _path: string, + _data: string | Buffer, + _options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, + ): Promise { + return Promise.reject(notImplemented("appendFile")); + } + + appendFileSync( + _path: string, + _data: string | Buffer, + _options?: { encoding?: BufferEncoding; mode?: number } | BufferEncoding, + ): void { + throw notImplemented("appendFileSync"); + } + + exists(path: string): Promise { + return Promise.resolve(this.existsSync(path)); + } + + existsSync(path: string): boolean { + try { + const { path: canonical } = canonicalizePath(path); + if (getPendingWriteBufferByPath(this.db, canonical) !== undefined) return true; + return resolveInode(this.db, path) !== null; + } catch { + return false; + } + } + + copyFile(_src: string, _dest: string, _mode?: number): Promise { + return Promise.reject(notImplemented("copyFile")); + } + + copyFileSync(_src: string, _dest: string, _mode?: number): void { + throw notImplemented("copyFileSync"); + } + + internalModuleStat(_path: string): number { + // Used by node:vfs module-resolution hooks. The computerd driver doesn't + // need it; if this provider is ever mounted via `vfs.mount()` we'll + // need to return 0 for files, 1 for dirs, -1 for not-found. + throw notImplemented("internalModuleStat"); + } + + realpath(path: string, _options?: { encoding?: BufferEncoding }): Promise { + return Promise.resolve(this.realpathSync(path)); + } + + realpathSync(path: string, _options?: { encoding?: BufferEncoding }): string { + const { path: canonical } = canonicalizePath(path); + if (resolveInode(this.db, canonical) === null) { + throw createWorkspaceError("ENOENT", `no such path: ${canonical}`, canonical); + } + return canonical; + } + + access(path: string, _mode?: number): Promise { + this.accessSync(path); + return Promise.resolve(); + } + + accessSync(path: string, _mode?: number): void { + if (resolveInode(this.db, path) === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + } + + // -- File descriptors ---------------------------------------------- + + closeSync(fd: number): void { + if (!this.#fds.delete(fd)) { + throw createWorkspaceError("EBADF", `unknown fd ${fd}`); + } + } + + readSync( + fd: number, + buffer: Buffer | Uint8Array, + offset: number, + length: number, + position: number | null, + ): number { + const state = this.#fdOrThrow(fd); + if (!state.readable) { + throw createWorkspaceError("EBADF", `fd ${fd} is not readable`); + } + const startAt = position ?? state.position; + const slice = readRangeSyncImpl(this.db, state.path, startAt, length); + const view = + buffer instanceof Buffer + ? buffer + : Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength); + view.set(slice, offset); + if (position === null || position === undefined) { + state.position = startAt + slice.byteLength; + } + return slice.byteLength; + } + + readRangeSync(path: string, offset: number, length: number): Buffer { + const slice = readRangeSyncImpl(this.db, path, offset, length); + return Buffer.from(slice.buffer, slice.byteOffset, slice.byteLength); + } + + writeSync( + fd: number, + buffer: Buffer | Uint8Array, + offset: number = 0, + length: number = buffer.byteLength - offset, + position: number | null = null, + ): number { + const state = this.#fdOrThrow(fd); + if (!state.writable) { + throw createWorkspaceError("EBADF", `fd ${fd} is not writable`); + } + // Append needs the current EOF, so stat only then. A non-append + // write of >0 bytes doesn't need it: writeRangeSyncImpl resolves the + // path and raises ENOENT/EISDIR. A zero-length write short-circuits + // before that resolve, so keep an explicit existence check for it. + let startAt: number; + if (state.append) { + startAt = this.statSync(state.path).size; + } else { + if (length === 0) { + this.statSync(state.path); + } + startAt = position ?? state.position; + } + const view = + buffer instanceof Buffer + ? new Uint8Array(buffer.buffer, buffer.byteOffset + offset, length) + : new Uint8Array(buffer.buffer, buffer.byteOffset + offset, length); + writeRangeSyncImpl(this.db, state.path, view, startAt, {}, this.now); + if (position === null || position === undefined) { + state.position = startAt + length; + } + return length; + } + + fstatSync(fd: number, _options?: { bigint?: boolean }): VirtualStatsLike { + const state = this.#fdOrThrow(fd); + return this.statSync(state.path); + } + + truncateSync(path: string, len: number): void { + const node = resolveInode(this.db, path); + if (node === null) { + throw createWorkspaceError("ENOENT", `no such path: ${path}`, path); + } + if (node.type !== "file") { + throw createWorkspaceError("EISDIR", `path is a directory: ${path}`, path); + } + truncateFileSyncImpl(this.db, path, len, this.now); + } + + ftruncateSync(fd: number, len: number): void { + const state = this.#fdOrThrow(fd); + this.truncateSync(state.path, len); + } + + #fdOrThrow(fd: number): FdState { + const state = this.#fds.get(fd); + if (state === undefined) { + throw createWorkspaceError("EBADF", `unknown fd ${fd}`); + } + return state; + } + + // -- Symlinks ------------------------------------------------------ + + readlink(path: string, _options?: { encoding?: BufferEncoding }): Promise { + return Promise.resolve(this.readlinkSync(path)); + } + + readlinkSync(path: string, _options?: { encoding?: BufferEncoding }): string { + return readlinkImpl(this.db, path); + } + + symlink(target: string, path: string, _type?: string): Promise { + this.symlinkSync(target, path); + return Promise.resolve(); + } + + symlinkSync(target: string, path: string, _type?: string): void { + symlinkImpl(this.db, target, path, this.now); + } + + // -- Watch ---------------------------------------------------------- + // + // The watcher polls vfs_meta.rev on a timer. Each tick + // coalesceChanges yields every path touched since the last + // observed rev; we filter by the watched directory (and + // recursive flag) and emit one 'change' event per path. Cheap + // because coalesceChanges is one indexed range scan on + // vfs_nodes.rev plus a path walk per touched inode. + // + // Event types follow node's fs.watch convention: + // - 'rename' for deletes (path went away) + // - 'change' for everything else (file/dir/symlink mutation) + // We don't distinguish first-time creation from in-place edit + // — the cost is a per-watcher state map that's bigger than + // the signal is worth. Callers that need rename-vs-change + // semantics can stat the path themselves. + + watch(path: string, options: WatchOptions = {}): WatchHandle { + return createWatcher(this.db, path, options, this.watchIntervalMs); + } + + watchAsync(path: string, options: WatchOptions = {}): AsyncIterable { + return createWatchAsyncIterable(this.watch(path, options)); + } + + // watchFile / unwatchFile fire on stat changes at a single path + // (not the directory under it). Different semantics from watch(); + // editors typically use watch() instead. Leave as ENOSYS until a + // real call site shows up. + watchFile( + _path: string, + _options?: unknown, + _listener?: (curr: VirtualStatsLike, prev: VirtualStatsLike) => void, + ): unknown { + throw notImplemented("watchFile"); + } + + unwatchFile( + _path: string, + _listener?: (curr: VirtualStatsLike, prev: VirtualStatsLike) => void, + ): void { + throw notImplemented("unwatchFile"); + } +} + +function notImplemented(method: string) { + return createWorkspaceError("ENOSYS", `SQLiteWorkspaceProvider.${method} is not implemented yet`); +} + +// -- VirtualStats / VirtualDirent shim ------------------------------ +// +// @platformatic/vfs callers (and FUSE drivers built on top) consult +// the full Node-style stat shape. Most fields don't map onto our +// content-addressed store, so they get sensible constants. The fields +// that do map — mode, size, mtime, ino — are populated for real. + +interface StatsInputs { + mode: number; + size: number; + mtimeMs: number; + ino: number; + isFile: boolean; + isDirectory: boolean; + isSymbolicLink: boolean; + nlink: number; +} + +// POSIX mode-bit constants. Linux FUSE rejects a stat whose mode +// has no S_IF* bits set with EIO — it can't decide whether +// the inode is a regular file, a directory, or a symlink. +const S_IFREG = 0o100000; +const S_IFDIR = 0o040000; +const S_IFLNK = 0o120000; + +function fileTypeBits(input: StatsInputs): number { + if (input.isDirectory) return S_IFDIR; + if (input.isSymbolicLink) return S_IFLNK; + if (input.isFile) return S_IFREG; + return 0; +} + +function linkCount(db: Database, inode: number): number { + const count = db.scalar("SELECT COUNT(*) FROM vfs_dirents WHERE child_inode = ?", inode); + return Math.max(1, count ?? 0); +} + +function fileSize(db: Database, inode: number): number { + const buffered = getWriteBuffer(db, inode); + if (buffered?.dirty) { + return buffered.size; + } + return db.scalar("SELECT size FROM vfs_nodes WHERE inode = ?", inode) ?? 0; +} + +function wrapStats(input: StatsInputs): VirtualStatsLike { + const mtime = new Date(input.mtimeMs); + return { + dev: 0, + mode: (input.mode & 0o7777) | fileTypeBits(input), + nlink: input.nlink, + uid: 0, + gid: 0, + rdev: 0, + blksize: 4096, + ino: input.ino, + size: input.size, + blocks: Math.ceil(input.size / 512), + atimeMs: input.mtimeMs, + mtimeMs: input.mtimeMs, + ctimeMs: input.mtimeMs, + birthtimeMs: input.mtimeMs, + atime: mtime, + mtime, + ctime: mtime, + birthtime: mtime, + isFile: () => input.isFile, + isDirectory: () => input.isDirectory, + isSymbolicLink: () => input.isSymbolicLink, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isFIFO: () => false, + isSocket: () => false, + }; +} + +interface DirentInput { + name: string; + parentPath: string; + isFile: boolean; + isDirectory: boolean; +} + +function wrapDirent(input: DirentInput): VirtualDirentLike { + const fullPath = + input.parentPath === "/" ? `/${input.name}` : `${input.parentPath}/${input.name}`; + return { + name: input.name, + parentPath: input.parentPath, + path: fullPath, + isFile: () => input.isFile, + isDirectory: () => input.isDirectory, + isSymbolicLink: () => false, + isBlockDevice: () => false, + isCharacterDevice: () => false, + isFIFO: () => false, + isSocket: () => false, + }; +} + +interface ParsedFlags { + read: boolean; + write: boolean; + create: boolean; + truncate: boolean; + append: boolean; + exclusive: boolean; +} + +// Translate Node's fs flag strings into the boolean flag set the fd +// table uses. Mirrors the documented behaviour of fs.open(flags) at +// https://nodejs.org/api/fs.html#file-system-flags. +function parseFlags(flags: string): ParsedFlags { + switch (flags) { + case "r": + return { + read: true, + write: false, + create: false, + truncate: false, + append: false, + exclusive: false, + }; + case "r+": + return { + read: true, + write: true, + create: false, + truncate: false, + append: false, + exclusive: false, + }; + case "w": + return { + read: false, + write: true, + create: true, + truncate: true, + append: false, + exclusive: false, + }; + case "w+": + return { + read: true, + write: true, + create: true, + truncate: true, + append: false, + exclusive: false, + }; + case "wx": + return { + read: false, + write: true, + create: true, + truncate: false, + append: false, + exclusive: true, + }; + case "wx+": + return { + read: true, + write: true, + create: true, + truncate: false, + append: false, + exclusive: true, + }; + case "a": + return { + read: false, + write: true, + create: true, + truncate: false, + append: true, + exclusive: false, + }; + case "a+": + return { + read: true, + write: true, + create: true, + truncate: false, + append: true, + exclusive: false, + }; + case "ax": + return { + read: false, + write: true, + create: true, + truncate: false, + append: true, + exclusive: true, + }; + case "ax+": + return { + read: true, + write: true, + create: true, + truncate: false, + append: true, + exclusive: true, + }; + default: + throw createWorkspaceError("EINVAL", `unsupported fs flag: ${flags}`); + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/provider.watch.test.ts b/spikes/349-dofs/vendor/dofs/src/provider.watch.test.ts new file mode 100644 index 00000000..6f75e484 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/provider.watch.test.ts @@ -0,0 +1,158 @@ +import type { EventEmitter } from "node:events"; +import { describe, expect, it } from "vitest"; +import { withDB } from "./fs/with-db.js"; +import { SQLiteWorkspaceProvider } from "./provider.js"; + +interface WatchEmitter extends EventEmitter { + close(): void; +} + +interface WatchEvent { + eventType: "rename" | "change"; + filename: string; +} + +async function withProvider(fn: (p: SQLiteWorkspaceProvider) => T | Promise): Promise { + return withDB((db) => + fn(new SQLiteWorkspaceProvider(db, { now: () => 1000, watchIntervalMs: 25 })), + ); +} + +async function nextEvents( + watcher: WatchEmitter, + count: number, + timeoutMs = 1500, +): Promise { + const events: WatchEvent[] = []; + const done = new Promise((resolve, reject) => { + const onChange = (eventType: "rename" | "change", filename: string) => { + events.push({ eventType, filename }); + if (events.length >= count) { + watcher.off("change", onChange); + resolve(); + } + }; + watcher.on("change", onChange); + setTimeout(() => { + watcher.off("change", onChange); + reject(new Error(`timed out waiting for ${count} events; got ${events.length}`)); + }, timeoutMs); + }); + await done; + return events; +} + +describe("SQLiteWorkspaceProvider — watch", () => { + it("supportsWatch is true", async () => { + await withProvider((p) => { + expect(p.supportsWatch).toBe(true); + }); + }); + + it("watch(dir) fires for a write to a direct child", async () => { + await withProvider(async (p) => { + p.mkdirSync("/d", {}); + const w = p.watch("/d", {}) as WatchEmitter; + try { + // Write happens after the watcher captures the baseline rev. + // Give the poll loop one tick to record the initial position. + await new Promise((r) => setTimeout(r, 30)); + p.writeFileSync("/d/a.txt", "hello"); + const events = await nextEvents(w, 1); + expect(events[0]).toMatchObject({ filename: "a.txt" }); + } finally { + w.close(); + } + }); + }); + + it("watch(dir, { recursive: true }) fires for nested writes", async () => { + await withProvider(async (p) => { + p.mkdirSync("/d", {}); + p.mkdirSync("/d/sub", {}); + const w = p.watch("/d", { recursive: true }) as WatchEmitter; + try { + await new Promise((r) => setTimeout(r, 30)); + p.writeFileSync("/d/sub/deep.txt", "yo"); + const events = await nextEvents(w, 1); + expect(events[0].filename).toBe("sub/deep.txt"); + } finally { + w.close(); + } + }); + }); + + it("watch(dir) does not fire for unrelated writes", async () => { + await withProvider(async (p) => { + p.mkdirSync("/watched", {}); + p.mkdirSync("/other", {}); + const w = p.watch("/watched", {}) as WatchEmitter; + const seen: string[] = []; + const onChange = (_t: string, name: string) => seen.push(name); + w.on("change", onChange); + try { + await new Promise((r) => setTimeout(r, 30)); + p.writeFileSync("/other/x.txt", "no"); + await new Promise((r) => setTimeout(r, 100)); + expect(seen).toEqual([]); + } finally { + w.off("change", onChange); + w.close(); + } + }); + }); + + it("watch fires rename for a delete and change for a write", async () => { + await withProvider(async (p) => { + p.mkdirSync("/d", {}); + p.writeFileSync("/d/a.txt", "first"); + const w = p.watch("/d", {}) as WatchEmitter; + try { + await new Promise((r) => setTimeout(r, 30)); + p.writeFileSync("/d/a.txt", "second"); + const first = await nextEvents(w, 1); + expect(first[0].eventType).toBe("change"); + // Separate tick so coalescing doesn't collapse the + // write and the delete into a single delete entry. + await new Promise((r) => setTimeout(r, 50)); + p.unlinkSync("/d/a.txt"); + const second = await nextEvents(w, 1); + expect(second[0].eventType).toBe("rename"); + } finally { + w.close(); + } + }); + }); + + it("watch.close() stops the poll loop", async () => { + await withProvider(async (p) => { + p.mkdirSync("/d", {}); + const w = p.watch("/d", {}) as WatchEmitter; + w.close(); + const seen: string[] = []; + w.on("change", (_t, name) => seen.push(name)); + await new Promise((r) => setTimeout(r, 50)); + p.writeFileSync("/d/a.txt", "noop"); + await new Promise((r) => setTimeout(r, 100)); + expect(seen).toEqual([]); + }); + }); + + it("watchAsync yields events via for-await", async () => { + await withProvider(async (p) => { + p.mkdirSync("/d", {}); + const it = p.watchAsync("/d", {}) as AsyncIterable & { + return(): Promise; + }; + const iter = it[Symbol.asyncIterator]() as AsyncIterator; + try { + await new Promise((r) => setTimeout(r, 30)); + p.writeFileSync("/d/a.txt", "x"); + const { value } = await iter.next(); + expect(value).toMatchObject({ filename: "a.txt" }); + } finally { + await iter.return?.(undefined); + } + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/rev.test.ts b/spikes/349-dofs/vendor/dofs/src/rev.test.ts new file mode 100644 index 00000000..d24bdfe6 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/rev.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; + +import { withDB } from "./fs/with-db.js"; +import { incrementRev } from "./rev.js"; + +describe("incrementRev", () => { + it("returns the new rev value and persists it to vfs_meta", async () => { + await withDB( + (db) => { + // initializeSchema seeds rev = 1. + const next = incrementRev(db); + expect(next).toBe(2); + + const stored = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rev"); + expect(stored).toBe(2); + }, + { now: () => 0 }, + ); + }); + + it("issues monotonically increasing revs inside a single transaction", async () => { + await withDB( + (db) => { + let a: number | undefined; + let b: number | undefined; + db.transactionSync(() => { + a = incrementRev(db); + b = incrementRev(db); + }); + expect(a).toBe(2); + expect(b).toBe(3); + const stored = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rev"); + expect(stored).toBe(3); + }, + { now: () => 0 }, + ); + }); + + it("rolls back if the surrounding transaction aborts", async () => { + await withDB( + (db) => { + expect(() => { + db.transactionSync(() => { + incrementRev(db); + throw new Error("abort"); + }); + }).toThrow("abort"); + const stored = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rev"); + expect(stored).toBe(1); + }, + { now: () => 0 }, + ); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/rev.ts b/spikes/349-dofs/vendor/dofs/src/rev.ts new file mode 100644 index 00000000..df7f6d3c --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/rev.ts @@ -0,0 +1,21 @@ +import type { Database } from "./storage.js"; + +// Atomic monotonic rev counter. Every FS mutation (mkdir, writeFile, +// rm, ...) calls incrementRev once per transaction and stamps the returned +// value into vfs_nodes.rev. The sync layer reads vfs_meta.rev as +// currentRev and consumes vfs_changes.rev for tombstones. +// +// Must be called inside a transactionSync — the UPDATE and SELECT +// otherwise race with concurrent mutations. The DO single-writer model +// makes that unlikely in practice, but the contract is "wrap me". +export function incrementRev(db: Database): number { + // RETURNING folds the read into the same statement so each mutation + // pays one round-trip instead of two. SQLite has supported it since + // 3.35; both node:sqlite and Cloudflare DO SqlStorage are on newer + // versions. + const row = db.one<{ v: number }>("UPDATE vfs_meta SET v = v + 1 WHERE k = 'rev' RETURNING v"); + if (row === undefined) { + throw new Error("vfs_meta.rev row missing; was initializeSchema run?"); + } + return row.v; +} diff --git a/spikes/349-dofs/vendor/dofs/src/schema/core.ts b/spikes/349-dofs/vendor/dofs/src/schema/core.ts new file mode 100644 index 00000000..bc77dd62 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/schema/core.ts @@ -0,0 +1,81 @@ +// Filesystem-side tables. These hold the inode graph and the +// content-addressed blob store. See docs/03_filesystem_schema.md. + +// Bumped to 2 when `_vfs_mounts.mode` landed (read-only mount +// enforcement at the data layer). Bumped to 3 when `vfs_nodes` +// gained a cached `size` column so stat() doesn't have to SUM +// chunks on every call. Bumped to 4 when `_vfs_watermark` gained +// a `backend` column so a single workspace can host more than +// one backend with independent sync cursors. Bumped to 5 when +// `vfs_dirents` and `vfs_chunks` became WITHOUT ROWID: their +// composite-PK lookups now read straight from the PK b-tree leaf +// with no rowid indirection, and `child_inode` lives in the +// dirents leaf so the (parent, name) resolve read is covering +// (no separate index needed). See `schema/migrations.ts` for the +// migration list; `sync.ts` carries the fresh-install DDL. +export const SCHEMA_VERSION = 5; +export const ROOT_INODE = 1; + +export const CORE_STATEMENTS = [ + `CREATE TABLE IF NOT EXISTS vfs_meta ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS vfs_nodes ( + inode INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), + mode INTEGER NOT NULL DEFAULT 493, + mtime INTEGER NOT NULL, + rev INTEGER NOT NULL DEFAULT 0, + mount_root TEXT, + stub_size INTEGER, + manifest_hash BLOB, + link_target TEXT, + size INTEGER NOT NULL DEFAULT 0 + )`, + // WITHOUT ROWID: the row lives in the (parent_inode, name) PK + // b-tree leaf, so resolving a path segment reads child_inode + // directly from the leaf — no autoindex -> rowid hop, and no + // separate covering index. Legal here because the PK is composite + // and the table has no AUTOINCREMENT. Existing databases are + // rebuilt by the v4 -> v5 migration in schema/migrations.ts; keep + // this DDL and that migrator's CREATE in lockstep. + `CREATE TABLE IF NOT EXISTS vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + ) WITHOUT ROWID`, + `CREATE INDEX IF NOT EXISTS vfs_dirents_by_child ON vfs_dirents(child_inode)`, + `CREATE INDEX IF NOT EXISTS vfs_nodes_by_rev ON vfs_nodes(rev)`, + // gc/manifests checks every manifest row against vfs_nodes via a + // correlated NOT EXISTS (manifest_hash = ?). Without this index + // gc full-scans vfs_nodes per candidate manifest — O(N×M). + // Partial because the column is null on every dir and symlink + // node, and on files until they get their first content write. + `CREATE INDEX IF NOT EXISTS vfs_nodes_by_manifest_hash + ON vfs_nodes(manifest_hash) WHERE manifest_hash IS NOT NULL`, + `CREATE TABLE IF NOT EXISTS vfs_blobs ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + last_seen INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS vfs_blob_bytes ( + hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE, + bytes BLOB NOT NULL + )`, + // WITHOUT ROWID: clustered on (inode, idx) so a file's chunks are + // stored and scanned in index order straight from the PK leaf. + // Legal here — composite PK, no AUTOINCREMENT. The bytes live in + // vfs_blob_bytes (content-addressed), so these rows stay small, + // which is what WITHOUT ROWID wants. Rebuilt for existing DBs by + // the v4 -> v5 migration; keep in lockstep with that migrator. + `CREATE TABLE IF NOT EXISTS vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + ) WITHOUT ROWID`, + `CREATE INDEX IF NOT EXISTS vfs_chunks_by_hash ON vfs_chunks(hash)`, +] as const; diff --git a/spikes/349-dofs/vendor/dofs/src/schema/index.test.ts b/spikes/349-dofs/vendor/dofs/src/schema/index.test.ts new file mode 100644 index 00000000..f07b597e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/schema/index.test.ts @@ -0,0 +1,457 @@ +import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; +import { describe, expect, it } from "vitest"; + +import { Database } from "../storage.js"; +import { RecordingStorage } from "../testing-recording.js"; +import { SCHEMA_VERSION } from "./core.js"; +import { initializeSchema } from "./index.js"; + +describe("initializeSchema", () => { + it("lazily initializes the documented schema on first use", () => { + const storage = new RecordingStorage(); + const db = new Database(storage); + + initializeSchema(db, () => 1234); + + const executed = storage.statements.map((statement) => statement.query); + expect(executed).toEqual( + expect.arrayContaining([ + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_meta"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_nodes"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_dirents"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_blobs"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_blob_bytes"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_chunks"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_manifests"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS vfs_changes"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS _vfs_watermark"), + expect.stringContaining("CREATE TABLE IF NOT EXISTS _vfs_mounts"), + ]), + ); + expect(storage.statements).toContainEqual( + expect.objectContaining({ + query: expect.stringContaining("INSERT OR IGNORE INTO vfs_nodes"), + bindings: [1, 493, 1234], + }), + ); + }); + + it("rejects a newer on-disk schema version", () => { + const storage = new RecordingStorage({ schemaVersion: 999 }); + const db = new Database(storage); + + expect(() => initializeSchema(db, () => 0)).toThrow( + /Unsupported workspace filesystem schema version 999/, + ); + }); + + it("stamps the current SCHEMA_VERSION in vfs_meta on a fresh DB", () => { + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + initializeSchema(db, () => 0); + + const row = db.one<{ v: number }>("SELECT v FROM vfs_meta WHERE k = ?", "schema_version"); + expect(row?.v).toBe(SCHEMA_VERSION); + }); + + it("creates _vfs_mounts.mode on a fresh DB with the default and CHECK", () => { + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + initializeSchema(db, () => 0); + + // Defaults to read-only when the column is omitted. + db.run("INSERT INTO _vfs_mounts (root, kind) VALUES (?, ?)", "/m1", "r2"); + const row = db.one<{ mode: string }>("SELECT mode FROM _vfs_mounts WHERE root = ?", "/m1"); + expect(row?.mode).toBe("read-only"); + + // Explicit read-write is accepted. + db.run( + "INSERT INTO _vfs_mounts (root, kind, mode) VALUES (?, ?, ?)", + "/m2", + "r2", + "read-write", + ); + const row2 = db.one<{ mode: string }>("SELECT mode FROM _vfs_mounts WHERE root = ?", "/m2"); + expect(row2?.mode).toBe("read-write"); + + // The CHECK constraint rejects anything else. + expect(() => + db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES (?, ?, ?)", "/m3", "r2", "bogus"), + ).toThrow(/CHECK constraint/); + }); + + it("upgrades a v1 database to the current SCHEMA_VERSION", () => { + // Stage a database at the old shape: _vfs_mounts without the + // mode column, vfs_meta.schema_version = 1. The baseline DDL + // run by initializeSchema is "IF NOT EXISTS" so it won't touch + // the existing _vfs_mounts; the migration must. + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + db.transactionSync(() => { + db.run( + `CREATE TABLE vfs_meta ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + ); + db.run( + `CREATE TABLE _vfs_mounts ( + root TEXT PRIMARY KEY, + kind TEXT NOT NULL, + indexed INTEGER NOT NULL DEFAULT 0 + )`, + ); + db.run("INSERT INTO _vfs_mounts (root, kind, indexed) VALUES (?, ?, ?)", "/m1", "r2", 1); + db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 1); + }); + + initializeSchema(db, () => 0); + + // Version bumped. + const versionRow = db.one<{ v: number }>( + "SELECT v FROM vfs_meta WHERE k = ?", + "schema_version", + ); + expect(versionRow?.v).toBe(SCHEMA_VERSION); + + // Existing row preserved and stamped with the conservative + // default so a re-index pass has to opt back into read-write. + const row = db.one<{ root: string; mode: string; indexed: number }>( + "SELECT root, mode, indexed FROM _vfs_mounts WHERE root = ?", + "/m1", + ); + expect(row).toEqual({ root: "/m1", mode: "read-only", indexed: 1 }); + + // Post-migration the CHECK constraint is live. + expect(() => + db.run("INSERT INTO _vfs_mounts (root, kind, mode) VALUES (?, ?, ?)", "/m2", "r2", "bogus"), + ).toThrow(/CHECK constraint/); + }); + + it("backfills vfs_nodes.size from chunk sums on the v2 -> v3 upgrade", () => { + // Stage a database at the v2 shape: vfs_nodes without the + // `size` column, schema_version = 2. The migration adds the + // column with a default of 0 and then UPDATEs it from the + // SUM of vfs_chunks.size for each file inode. + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + db.transactionSync(() => { + db.run( + `CREATE TABLE vfs_meta ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + ); + db.run( + `CREATE TABLE vfs_nodes ( + inode INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), + mode INTEGER NOT NULL DEFAULT 493, + mtime INTEGER NOT NULL, + rev INTEGER NOT NULL DEFAULT 0, + mount_root TEXT, + stub_size INTEGER, + manifest_hash BLOB, + link_target TEXT + )`, + ); + db.run( + `CREATE TABLE vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + )`, + ); + // A live file with two chunks summing to 7 bytes, a live dir, + // and a live file with no chunks (empty file). + db.run( + `INSERT INTO vfs_nodes (inode, type, mode, mtime, rev) VALUES + (1, 'dir', 493, 0, 0), + (2, 'file', 420, 0, 0), + (3, 'file', 420, 0, 0)`, + ); + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + 2, + 0, + new Uint8Array(32), + 3, + ); + db.run( + "INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", + 2, + 1, + new Uint8Array(32), + 4, + ); + db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 2); + }); + + initializeSchema(db, () => 0); + + const sizes = db.all<{ inode: number; size: number }>( + "SELECT inode, size FROM vfs_nodes ORDER BY inode", + ); + expect(sizes).toEqual([ + { inode: 1, size: 0 }, + { inode: 2, size: 7 }, + { inode: 3, size: 0 }, + ]); + }); + + it("upgrades a v3 database, backfilling _vfs_watermark with the default backend", () => { + // Stage a database at the v3 shape: _vfs_watermark with the + // old single-column primary key, two existing rows; vfs_nodes + // already has the size column from the v2 -> v3 migration. + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + db.transactionSync(() => { + db.run( + `CREATE TABLE vfs_meta ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + ); + db.run( + `CREATE TABLE _vfs_watermark ( + k TEXT PRIMARY KEY, + v INTEGER NOT NULL + )`, + ); + db.run("INSERT INTO _vfs_watermark (k, v) VALUES (?, ?)", "pushRev", 42); + db.run("INSERT INTO _vfs_watermark (k, v) VALUES (?, ?)", "fetchRev", 17); + db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 3); + }); + + initializeSchema(db, () => 0); + + // Version bumped. + const versionRow = db.one<{ v: number }>( + "SELECT v FROM vfs_meta WHERE k = ?", + "schema_version", + ); + expect(versionRow?.v).toBe(SCHEMA_VERSION); + + // Existing rows preserved under the default backend slot. + const push = db.one<{ k: string; backend: string; v: number }>( + "SELECT k, backend, v FROM _vfs_watermark WHERE k = ?", + "pushRev", + ); + expect(push).toEqual({ k: "pushRev", backend: "default", v: 42 }); + const fetch = db.one<{ k: string; backend: string; v: number }>( + "SELECT k, backend, v FROM _vfs_watermark WHERE k = ?", + "fetchRev", + ); + expect(fetch).toEqual({ k: "fetchRev", backend: "default", v: 17 }); + + // The composite PK is live: same key under a different backend + // is allowed and doesn't collide with the migrated row. + db.run("INSERT INTO _vfs_watermark (k, backend, v) VALUES (?, ?, ?)", "pushRev", "worker", 99); + const worker = db.one<{ v: number }>( + "SELECT v FROM _vfs_watermark WHERE k = ? AND backend = ?", + "pushRev", + "worker", + ); + expect(worker?.v).toBe(99); + }); + + it("rebuilds vfs_dirents and vfs_chunks as WITHOUT ROWID on the v4 -> v5 upgrade, preserving all data", () => { + // Stage a v4-shape database: vfs_dirents and vfs_chunks are plain + // rowid tables carrying their secondary indexes; vfs_nodes already + // has the size column. Populate a representative graph — nested + // dirs, a hardlink (one inode, two names), multi-chunk files, and + // content dedup (distinct files sharing blob hashes) — so the + // rebuild is proven lossless, not merely structurally correct. + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + const hashA = new Uint8Array(32).fill(0xaa); + const hashB = new Uint8Array(32).fill(0xbb); + + db.transactionSync(() => { + db.run(`CREATE TABLE vfs_meta (k TEXT PRIMARY KEY, v INTEGER NOT NULL)`); + db.run( + `CREATE TABLE vfs_nodes ( + inode INTEGER PRIMARY KEY AUTOINCREMENT, + type TEXT NOT NULL CHECK(type IN ('file','dir','symlink')), + mode INTEGER NOT NULL DEFAULT 493, + mtime INTEGER NOT NULL, + rev INTEGER NOT NULL DEFAULT 0, + mount_root TEXT, + stub_size INTEGER, + manifest_hash BLOB, + link_target TEXT, + size INTEGER NOT NULL DEFAULT 0 + )`, + ); + // Pre-migration shape: rowid tables plus their secondary indexes. + db.run( + `CREATE TABLE vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + )`, + ); + db.run(`CREATE INDEX vfs_dirents_by_child ON vfs_dirents(child_inode)`); + db.run( + `CREATE TABLE vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + )`, + ); + db.run(`CREATE INDEX vfs_chunks_by_hash ON vfs_chunks(hash)`); + db.run( + `CREATE TABLE vfs_blobs ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + last_seen INTEGER NOT NULL + )`, + ); + db.run( + `CREATE TABLE vfs_blob_bytes ( + hash BLOB PRIMARY KEY REFERENCES vfs_blobs(hash) ON DELETE CASCADE, + bytes BLOB NOT NULL + )`, + ); + + // Graph: /(1) -> a(2) -> { f1(3), f2(4), b(5) }, b(5) -> deep(6). + // Hardlink: /a/hard is a second name for inode 3. + db.run( + `INSERT INTO vfs_nodes (inode, type, mode, mtime, rev, size) VALUES + (1, 'dir', 493, 0, 0, 0), + (2, 'dir', 493, 0, 1, 0), + (3, 'file', 420, 0, 2, 10), + (4, 'file', 420, 0, 3, 5), + (5, 'dir', 493, 0, 4, 0), + (6, 'file', 420, 0, 5, 5)`, + ); + db.run( + `INSERT INTO vfs_dirents (parent_inode, name, child_inode) VALUES + (1, 'a', 2), + (2, 'f1', 3), + (2, 'f2', 4), + (2, 'hard', 3), + (2, 'b', 5), + (5, 'deep', 6)`, + ); + // f1(3): hashA + hashB. f2(4): hashA (dedup). deep(6): hashB (dedup). + // -> 4 chunk rows referencing 2 distinct blobs. + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 3, 0, hashA, 5); + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 3, 1, hashB, 5); + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 4, 0, hashA, 5); + db.run("INSERT INTO vfs_chunks (inode, idx, hash, size) VALUES (?, ?, ?, ?)", 6, 0, hashB, 5); + db.run("INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?)", hashA, 5, 0); + db.run("INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?)", hashB, 5, 0); + db.run( + "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?)", + hashA, + new Uint8Array(5).fill(1), + ); + db.run( + "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?)", + hashB, + new Uint8Array(5).fill(2), + ); + + db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", 4); + }); + + // Snapshot the two rebuilt tables before migrating. + const direntsBefore = db.all<{ parent_inode: number; name: string; child_inode: number }>( + "SELECT parent_inode, name, child_inode FROM vfs_dirents ORDER BY parent_inode, name", + ); + const chunksBefore = db.all<{ inode: number; idx: number; hash: Uint8Array; size: number }>( + "SELECT inode, idx, hash, size FROM vfs_chunks ORDER BY inode, idx", + ); + + initializeSchema(db, () => 0); + + // (a) Version bumped. + expect(db.one<{ v: number }>("SELECT v FROM vfs_meta WHERE k = ?", "schema_version")?.v).toBe( + SCHEMA_VERSION, + ); + + const tableSql = (name: string): string => + db.one<{ sql: string }>( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", + name, + )?.sql ?? ""; + + // (b) Both targets are now WITHOUT ROWID; vfs_blob_bytes is untouched. + expect(tableSql("vfs_dirents").toUpperCase()).toContain("WITHOUT ROWID"); + expect(tableSql("vfs_chunks").toUpperCase()).toContain("WITHOUT ROWID"); + expect(tableSql("vfs_blob_bytes").toUpperCase()).not.toContain("WITHOUT ROWID"); + + // (c) Both secondary indexes survived the rebuild. + const indexNames = db + .all<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'index'") + .map((r) => r.name); + expect(indexNames).toContain("vfs_dirents_by_child"); + expect(indexNames).toContain("vfs_chunks_by_hash"); + + // The rebuild's temp tables are dropped — no leftovers. + const tableNames = db + .all<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table'") + .map((r) => r.name); + expect(tableNames).not.toContain("vfs_dirents_v4"); + expect(tableNames).not.toContain("vfs_chunks_v4"); + + // (d) Data survived byte-for-byte. + expect( + db.all("SELECT parent_inode, name, child_inode FROM vfs_dirents ORDER BY parent_inode, name"), + ).toEqual(direntsBefore); + expect(db.all("SELECT inode, idx, hash, size FROM vfs_chunks ORDER BY inode, idx")).toEqual( + chunksBefore, + ); + // Hardlink preserved: inode 3 still reached by both names via the + // recreated child index. + expect( + db.all<{ parent_inode: number; name: string }>( + "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ? ORDER BY name", + 3, + ), + ).toEqual([ + { parent_inode: 2, name: "f1" }, + { parent_inode: 2, name: "hard" }, + ]); + // Dedup intact: 4 chunk rows, 2 distinct blobs. + expect(db.one<{ c: number }>("SELECT COUNT(*) AS c FROM vfs_chunks")?.c).toBe(4); + expect(db.one<{ c: number }>("SELECT COUNT(*) AS c FROM vfs_blobs")?.c).toBe(2); + expect(db.one<{ c: number }>("SELECT COUNT(DISTINCT hash) AS c FROM vfs_chunks")?.c).toBe(2); + + // (e) A fresh install lands the identical table shape (modulo + // whitespace) as the migrated database. + const fresh = new Database(new SQLiteTestStorage()); + initializeSchema(fresh, () => 0); + const freshSql = (name: string): string => + fresh.one<{ sql: string }>( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", + name, + )?.sql ?? ""; + const norm = (sql: string): string => sql.replace(/\s+/g, " ").trim().toUpperCase(); + expect(norm(tableSql("vfs_dirents"))).toBe(norm(freshSql("vfs_dirents"))); + expect(norm(tableSql("vfs_chunks"))).toBe(norm(freshSql("vfs_chunks"))); + }); + + it("is idempotent across repeat calls", () => { + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + initializeSchema(db, () => 0); + expect(() => initializeSchema(db, () => 0)).not.toThrow(); + + const row = db.one<{ v: number }>("SELECT v FROM vfs_meta WHERE k = ?", "schema_version"); + expect(row?.v).toBe(SCHEMA_VERSION); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/schema/index.ts b/spikes/349-dofs/vendor/dofs/src/schema/index.ts new file mode 100644 index 00000000..e0147e96 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/schema/index.ts @@ -0,0 +1,82 @@ +import { createWorkspaceError } from "../errors.js"; +import type { Database } from "../storage.js"; +import { CORE_STATEMENTS, ROOT_INODE, SCHEMA_VERSION } from "./core.js"; +import { runMigrations } from "./migrations.js"; +import { SYNC_STATEMENTS } from "./sync.js"; + +export { ROOT_INODE, SCHEMA_VERSION } from "./core.js"; + +interface MetaRow { + v: number; +} + +export function initializeSchema(db: Database, now: () => number): void { + db.transactionSync(() => { + // 1. Baseline DDL. Every statement is "CREATE TABLE IF NOT + // EXISTS" / "CREATE INDEX IF NOT EXISTS" so this is a no-op + // on already-initialized databases. Fresh databases come out + // of this step at the latest column shape (SCHEMA_VERSION). + for (const statement of CORE_STATEMENTS) { + db.run(statement); + } + for (const statement of SYNC_STATEMENTS) { + db.run(statement); + } + + // 2. Read the on-disk schema version. Absent → 0 (very first + // boot of this database). The baseline above just created + // every table at the latest shape, so a 0 → SCHEMA_VERSION + // jump has nothing to migrate. + const storedVersion = db.one( + "SELECT v FROM vfs_meta WHERE k = ?", + "schema_version", + )?.v; + const onDiskVersion = storedVersion ?? 0; + + if (onDiskVersion > SCHEMA_VERSION) { + throw createWorkspaceError( + "EIO", + `Unsupported workspace filesystem schema version ${onDiskVersion}`, + ); + } + + // 3. Migrate. Skip when the database is fresh (0) — the + // baseline DDL already shipped the latest shape. Otherwise + // dispatch each registered migrator until we hit the + // target. + if (onDiskVersion > 0 && onDiskVersion < SCHEMA_VERSION) { + runMigrations(db, onDiskVersion, SCHEMA_VERSION); + } + + // 4. Stamp the version and seed the boot rows. Both shapes + // (insert-if-missing, then update) keep this idempotent so + // repeat calls do nothing. + db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "schema_version", SCHEMA_VERSION); + db.run("UPDATE vfs_meta SET v = ? WHERE k = ?", SCHEMA_VERSION, "schema_version"); + db.run("INSERT OR IGNORE INTO vfs_meta (k, v) VALUES (?, ?)", "rev", 1); + db.run( + "INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", + "pushRev", + 0, + ); + db.run( + "INSERT OR IGNORE INTO _vfs_watermark (k, backend, v) VALUES (?, 'default', ?)", + "fetchRev", + 0, + ); + db.run( + "INSERT OR IGNORE INTO _vfs_fetch_cursor (k, backend, path) VALUES (?, 'default', ?)", + "fetch", + null, + ); + + db.run( + `INSERT OR IGNORE INTO vfs_nodes + (inode, type, mode, mtime, rev) + VALUES (?, 'dir', ?, ?, 0)`, + ROOT_INODE, + 0o755, + now(), + ); + }); +} diff --git a/spikes/349-dofs/vendor/dofs/src/schema/migrations.ts b/spikes/349-dofs/vendor/dofs/src/schema/migrations.ts new file mode 100644 index 00000000..d2425ada --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/schema/migrations.ts @@ -0,0 +1,169 @@ +// Schema migration runner. +// +// The schema's "CREATE TABLE IF NOT EXISTS" baseline handles fresh +// databases. When a schema column changes shape — added, dropped, +// renamed, retyped — IF NOT EXISTS does nothing and the older +// rows stay incompatible. Migrations close that gap. +// +// Shape: an ordered list of `(from, to, migrator)` tuples. The +// runner reads `vfs_meta.schema_version` (defaulting to 0 when the +// row is absent), picks every migration whose `from === current`, +// runs it, advances `current`, and repeats until `current >= +// SCHEMA_VERSION`. The whole pass runs inside the caller's +// transactionSync so a partial migration rolls back. +// +// Each migrator is a `(db: Database) => void` and may assume the +// previous version's schema is in place. Migrators land schema +// changes only; they don't touch user data unless the column shape +// requires it. + +import type { Database } from "../storage.js"; + +export interface Migration { + readonly from: number; + readonly to: number; + readonly migrator: (db: Database) => void; +} + +// v1 → v2 — add `_vfs_mounts.mode` so dofs can enforce read-only +// mounts at the data layer. Existing rows default to 'read-only'; +// the workspace re-stamps them with the registered mount's mode on +// the next index pass. +// +// The CHECK constraint is duplicated in `sync.ts`'s fresh-install +// DDL; both paths must keep the same allowed set. +function v1_to_v2_add_mounts_mode(db: Database): void { + db.run( + `ALTER TABLE _vfs_mounts + ADD COLUMN mode TEXT NOT NULL DEFAULT 'read-only' + CHECK(mode IN ('read-only', 'read-write'))`, + ); +} + +// v2 → v3 — denormalise file size onto vfs_nodes so stat doesn't +// have to SUM the chunk rows on every call. The column is +// backfilled from existing vfs_chunks; later writes maintain it. +function v2_to_v3_add_size_column(db: Database): void { + const hasColumn = db + .all<{ name: string }>("PRAGMA table_info(vfs_nodes)") + .some((column) => column.name === "size"); + if (!hasColumn) { + db.run("ALTER TABLE vfs_nodes ADD COLUMN size INTEGER NOT NULL DEFAULT 0"); + } + db.run( + `UPDATE vfs_nodes + SET size = COALESCE( + (SELECT SUM(size) FROM vfs_chunks WHERE vfs_chunks.inode = vfs_nodes.inode), + 0 + ) + WHERE type = 'file'`, + ); +} + +// v3 → v4 — add a `backend` column to `_vfs_watermark` so a +// workspace can host more than one backend with independent sync +// cursors. SQLite's ALTER TABLE can't change a primary key; copy +// existing rows into a fresh table with the composite +// (k, backend) primary key, then swap the tables. +// +// Existing rows land under the `default` backend id, which the +// dofs sync helpers also use as the fallback when a caller +// doesn't pass an id. Pre-multi-backend workspaces keep their +// pushRev / fetchRev cursors intact through the upgrade. +function v3_to_v4_watermark_backend_column(db: Database): void { + db.run(`ALTER TABLE _vfs_watermark RENAME TO _vfs_watermark_v3`); + db.run( + `CREATE TABLE _vfs_watermark ( + k TEXT NOT NULL, + backend TEXT NOT NULL DEFAULT 'default', + v INTEGER NOT NULL, + PRIMARY KEY (k, backend) + )`, + ); + db.run( + `INSERT INTO _vfs_watermark (k, backend, v) + SELECT k, 'default', v FROM _vfs_watermark_v3`, + ); + db.run(`DROP TABLE _vfs_watermark_v3`); +} + +// v4 → v5 — rebuild `vfs_dirents` and `vfs_chunks` as WITHOUT ROWID. +// SQLite can't convert a table to WITHOUT ROWID in place, so for each +// table: rename it aside, create the WITHOUT ROWID replacement, copy +// the rows, drop the old table. +// +// Both targets are FK-inert (neither is an FK parent or child; the +// schema's only foreign key is vfs_blob_bytes -> vfs_blobs) and have +// composite primary keys with no AUTOINCREMENT, so WITHOUT ROWID is +// legal and sqlite_sequence is untouched. `vfs_blob_bytes` is left +// alone on purpose — it holds the large blob payloads and the FK. +// +// A RENAME carries the table's secondary index along to the temp +// name, and the following DROP takes the index with it. The baseline +// `CREATE INDEX IF NOT EXISTS` in initializeSchema already ran, before +// migrations, and does not re-run — so this migrator must recreate +// vfs_dirents_by_child and vfs_chunks_by_hash itself, or upgraded +// databases silently lose them. Keep the CREATE bodies in lockstep +// with the fresh-install DDL in core.ts. +function v4_to_v5_without_rowid(db: Database): void { + // vfs_dirents + db.run(`ALTER TABLE vfs_dirents RENAME TO vfs_dirents_v4`); + db.run( + `CREATE TABLE vfs_dirents ( + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL, + child_inode INTEGER NOT NULL, + PRIMARY KEY (parent_inode, name) + ) WITHOUT ROWID`, + ); + db.run( + `INSERT INTO vfs_dirents (parent_inode, name, child_inode) + SELECT parent_inode, name, child_inode FROM vfs_dirents_v4`, + ); + db.run(`DROP TABLE vfs_dirents_v4`); + db.run(`CREATE INDEX vfs_dirents_by_child ON vfs_dirents(child_inode)`); + + // vfs_chunks + db.run(`ALTER TABLE vfs_chunks RENAME TO vfs_chunks_v4`); + db.run( + `CREATE TABLE vfs_chunks ( + inode INTEGER NOT NULL, + idx INTEGER NOT NULL, + hash BLOB NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (inode, idx) + ) WITHOUT ROWID`, + ); + db.run( + `INSERT INTO vfs_chunks (inode, idx, hash, size) + SELECT inode, idx, hash, size FROM vfs_chunks_v4`, + ); + db.run(`DROP TABLE vfs_chunks_v4`); + db.run(`CREATE INDEX vfs_chunks_by_hash ON vfs_chunks(hash)`); +} + +export const MIGRATIONS: readonly Migration[] = [ + { from: 1, to: 2, migrator: v1_to_v2_add_mounts_mode }, + { from: 2, to: 3, migrator: v2_to_v3_add_size_column }, + { from: 3, to: 4, migrator: v3_to_v4_watermark_backend_column }, + { from: 4, to: 5, migrator: v4_to_v5_without_rowid }, +] as const; + +// Apply every migration whose `from` matches the current version, +// in order, until we reach the target. The caller has already +// wrapped this in a transactionSync; failures here roll the whole +// initializeSchema call back. +export function runMigrations(db: Database, current: number, target: number): number { + let version = current; + while (version < target) { + const next = MIGRATIONS.find((m) => m.from === version); + if (next === undefined) { + // No migration registered for this jump. This is a bug — the + // version was bumped without a matching migration. + throw new Error(`dofs schema: no migration registered for v${version} -> v${target}`); + } + next.migrator(db); + version = next.to; + } + return version; +} diff --git a/spikes/349-dofs/vendor/dofs/src/schema/sync.ts b/spikes/349-dofs/vendor/dofs/src/schema/sync.ts new file mode 100644 index 00000000..fdbac101 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/schema/sync.ts @@ -0,0 +1,59 @@ +// Sync-protocol tables. Populated by the sync module; the FS module +// only writes to vfs_changes (via sync/changes.ts) on rm. The rest +// of these tables stay empty until the sync task is implemented. + +export const SYNC_STATEMENTS = [ + `CREATE TABLE IF NOT EXISTS vfs_manifests ( + hash BLOB PRIMARY KEY, + size INTEGER NOT NULL, + encoded BLOB NOT NULL, + last_seen INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE TABLE IF NOT EXISTS vfs_changes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rev INTEGER NOT NULL, + path TEXT NOT NULL, + op TEXT NOT NULL CHECK(op IN ('delete')) + )`, + `CREATE INDEX IF NOT EXISTS vfs_changes_by_rev ON vfs_changes(rev)`, + // changes.ts looks up the latest op for a path via + // `WHERE path = ? ORDER BY id DESC LIMIT 1`. Without the index + // SQLite falls back to a full scan; with (path, id DESC) the + // lookup is O(log n) and the ORDER BY drains straight from the + // index. Used on every recordDelete and on every push-tick that + // processes tombstones. + `CREATE INDEX IF NOT EXISTS vfs_changes_by_path ON vfs_changes(path, id DESC)`, + // Watermarks are keyed by (k, backend) so a workspace hosting + // multiple backends keeps each backend's sync cursors + // independent. The `backend` column was added at schema v3; + // `schema/migrations.ts` owns the ALTER for existing + // databases. Fresh installs land the composite key directly. + `CREATE TABLE IF NOT EXISTS _vfs_watermark ( + k TEXT NOT NULL, + backend TEXT NOT NULL DEFAULT 'default', + v INTEGER NOT NULL, + PRIMARY KEY (k, backend) + )`, + // The fetch cursor's same-rev `path` component, keyed by + // (k, backend) so each backend resumes a partially-drained rev + // independently. The rev component lives in _vfs_watermark under + // 'fetchRev'; this table only holds the in-rev path. `backend` + // mirrors _vfs_watermark and defaults to 'default'. + `CREATE TABLE IF NOT EXISTS _vfs_fetch_cursor ( + k TEXT NOT NULL CHECK(k = 'fetch'), + backend TEXT NOT NULL DEFAULT 'default', + path TEXT, + PRIMARY KEY (k, backend) + )`, + // The `mode` column was added at schema v2; `schema/migrations.ts` + // owns the ALTER for existing databases. Keep the CHECK + // constraint here aligned with the migration's CHECK so fresh + // installs and upgrades enforce the same allowed set. + `CREATE TABLE IF NOT EXISTS _vfs_mounts ( + root TEXT PRIMARY KEY, + kind TEXT NOT NULL, + indexed INTEGER NOT NULL DEFAULT 0, + mode TEXT NOT NULL DEFAULT 'read-only' + CHECK(mode IN ('read-only', 'read-write')) + )`, +] as const; diff --git a/spikes/349-dofs/vendor/dofs/src/storage.ts b/spikes/349-dofs/vendor/dofs/src/storage.ts new file mode 100644 index 00000000..becce5a7 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/storage.ts @@ -0,0 +1,115 @@ +import type { DurableObjectStorageLike, SQLStorageLike } from "./types.js"; + +export class Database { + readonly sql: SQLStorageLike; + readonly transactionSync: (closure: () => T) => T; + // Depth counter so reentrant transactionSync() calls work. The + // outer call uses the storage adapter's transactionSync (or + // BEGIN/COMMIT under the hood); nested calls use SAVEPOINTs + // through sql.exec directly. SQLite forbids a real BEGIN inside + // an active transaction. + #txDepth = 0; + + constructor(storage: DurableObjectStorageLike) { + this.sql = storage.sql; + this.transactionSync = (closure: () => T): T => { + if (this.#txDepth > 0) { + // Reentrant call: use a savepoint. SQLite's RELEASE on a + // savepoint inside an outer transaction commits the inner + // work without ending the outer one. + const sp = `_t${this.#txDepth}`; + this.sql.exec(`SAVEPOINT ${sp}`); + this.#txDepth++; + try { + const result = closure(); + this.sql.exec(`RELEASE ${sp}`); + return result; + } catch (error) { + this.sql.exec(`ROLLBACK TO ${sp}`); + this.sql.exec(`RELEASE ${sp}`); + throw error; + } finally { + this.#txDepth--; + } + } + // Outer call: hand off to the storage adapter so the DO + // runtime's transaction semantics apply. + this.#txDepth++; + try { + if (storage.transactionSync !== undefined) { + return storage.transactionSync(closure); + } + if (storage.transaction !== undefined) { + const result = storage.transaction(closure); + if ( + result !== undefined && + result !== null && + typeof result === "object" && + "then" in result + ) { + throw new Error("Durable Object storage adapter requires synchronous transactions"); + } + return result; + } + return closure(); + } finally { + this.#txDepth--; + } + }; + } + + // True while a transactionSync closure is on the stack. The resolve + // cache uses this to refuse populating entries mid-transaction, so a + // rolled-back mutation can never leave the cache reflecting + // uncommitted state. (Invalidation still runs freely inside a + // transaction — dropping an entry is always safe.) + // + // Invariant: #txDepth only tracks transactionSync. A raw + // BEGIN/SAVEPOINT issued through run() would open a transaction this + // flag can't see, letting the cache populate mid-transaction and + // survive a rollback — so transactionSync is the only sanctioned way + // to open one. + get inTransaction(): boolean { + return this.#txDepth > 0; + } + + run(query: string, ...bindings: unknown[]): void { + this.sql.exec(query, ...bindings); + } + + all(query: string, ...bindings: unknown[]): Row[] { + const rows = this.sql.exec(query, ...bindings).toArray(); + return rows.map((row) => normalizeRow(row as Record)) as Row[]; + } + + one(query: string, ...bindings: unknown[]): Row | undefined { + return this.all(query, ...bindings)[0]; + } + + scalar(query: string, ...bindings: unknown[]): T | undefined { + const row = this.one>(query, ...bindings); + if (row === undefined) { + return undefined; + } + + const [value] = Object.values(row); + return value; + } +} + +// Cloudflare's DO SqlStorage returns BLOB columns as ArrayBuffer, +// whereas node:sqlite returns Uint8Array. Normalise to Uint8Array so +// the rest of the code only has to handle one shape. +function normalizeRow(row: Record): Record { + // node:sqlite hands back rows with a null prototype; the DO SQL + // flavour returns ArrayBuffer for BLOB columns. Re-key into a plain + // {} so consumers get Object.prototype-shaped rows (capnweb's + // serializer keys off Object.prototype to detect "object") and + // convert any ArrayBuffer to Uint8Array in the same pass. + const out: Record = {}; + for (const key of Object.keys(row)) { + const value = row[key]; + out[key] = value instanceof ArrayBuffer ? new Uint8Array(value) : value; + } + return out; +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/apply.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/apply.test.ts new file mode 100644 index 00000000..5ced0d0e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/apply.test.ts @@ -0,0 +1,1120 @@ +import { describe, expect, it } from "vitest"; + +import { link } from "../fs/link.js"; +import { mkdir } from "../fs/mkdir.js"; +import { invalidateReadOnlyMountCache } from "../fs/mount-guard.js"; +import { readFile } from "../fs/readFile.js"; +import { readlink } from "../fs/readlink.js"; +import { rename } from "../fs/rename.js"; +import { resolveInode } from "../fs/resolve.js"; +import { rm } from "../fs/rm.js"; +import { symlink } from "../fs/symlink.js"; +import { withDB, withTwoDBs } from "../fs/with-db.js"; +import { writeFile, writeFileSync } from "../fs/writeFile.js"; +import { applyChanges, applyChangesSync } from "./apply.js"; +import type { ChangeEntry } from "./changes.js"; +import { coalesceChanges } from "./coalesce.js"; +import { fetchObjects } from "./fetch.js"; +import { currentRev } from "./watermarks.js"; + +async function drain(it: AsyncIterable): Promise { + const out: T[] = []; + for await (const x of it) out.push(x); + return out; +} + +function hex(bytes: Uint8Array): string { + let s = ""; + for (let i = 0; i < bytes.byteLength; i++) s += bytes[i].toString(16).padStart(2, "0"); + return s; +} + +function deepChildPath(depth: number): string { + return `/dst/${Array.from({ length: depth }, () => "d").join("/")}/old.txt`; +} + +async function collectObjects( + db: import("../storage.js").Database, + entries: ChangeEntry[], +): Promise> { + const hashes: Uint8Array[] = []; + const seen = new Set(); + for (const e of entries) { + if (e.kind !== "file") continue; + for (const c of e.chunks) { + const k = hex(c.hash); + if (!seen.has(k)) { + seen.add(k); + hashes.push(c.hash); + } + } + } + const out = new Map(); + for await (const { hash, bytes } of fetchObjects(db, hashes)) { + out.set(hex(hash), bytes); + } + return out; +} + +describe("applyChanges", () => { + it("converges across a mixed stream", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/a.txt", "alpha", {}, () => 1); + await writeFile(a, "/b.txt", "beta", {}, () => 2); + const entries = await drain(coalesceChanges(a, 0)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await applyChanges(b, entries, objects); + expect(await readFile(b, "/a.txt", "utf8")).toBe("alpha"); + expect(await readFile(b, "/b.txt", "utf8")).toBe("beta"); + }, + ); + }); + + it("commits in batches capped by byte budget", async () => { + // Force many small files; with a tiny byte budget the apply + // path should still converge, just across more batches. We + // verify convergence rather than batch count (batch count is + // an implementation detail). + await withTwoDBs( + async (a) => { + for (let i = 0; i < 10; i++) { + await writeFile(a, `/f${i}.txt`, `payload ${i}`, {}, () => 100 + i); + } + const entries = await drain(coalesceChanges(a, 0)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await applyChanges(b, entries, objects, { maxBytesPerBatch: 16 }); + for (let i = 0; i < 10; i++) { + expect(await readFile(b, `/f${i}.txt`, "utf8")).toBe(`payload ${i}`); + } + }, + ); + }); + + it("applies a file rename over an existing file", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/src", "new", {}, () => 1); + await writeFile(a, "/dst", "old", {}, () => 2); + const cursor = currentRev(a); + + rename(a, "/src", "/dst"); + + const entries = await drain(coalesceChanges(a, cursor)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await writeFile(b, "/src", "new", {}, () => 1); + await writeFile(b, "/dst", "old", {}, () => 2); + + await applyChanges(b, entries, objects); + + expect(resolveInode(b, "/src")).toBeNull(); + expect(await readFile(b, "/dst", "utf8")).toBe("new"); + }, + ); + }); + + it("applies a directory rename over an existing empty directory", async () => { + await withTwoDBs( + async (a) => { + mkdir(a, "/src", { mode: 0o700, recursive: true }, () => 1); + await writeFile(a, "/src/inside", "x", {}, () => 2); + mkdir(a, "/dst", { mode: 0o755 }, () => 3); + const cursor = currentRev(a); + + rename(a, "/src", "/dst"); + + const entries = await drain(coalesceChanges(a, cursor)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + mkdir(b, "/src", { mode: 0o700, recursive: true }, () => 1); + await writeFile(b, "/src/inside", "x", {}, () => 2); + mkdir(b, "/dst", { mode: 0o755 }, () => 3); + + await applyChanges(b, entries, objects); + + expect(resolveInode(b, "/src")).toBeNull(); + expect(resolveInode(b, "/dst", { followSymlinks: false })).toMatchObject({ + type: "dir", + mode: 0o700, + mtime: 1, + }); + expect(await readFile(b, "/dst/inside", "utf8")).toBe("x"); + }, + ); + }); + + it("applyChangesSync updates existing directory metadata", async () => { + await withDB((db) => { + mkdir(db, "/dst", { mode: 0o755 }, () => 3); + + applyChangesSync( + db, + [{ kind: "dir", rev: 99, path: "/dst", mode: 0o700, mtime: 1 }], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ + type: "dir", + mode: 0o700, + mtime: 1, + }); + }); + }); + + it("does not update existing directory mtime for upstream entries with matching mode", async () => { + await withDB(async (db) => { + mkdir(db, "/dst", { mode: 0o700 }, () => 3); + + await applyChanges( + db, + [{ kind: "dir", rev: 99, path: "/dst", mode: 0o700, mtime: 1 }], + new Map(), + { source: "upstream" }, + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ + type: "dir", + mode: 0o700, + mtime: 3, + }); + }); + }); + + it("applies a file rename over an existing symlink", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/target", "target", {}, () => 1); + await writeFile(a, "/src", "new", {}, () => 2); + symlink(a, "/target", "/dst", () => 3); + const cursor = currentRev(a); + + rename(a, "/src", "/dst"); + + const entries = await drain(coalesceChanges(a, cursor)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await writeFile(b, "/target", "target", {}, () => 1); + await writeFile(b, "/src", "new", {}, () => 2); + symlink(b, "/target", "/dst", () => 3); + + await applyChanges(b, entries, objects); + + expect(resolveInode(b, "/src")).toBeNull(); + expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("file"); + expect(await readFile(b, "/dst", "utf8")).toBe("new"); + expect(await readFile(b, "/target", "utf8")).toBe("target"); + }, + ); + }); + + it("applies a symlink rename over an existing file", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/target", "target", {}, () => 1); + symlink(a, "/target", "/src", () => 2); + await writeFile(a, "/dst", "old", {}, () => 3); + const cursor = currentRev(a); + + rename(a, "/src", "/dst"); + + const entries = await drain(coalesceChanges(a, cursor)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await writeFile(b, "/target", "target", {}, () => 1); + symlink(b, "/target", "/src", () => 2); + await writeFile(b, "/dst", "old", {}, () => 3); + + await applyChanges(b, entries, objects); + + expect(resolveInode(b, "/src", { followSymlinks: false })).toBeNull(); + expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("symlink"); + expect(readlink(b, "/dst")).toBe("/target"); + expect(await readFile(b, "/target", "utf8")).toBe("target"); + }, + ); + }); + + it("applies a symlink rename over an existing symlink", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/target", "target", {}, () => 1); + await writeFile(a, "/other", "other", {}, () => 2); + symlink(a, "/target", "/src", () => 3); + symlink(a, "/other", "/dst", () => 4); + const cursor = currentRev(a); + + rename(a, "/src", "/dst"); + + const entries = await drain(coalesceChanges(a, cursor)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await writeFile(b, "/target", "target", {}, () => 1); + await writeFile(b, "/other", "other", {}, () => 2); + symlink(b, "/target", "/src", () => 3); + symlink(b, "/other", "/dst", () => 4); + + await applyChanges(b, entries, objects); + + expect(resolveInode(b, "/src", { followSymlinks: false })).toBeNull(); + expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("symlink"); + expect(readlink(b, "/dst")).toBe("/target"); + expect(await readFile(b, "/target", "utf8")).toBe("target"); + expect(await readFile(b, "/other", "utf8")).toBe("other"); + }, + ); + }); + + it("applies a directory rename containing a symlink without deleting its target", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/target", "target", {}, () => 1); + mkdir(a, "/src", {}, () => 2); + symlink(a, "/target", "/src/link", () => 3); + const cursor = currentRev(a); + + rename(a, "/src", "/dst"); + + const entries = await drain(coalesceChanges(a, cursor)); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await writeFile(b, "/target", "target", {}, () => 1); + mkdir(b, "/src", {}, () => 2); + symlink(b, "/target", "/src/link", () => 3); + + await applyChanges(b, entries, objects); + + expect(resolveInode(b, "/src/link", { followSymlinks: false })).toBeNull(); + expect(resolveInode(b, "/dst/link", { followSymlinks: false })?.type).toBe("symlink"); + expect(readlink(b, "/dst/link")).toBe("/target"); + expect(await readFile(b, "/target", "utf8")).toBe("target"); + }, + ); + }); + + it("applies a file over an existing directory subtree", async () => { + await withDB(async (db) => { + mkdir(db, "/dst/sub", { recursive: true }, () => 1); + await writeFile(db, "/dst/sub/old.txt", "old", {}, () => 2); + + await applyChanges( + db, + [ + { + kind: "file", + rev: 99, + path: "/dst", + mode: 0o644, + mtime: 3, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("file"); + expect(resolveInode(db, "/dst/sub/old.txt")).toBeNull(); + }); + }); + + it("applies a file over a deeply nested existing directory subtree", async () => { + await withDB(async (db) => { + const oldPath = deepChildPath(12_000); + mkdir(db, oldPath.slice(0, -"/old.txt".length), { recursive: true }, () => 1); + await writeFile(db, oldPath, "old", {}, () => 2); + + await applyChanges( + db, + [ + { + kind: "file", + rev: 99, + path: "/dst", + mode: 0o644, + mtime: 3, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("file"); + expect(resolveInode(db, oldPath, { followSymlinks: false })).toBeNull(); + }); + }); + + it("applies a symlink over an existing directory subtree", async () => { + await withDB((db) => { + mkdir(db, "/dst/sub", { recursive: true }, () => 1); + writeFileSync(db, "/dst/sub/old.txt", new Uint8Array(), {}, () => 2); + + applyChangesSync( + db, + [ + { + kind: "symlink", + rev: 99, + path: "/dst", + mode: 0o777, + mtime: 3, + target: "/target", + }, + ], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("symlink"); + expect(readlink(db, "/dst")).toBe("/target"); + expect(resolveInode(db, "/dst/sub/old.txt")).toBeNull(); + }); + }); + + it("applies a symlink over one hardlink without removing sibling hardlinks", async () => { + await withDB(async (db) => { + await writeFile(db, "/dst", "shared", {}, () => 1); + link(db, "/dst", "/other"); + + await applyChanges( + db, + [ + { + kind: "symlink", + rev: 99, + path: "/dst", + mode: 0o777, + mtime: 2, + target: "/target", + }, + ], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("symlink"); + expect(readlink(db, "/dst")).toBe("/target"); + expect(await readFile(db, "/other", "utf8")).toBe("shared"); + }); + }); + + it("applies a directory over one hardlink without removing sibling hardlinks", async () => { + await withDB(async (db) => { + await writeFile(db, "/dst", "shared", {}, () => 1); + link(db, "/dst", "/other"); + + await applyChanges( + db, + [{ kind: "dir", rev: 99, path: "/dst", mode: 0o755, mtime: 2 }], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("dir"); + expect(await readFile(db, "/other", "utf8")).toBe("shared"); + }); + }); + + it("applies a file over a directory whose subtree hardlinks a file outside it", async () => { + await withDB(async (db) => { + await mkdir(db, "/dir", { mode: 0o755 }, () => 1); + await writeFile(db, "/dir/inner", "shared", {}, () => 1); + // /outside is a second name for /dir/inner; replacing /dir must + // walk the subtree by name and keep /outside alive. + link(db, "/dir/inner", "/outside"); + + await applyChanges( + db, + [ + { + kind: "file", + rev: 99, + path: "/dir", + mode: 0o644, + mtime: 2, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(resolveInode(db, "/dir", { followSymlinks: false })?.type).toBe("file"); + expect(resolveInode(db, "/dir/inner", { followSymlinks: false })).toBeNull(); + expect(await readFile(db, "/outside", "utf8")).toBe("shared"); + }); + }); + + it("applyChangesSync applies a symlink over a deeply nested existing directory subtree", async () => { + await withDB((db) => { + const oldPath = deepChildPath(12_000); + mkdir(db, oldPath.slice(0, -"/old.txt".length), { recursive: true }, () => 1); + writeFileSync(db, oldPath, new Uint8Array(), {}, () => 2); + + applyChangesSync( + db, + [ + { + kind: "symlink", + rev: 99, + path: "/dst", + mode: 0o777, + mtime: 3, + target: "/target", + }, + ], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })?.type).toBe("symlink"); + expect(readlink(db, "/dst")).toBe("/target"); + expect(resolveInode(db, oldPath, { followSymlinks: false })).toBeNull(); + }); + }); + + it("applies a directory over an existing file", async () => { + await withDB(async (db) => { + await writeFile(db, "/dst", "old", {}, () => 1); + + await applyChanges( + db, + [{ kind: "dir", rev: 99, path: "/dst", mode: 0o755, mtime: 2 }], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ + type: "dir", + mode: 0o755, + mtime: 2, + }); + }); + }); + + it("applies a directory over an existing symlink", async () => { + await withDB((db) => { + symlink(db, "/target", "/dst", () => 1); + + applyChangesSync( + db, + [{ kind: "dir", rev: 99, path: "/dst", mode: 0o755, mtime: 2 }], + new Map(), + ); + + expect(resolveInode(db, "/dst", { followSymlinks: false })).toMatchObject({ + type: "dir", + mode: 0o755, + mtime: 2, + }); + }); + }); + + it("applies a coalesced file-to-directory replacement", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/dst", "old", {}, () => 1); + const cursor = currentRev(a); + rm(a, "/dst", { force: true }); + mkdir(a, "/dst", {}, () => 2); + const entries = await drain(coalesceChanges(a, cursor)); + expect(entries).toEqual([expect.objectContaining({ kind: "dir", path: "/dst" })]); + return { entries, objects: await collectObjects(a, entries) }; + }, + async (b, { entries, objects }) => { + await writeFile(b, "/dst", "old", {}, () => 1); + + await applyChanges(b, entries, objects); + + expect(resolveInode(b, "/dst", { followSymlinks: false })?.type).toBe("dir"); + }, + ); + }); + + it("handles delete entries", async () => { + await withDB(async (db) => { + await writeFile(db, "/gone.txt", "bye", {}, () => 1); + await applyChanges(db, [{ kind: "delete", rev: 99, path: "/gone.txt" }], new Map()); + expect(resolveInode(db, "/gone.txt")).toBeNull(); + }); + }); +}); + +describe("applyChanges loopback suppression", () => { + it("does not advance pushRev locally on upstream apply", async () => { + await withDB(async (db) => { + // Pre-existing local state: a write the container already + // pushed. pushRev sits at currentRev. + await writeFile(db, "/local.txt", "local", {}, () => 1); + const { currentRev, readWatermark, writeWatermark } = await import("./watermarks.js"); + writeWatermark(db, "pushRev", currentRev(db)); + const beforePushRev = readWatermark(db, "pushRev"); + expect(beforePushRev).toBeGreaterThan(0); + + // Apply an entry as if it came from upstream. The apply's + // writeFile bumps the local rev counter, but pushRev must + // *not* advance with it — advancing locally would move our + // pushRev past entries the remote does not know we have + // shipped, breaking the cross-side invariant on the next + // pull. The next pushOnce re-ships these rev bumps and the + // receiver's alreadyApplied() check drops them. + await applyChanges( + db, + [ + { + kind: "file", + rev: 100, + path: "/from-upstream.txt", + mode: 0o644, + mtime: 2, + size: 0, + chunks: [], + }, + ], + new Map(), + { source: "upstream" }, + ); + + const afterCurrent = currentRev(db); + const afterPushRev = readWatermark(db, "pushRev"); + expect(afterCurrent).toBeGreaterThan(beforePushRev); + expect(afterPushRev).toBe(beforePushRev); + }); + }); + + it("source=local (default) does not advance pushRev", async () => { + await withDB(async (db) => { + const { readWatermark } = await import("./watermarks.js"); + await applyChanges( + db, + [ + { + kind: "file", + rev: 100, + path: "/local.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + expect(readWatermark(db, "pushRev")).toBe(0); + }); + }); + + it("upstream entries surface on the next coalesce and rely on receiver-side alreadyApplied", async () => { + await withDB(async (db) => { + const { coalesceChanges } = await import("./coalesce.js"); + const { currentRev, readWatermark, writeWatermark } = await import("./watermarks.js"); + // Seed pushRev at the current point. + writeWatermark(db, "pushRev", currentRev(db)); + + // Upstream sends a file. After apply, pushRev stays where it + // was (the local advance was unsound — see the test above). + // The next coalesceChanges(db, pushRev) re-emits the entry; + // the receiver's alreadyApplied() check drops it. One extra + // round trip per apply, watermarks stay in lockstep. + await applyChanges( + db, + [ + { + kind: "file", + rev: 100, + path: "/upstream.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + { source: "upstream" }, + ); + const cursor = readWatermark(db, "pushRev"); + const drained = []; + for await (const e of coalesceChanges(db, cursor)) drained.push(e); + expect(drained.map((e) => e.path)).toContain("/upstream.txt"); + }); + }); +}); + +describe("applyChanges loopback suppression — F1", () => { + // Regression for F1: when local writes are sitting at + // rev > pushRev (i.e. queued for the next push) and an + // upstream pull arrives, the old code advanced pushRev to + // currentRev unconditionally. That stranded the local + // writes — the next pushOnce skipped them as already- + // shipped. Fix: only advance pushRev when the existing + // value already covers everything that existed before this + // apply. + it("does not advance pushRev past unpushed local writes", async () => { + await withDB(async (db) => { + const { currentRev, readWatermark } = await import("./watermarks.js"); + // Simulate an unpushed local write: pushRev stays at + // its initial value (1) but currentRev climbs. + await writeFile(db, "/local.txt", new Uint8Array([1, 2, 3]), { mode: 0o644 }, () => 1); + const revBeforeApply = currentRev(db); + const pushRevBefore = readWatermark(db, "pushRev"); + expect(pushRevBefore).toBeLessThan(revBeforeApply); + + // Upstream sends an entry. alreadyApplied skips it + // (we don't have it locally, so it actually writes — + // pick a path that won't conflict). + await applyChanges( + db, + [ + { + kind: "file", + rev: 100, + path: "/from-upstream.txt", + mode: 0o644, + mtime: 2, + size: 0, + chunks: [], + }, + ], + new Map(), + { source: "upstream" }, + ); + // pushRev must NOT have jumped past the unpushed + // local write. The local write is at revBeforeApply; + // we want pushRev still < revBeforeApply so the next + // pushOnce drains it. + const pushRevAfter = readWatermark(db, "pushRev"); + expect(pushRevAfter).toBeLessThan(revBeforeApply); + // The local write should still appear in coalesce. + const drained = []; + for await (const e of coalesceChanges(db, pushRevAfter)) drained.push(e); + const paths = drained.map((e) => (e.kind === "delete" ? e.path : e.path)); + expect(paths).toContain("/local.txt"); + }); + }); + + it("leaves pushRev alone even when the caller had no unpushed locals", async () => { + await withDB(async (db) => { + const { currentRev, readWatermark, writeWatermark } = await import("./watermarks.js"); + // pushRev already caught up to currentRev: caller has no + // pending local writes. The old apply path advanced pushRev + // here as an optimization; we no longer do that because it + // desynced our pushRev from the remote's fetchRev (echoed + // back as appliedPushRev on the wire). The next pushOnce + // re-ships the apply's rev bump and the receiver's + // alreadyApplied() check drops it. + const before = currentRev(db); + writeWatermark(db, "pushRev", before); + await applyChanges( + db, + [ + { + kind: "file", + rev: 100, + path: "/from-upstream.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + { source: "upstream" }, + ); + const after = currentRev(db); + expect(after).toBeGreaterThan(before); + expect(readWatermark(db, "pushRev")).toBe(before); + }); + }); +}); + +describe("applyChanges with read-only mount roots", () => { + function stageReadOnly(db: import("../storage.js").Database, root: string): void { + db.run( + "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-only')", + root, + "test", + ); + invalidateReadOnlyMountCache(db); + } + + it("skips a write entry under a read-only mount and reports it", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageReadOnly(db, "/workspace/r2"); + + const result = await applyChanges( + db, + [ + { + kind: "file", + rev: 100, + path: "/workspace/r2/hello.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(result.applied).toBe(0); + expect(result.skipped).toEqual([ + { + path: "/workspace/r2/hello.txt", + mountRoot: "/workspace/r2", + op: "write", + reason: "read-only", + }, + ]); + // The skipped path is not on disk. + expect(resolveInode(db, "/workspace/r2/hello.txt")).toBeNull(); + }); + }); + + it("skips a delete entry under a read-only mount and reports op:delete", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageReadOnly(db, "/workspace/r2"); + + const result = await applyChanges( + db, + [{ kind: "delete", rev: 101, path: "/workspace/r2/gone.txt" }], + new Map(), + ); + + expect(result.applied).toBe(0); + expect(result.skipped).toEqual([ + { + path: "/workspace/r2/gone.txt", + mountRoot: "/workspace/r2", + op: "delete", + reason: "read-only", + }, + ]); + }); + }); + + it("skips an entry whose path is exactly the mount root", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageReadOnly(db, "/workspace/r2"); + + const result = await applyChanges( + db, + [ + { + kind: "dir", + rev: 102, + path: "/workspace/r2", + mode: 0o755, + mtime: 1, + }, + ], + new Map(), + ); + + expect(result.applied).toBe(0); + expect(result.skipped[0]?.mountRoot).toBe("/workspace/r2"); + }); + }); + + it("applies entries that lie outside any mount and reports an empty skipped list", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageReadOnly(db, "/workspace/r2"); + + mkdir(db, "/scratch", { recursive: true }, () => 0); + + const result = await applyChanges( + db, + [ + { + kind: "file", + rev: 103, + path: "/scratch/ok.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(result.applied).toBe(1); + expect(result.skipped).toEqual([]); + expect(resolveInode(db, "/scratch/ok.txt")).not.toBeNull(); + }); + }); + + it("does not skip entries under a read-write mount", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/rw", { recursive: true }, () => 0); + db.run( + "INSERT INTO _vfs_mounts (root, kind, indexed, mode) VALUES (?, ?, 1, 'read-write')", + "/workspace/rw", + "test", + ); + invalidateReadOnlyMountCache(db); + + const result = await applyChanges( + db, + [ + { + kind: "file", + rev: 104, + path: "/workspace/rw/ok.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(result.applied).toBe(1); + expect(result.skipped).toEqual([]); + }); + }); + + it("folds skip + apply across a mixed batch", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + mkdir(db, "/scratch", { recursive: true }, () => 0); + stageReadOnly(db, "/workspace/r2"); + + const result = await applyChanges( + db, + [ + { + kind: "file", + rev: 200, + path: "/scratch/a.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + { + kind: "file", + rev: 201, + path: "/workspace/r2/blocked.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + { + kind: "file", + rev: 202, + path: "/scratch/b.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(result.applied).toBe(2); + expect(result.skipped).toEqual([ + { + path: "/workspace/r2/blocked.txt", + mountRoot: "/workspace/r2", + op: "write", + reason: "read-only", + }, + ]); + }); + }); + + it("applyChangesSync emits the same SkippedEntry shape", async () => { + await withDB(async (db) => { + mkdir(db, "/workspace/r2", { recursive: true }, () => 0); + stageReadOnly(db, "/workspace/r2"); + + const result = applyChangesSync( + db, + [ + { + kind: "file", + rev: 300, + path: "/workspace/r2/blocked.txt", + mode: 0o644, + mtime: 1, + size: 0, + chunks: [], + }, + ], + new Map(), + ); + + expect(result.applied).toBe(0); + expect(result.skipped).toEqual([ + { + path: "/workspace/r2/blocked.txt", + mountRoot: "/workspace/r2", + op: "write", + reason: "read-only", + }, + ]); + }); + }); +}); + +describe("applyChanges mtime propagation (auto_cache contract)", () => { + // The FUSE driver mounts with COMPUTERD_FUSE_AUTO_CACHE=1 in the + // production-safe profile. auto_cache tells the kernel to keep + // file data in the page cache until the file is reopened with a + // different mtime or size; the kernel then drops the cached + // pages and re-reads through FUSE. The whole story rests on + // mtime moving forward whenever the bytes change, including the + // tricky cases where size stays the same. + // + // These tests pin that contract on the sync apply path. If a + // future apply refactor stops propagating mtime, a container + // with auto_cache enabled would keep serving the old bytes + // after a remote push and the cache-coherency story would + // silently break. + + it("bumps destination mtime when a same-size file changes content", async () => { + await withTwoDBs( + async (a) => { + // Source: write the first version, push, then overwrite + // with bytes of the same length but a different value + // and a strictly later mtime. + await writeFile(a, "/note.txt", "alpha", {}, () => 1000); + const first = await drain(coalesceChanges(a, 0)); + const firstObjects = await collectObjects(a, first); + + await writeFile(a, "/note.txt", "OMEGA", {}, () => 2000); + const second = await drain(coalesceChanges(a, Math.max(...first.map((e) => e.rev)))); + const secondObjects = await collectObjects(a, second); + return { first, firstObjects, second, secondObjects }; + }, + async (b, { first, firstObjects, second, secondObjects }) => { + await applyChanges(b, first, firstObjects); + const beforeInode = resolveInode(b, "/note.txt"); + expect(beforeInode).not.toBeNull(); + expect(beforeInode?.mtime).toBe(1000); + expect(await readFile(b, "/note.txt", "utf8")).toBe("alpha"); + + await applyChanges(b, second, secondObjects); + const afterInode = resolveInode(b, "/note.txt"); + expect(afterInode).not.toBeNull(); + // Bytes changed: the kernel must see a strictly newer + // mtime so auto_cache drops the page cache on reopen. + expect(afterInode?.mtime).toBeGreaterThan(beforeInode?.mtime ?? 0); + expect(afterInode?.mtime).toBe(2000); + expect(await readFile(b, "/note.txt", "utf8")).toBe("OMEGA"); + }, + ); + }); + + it("skips an apply when bytes are identical even if the source mtime is newer", async () => { + // The mirror of the test above. If a sender pushes the same + // content with a fresher mtime, the apply path takes the + // alreadyApplied fast path and does not touch the local row. + // The local mtime stays put. This is by design: auto_cache + // only needs to invalidate when bytes change. A pure mtime + // bump would still be safe (the kernel would invalidate and + // re-read identical bytes), but it would burn a local rev + // for nothing. + await withTwoDBs( + async (a) => { + await writeFile(a, "/same.txt", "static", {}, () => 1000); + const first = await drain(coalesceChanges(a, 0)); + const firstObjects = await collectObjects(a, first); + // Re-stamp the same bytes with a newer mtime on the + // source. This isn't something writeFile normally + // produces (it bumps rev and emits the same chunks), but + // it's the worst case a future sender might present. + const reissued = first.map((entry): ChangeEntry => { + if (entry.kind !== "file" || entry.path !== "/same.txt") return entry; + return { ...entry, mtime: 9999, rev: entry.rev + 1000 }; + }); + return { first, firstObjects, reissued }; + }, + async (b, { first, firstObjects, reissued }) => { + // First apply is from upstream too, so the local rev + // counter doesn't claim the bytes as a local write. + await applyChanges(b, first, firstObjects, { source: "upstream" }); + const beforeInode = resolveInode(b, "/same.txt"); + expect(beforeInode?.mtime).toBe(1000); + + // Reapply with a fresh mtime but identical chunks. The + // upstream-source guard runs alreadyApplied(), which + // matches on manifest hash and drops the entry. + await applyChanges(b, reissued, firstObjects, { source: "upstream" }); + const afterInode = resolveInode(b, "/same.txt"); + // alreadyApplied caught the no-op; the mtime stays at + // 1000 because the row never moved. auto_cache's safety + // story is unaffected — the bytes are still the same, + // so a stale page cache would still return correct bytes. + expect(afterInode?.mtime).toBe(1000); + }, + ); + }); + + it("bumps destination mtime when a file shrinks", async () => { + // Size change alone is enough to invalidate auto_cache, but + // the kernel still consults mtime first. Make sure the + // shrinking case carries the new mtime through so a fast + // mtime-cache check at the kernel layer can short-circuit. + await withTwoDBs( + async (a) => { + await writeFile(a, "/shrink.txt", "original-content", {}, () => 1000); + const first = await drain(coalesceChanges(a, 0)); + const firstObjects = await collectObjects(a, first); + + await writeFile(a, "/shrink.txt", "x", {}, () => 2000); + const second = await drain(coalesceChanges(a, Math.max(...first.map((e) => e.rev)))); + const secondObjects = await collectObjects(a, second); + return { first, firstObjects, second, secondObjects }; + }, + async (b, { first, firstObjects, second, secondObjects }) => { + await applyChanges(b, first, firstObjects); + await applyChanges(b, second, secondObjects); + const inode = resolveInode(b, "/shrink.txt"); + expect(inode?.mtime).toBe(2000); + expect(await readFile(b, "/shrink.txt", "utf8")).toBe("x"); + }, + ); + }); + + it("bumps mtime on the sync-style synchronous apply path too", async () => { + // applyChangesSync is the alternate entry point used by + // SyncRPC.fetch's commit phase. It walks the same + // alreadyApplied / writeFileSync path as applyChanges, but + // the two paths are easy to drift apart in a refactor. Lock + // the same mtime-propagation contract on both. + await withTwoDBs( + async (a) => { + await writeFile(a, "/sync.txt", "first", {}, () => 1000); + const first = await drain(coalesceChanges(a, 0)); + const firstObjects = await collectObjects(a, first); + + await writeFile(a, "/sync.txt", "SECOND", {}, () => 2000); + const second = await drain(coalesceChanges(a, Math.max(...first.map((e) => e.rev)))); + const secondObjects = await collectObjects(a, second); + return { first, firstObjects, second, secondObjects }; + }, + async (b, { first, firstObjects, second, secondObjects }) => { + applyChangesSync(b, first, firstObjects); + expect(resolveInode(b, "/sync.txt")?.mtime).toBe(1000); + applyChangesSync(b, second, secondObjects); + expect(resolveInode(b, "/sync.txt")?.mtime).toBe(2000); + expect(await readFile(b, "/sync.txt", "utf8")).toBe("SECOND"); + }, + ); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/apply.ts b/spikes/349-dofs/vendor/dofs/src/sync/apply.ts new file mode 100644 index 00000000..4986c98e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/apply.ts @@ -0,0 +1,485 @@ +import { mkdir } from "../fs/mkdir.js"; +import { readOnlyRootFor } from "../fs/mount-guard.js"; +import { resolveInode } from "../fs/resolve.js"; +import { invalidateResolveSubtree } from "../fs/resolveCache.js"; +import { rm } from "../fs/rm.js"; +import { symlink } from "../fs/symlink.js"; +import { unlinkDirent } from "../fs/unlink.js"; +import { writeFile, writeFileSync } from "../fs/writeFile.js"; +import { canonicalizePath } from "../path.js"; +import { incrementRev } from "../rev.js"; +import type { Database } from "../storage.js"; +import type { ChangeEntry } from "./changes.js"; +import { computeManifestHash } from "./manifests.js"; + +// One container-side change that landed under a read-only mount and +// was therefore skipped rather than applied. Callers (the workspace +// pull surface, the shell exec bracket) surface these so the user +// learns the mount stayed authoritative. +export interface SkippedEntry { + // Absolute VFS path the change targeted. + path: string; + // Mount root that owns the path (the one whose mode is + // read-only). Lets callers group skipped entries by mount. + mountRoot: string; + // 'write' covers file / dir / symlink create-or-update; 'delete' + // covers tombstones. The single field is enough for callers to + // decide messaging. + op: "write" | "delete"; + // Open shape: future skip reasons can join this union without + // breaking callers that match on 'read-only' today. + reason: "read-only"; +} + +// Return shape of applyChanges / applyChangesSync. Existing callers +// that only wanted a count read `result.applied`; new callers can +// surface `result.skipped`. +export interface ApplyResult { + // Entries written through writeFile / mkdir / symlink / rm. + applied: number; + // Entries dropped because they targeted a read-only mount root. + // Empty when no such mounts are registered or the stream stayed + // clear of them. + skipped: SkippedEntry[]; +} + +export interface ApplyOptions { + // Soft cap on bytes written per transactionSync batch. Default 64 + // MiB; matches docs/02_sync_protocol.md. The cap is advisory: a + // single large file is always one batch. + maxBytesPerBatch?: number; + // Soft cap on entries per batch. Default 1024 paths. + maxPathsPerBatch?: number; + // Where the entries came from. 'local' (default) treats the apply + // path like any other mutation: writeFile/mkdir/etc bump + // vfs_meta.rev and the push loop later ships those new revs + // upstream. 'upstream' is informational: the apply still bumps + // rev so readers see fresh data, and the next pushOnce ships + // those rev bumps back to the sender. Loop convergence is the + // receiver's job — the apply path on the original sender uses + // alreadyApplied() to drop the redundant entries without bumping + // rev further, bounding the echo at one extra round trip per + // upstream apply. + source?: "local" | "upstream"; + // Backend id whose watermark row this apply should touch. The + // DO hosts independent sync cursors per backend; threading the + // id through here keeps a pull from backend A from bumping + // backend B's pushRev. Defaults to the dofs `default` slot, + // which is fine for the container backend the package shipped + // with first. + backend?: string; +} + +const DEFAULT_MAX_BYTES = 64 * 1024 * 1024; +const DEFAULT_MAX_PATHS = 1024; + +type NodeType = "file" | "dir" | "symlink"; + +function hex(bytes: Uint8Array): string { + let s = ""; + for (let i = 0; i < bytes.byteLength; i++) s += bytes[i].toString(16).padStart(2, "0"); + return s; +} + +function removeReplaceableFinalEntry( + db: Database, + path: string, + incomingKind: "file" | "symlink", +): void { + const existing = resolveInode(db, path, { followSymlinks: false }); + if (existing === null) return; + if (incomingKind === "file" && existing.type === "file") return; + + removeInodeTreeAtPath(db, path, existing.inode, existing.type); +} + +// Structural conflict cleanup for upstream applies. This removes +// the local shape without recording tombstones because the incoming +// entry is the authoritative state for this path. +function removeInodeTreeAtPath(db: Database, path: string, inode: number, type: NodeType): void { + // Structural subtree removal that bypasses rm() and calls unlinkDirent + // directly, so it must drop cached resolutions itself. One subtree + // drop at the root covers every descendant the walk unlinks. + // Canonicalize to the exact key readers cache under (every other hook + // already passes a canonical path; this one takes an entry path). + invalidateResolveSubtree(db, canonicalizePath(path).path); + const root = direntForPath(db, path, inode); + const stack: Array<{ + path: string; + parentInode: number; + name: string; + inode: number; + type: NodeType; + expanded: boolean; + }> = [ + { + path, + parentInode: root.parentInode, + name: root.name, + inode, + type, + expanded: false, + }, + ]; + + while (stack.length > 0) { + const current = stack.pop(); + if (current === undefined) break; + + if (current.type === "dir" && !current.expanded) { + const children = db.all<{ name: string; child_inode: number; type: NodeType }>( + `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type + FROM vfs_dirents d + JOIN vfs_nodes n ON n.inode = d.child_inode + WHERE d.parent_inode = ?`, + current.inode, + ); + stack.push({ ...current, expanded: true }); + for (let i = children.length - 1; i >= 0; i--) { + const child = children[i]; + const childPath = current.path === "/" ? `/${child.name}` : `${current.path}/${child.name}`; + stack.push({ + path: childPath, + parentInode: current.inode, + name: child.name, + inode: child.child_inode, + type: child.type, + expanded: false, + }); + } + continue; + } + + // Unlink this one name and reap the inode only when its last link + // disappears, so a sibling hardlink (inside or outside the subtree) + // keeps the file alive. (parent, name) is unique, so this removes + // exactly the dirent the walk is visiting. + unlinkDirent(db, current.parentInode, current.name, current.inode, current.type); + } +} + +function direntForPath( + db: Database, + path: string, + inode: number, +): { parentInode: number; name: string } { + const { parts, path: canonical } = canonicalizePath(path); + if (parts.length === 0) { + throw new Error(`applyChanges: cannot structurally replace root ${canonical}`); + } + const name = parts[parts.length - 1]; + const parentPath = parts.length === 1 ? "/" : `/${parts.slice(0, -1).join("/")}`; + const parent = resolveInode(db, parentPath, { followSymlinks: false }); + if (parent === null || parent.type !== "dir") { + throw new Error(`applyChanges: parent missing for structural replacement ${canonical}`); + } + const child = db.scalar( + "SELECT child_inode FROM vfs_dirents WHERE parent_inode = ? AND name = ?", + parent.inode, + name, + ); + if (child !== inode) { + throw new Error(`applyChanges: dirent mismatch for structural replacement ${canonical}`); + } + return { parentInode: parent.inode, name }; +} + +function applyDirectoryEntry(db: Database, entry: Extract): void { + const mode = entry.mode & 0o7777; + const existing = resolveInode(db, entry.path, { followSymlinks: false }); + if (existing === null) { + mkdir(db, entry.path, { mode, recursive: true }, () => entry.mtime); + return; + } + if (existing.type !== "dir") { + removeInodeTreeAtPath(db, entry.path, existing.inode, existing.type); + mkdir(db, entry.path, { mode, recursive: true }, () => entry.mtime); + return; + } + + db.transactionSync(() => { + const rev = incrementRev(db); + db.run( + "UPDATE vfs_nodes SET mode = ?, mtime = ?, rev = ? WHERE inode = ?", + mode, + entry.mtime, + rev, + existing.inode, + ); + }); +} + +// Drive a ChangeEntry stream against `db`, batching writes so peak +// memory stays bounded and a crash mid-apply leaves the DB in a +// consistent state. Each batch runs inside a single transactionSync +// from the underlying FS helpers — mkdir, writeFile, symlink, +// rm all wrap their own transactionSync, so a batch is in practice +// a sequence of independently-committed mutations rather than one +// fat transaction. The bounded-batch contract still holds because +// fetchRev only advances after the stream drains. +// +// `objects` is a hash-keyed map of chunk bytes the sender shipped +// via pushObjects / fetchObjects. File entries reassemble their +// chunks from this map; missing entries throw. +export async function applyChanges( + db: Database, + entries: Iterable | AsyncIterable, + objects: Map, + options: ApplyOptions = {}, +): Promise { + const maxBytes = options.maxBytesPerBatch ?? DEFAULT_MAX_BYTES; + const maxPaths = options.maxPathsPerBatch ?? DEFAULT_MAX_PATHS; + + let bytesInBatch = 0; + let pathsInBatch = 0; + let applied = 0; + const skipped: SkippedEntry[] = []; + const flush = () => { + bytesInBatch = 0; + pathsInBatch = 0; + }; + + for await (const entry of entries) { + // Idempotent skip: if the entry already matches the local + // state, drop it on the floor. The check is what stops a + // pull from bumping vfs_meta.rev for entries that are + // already in place, which in turn stops the next push from + // re-shipping them. + if (options.source === "upstream" && entry.kind !== "delete") { + if (alreadyApplied(db, entry)) continue; + } + // Read-only mount guard. Entries under a registered read-only + // mount root are surfaced via the return value and not applied. + // The owning workspace's surface (Workspace.pull, exec()) folds + // these into its own return so callers see what stayed + // authoritative on the mount. + const blockingRoot = readOnlyRootFor(db, entry.path); + if (blockingRoot !== undefined) { + skipped.push({ + path: entry.path, + mountRoot: blockingRoot, + op: entry.kind === "delete" ? "delete" : "write", + reason: "read-only", + }); + continue; + } + if (entry.kind === "delete") { + try { + rm(db, entry.path, { recursive: true, force: true }); + } catch { + // Already gone is fine — idempotent apply. + } + applied++; + pathsInBatch++; + if (pathsInBatch >= maxPaths) flush(); + continue; + } + if (entry.kind === "dir") { + applyDirectoryEntry(db, entry); + applied++; + pathsInBatch++; + if (pathsInBatch >= maxPaths) flush(); + continue; + } + if (entry.kind === "symlink") { + removeReplaceableFinalEntry(db, entry.path, "symlink"); + symlink(db, entry.target, entry.path, () => entry.mtime); + applied++; + pathsInBatch++; + if (pathsInBatch >= maxPaths) flush(); + continue; + } + // file: assemble chunk bytes. First check the in-memory map + // (the streaming hand-off); fall back to vfs_blob_bytes (the + // staged-via-pushObjects path). + const parts: Uint8Array[] = []; + let total = 0; + for (const c of entry.chunks) { + const k = hex(c.hash); + let bytes = objects.get(k); + if (bytes === undefined) { + const row = db.one<{ bytes: Uint8Array }>( + "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", + c.hash, + ); + bytes = row?.bytes; + } + if (bytes === undefined) { + throw new Error(`applyChanges: missing object ${k} for ${entry.path}`); + } + parts.push(bytes); + total += bytes.byteLength; + } + const buf = new Uint8Array(total); + let off = 0; + for (const p of parts) { + buf.set(p, off); + off += p.byteLength; + } + removeReplaceableFinalEntry(db, entry.path, "file"); + await writeFile(db, entry.path, buf, { mode: entry.mode }, () => entry.mtime); + applied++; + bytesInBatch += total; + pathsInBatch++; + if (bytesInBatch >= maxBytes || pathsInBatch >= maxPaths) flush(); + } + + // Loopback suppression used to advance pushRev locally after an + // upstream apply so the next push tick wouldn't re-ship the rev + // bumps the apply produced. That optimization is unsound: it + // moves the *local* pushRev past entries the remote does not + // know we have shipped, while the remote's fetchRev (echoed back + // as appliedPushRev on every fetchChanges) stays where it was. + // The cross-side invariant check in pullOnce then trips on the + // very next pull and the post-drain pullOnce in the exec bracket + // swallows the error, leaving every subsequent container-side + // write invisible to the host until something reconciles. + // + // The bounded "redundant round-trip" the old comment promised is + // still bounded, and the receiver's alreadyApplied() check still + // suppresses the entries on the next pushOnce. We just pay one + // extra push per upstream apply to keep the two sides in lockstep. + return { applied, skipped }; +} + +// Synchronous variant of applyChanges. Same semantics; takes an +// in-memory entry array instead of an iterable. Used on the push +// receiver so the whole batch can run inside a single transactionSync +// and a mid-stream failure rolls back every prior entry. +// +// Stays separate from applyChanges so the streaming pull path +// (which can't hold a sync transaction across network I/O) keeps +// its async semantics. +export function applyChangesSync( + db: Database, + entries: readonly ChangeEntry[], + objects: Map, + options: ApplyOptions = {}, +): ApplyResult { + const maxBytes = options.maxBytesPerBatch ?? DEFAULT_MAX_BYTES; + const maxPaths = options.maxPathsPerBatch ?? DEFAULT_MAX_PATHS; + + let bytesInBatch = 0; + let pathsInBatch = 0; + let applied = 0; + const skipped: SkippedEntry[] = []; + const flush = () => { + bytesInBatch = 0; + pathsInBatch = 0; + }; + + for (const entry of entries) { + if (options.source === "upstream" && entry.kind !== "delete") { + if (alreadyApplied(db, entry)) continue; + } + const blockingRoot = readOnlyRootFor(db, entry.path); + if (blockingRoot !== undefined) { + skipped.push({ + path: entry.path, + mountRoot: blockingRoot, + op: entry.kind === "delete" ? "delete" : "write", + reason: "read-only", + }); + continue; + } + if (entry.kind === "delete") { + try { + rm(db, entry.path, { recursive: true, force: true }); + } catch { + // Already gone is fine — idempotent apply. + } + applied++; + pathsInBatch++; + if (pathsInBatch >= maxPaths) flush(); + continue; + } + if (entry.kind === "dir") { + applyDirectoryEntry(db, entry); + applied++; + pathsInBatch++; + if (pathsInBatch >= maxPaths) flush(); + continue; + } + if (entry.kind === "symlink") { + removeReplaceableFinalEntry(db, entry.path, "symlink"); + symlink(db, entry.target, entry.path, () => entry.mtime); + applied++; + pathsInBatch++; + if (pathsInBatch >= maxPaths) flush(); + continue; + } + const parts: Uint8Array[] = []; + let total = 0; + for (const c of entry.chunks) { + const k = hex(c.hash); + let bytes = objects.get(k); + if (bytes === undefined) { + const row = db.one<{ bytes: Uint8Array }>( + "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", + c.hash, + ); + bytes = row?.bytes; + } + if (bytes === undefined) { + throw new Error(`applyChanges: missing object ${k} for ${entry.path}`); + } + parts.push(bytes); + total += bytes.byteLength; + } + const buf = new Uint8Array(total); + let off = 0; + for (const p of parts) { + buf.set(p, off); + off += p.byteLength; + } + removeReplaceableFinalEntry(db, entry.path, "file"); + writeFileSync(db, entry.path, buf, { mode: entry.mode }, () => entry.mtime); + applied++; + bytesInBatch += total; + pathsInBatch++; + if (bytesInBatch >= maxBytes || pathsInBatch >= maxPaths) flush(); + } + + // See applyChanges() for why pushRev no longer advances locally + // on upstream applies. The receiver's alreadyApplied() check + // suppresses the redundant entries on the next pushOnce; one + // extra push per apply keeps the cross-side invariant intact. + + return { applied, skipped }; +} + +// Compare an entry against the local node graph. Returns true when +// the entry would be a no-op apply: the manifest hash (files), mode +// (dirs), or mode + symlink target (symlinks) already matches. +function alreadyApplied(db: Database, entry: Exclude): boolean { + const live = resolveInode(db, entry.path, { followSymlinks: false }); + if (live === null) return false; + + if (entry.kind === "file") { + if (live.type !== "file") return false; + const row = db.one<{ manifest_hash: Uint8Array | null }>( + "SELECT manifest_hash FROM vfs_nodes WHERE inode = ?", + live.inode, + ); + if (!row?.manifest_hash) return false; + const wanted = computeManifestHash(entry.chunks); + return uint8Equal(row.manifest_hash, wanted); + } + if (entry.kind === "dir") { + return live.type === "dir" && (live.mode & 0o7777) === (entry.mode & 0o7777); + } + // symlink + return ( + live.type === "symlink" && + live.linkTarget === entry.target && + (live.mode & 0o7777) === (entry.mode & 0o7777) + ); +} + +function uint8Equal(a: Uint8Array, b: Uint8Array): boolean { + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) { + if (a[i] !== b[i]) return false; + } + return true; +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/blobs.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/blobs.test.ts new file mode 100644 index 00000000..e9568596 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/blobs.test.ts @@ -0,0 +1,99 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { gc } from "../fs/gc.js"; +import { withDB } from "../fs/with-db.js"; +import { writeFile } from "../fs/writeFile.js"; +import { Database } from "../storage.js"; +import { stageBlob } from "./blobs.js"; + +function sha256(bytes: Uint8Array): Uint8Array { + return new Uint8Array(createHash("sha256").update(bytes).digest()); +} + +describe("stageBlob", () => { + it("lands a chunk into vfs_blobs + vfs_blob_bytes", async () => { + await withDB(async (db) => { + const bytes = new TextEncoder().encode("payload"); + const hash = sha256(bytes); + stageBlob(db, hash, bytes, 1234); + const blob = db.one<{ size: number; last_seen: number }>( + "SELECT size, last_seen FROM vfs_blobs WHERE hash = ?", + hash, + ); + expect(blob?.size).toBe(7); + expect(blob?.last_seen).toBe(1234); + const row = db.one<{ bytes: Uint8Array }>( + "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", + hash, + ); + expect(row?.bytes).toEqual(bytes); + }); + }); + + it("rolls back metadata and bytes when storing the bytes fails", async () => { + await withDB(async (db) => { + const bytes = new TextEncoder().encode("payload"); + const hash = sha256(bytes); + const failingDb = new Database({ + sql: { + exec: (query: string, ...bindings: unknown[]) => { + if (query.startsWith("INSERT INTO vfs_blob_bytes")) { + throw new Error("injected bytes failure"); + } + return db.sql.exec(query, ...bindings); + }, + }, + transactionSync: (closure) => db.transactionSync(closure), + }); + + expect(() => stageBlob(failingDb, hash, bytes, 1234)).toThrow("injected bytes failure"); + expect(db.scalar("SELECT COUNT(*) FROM vfs_blobs")).toBe(0); + expect(db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes")).toBe(0); + }); + }); + + it("is idempotent: a second call refreshes last_seen but leaves bytes alone", async () => { + await withDB(async (db) => { + const bytes = new TextEncoder().encode("same"); + const hash = sha256(bytes); + stageBlob(db, hash, bytes, 100); + stageBlob(db, hash, bytes, 200); + const blob = db.one<{ last_seen: number }>( + "SELECT last_seen FROM vfs_blobs WHERE hash = ?", + hash, + ); + expect(blob?.last_seen).toBe(200); + // Still one row. + const count = db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes"); + expect(count).toBe(1); + }); + }); + + it("a staged-but-unreferenced blob is reaped by gc outside the safety window", async () => { + await withDB(async (db) => { + const bytes = new TextEncoder().encode("orphan"); + const hash = sha256(bytes); + stageBlob(db, hash, bytes, 100); + // Inside the safety window, gc preserves it. + expect(gc(db, { now: () => 200, safetyWindowMs: 1000 }).blobsFreed).toBe(0); + // Outside, gc reaps it. + expect(gc(db, { now: () => 5000, safetyWindowMs: 1000 }).blobsFreed).toBe(1); + }); + }); + + it("dedups against a blob already written by writeFile", async () => { + await withDB(async (db) => { + const bytes = new TextEncoder().encode("shared"); + const hash = sha256(bytes); + await writeFile(db, "/a.txt", "shared", {}, () => 1); + // The writeFile path already wrote the blob with that hash. + const before = db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes"); + stageBlob(db, hash, bytes, 5000); + const after = db.scalar("SELECT COUNT(*) FROM vfs_blob_bytes"); + expect(after).toBe(before); + // last_seen on the blob row has been bumped. + const ls = db.scalar("SELECT last_seen FROM vfs_blobs WHERE hash = ?", hash); + expect(ls).toBe(5000); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/blobs.ts b/spikes/349-dofs/vendor/dofs/src/sync/blobs.ts new file mode 100644 index 00000000..d51fabe5 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/blobs.ts @@ -0,0 +1,32 @@ +import { clearBlobCache } from "../fs/blobCache.js"; +import type { Database } from "../storage.js"; + +// Stage a chunk directly into vfs_blobs + vfs_blob_bytes without +// creating a node or a manifest. The receiver-side push path uses +// this to land bytes the sender shipped via pushObjects so a +// subsequent applyChanges call can find them by hash. +// +// Idempotent: a second call with the same hash refreshes +// last_seen so the bytes don't get reaped by an interleaved gc. +// Conflict updates also repair incomplete or size-mismatched rows +// left by an interrupted or corrupt write. +// +// Callers are expected to have verified that hash === sha256(bytes) +// before calling. The function trusts the caller; a mismatched +// pair would silently land under the wrong key. +export function stageBlob(db: Database, hash: Uint8Array, bytes: Uint8Array, now: number): void { + db.transactionSync(() => { + db.run( + "INSERT INTO vfs_blobs (hash, size, last_seen) VALUES (?, ?, ?) ON CONFLICT(hash) DO UPDATE SET size = excluded.size, last_seen = excluded.last_seen", + hash, + bytes.byteLength, + now, + ); + db.run( + "INSERT INTO vfs_blob_bytes (hash, bytes) VALUES (?, ?) ON CONFLICT(hash) DO UPDATE SET bytes = excluded.bytes", + hash, + bytes, + ); + }); + clearBlobCache(db); +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/changes.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/changes.test.ts new file mode 100644 index 00000000..e685b6c4 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/changes.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { mkdir } from "../fs/mkdir.js"; +import { rm } from "../fs/rm.js"; +import { symlink } from "../fs/symlink.js"; +import { withDB } from "../fs/with-db.js"; +import { writeFile } from "../fs/writeFile.js"; +import { materialiseChange } from "./changes.js"; + +describe("materialiseChange", () => { + it("returns a file entry with chunk hashes for a written file", async () => { + await withDB(async (db) => { + await writeFile(db, "/hello.txt", "hello world", { mode: 0o644 }, () => 1234); + const entry = materialiseChange(db, "/hello.txt"); + expect(entry).toMatchObject({ + kind: "file", + path: "/hello.txt", + mode: 0o644, + mtime: 1234, + size: 11, + }); + if (entry?.kind !== "file") throw new Error("expected file"); + expect(entry.rev).toBeGreaterThan(0); + if (entry?.kind !== "file") throw new Error("expected file"); + expect(entry.chunks).toHaveLength(1); + expect(entry.chunks[0].size).toBe(11); + expect(entry.chunks[0].hash).toBeInstanceOf(Uint8Array); + expect(entry.chunks[0].hash.byteLength).toBe(32); + }); + }); + + it("returns a dir entry for a created directory", async () => { + await withDB(async (db) => { + mkdir(db, "/sub", { mode: 0o755 }, () => 999); + expect(materialiseChange(db, "/sub")).toEqual({ + kind: "dir", + rev: expect.any(Number), + path: "/sub", + mode: 0o755, + mtime: 999, + }); + }); + }); + + it("returns a symlink entry for a symlink", async () => { + await withDB(async (db) => { + await writeFile(db, "/target.txt", "x", {}, () => 1); + symlink(db, "/target.txt", "/link", () => 2); + expect(materialiseChange(db, "/link")).toEqual({ + kind: "symlink", + rev: expect.any(Number), + path: "/link", + target: "/target.txt", + // chmod on a symlink itself is platform-specific; we record + // whatever symlink() stamped on vfs_nodes. + mode: expect.any(Number), + mtime: 2, + }); + }); + }); + + it("returns a delete entry for a tombstoned path", async () => { + await withDB(async (db) => { + await writeFile(db, "/gone.txt", "bye", {}, () => 1); + rm(db, "/gone.txt", {}); + expect(materialiseChange(db, "/gone.txt")).toEqual({ + kind: "delete", + rev: expect.any(Number), + path: "/gone.txt", + }); + }); + }); + + it("returns null for a path that was never written and has no tombstone", async () => { + await withDB(async (db) => { + expect(materialiseChange(db, "/never")).toBeNull(); + }); + }); + + it("size reflects the sum of chunk sizes", async () => { + await withDB(async (db) => { + // Force two chunks: 600 KiB > CHUNK_SIZE (512 KiB). + const bytes = new Uint8Array(600 * 1024); + for (let i = 0; i < bytes.byteLength; i++) bytes[i] = i & 0xff; + await writeFile(db, "/big.bin", bytes, {}, () => 1); + const entry = materialiseChange(db, "/big.bin"); + if (entry?.kind !== "file") throw new Error("expected file"); + expect(entry.chunks).toHaveLength(2); + expect(entry.size).toBe(600 * 1024); + expect(entry.chunks[0].size + entry.chunks[1].size).toBe(600 * 1024); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/changes.ts b/spikes/349-dofs/vendor/dofs/src/sync/changes.ts new file mode 100644 index 00000000..2cef2326 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/changes.ts @@ -0,0 +1,105 @@ +import { resolveInode } from "../fs/resolve.js"; +import { canonicalizePath } from "../path.js"; +import type { Database } from "../storage.js"; + +// Record a tombstone for a deleted path so the next push to the +// container learns the path is gone. Called by fs/rm inside the same +// transaction that bumped rev and removed the inode rows; the caller +// passes the post-bump rev value. +export function recordDelete(db: Database, rev: number, path: string): void { + db.run("INSERT INTO vfs_changes (rev, path, op) VALUES (?, ?, 'delete')", rev, path); +} + +// One row of the sync wire. The DO pushes these to the container +// and the container fetches them back. Bytes are never inline: +// file entries carry chunk hashes and the receiver does its own +// hasObjects probe + fetchObjects pull for the bytes it lacks. +// +// `rev` is the sender's currentRev at the moment this entry was +// stamped — vfs_nodes.rev for live mutations, vfs_changes.rev for +// tombstones. The puller uses it as a per-entry cursor so it can +// advance fetchRev per committed batch instead of waiting for the +// whole stream to drain. +export type ChangeEntry = + | { + kind: "file"; + rev: number; + path: string; + mode: number; + mtime: number; + size: number; + chunks: { hash: Uint8Array; size: number }[]; + } + | { kind: "dir"; rev: number; path: string; mode: number; mtime: number } + | { + kind: "symlink"; + rev: number; + path: string; + target: string; + mode: number; + mtime: number; + } + | { kind: "delete"; rev: number; path: string }; + +// Read the current state of `path` and turn it into a wire entry. +// Returns null when the path was never touched (no live inode and no +// tombstone in vfs_changes). Live inodes win over tombstones, which +// handles the delete-then-recreate case correctly. +// +// Symlinks are returned as symlink entries; we never follow them on +// the sync wire. Callers that want "the file the link points at" +// resolve it themselves after applying the symlink entry. +export function materialiseChange(db: Database, path: string): ChangeEntry | null { + const canonical = canonicalizePath(path).path; + const live = resolveInode(db, canonical, { followSymlinks: false }); + if (live !== null) { + // Read the rev stamped on this inode. Used as the per-entry + // cursor on the sync wire; coalesceChanges yields entries in + // ascending rev order so the puller can checkpoint per batch. + const revRow = db.one<{ rev: number }>("SELECT rev FROM vfs_nodes WHERE inode = ?", live.inode); + const rev = revRow?.rev ?? 0; + if (live.type === "dir") { + return { kind: "dir", rev, path: canonical, mode: live.mode, mtime: live.mtime }; + } + if (live.type === "symlink") { + return { + kind: "symlink", + rev, + path: canonical, + target: live.linkTarget ?? "", + mode: live.mode, + mtime: live.mtime, + }; + } + // file: collect chunk rows in index order. Each row carries hash + // and size so the receiver can probe hasObjects without a + // separate manifest lookup. An empty file has zero chunk rows + // and reports size 0. + const chunks = db.all<{ hash: Uint8Array; size: number }>( + "SELECT hash, size FROM vfs_chunks WHERE inode = ? ORDER BY idx", + live.inode, + ); + let size = 0; + for (const c of chunks) size += c.size; + return { + kind: "file", + rev, + path: canonical, + mode: live.mode, + mtime: live.mtime, + size, + chunks, + }; + } + // No live inode — check for a tombstone. The last row wins if the + // path was deleted and never recreated; an indexed scan by path is + // cheap because vfs_changes is bounded by the watermark window. + const tomb = db.one<{ rev: number; op: string }>( + "SELECT rev, op FROM vfs_changes WHERE path = ? ORDER BY id DESC LIMIT 1", + canonical, + ); + if (tomb?.op === "delete") { + return { kind: "delete", rev: tomb.rev, path: canonical }; + } + return null; +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/coalesce.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/coalesce.test.ts new file mode 100644 index 00000000..3bbc9749 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/coalesce.test.ts @@ -0,0 +1,342 @@ +import { describe, expect, it } from "vitest"; + +import { link } from "../fs/link.js"; +import { mkdir } from "../fs/mkdir.js"; +import { rename } from "../fs/rename.js"; +import { rm } from "../fs/rm.js"; +import { symlink } from "../fs/symlink.js"; +import { withDB } from "../fs/with-db.js"; +import { writeFile, writeFileSync } from "../fs/writeFile.js"; +import { coalesceChanges } from "./coalesce.js"; +import { currentRev } from "./watermarks.js"; + +// Drain an async iterable into an array. Tests stay synchronous-looking +// while the production code can stream. +async function drain(it: AsyncIterable): Promise { + const out: T[] = []; + for await (const x of it) out.push(x); + return out; +} + +describe("coalesceChanges", () => { + it("yields nothing for an empty rev window", async () => { + await withDB(async (db) => { + const entries = await drain(coalesceChanges(db, 1_000_000)); + expect(entries).toEqual([]); + }); + }); + + it("yields one entry per touched path after the cursor", async () => { + await withDB(async (db) => { + mkdir(db, "/d", { mode: 0o755 }, () => 1); + await writeFile(db, "/d/a.txt", "alpha", {}, () => 2); + const entries = await drain(coalesceChanges(db, 0)); + // root mkdir is implicit (already exists); we should see /d and /d/a.txt. + const paths = entries.map((e) => e.path).sort(); + expect(paths).toContain("/d"); + expect(paths).toContain("/d/a.txt"); + }); + }); + + it("coalesces five rewrites of the same path into one entry", async () => { + await withDB(async (db) => { + for (let i = 0; i < 5; i++) { + await writeFile(db, "/log.txt", `pass ${i}`, {}, () => 100 + i); + } + const entries = await drain(coalesceChanges(db, 0)); + const log = entries.filter((e) => e.path === "/log.txt"); + expect(log).toHaveLength(1); + // Latest state wins: the entry carries the size of "pass 4" (6 bytes). + expect(log[0]).toMatchObject({ kind: "file", size: 6 }); + }); + }); + + it("emits delete entries for tombstoned paths", async () => { + await withDB(async (db) => { + await writeFile(db, "/gone.txt", "x", {}, () => 1); + rm(db, "/gone.txt", {}); + const entries = await drain(coalesceChanges(db, 0)); + const gone = entries.find((e) => e.path === "/gone.txt"); + expect(gone).toEqual({ kind: "delete", rev: expect.any(Number), path: "/gone.txt" }); + }); + }); + + it("emits resolved delete paths for removes through intermediate symlinks", async () => { + await withDB(async (db) => { + mkdir(db, "/real", {}, () => 1); + await writeFile(db, "/real/file.txt", "x", {}, () => 2); + symlink(db, "/real", "/link", () => 3); + + rm(db, "/link/file.txt", {}); + + const entries = await drain(coalesceChanges(db, 0)); + expect(entries).toContainEqual({ + kind: "delete", + rev: expect.any(Number), + path: "/real/file.txt", + }); + expect(entries).not.toContainEqual({ + kind: "delete", + rev: expect.any(Number), + path: "/link/file.txt", + }); + }); + }); + + it("delete-then-recreate yields a single live entry, not a delete", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "first", {}, () => 1); + rm(db, "/x.txt", {}); + await writeFile(db, "/x.txt", "second", {}, () => 2); + const entries = await drain(coalesceChanges(db, 0)); + const x = entries.filter((e) => e.path === "/x.txt"); + expect(x).toHaveLength(1); + expect(x[0].kind).toBe("file"); + }); + }); + + it("cursor rev filters out changes the receiver has already seen", async () => { + await withDB(async (db) => { + await writeFile(db, "/old.txt", "old", {}, () => 1); + // Read current rev counter to use as the cursor. + const cursor = db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; + await writeFile(db, "/new.txt", "new", {}, () => 2); + const entries = await drain(coalesceChanges(db, cursor)); + const paths = entries.map((e) => e.path); + expect(paths).toContain("/new.txt"); + expect(paths).not.toContain("/old.txt"); + }); + }); + + it("yields entries in ascending rev order", async () => { + // pullOnce uses entry.rev as a per-batch checkpoint cursor. The + // contract is monotonicity: if entry N has rev R, every entry + // already emitted has rev <= R. Without that, the puller can't + // advance fetchRev mid-stream without risking a skip on the next + // resume. + await withDB(async (db) => { + await writeFile(db, "/a.txt", "a", {}, () => 1); + mkdir(db, "/d", {}, () => 1); + await writeFile(db, "/d/b.txt", "b", {}, () => 1); + await writeFile(db, "/c.txt", "c", {}, () => 1); + rm(db, "/a.txt", {}); + await writeFile(db, "/d/b.txt", "b2", {}, () => 2); + const entries = await drain(coalesceChanges(db, 0)); + const revs = entries.map((e) => e.rev); + for (let i = 1; i < revs.length; i++) { + expect(revs[i]).toBeGreaterThanOrEqual(revs[i - 1]); + } + }); + }); + + it("resumes after a path within the same rev", async () => { + await withDB(async (db) => { + mkdir(db, "/src", {}, () => 1); + await writeFile(db, "/src/a.txt", "a", {}, () => 2); + await writeFile(db, "/src/b.txt", "b", {}, () => 3); + const beforeRename = currentRev(db); + + rename(db, "/src", "/dst"); + const renameRev = currentRev(db); + expect(renameRev).toBeGreaterThan(beforeRename); + + const allSameRev = await drain(coalesceChanges(db, { rev: beforeRename, path: null })); + const paths = allSameRev.map((entry) => entry.path); + expect(paths).toEqual([...paths].sort()); + expect(new Set(allSameRev.map((entry) => entry.rev))).toEqual(new Set([renameRev])); + + const resumed = await drain(coalesceChanges(db, { rev: renameRev, path: "/dst/a.txt" })); + expect(resumed.map((entry) => entry.path)).toEqual( + paths.filter((path) => path > "/dst/a.txt"), + ); + }); + }); + + it("skips a whole rev when the cursor path is null", async () => { + await withDB(async (db) => { + mkdir(db, "/src", {}, () => 1); + await writeFile(db, "/src/a.txt", "a", {}, () => 2); + const beforeRename = currentRev(db); + + rename(db, "/src", "/dst"); + const renameRev = currentRev(db); + expect(await drain(coalesceChanges(db, { rev: beforeRename, path: null }))).not.toEqual([]); + expect(await drain(coalesceChanges(db, { rev: renameRev, path: null }))).toEqual([]); + }); + }); + + it("orders entries deterministically by rev then path", async () => { + await withDB(async (db) => { + mkdir(db, "/src", {}, () => 1); + await writeFile(db, "/src/b.txt", "b", {}, () => 2); + await writeFile(db, "/src/a.txt", "a", {}, () => 3); + rename(db, "/src", "/dst"); + + const entries = await drain(coalesceChanges(db, { rev: 0, path: null })); + const pairs = entries.map((entry) => [entry.rev, entry.path] as const); + expect(pairs).toEqual( + [...pairs].sort((a, b) => { + if (a[0] !== b[0]) return a[0] - b[0]; + return a[1].localeCompare(b[1]); + }), + ); + }); + }); + + it("excludes entries newer than the through rev", async () => { + await withDB(async (db) => { + await writeFile(db, "/included.txt", "included", {}, () => 1); + const throughRev = currentRev(db); + await writeFile(db, "/excluded.txt", "excluded", {}, () => 2); + + const entries = await drain( + coalesceChanges(db, { rev: 0, path: null }, { through: { rev: throughRev, path: null } }), + ); + + expect(entries.map((entry) => entry.path)).toContain("/included.txt"); + expect(entries.map((entry) => entry.path)).not.toContain("/excluded.txt"); + }); + }); + + it("excludes same-rev entries after the through path", async () => { + await withDB(async (db) => { + mkdir(db, "/src", {}, () => 1); + await writeFile(db, "/src/a.txt", "a", {}, () => 2); + await writeFile(db, "/src/b.txt", "b", {}, () => 3); + const beforeRename = currentRev(db); + + rename(db, "/src", "/dst"); + const renameRev = currentRev(db); + + const entries = await drain( + coalesceChanges( + db, + { rev: beforeRename, path: null }, + { + through: { rev: renameRev, path: "/dst/a.txt" }, + }, + ), + ); + + expect(entries.map((entry) => entry.path)).toEqual(["/dst", "/dst/a.txt"]); + }); + }); + + it("skips an entry that materializes beyond the through cursor", async () => { + await withDB(async (db) => { + await writeFile(db, "/file.txt", "first", {}, () => 1); + const throughRev = currentRev(db); + + const originalOne = db.one.bind(db); + let rewrote = false; + db.one = ((query: string, ...bindings: unknown[]) => { + if (!rewrote && query === "SELECT rev FROM vfs_nodes WHERE inode = ?") { + rewrote = true; + writeFileSync(db, "/file.txt", new TextEncoder().encode("second"), {}, () => 2); + } + return originalOne(query, ...bindings); + }) as typeof db.one; + + const entries = await drain( + coalesceChanges(db, { rev: 0, path: null }, { through: { rev: throughRev, path: null } }), + ); + + expect(rewrote).toBe(true); + expect(entries).toEqual([]); + }); + }); +}); + +describe("coalesceChanges (ignore)", () => { + it("drops entries whose path contains an ignored segment", async () => { + await withDB(async (db) => { + mkdir(db, "/src", {}, () => 0); + mkdir(db, "/node_modules", {}, () => 0); + mkdir(db, "/node_modules/lodash", {}, () => 0); + mkdir(db, "/a", {}, () => 0); + mkdir(db, "/a/node_modules", {}, () => 0); + mkdir(db, "/a/node_modules/p", {}, () => 0); + await writeFile(db, "/src/index.ts", "x", {}, () => 1); + await writeFile(db, "/node_modules/lodash/index.js", "y", {}, () => 2); + await writeFile(db, "/a/node_modules/p/q.js", "z", {}, () => 3); + const entries = await drain(coalesceChanges(db, 0, { ignore: ["node_modules"] })); + const paths = entries.map((e) => e.path); + expect(paths).toContain("/src/index.ts"); + for (const p of paths) { + expect(p.includes("node_modules")).toBe(false); + } + }); + }); + + it("an empty ignore list is the default behaviour", async () => { + await withDB(async (db) => { + mkdir(db, "/node_modules", {}, () => 0); + await writeFile(db, "/node_modules/x.js", "x", {}, () => 1); + const entries = await drain(coalesceChanges(db, 0)); + expect(entries.some((e) => e.path === "/node_modules/x.js")).toBe(true); + }); + }); + + it("emits an entry for every hardlink name of a touched inode", async () => { + await withDB(async (db) => { + await writeFile(db, "/a", "shared", {}, () => 1); + link(db, "/a", "/b"); + + const entries = await drain(coalesceChanges(db, 0)); + const files = entries.filter((e) => e.kind === "file").map((e) => e.path); + // Both names share one inode; pathOf would pick only one of them. + // The wire has to carry both so the receiver materialises each. + expect(files).toContain("/a"); + expect(files).toContain("/b"); + }); + }); + + it("defers a path whose live state raced past the snapshot, then redelivers it", async () => { + // Pins the documented cursor contract (docs/02_sync_protocol.md): + // `through` is a resume bound, not a point-in-time snapshot. A path + // deleted at the snapshot rev but recreated above it is dropped from + // the bounded scan, because materialiseChange reads the live state + // and inCursorWindow filters it out. The omission is not loss — the + // recreate's rev is above the snapshot, so the next scan redelivers + // the path. Convergence holds without the store keeping history. + await withDB(async (db) => { + await writeFile(db, "/x", "v1", {}, () => 1); + rm(db, "/x", {}); + const snapshot = currentRev(db); + await writeFile(db, "/x", "v2", {}, () => 2); + + // Bounded by the snapshot: the rev-`snapshot` delete is a + // candidate, but the live entry now sits above the window, so /x + // is omitted from this snapshot rather than frozen at the delete. + const bounded = await drain( + coalesceChanges(db, 0, { through: { rev: snapshot, path: null } }), + ); + expect(bounded.some((e) => e.path === "/x")).toBe(false); + + // The next scan resumes after the snapshot rev and redelivers /x + // at its current state. The rev that caused the drop is above the + // snapshot, so a later window always covers it. + const next = await drain(coalesceChanges(db, snapshot)); + expect(next.find((e) => e.path === "/x")).toMatchObject({ kind: "file", path: "/x" }); + }); + }); + + it("emits the new name when a hardlinked file is renamed", async () => { + await withDB(async (db) => { + await writeFile(db, "/a", "shared", {}, () => 1); + link(db, "/a", "/b"); + // /a and /b now share an inode. Renaming /a to /c leaves the + // inode named /b and /c; pathOf might resolve it to /b and never + // emit /c, dropping the renamed name on the wire. + const baseline = currentRev(db); + rename(db, "/a", "/c"); + + const entries = await drain(coalesceChanges(db, baseline)); + const live = entries.filter((e) => e.kind !== "delete").map((e) => e.path); + const deletes = entries.filter((e) => e.kind === "delete").map((e) => e.path); + expect(live).toContain("/c"); + expect(live).toContain("/b"); + expect(deletes).toContain("/a"); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/coalesce.ts b/spikes/349-dofs/vendor/dofs/src/sync/coalesce.ts new file mode 100644 index 00000000..bfd4b304 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/coalesce.ts @@ -0,0 +1,141 @@ +import type { Database } from "../storage.js"; +import { type ChangeEntry, materialiseChange } from "./changes.js"; +import { isIgnored } from "./ignore.js"; +import { pathsOf } from "./paths.js"; +import { type ChangeCursor, compareChangeCursors } from "./watermarks.js"; + +// Yield one ChangeEntry per path touched after `after`. Per-path +// coalescing: five rewrites of the same path between watermarks +// produce one entry (the latest state wins). Tombstoned paths get a +// delete entry unless they have been recreated, in which case the +// live entry wins. +// +// Entries are emitted in ascending rev order. pullOnce relies on +// this so it can advance fetchRev per committed batch — if entry N +// has rev R, every entry already emitted has rev <= R, so +// checkpointing fetchRev at R is safe. +// +// Streaming, not buffering at the wire: the per-path coalesce step +// holds at most one slot per dirty path in memory (sorted by rev), +// which is the same bound the live + tombstone scans already pay. +export interface CoalesceOptions { + // Path-segment patterns to drop before yielding. The wire never + // carries entries under an ignored segment. + ignore?: string[]; + // Internal snapshot bound. When set, entries are limited to the + // cursor window `after < entry <= through`. The bound is applied to + // each path's *live* rev at materialise time, not to a frozen + // snapshot: a path whose state moves past `through` after the scan + // (a concurrent rewrite or delete) is dropped here and redelivered + // under a later cursor, since the cursor never advances past the rev + // that caused the drop. See the redelivery note on the yield loop. + through?: ChangeCursor; +} + +export async function* coalesceChanges( + db: Database, + after: ChangeCursor | number, + options: CoalesceOptions = {}, +): AsyncIterable { + const ignore = options.ignore ?? []; + const cursor = typeof after === "number" ? { rev: after, path: null } : after; + const through = options.through; + + // Build the per-path candidate set in two passes, keeping the + // highest rev seen for each path. A live mutation that landed + // after a tombstone wins; a tombstone that landed after a write + // wins; the highest rev is the rev we stamp on the wire and the + // rev pullOnce checkpoints to. + type Candidate = { path: string; rev: number }; + const candidates = new Map(); + + // Live mutations: every mkdir / writeFile / symlink bumps + // vfs_nodes.rev. The by_rev index makes this a range scan. + const lowerRev = cursor.path === null ? cursor.rev : cursor.rev - 1; + const touched = + through === undefined + ? db.all<{ inode: number; rev: number }>( + "SELECT inode, rev FROM vfs_nodes WHERE rev > ? ORDER BY rev", + lowerRev, + ) + : db.all<{ inode: number; rev: number }>( + "SELECT inode, rev FROM vfs_nodes WHERE rev > ? AND rev <= ? ORDER BY rev", + lowerRev, + through.rev, + ); + for (const { inode, rev } of touched) { + // One inode can carry several hardlink names; every name has to + // become a candidate so the wire materialises each, not just the + // arbitrary one pathOf would return. + for (const path of pathsOf(db, inode)) { + if (!inCursorWindow({ rev, path }, cursor, through)) continue; + if (isIgnored(path, ignore)) continue; + const prior = candidates.get(path); + if (prior === undefined || rev > prior.rev) { + candidates.set(path, { path, rev }); + } + } + } + + // Tombstones: each rm appends a row to vfs_changes with the + // post-bump rev. The highest rev per path wins (a path can be + // deleted-recreated-deleted; we want the last rm's rev). + const tombs = + through === undefined + ? db.all<{ path: string; rev: number }>( + "SELECT path, MAX(rev) AS rev FROM vfs_changes WHERE rev > ? AND op = 'delete' GROUP BY path", + lowerRev, + ) + : db.all<{ path: string; rev: number }>( + "SELECT path, MAX(rev) AS rev FROM vfs_changes WHERE rev > ? AND rev <= ? AND op = 'delete' GROUP BY path", + lowerRev, + through.rev, + ); + for (const { path, rev } of tombs) { + if (!inCursorWindow({ rev, path }, cursor, through)) continue; + if (isIgnored(path, ignore)) continue; + const prior = candidates.get(path); + if (prior === undefined || rev > prior.rev) { + candidates.set(path, { path, rev }); + } + } + + // Sort by rev ascending so pullOnce can checkpoint per batch. + // Ties on rev (same transactionSync touching multiple paths) + // break on path so the wire order is deterministic. + const ordered = Array.from(candidates.values()).sort((a, b) => { + if (a.rev !== b.rev) return a.rev - b.rev; + return a.path < b.path ? -1 : a.path > b.path ? 1 : 0; + }); + + // materialiseChange reads each path's *current* state, not the state + // it held at the candidate's rev — the VFS keeps no content history. + // If a path was rewritten or deleted again after the scan, its live + // rev can now sit above `through`; inCursorWindow drops it. The + // dropped change is not lost: the rev that pushed it past `through` + // is, by definition, greater than `through.rev` (the cursor the + // puller persists), so the next pull's `after < entry` scan finds the + // path again and delivers its then-current state. The consequence is + // that a `{rev, null}` cursor means "every change committed at or + // before rev has been *offered*"; it does not guarantee the receiver + // tree byte-matches the rev snapshot for a path that raced ahead. + // Convergence is preserved because the cursor never advances past the + // racing rev. See docs/02_sync_protocol.md. + for (const { path } of ordered) { + const entry = materialiseChange(db, path); + if (entry !== null && inCursorWindow(entry, cursor, through)) { + yield entry; + } + } +} + +function inCursorWindow( + entry: ChangeCursor & { path: string }, + after: ChangeCursor, + through?: ChangeCursor, +): boolean { + return ( + compareChangeCursors(entry, after) > 0 && + (through === undefined || compareChangeCursors(entry, through) <= 0) + ); +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/fetch.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/fetch.test.ts new file mode 100644 index 00000000..01b295e2 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/fetch.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from "vitest"; + +import { withDB } from "../fs/with-db.js"; +import { createFileSync, writeFile, writeRangeSync } from "../fs/writeFile.js"; +import { coalesceChanges } from "./coalesce.js"; +import { fetchChanges, fetchObjects, hasObjects } from "./fetch.js"; + +async function drain(it: AsyncIterable): Promise { + const out: T[] = []; + for await (const x of it) out.push(x); + return out; +} + +describe("fetch wire", () => { + it("fetchChanges yields the same entries as coalesceChanges", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "alpha", {}, () => 1); + await writeFile(db, "/b.txt", "beta", {}, () => 2); + const viaCoalesce = await drain(coalesceChanges(db, 0)); + const viaFetch = await drain(fetchChanges(db, 0)); + expect(viaFetch).toEqual(viaCoalesce); + }); + }); + + it("fetchChanges and fetchObjects include small direct writes", async () => { + await withDB(async (db) => { + createFileSync(db, "/inline.txt", {}, () => 1); + writeRangeSync(db, "/inline.txt", new TextEncoder().encode("inline direct"), 0, {}, () => 2); + + const entries = await drain(fetchChanges(db, 0)); + const file = entries.find((entry) => entry.kind === "file" && entry.path === "/inline.txt"); + expect(file).toMatchObject({ kind: "file", size: "inline direct".length }); + expect(file?.kind === "file" ? file.chunks : []).toHaveLength(1); + const hash = file?.kind === "file" ? file.chunks[0].hash : new Uint8Array(); + const objects = await drain(fetchObjects(db, [hash])); + expect(objects).toHaveLength(1); + expect(new TextDecoder().decode(objects[0].bytes)).toBe("inline direct"); + }); + }); + + it("fetchChanges resumes from a rev/path cursor", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "alpha", {}, () => 1); + await writeFile(db, "/b.txt", "beta", {}, () => 2); + const entries = await drain(fetchChanges(db, { rev: 0, path: null })); + const first = entries[0]; + const resumed = await drain(fetchChanges(db, { rev: first.rev, path: first.path })); + expect(resumed).toEqual(entries.slice(1)); + }); + }); + + it("fetchObjects yields each hash exactly once", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "shared", {}, () => 1); + await writeFile(db, "/b.txt", "shared", {}, () => 2); + // Pick the single chunk hash from the file entry. + const entries = await drain(fetchChanges(db, 0)); + const hashes: Uint8Array[] = []; + for (const e of entries) { + if (e.kind === "file") hashes.push(...e.chunks.map((c) => c.hash)); + } + // Two files, same content, so two references to one hash. + expect(hashes).toHaveLength(2); + const seen = new Set(); + for await (const { hash, bytes } of fetchObjects(db, [hashes[0]])) { + seen.add(Array.from(hash).join(",")); + expect(new TextDecoder().decode(bytes)).toBe("shared"); + } + expect(seen.size).toBe(1); + }); + }); +}); + +describe("hasObjects", () => { + it("returns the subset of inputs the receiver already holds", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "alpha", {}, () => 1); + const entries = await drain(fetchChanges(db, 0)); + const known = (entries.find((e) => e.kind === "file") as { chunks: { hash: Uint8Array }[] }) + .chunks[0].hash; + const unknown = new Uint8Array(32); + unknown.fill(0xff); + const got = hasObjects(db, [known, unknown]); + expect(got).toHaveLength(1); + expect(got[0]).toEqual(known); + }); + }); + + it("treats blob metadata without bytes as missing", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "alpha", {}, () => 1); + const entries = await drain(fetchChanges(db, 0)); + const known = ( + entries.find((e) => e.kind === "file") as { + chunks: { hash: Uint8Array }[]; + } + ).chunks[0].hash; + db.run("DELETE FROM vfs_blob_bytes WHERE hash = ?", known); + + expect(hasObjects(db, [known])).toEqual([]); + }); + }); + + it("treats blob bytes with the wrong length as missing", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "alpha", {}, () => 1); + const entries = await drain(fetchChanges(db, 0)); + const known = ( + entries.find((e) => e.kind === "file") as { + chunks: { hash: Uint8Array }[]; + } + ).chunks[0].hash; + db.run("UPDATE vfs_blob_bytes SET bytes = ? WHERE hash = ?", new Uint8Array([1]), known); + + expect(hasObjects(db, [known])).toEqual([]); + }); + }); + + it("returns an empty array when nothing matches", async () => { + await withDB(async (db) => { + const zero = new Uint8Array(32); + expect(hasObjects(db, [zero])).toEqual([]); + }); + }); + + it("returns an empty array when no hashes are passed", async () => { + await withDB(async (db) => { + expect(hasObjects(db, [])).toEqual([]); + }); + }); + + it("preserves input order and duplicates across mixed inputs", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "alpha", {}, () => 1); + await writeFile(db, "/b.txt", "beta", {}, () => 2); + const entries = await drain(fetchChanges(db, 0)); + const hashOf = (path: string): Uint8Array => { + const e = entries.find((x) => x.kind === "file" && x.path === path); + return e?.kind === "file" ? e.chunks[0].hash : new Uint8Array(); + }; + const a = hashOf("/a.txt"); + const b = hashOf("/b.txt"); + const missing = new Uint8Array(32).fill(0xff); + + // All present, returned in input order. + expect(hasObjects(db, [a, b])).toEqual([a, b]); + // Mixed: only present hashes, still in input order. + expect(hasObjects(db, [missing, b, a])).toEqual([b, a]); + // A duplicated present hash keeps every occurrence. + expect(hasObjects(db, [a, a, missing])).toEqual([a, a]); + // Duplicated absent hashes drop entirely. + expect(hasObjects(db, [missing, missing])).toEqual([]); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/fetch.ts b/spikes/349-dofs/vendor/dofs/src/sync/fetch.ts new file mode 100644 index 00000000..317540fe --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/fetch.ts @@ -0,0 +1,64 @@ +import type { Database } from "../storage.js"; +import type { ChangeEntry } from "./changes.js"; +import { coalesceChanges } from "./coalesce.js"; +import { pushObjects } from "./push.js"; +import type { ChangeCursor } from "./watermarks.js"; + +// The fetch wire is the mirror of the push wire: same SQL, +// opposite direction. The DO calls fetchChanges / fetchObjects on +// the container; the container calls push / pushObjects on the DO. +// Both names exist so call sites read in their own direction. + +export function fetchChanges( + db: Database, + after: ChangeCursor | number, + options: { ignore?: string[] } = {}, +): AsyncIterable { + return coalesceChanges(db, after, options); +} + +export function fetchObjects( + db: Database, + hashes: Uint8Array[], +): AsyncIterable<{ hash: Uint8Array; bytes: Uint8Array }> { + return pushObjects(db, hashes); +} + +// Stable hex key for JS-side membership tests. Only content matters +// here; the SQL match is on the raw hash blob. +function toHex(bytes: Uint8Array): string { + let out = ""; + for (const b of bytes) out += b.toString(16).padStart(2, "0"); + return out; +} + +// Largest hash list bound into one IN (…) probe. Comfortably under +// SQLite's bound-parameter limit, so a large probe splits into a few +// index-backed lookups instead of one oversized statement. +const PROBE_BATCH = 256; + +// Subset-test the input hashes against complete local objects. A +// metadata row alone is not enough: the payload must exist and its +// byte length must match the advertised blob size. +// +// Matches the raw hash blobs through an IN (…) list so the lookup +// rides the primary-key index. Present hashes are returned in input +// order, preserving any duplicates the caller passed. +export function hasObjects(db: Database, hashes: Uint8Array[]): Uint8Array[] { + if (hashes.length === 0) return []; + const present = new Set(); + for (let i = 0; i < hashes.length; i += PROBE_BATCH) { + const window = hashes.slice(i, i + PROBE_BATCH); + const placeholders = window.map(() => "?").join(", "); + const rows = db.all<{ hash: Uint8Array }>( + `SELECT b.hash + FROM vfs_blobs b + JOIN vfs_blob_bytes bb ON bb.hash = b.hash + WHERE b.hash IN (${placeholders}) + AND length(bb.bytes) = b.size`, + ...window, + ); + for (const row of rows) present.add(toHex(row.hash)); + } + return hashes.filter((h) => present.has(toHex(h))); +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/ignore.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/ignore.test.ts new file mode 100644 index 00000000..37975760 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/ignore.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; + +import { isIgnored } from "./ignore.js"; + +describe("isIgnored", () => { + const list = ["node_modules", ".next", "target"]; + + it("returns false for paths that don't intersect the list", () => { + expect(isIgnored("/src/index.ts", list)).toBe(false); + expect(isIgnored("/README.md", list)).toBe(false); + expect(isIgnored("/", list)).toBe(false); + }); + + it("matches the segment exactly, not as a substring", () => { + expect(isIgnored("/node_modules", list)).toBe(true); + expect(isIgnored("/node_modules_old", list)).toBe(false); + expect(isIgnored("/my_node_modules", list)).toBe(false); + }); + + it("matches anywhere in the path", () => { + expect(isIgnored("/a/b/node_modules", list)).toBe(true); + expect(isIgnored("/a/b/node_modules/c.js", list)).toBe(true); + expect(isIgnored("/packages/x/node_modules/y/index.js", list)).toBe(true); + }); + + it("matches nested ignored dirs too", () => { + expect(isIgnored("/a/.next/cache", list)).toBe(true); + expect(isIgnored("/rust/target/debug/foo", list)).toBe(true); + }); + + it("returns false for an empty list", () => { + expect(isIgnored("/anywhere/node_modules", [])).toBe(false); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/ignore.ts b/spikes/349-dofs/vendor/dofs/src/sync/ignore.ts new file mode 100644 index 00000000..91f7b841 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/ignore.ts @@ -0,0 +1,24 @@ +// Path segment matcher for the container-side ignore list. The +// container uses this to drop paths from coalesceChanges before +// they hit the wire; the DO's Workspace.fs surface uses the same +// helper to make ignored paths invisible to API consumers. +// +// Matching is whole-segment: "node_modules" matches the segment +// node_modules anywhere in the path but does not match +// node_modules_old or my_node_modules. Patterns are plain strings, +// not globs; we can extend to globs later if a real case demands it. + +export const DEFAULT_IGNORE = ["node_modules"]; + +export function isIgnored(path: string, patterns: string[]): boolean { + if (patterns.length === 0) return false; + // canonicalizePath strips the trailing slash and leaves a leading + // "/" for non-root paths; split skips the empty leading segment. + const segments = path.split("/").filter((s) => s.length > 0); + for (const segment of segments) { + for (const p of patterns) { + if (segment === p) return true; + } + } + return false; +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/invariant.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/invariant.test.ts new file mode 100644 index 00000000..7e40d9f3 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/invariant.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; + +import { assertAppliedPushCursor } from "./invariant.js"; + +describe("assertAppliedPushCursor", () => { + it("passes when applied covers pushed", () => { + expect(() => + assertAppliedPushCursor({ rev: 10, path: null }, { rev: 10, path: null }), + ).not.toThrow(); + expect(() => + assertAppliedPushCursor({ rev: 11, path: "/partial.txt" }, { rev: 10, path: null }), + ).not.toThrow(); + }); + + it("passes at zero", () => { + expect(() => + assertAppliedPushCursor({ rev: 0, path: null }, { rev: 0, path: null }), + ).not.toThrow(); + }); + + it("throws when the receiver only partially applied the pushed rev", () => { + expect(() => + assertAppliedPushCursor({ rev: 10, path: "/partial.txt" }, { rev: 10, path: null }), + ).toThrowError(/appliedPushCursor.*pushCursor/i); + }); + + it("throws when the receiver is behind the sender's push cursor", () => { + expect(() => + assertAppliedPushCursor({ rev: 5, path: null }, { rev: 10, path: null }), + ).toThrowError(/appliedPushCursor.*pushCursor/i); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/invariant.ts b/spikes/349-dofs/vendor/dofs/src/sync/invariant.ts new file mode 100644 index 00000000..036fd0e7 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/invariant.ts @@ -0,0 +1,29 @@ +import { type ChangeCursor, compareChangeCursors } from "./watermarks.js"; + +// Cross-side invariant: every fetchChanges and push response carries +// the receiver's current applied push cursor. The sender asserts that +// cursor covers its local push cursor on every response. +// +// The two sides never share a single clock, but echoing the largest +// applied sender cursor makes the "receiver is caught up with our +// pushes" invariant inspectable on the wire instead of load-bearing +// in-process state. A regression in the suppress-dirty-tracking apply +// path trips the assertion immediately rather than corrupting data +// silently. +// +// Throwing an Error is the right escalation: a violation means the +// protocol is broken; the connection should tear down and rebuild +// rather than soldiering on with stale state. + +export function assertAppliedPushCursor( + appliedPushCursor: ChangeCursor, + pushCursor: ChangeCursor, +): void { + if (compareChangeCursors(appliedPushCursor, pushCursor) < 0) { + throw new Error( + `cross-side invariant violated: appliedPushCursor (${JSON.stringify( + appliedPushCursor, + )}) < pushCursor (${JSON.stringify(pushCursor)})`, + ); + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/manifests.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/manifests.test.ts new file mode 100644 index 00000000..15a649d5 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/manifests.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; + +import { withDB } from "../fs/with-db.js"; +import { writeFile } from "../fs/writeFile.js"; + +describe("manifests", () => { + it("writeFile sets vfs_nodes.manifest_hash to a non-null hash", async () => { + await withDB(async (db) => { + await writeFile(db, "/hello.txt", "hello", {}, () => 1); + const row = db.one<{ manifest_hash: Uint8Array | null }>( + "SELECT manifest_hash FROM vfs_nodes WHERE inode = (SELECT child_inode FROM vfs_dirents WHERE parent_inode = 1 AND name = 'hello.txt')", + ); + expect(row?.manifest_hash).toBeInstanceOf(Uint8Array); + expect(row?.manifest_hash?.byteLength).toBe(32); + }); + }); + + it("identical content at two paths shares one manifest row", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "same bytes", {}, () => 1); + await writeFile(db, "/b.txt", "same bytes", {}, () => 1); + const count = db.scalar("SELECT COUNT(*) AS n FROM vfs_manifests"); + expect(count).toBe(1); + // Both inodes point at the same manifest_hash. + const hashes = db.all<{ manifest_hash: Uint8Array | null }>( + "SELECT manifest_hash FROM vfs_nodes WHERE type = 'file' ORDER BY inode", + ); + expect(hashes).toHaveLength(2); + expect(hashes[0].manifest_hash).toEqual(hashes[1].manifest_hash); + }); + }); + + it("different content produces different manifest hashes", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.txt", "one", {}, () => 1); + await writeFile(db, "/b.txt", "two", {}, () => 1); + const count = db.scalar("SELECT COUNT(*) AS n FROM vfs_manifests"); + expect(count).toBe(2); + }); + }); + + it("vfs_manifests row carries the chunk list as JSON", async () => { + await withDB(async (db) => { + await writeFile(db, "/hi.txt", "hi", {}, () => 1); + const row = db.one<{ encoded: Uint8Array; size: number }>( + "SELECT encoded, size FROM vfs_manifests LIMIT 1", + ); + expect(row).toBeDefined(); + expect(row?.size).toBe(2); + const decoded = JSON.parse(new TextDecoder().decode(row?.encoded)); + expect(decoded.version).toBe(1); + expect(decoded.chunks).toHaveLength(1); + expect(decoded.chunks[0].size).toBe(2); + expect(typeof decoded.chunks[0].hash).toBe("string"); + expect(decoded.chunks[0].hash).toMatch(/^[0-9a-f]{64}$/); + }); + }); + + it("overwriting a file updates manifest_hash and may leave the old manifest orphaned", async () => { + await withDB(async (db) => { + await writeFile(db, "/x.txt", "first", {}, () => 1); + const before = db.one<{ manifest_hash: Uint8Array | null }>( + "SELECT manifest_hash FROM vfs_nodes WHERE type = 'file'", + ); + await writeFile(db, "/x.txt", "second", {}, () => 2); + const after = db.one<{ manifest_hash: Uint8Array | null }>( + "SELECT manifest_hash FROM vfs_nodes WHERE type = 'file'", + ); + expect(after?.manifest_hash).not.toEqual(before?.manifest_hash); + // Two manifest rows now exist; the old one is orphaned and will + // be reaped by gc(). + const count = db.scalar("SELECT COUNT(*) AS n FROM vfs_manifests"); + expect(count).toBe(2); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/manifests.ts b/spikes/349-dofs/vendor/dofs/src/sync/manifests.ts new file mode 100644 index 00000000..98dfcfbc --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/manifests.ts @@ -0,0 +1,74 @@ +import { createHash } from "node:crypto"; + +import type { Database } from "../storage.js"; + +// A manifest names the ordered chunk list for a single file. Two +// files whose bytes chunk identically share one manifest row, which +// is what lets the sync wire say "this file is the same as the one +// I just sent you" by hash alone. +// +// Encoding is JSON for now — readable, debuggable, and structurally +// identical to casync's `.caidx`. A future commit can swap the +// encoding to the `.caidx` byte layout without a schema change. + +export interface ManifestChunk { + hash: Uint8Array; + size: number; +} + +export const MANIFEST_VERSION = 1; + +interface EncodedManifest { + version: number; + chunks: { hash: string; size: number }[]; +} + +function toHex(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.byteLength; i++) { + out += bytes[i].toString(16).padStart(2, "0"); + } + return out; +} + +function sha256(bytes: Uint8Array): Uint8Array { + return new Uint8Array(createHash("sha256").update(bytes).digest()); +} + +// Serialize a chunk list into the canonical manifest bytes. The +// hash is taken over these bytes and the same bytes are stored, so +// producing them once keeps the two in step. +function encodeManifest(chunks: ManifestChunk[]): Uint8Array { + const encoded: EncodedManifest = { + version: MANIFEST_VERSION, + chunks: chunks.map((c) => ({ hash: toHex(c.hash), size: c.size })), + }; + return new TextEncoder().encode(JSON.stringify(encoded)); +} + +// Compute the manifest hash for a chunk list without touching the +// DB. Used by the apply path to short-circuit when an upstream +// entry already matches the local node — the manifest hash is +// content-addressed so identical chunks always produce the same +// hash. +export function computeManifestHash(chunks: ManifestChunk[]): Uint8Array { + return sha256(encodeManifest(chunks)); +} + +// Build a manifest row for the given chunk list. Idempotent: a +// second call with the same chunks no-ops on the UNIQUE(hash). The +// returned hash is what the caller writes onto +// `vfs_nodes.manifest_hash`. +export function buildManifest(db: Database, chunks: ManifestChunk[], now: number): Uint8Array { + const bytes = encodeManifest(chunks); + const hash = sha256(bytes); + const size = chunks.reduce((acc, c) => acc + c.size, 0); + db.run( + "INSERT INTO vfs_manifests (hash, size, encoded, last_seen) VALUES (?, ?, ?, ?) ON CONFLICT(hash) DO UPDATE SET last_seen = excluded.last_seen", + hash, + size, + bytes, + now, + ); + return hash; +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/paths.ts b/spikes/349-dofs/vendor/dofs/src/sync/paths.ts new file mode 100644 index 00000000..d68a1e14 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/paths.ts @@ -0,0 +1,46 @@ +import { ROOT_INODE } from "../schema/index.js"; +import type { Database } from "../storage.js"; + +// Walk vfs_dirents from `inode` up to ROOT_INODE, gathering the path +// segments along the way. Returns null when the inode is unreachable. +export function pathOf(db: Database, inode: number): string | null { + if (inode === ROOT_INODE) return "/"; + const segments: string[] = []; + let current = inode; + // Bound the walk: a million levels deep is well past any real FS; + // anything beyond that is corruption and should not loop forever. + for (let i = 0; i < 1_000_000; i++) { + const row = db.one<{ parent_inode: number; name: string }>( + "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", + current, + ); + if (row === undefined) return null; + segments.push(row.name); + if (row.parent_inode === ROOT_INODE) { + segments.reverse(); + return `/${segments.join("/")}`; + } + current = row.parent_inode; + } + return null; +} + +// Every path that currently names `inode`. A file may carry several +// hardlink names; pathOf collapses them to one arbitrary name, which +// is wrong for the change stream — every name has to reach the wire so +// the receiver materialises each. Directories cannot be hardlinked, so +// each parent walk is unambiguous. +export function pathsOf(db: Database, inode: number): string[] { + if (inode === ROOT_INODE) return ["/"]; + const dirents = db.all<{ parent_inode: number; name: string }>( + "SELECT parent_inode, name FROM vfs_dirents WHERE child_inode = ?", + inode, + ); + const paths: string[] = []; + for (const { parent_inode, name } of dirents) { + const parent = pathOf(db, parent_inode); + if (parent === null) continue; + paths.push(parent === "/" ? `/${name}` : `${parent}/${name}`); + } + return paths; +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/push.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/push.test.ts new file mode 100644 index 00000000..3f22e455 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/push.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from "vitest"; +import { mkdir } from "../fs/mkdir.js"; +import { readFile } from "../fs/readFile.js"; +import { resolveInode } from "../fs/resolve.js"; +import { rm } from "../fs/rm.js"; +import { symlink } from "../fs/symlink.js"; +import { withDB, withTwoDBs } from "../fs/with-db.js"; +import { writeFile } from "../fs/writeFile.js"; +import type { Database } from "../storage.js"; +import type { ChangeEntry } from "./changes.js"; +import { coalesceChanges } from "./coalesce.js"; +import { pushObjects } from "./push.js"; + +// Minimal hand-rolled apply loop. We just need to prove that +// coalesceChanges + pushObjects together transfer enough information +// for the receiver to converge. +async function apply(db: Database, entries: ChangeEntry[], objects: Map) { + for (const entry of entries) { + if (entry.kind === "delete") { + try { + rm(db, entry.path, { recursive: true, force: true }); + } catch { + // Path may already be gone if the receiver was empty. + } + continue; + } + if (entry.kind === "dir") { + mkdir(db, entry.path, { mode: entry.mode, recursive: true }, () => entry.mtime); + continue; + } + if (entry.kind === "symlink") { + // Best-effort: writeFile to a path the symlink replaces won't + // round-trip through symlink. For this test we never overwrite + // a symlink with a non-symlink, so a fresh create is fine. + symlink(db, entry.target, entry.path, () => entry.mtime); + continue; + } + // file: assemble bytes from the chunks the sender shipped. + const parts: Uint8Array[] = []; + for (const c of entry.chunks) { + const key = hex(c.hash); + const bytes = objects.get(key); + if (bytes === undefined) throw new Error(`missing object for ${key}`); + parts.push(bytes); + } + const total = parts.reduce((acc, p) => acc + p.byteLength, 0); + const buf = new Uint8Array(total); + let off = 0; + for (const p of parts) { + buf.set(p, off); + off += p.byteLength; + } + await writeFile(db, entry.path, buf, { mode: entry.mode }, () => entry.mtime); + } +} + +function hex(bytes: Uint8Array): string { + let s = ""; + for (let i = 0; i < bytes.byteLength; i++) s += bytes[i].toString(16).padStart(2, "0"); + return s; +} + +async function drain(it: AsyncIterable): Promise { + const out: T[] = []; + for await (const x of it) out.push(x); + return out; +} + +describe("push", () => { + it("transfers a single file end-to-end", async () => { + await withTwoDBs( + async (a) => { + await writeFile(a, "/hello.txt", "hello world", { mode: 0o644 }, () => 100); + const entries = await drain(coalesceChanges(a, 0)); + const objects = await pull(a, collectHashes(entries)); + return { entries, objects }; + }, + async (b, { entries, objects }) => { + await apply(b, entries, objects); + const node = resolveInode(b, "/hello.txt"); + expect(node?.type).toBe("file"); + const got = await readFile(b, "/hello.txt", "utf8"); + expect(got).toBe("hello world"); + }, + ); + }); + + it("converges across mixed mutations", async () => { + await withTwoDBs( + async (a) => { + mkdir(a, "/d", { mode: 0o755 }, () => 1); + await writeFile(a, "/d/a.txt", "alpha", {}, () => 2); + await writeFile(a, "/d/b.txt", "beta", {}, () => 3); + symlink(a, "/d/a.txt", "/link", () => 4); + await writeFile(a, "/tmp.txt", "scratch", {}, () => 5); + rm(a, "/tmp.txt", {}); + const entries = await drain(coalesceChanges(a, 0)); + const objects = await pull(a, collectHashes(entries)); + return { entries, objects }; + }, + async (b, { entries, objects }) => { + await apply(b, entries, objects); + expect(await readFile(b, "/d/a.txt", "utf8")).toBe("alpha"); + expect(await readFile(b, "/d/b.txt", "utf8")).toBe("beta"); + expect(resolveInode(b, "/tmp.txt")).toBeNull(); + const linked = await readFile(b, "/link", "utf8"); + expect(linked).toBe("alpha"); + }, + ); + }); + + it("pushObjects yields each requested hash exactly once", async () => { + await withDB(async (a) => { + await writeFile(a, "/a.txt", "same", {}, () => 1); + await writeFile(a, "/b.txt", "same", {}, () => 2); + const entries = await drain(coalesceChanges(a, 0)); + const hashes = collectHashes(entries); + // Both files reuse one chunk hash; collectHashes already + // dedups, so we should see exactly one object on the wire. + expect(hashes).toHaveLength(1); + const objects = await pull(a, hashes); + expect(objects.size).toBe(1); + }); + }); + + it("pushObjects throws EUNKNOWN_HASH when a hash is not in vfs_blob_bytes", async () => { + await withDB(async (a) => { + const unknown = new Uint8Array(32); + unknown.fill(0xff); + let caught: unknown; + try { + for await (const _ of pushObjects(a, [unknown])) { + // drain + } + } catch (e) { + caught = e; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as { code?: string }).code).toBe("EUNKNOWN_HASH"); + }); + }); +}); + +function collectHashes(entries: ChangeEntry[]): Uint8Array[] { + const seen = new Set(); + const out: Uint8Array[] = []; + for (const e of entries) { + if (e.kind !== "file") continue; + for (const c of e.chunks) { + const key = hex(c.hash); + if (!seen.has(key)) { + seen.add(key); + out.push(c.hash); + } + } + } + return out; +} + +async function pull(db: Database, hashes: Uint8Array[]): Promise> { + const out = new Map(); + for await (const { hash, bytes } of pushObjects(db, hashes)) { + out.set(hex(hash), bytes); + } + return out; +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/push.ts b/spikes/349-dofs/vendor/dofs/src/sync/push.ts new file mode 100644 index 00000000..cf84a446 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/push.ts @@ -0,0 +1,27 @@ +import { createWorkspaceError } from "../errors.js"; +import type { Database } from "../storage.js"; + +// Stream chunk bytes by hash. The receiver collects these into the +// keyed map it uses when assembling files from ChangeEntry chunks. +// Missing hashes throw — the caller is supposed to have probed +// hasObjects() first to avoid asking for what the sender doesn't have. +// +// The push direction (DO → container) and the fetch direction +// (container → DO) both use this same shape; on the wire it is +// fetchObjects on one side and pushObjects on the other. Both names +// resolve to the same SQL. +export async function* pushObjects( + db: Database, + hashes: Uint8Array[], +): AsyncIterable<{ hash: Uint8Array; bytes: Uint8Array }> { + for (const hash of hashes) { + const row = db.one<{ bytes: Uint8Array }>( + "SELECT bytes FROM vfs_blob_bytes WHERE hash = ?", + hash, + ); + if (row === undefined) { + throw createWorkspaceError("EUNKNOWN_HASH", "pushObjects: missing blob for requested hash"); + } + yield { hash, bytes: row.bytes }; + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/sync/watermarks.test.ts b/spikes/349-dofs/vendor/dofs/src/sync/watermarks.test.ts new file mode 100644 index 00000000..7b9213b0 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/watermarks.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; + +import { withDB } from "../fs/with-db.js"; +import { writeFile } from "../fs/writeFile.js"; +import { + compareChangeCursors, + currentRev, + readFetchCursor, + readWatermark, + writeFetchCursor, + writeWatermark, +} from "./watermarks.js"; + +describe("watermarks", () => { + it("readWatermark returns 0 for a fresh DB", async () => { + await withDB(async (db) => { + expect(readWatermark(db, "pushRev")).toBe(0); + }); + }); + + it("writeWatermark persists across reads", async () => { + await withDB(async (db) => { + writeWatermark(db, "pushRev", 42); + expect(readWatermark(db, "pushRev")).toBe(42); + }); + }); + + it("persists the fetch cursor rev and path separately", async () => { + await withDB(async (db) => { + expect(readFetchCursor(db)).toEqual({ rev: 0, path: null }); + writeFetchCursor(db, { rev: 12, path: "/dir/file.txt" }); + expect(readFetchCursor(db)).toEqual({ rev: 12, path: "/dir/file.txt" }); + writeFetchCursor(db, { rev: 13, path: null }); + expect(readFetchCursor(db)).toEqual({ rev: 13, path: null }); + }); + }); + + it("does not persist an intermediate full-rev cursor when a partial cursor write fails", async () => { + await withDB(async (db) => { + writeFetchCursor(db, { rev: 12, path: null }); + + const originalRun = db.run.bind(db); + db.run = ((query: string, ...bindings: unknown[]) => { + if (query.includes("_vfs_fetch_cursor")) { + throw new Error("forced cursor path failure"); + } + return originalRun(query, ...bindings); + }) as typeof db.run; + + expect(() => writeFetchCursor(db, { rev: 13, path: "/partial.txt" })).toThrow( + "forced cursor path failure", + ); + expect(readFetchCursor(db)).toEqual({ rev: 12, path: null }); + }); + }); + + it("returns a fresh start cursor for a zero fetchRev", async () => { + await withDB(async (db) => { + const cursor = readFetchCursor(db); + cursor.rev = 99; + cursor.path = "/mutated.txt"; + + expect(readFetchCursor(db)).toEqual({ rev: 0, path: null }); + }); + }); + + it("orders partial cursors before full same-rev cursors", () => { + expect(compareChangeCursors({ rev: 0, path: null }, { rev: 0, path: null })).toBe(0); + expect(compareChangeCursors({ rev: 12, path: null }, { rev: 12, path: "/partial.txt" })).toBe( + 1, + ); + expect(compareChangeCursors({ rev: 12, path: "/partial.txt" }, { rev: 12, path: null })).toBe( + -1, + ); + expect(compareChangeCursors({ rev: 13, path: "/partial.txt" }, { rev: 12, path: null })).toBe( + 1, + ); + }); + + it("watermarks advance monotonically (the caller enforces this)", async () => { + await withDB(async (db) => { + writeWatermark(db, "pushRev", 5); + writeWatermark(db, "pushRev", 10); + expect(readWatermark(db, "pushRev")).toBe(10); + // Going backwards is allowed by the helper itself; the sync + // layer's batch-commit logic is what keeps the counter monotonic. + writeWatermark(db, "pushRev", 3); + expect(readWatermark(db, "pushRev")).toBe(3); + }); + }); + + it("currentRev reports the latest rev stamped on a mutation", async () => { + await withDB(async (db) => { + // initializeSchema seeds rev=1 (stamped on the root inode). + const base = currentRev(db); + expect(base).toBeGreaterThanOrEqual(1); + await writeFile(db, "/a.txt", "x", {}, () => 1); + const r1 = currentRev(db); + expect(r1).toBeGreaterThan(base); + await writeFile(db, "/b.txt", "y", {}, () => 2); + expect(currentRev(db)).toBeGreaterThan(r1); + }); + }); + + it("rejects unknown watermark keys at the type level via the helper signature", () => { + // Compile-time only: writeWatermark only accepts "pushRev". + // Fetch progress must go through readFetchCursor/writeFetchCursor. + expect(true).toBe(true); + }); + + describe("per-backend keying", () => { + it("writes under the default backend when the caller omits the id", async () => { + await withDB(async (db) => { + writeWatermark(db, "pushRev", 17); + // The omitted-id read sees the same row. + expect(readWatermark(db, "pushRev")).toBe(17); + // An explicit "default" id also sees it — same slot. + expect(readWatermark(db, "pushRev", "default")).toBe(17); + }); + }); + + it("keeps each backend's cursors independent", async () => { + await withDB(async (db) => { + writeWatermark(db, "pushRev", 10, "worker"); + writeWatermark(db, "pushRev", 20, "container"); + expect(readWatermark(db, "pushRev", "worker")).toBe(10); + expect(readWatermark(db, "pushRev", "container")).toBe(20); + // A push under "worker" doesn't disturb the "container" + // backend's cursor. + writeWatermark(db, "pushRev", 11, "worker"); + expect(readWatermark(db, "pushRev", "worker")).toBe(11); + expect(readWatermark(db, "pushRev", "container")).toBe(20); + }); + }); + + it("unknown backend id reads as 0", async () => { + await withDB(async (db) => { + writeWatermark(db, "pushRev", 5, "worker"); + expect(readWatermark(db, "pushRev", "never-registered")).toBe(0); + }); + }); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/sync/watermarks.ts b/spikes/349-dofs/vendor/dofs/src/sync/watermarks.ts new file mode 100644 index 00000000..084c3c68 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/sync/watermarks.ts @@ -0,0 +1,130 @@ +import type { Database } from "../storage.js"; + +// Watermarks owned by the local database. Keyed by (k, backend) so a +// single workspace can host more than one backend and each keeps its +// own sync cursors. The container's appliedPushRev lives in-memory on +// the container side; we don't store it here. +// +// pushRev — last DO-side rev successfully pushed to the backend. +// +// initializeSchema() seeds pushRev at 0 in _vfs_watermark for the +// default backend. The schema table is the durability surface; +// readers and writers always go through this module so the SQL +// stays in one place. +// +// `backend` defaults to DEFAULT_BACKEND_ID so older callers that +// only ran one backend (or ran the package against a schema before +// per-backend keying landed) keep working unchanged. The v3 → v4 +// schema migration backfills the column on existing rows with the +// same default. +// +// Fetch progress is a `{ rev, path }` cursor. Its rev component is +// still stored in _vfs_watermark for schema compatibility, but callers +// must use readFetchCursor() / writeFetchCursor() so rev and path stay +// consistent. +export type WatermarkKey = "pushRev"; + +export const DEFAULT_BACKEND_ID = "default"; + +// Cursor into the remote change stream. `path: null` means `rev` +// is fully drained and the next fetch resumes strictly after that +// rev. A string path means resume inside the same rev after that +// path. The empty string is a real path value, not a sentinel, and +// must not be used to mean "start of rev". +export type ChangeCursor = { rev: number; path: string | null }; + +export function readWatermark( + db: Database, + key: WatermarkKey, + backend: string = DEFAULT_BACKEND_ID, +): number { + return ( + db.scalar("SELECT v FROM _vfs_watermark WHERE k = ? AND backend = ?", key, backend) ?? 0 + ); +} + +function readFetchRev(db: Database, backend: string = DEFAULT_BACKEND_ID): number { + return ( + db.scalar( + "SELECT v FROM _vfs_watermark WHERE k = ? AND backend = ?", + "fetchRev", + backend, + ) ?? 0 + ); +} + +function writeWatermarkValue( + db: Database, + key: WatermarkKey | "fetchRev", + value: number, + backend: string = DEFAULT_BACKEND_ID, +): void { + db.run( + "INSERT INTO _vfs_watermark (k, backend, v) VALUES (?, ?, ?) " + + "ON CONFLICT(k, backend) DO UPDATE SET v = excluded.v", + key, + backend, + value, + ); +} + +function writeFetchCursorPath( + db: Database, + path: string | null, + backend: string = DEFAULT_BACKEND_ID, +): void { + db.run( + "INSERT INTO _vfs_fetch_cursor (k, backend, path) VALUES (?, ?, ?) " + + "ON CONFLICT(k, backend) DO UPDATE SET path = excluded.path", + "fetch", + backend, + path, + ); +} + +export function writeWatermark( + db: Database, + key: WatermarkKey, + value: number, + backend: string = DEFAULT_BACKEND_ID, +): void { + writeWatermarkValue(db, key, value, backend); +} + +export function readFetchCursor(db: Database, backend: string = DEFAULT_BACKEND_ID): ChangeCursor { + const rev = readFetchRev(db, backend); + if (rev === 0) return { rev: 0, path: null }; + const path = db.scalar( + "SELECT path FROM _vfs_fetch_cursor WHERE k = ? AND backend = ?", + "fetch", + backend, + ); + return { rev, path: path ?? null }; +} + +export function writeFetchCursor( + db: Database, + cursor: ChangeCursor, + backend: string = DEFAULT_BACKEND_ID, +): void { + db.transactionSync(() => { + writeWatermarkValue(db, "fetchRev", cursor.rev, backend); + writeFetchCursorPath(db, cursor.path, backend); + }); +} + +export function compareChangeCursors(a: ChangeCursor, b: ChangeCursor): number { + if (a.rev !== b.rev) return a.rev - b.rev; + if (a.path === b.path) return 0; + if (a.path === null) return 1; + if (b.path === null) return -1; + return a.path < b.path ? -1 : 1; +} + +// The latest rev stamped on any DO-side mutation. coalesceChanges +// reads this implicitly via vfs_nodes.rev; the sync layer exposes it +// to callers that want to record the rev component of their next +// cursor. +export function currentRev(db: Database): number { + return db.scalar("SELECT v FROM vfs_meta WHERE k = 'rev'") ?? 0; +} diff --git a/spikes/349-dofs/vendor/dofs/src/testing-recording.ts b/spikes/349-dofs/vendor/dofs/src/testing-recording.ts new file mode 100644 index 00000000..04f9cc81 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/testing-recording.ts @@ -0,0 +1,78 @@ +import type { DurableObjectStorageLike, SQLCursorLike } from "./types.js"; + +// Pure-JS test fixture. Records every SQL statement the production +// code emits, lets the test assert against the trace, and serves a +// tiny in-process subset of vfs_meta semantics so the FS scaffolding +// can boot without dragging node:sqlite in. +// +// Lives in its own file so the workerd test runner can import this +// fixture without loading the SQLiteTestStorage class that wraps +// node:sqlite (which workerd doesn't ship). + +export interface ExecutedStatement { + query: string; + bindings: unknown[]; +} + +class TestCursor implements SQLCursorLike { + private readonly rows: Row[]; + + constructor(rows: Row[]) { + this.rows = rows; + } + + toArray(): Row[] { + return this.rows; + } +} + +export class RecordingStorage implements DurableObjectStorageLike { + readonly statements: ExecutedStatement[] = []; + readonly sql = { + exec: >( + query: string, + ...bindings: unknown[] + ): SQLCursorLike => { + this.statements.push({ query, bindings }); + return new TestCursor(this.rowsFor(query, bindings)); + }, + }; + + private readonly meta = new Map(); + + constructor(seed?: { schemaVersion?: number; rev?: number }) { + if (seed?.schemaVersion !== undefined) { + this.meta.set("schema_version", seed.schemaVersion); + } + if (seed?.rev !== undefined) { + this.meta.set("rev", seed.rev); + } + } + + transactionSync(closure: () => T): T { + return closure(); + } + + private rowsFor(query: string, bindings: unknown[]): Row[] { + const normalized = query.replace(/\s+/g, " ").trim().toLowerCase(); + if (normalized === "select v from vfs_meta where k = ?") { + const key = String(bindings[0]); + const value = this.meta.get(key); + return value === undefined ? [] : ([{ v: value }] as Row[]); + } + + if (normalized.startsWith("insert or ignore into vfs_meta")) { + const key = String(bindings[0]); + const value = Number(bindings[1]); + if (!this.meta.has(key)) { + this.meta.set(key, value); + } + } + + if (normalized.startsWith("update vfs_meta set v = ? where k = ?")) { + this.meta.set(String(bindings[1]), Number(bindings[0])); + } + + return []; + } +} diff --git a/spikes/349-dofs/vendor/dofs/src/testing.test.ts b/spikes/349-dofs/vendor/dofs/src/testing.test.ts new file mode 100644 index 00000000..507e8c6b --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/testing.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import { initializeSchema, ROOT_INODE } from "./schema/index.js"; +import { Database } from "./storage.js"; +import { SQLiteTestStorage } from "./testing.js"; + +describe("SQLiteTestStorage", () => { + it("backs a real in-memory database that initializeSchema can apply", () => { + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + + initializeSchema(db, () => 1234); + + const row = db.one<{ inode: number; type: string; mtime: number }>( + "SELECT inode, type, mtime FROM vfs_nodes WHERE inode = ?", + ROOT_INODE, + ); + expect(row).toEqual({ inode: ROOT_INODE, type: "dir", mtime: 1234 }); + }); + + it("runs transactionSync atomically", () => { + const storage = new SQLiteTestStorage(); + const db = new Database(storage); + initializeSchema(db, () => 0); + + expect(() => { + db.transactionSync(() => { + db.run("INSERT INTO vfs_meta (k, v) VALUES (?, ?)", "rollback_probe", 1); + throw new Error("forced"); + }); + }).toThrow("forced"); + + const value = db.scalar("SELECT v FROM vfs_meta WHERE k = ?", "rollback_probe"); + expect(value).toBeUndefined(); + }); +}); diff --git a/spikes/349-dofs/vendor/dofs/src/testing.ts b/spikes/349-dofs/vendor/dofs/src/testing.ts new file mode 100644 index 00000000..3b42ec02 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/testing.ts @@ -0,0 +1,90 @@ +// Real-DB DurableObjectStorageLike for unit tests. Backed by Node's +// built-in node:sqlite running an in-memory database. Workers' DO SQL +// surface is a subset of this, so anything that works here works on +// the real platform too. +// +// This module imports node:sqlite at the top level and therefore +// cannot be loaded under workerd. RecordingStorage — the +// pure-JS fixture that also lives in dofs's testing surface +// — has moved to ./testing-recording.ts so it can be imported +// from workerd-runnable tests. We re-export it here so existing +// `import { RecordingStorage } from "@cloudflare/dofs/testing"` +// call sites keep working under node. + +import { DatabaseSync, type StatementSync } from "node:sqlite"; + +import type { DurableObjectStorageLike, SQLCursorLike } from "./types.js"; + +export type { ExecutedStatement } from "./testing-recording.js"; +export { RecordingStorage } from "./testing-recording.js"; + +class TestCursor implements SQLCursorLike { + private readonly rows: Row[]; + + constructor(rows: Row[]) { + this.rows = rows; + } + + toArray(): Row[] { + return this.rows; + } +} + +export class SQLiteTestStorage implements DurableObjectStorageLike { + private readonly db: DatabaseSync; + private readonly cache = new Map(); + readonly sql: { + exec: (query: string, ...bindings: unknown[]) => SQLCursorLike; + }; + + constructor() { + this.db = new DatabaseSync(":memory:"); + this.sql = { + exec: (query: string, ...bindings: unknown[]): SQLCursorLike => { + // node:sqlite refuses statements with trailing whitespace through + // prepare(); also we cache prepared statements per unique query + // string to keep the fixture fast. + const key = query; + let stmt = this.cache.get(key); + if (stmt === undefined) { + stmt = this.db.prepare(query); + this.cache.set(key, stmt); + } + const normalizedBindings = bindings.map(toSQLiteValue); + const rows = (stmt.all(...(normalizedBindings as never[])) as Row[]) ?? []; + return new TestCursor(rows); + }, + }; + } + + transactionSync(closure: () => T): T { + this.db.exec("BEGIN"); + try { + const result = closure(); + this.db.exec("COMMIT"); + return result; + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + close(): void { + // StatementSync instances are released when the database closes. + this.cache.clear(); + this.db.close(); + } +} + +// node:sqlite is strict about input shapes: it accepts strings, numbers, +// bigints, null, and Uint8Array but not undefined, Buffer subclasses +// other than Uint8Array, or booleans. Normalize. +function toSQLiteValue(value: unknown): string | number | bigint | null | Uint8Array { + if (value === undefined || value === null) return null; + if (typeof value === "boolean") return value ? 1 : 0; + if (value instanceof Uint8Array) return value; + if (typeof value === "string" || typeof value === "number" || typeof value === "bigint") { + return value; + } + throw new TypeError(`SQLiteTestStorage cannot bind value of type ${typeof value}`); +} diff --git a/spikes/349-dofs/vendor/dofs/src/types.ts b/spikes/349-dofs/vendor/dofs/src/types.ts new file mode 100644 index 00000000..2832820e --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/src/types.ts @@ -0,0 +1,16 @@ +export interface SQLCursorLike> { + toArray(): Row[]; +} + +export interface SQLStorageLike { + exec>( + query: string, + ...bindings: unknown[] + ): SQLCursorLike; +} + +export interface DurableObjectStorageLike { + sql: SQLStorageLike; + transaction?(closure: () => T | Promise): T | Promise; + transactionSync?(closure: () => T): T; +} diff --git a/spikes/349-dofs/vendor/dofs/tests/worker.ts b/spikes/349-dofs/vendor/dofs/tests/worker.ts new file mode 100644 index 00000000..ef00d508 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/tests/worker.ts @@ -0,0 +1,25 @@ +// Minimal worker + DO shim used by the workerd-backed test runner. +// The DO exists solely so vitest-pool-workers can hand a real +// DurableObjectStorage instance to test callbacks via +// runInDurableObject(). The DO doesn't expose any externally useful +// surface; it lives under tests/ so it stays outside the package's +// public exports. + +import { DurableObject } from "cloudflare:workers"; + +export interface TestBindings { + TestStorage: DurableObjectNamespace; +} + +export class TestStorage extends DurableObject { + // No methods of our own; tests reach in via runInDurableObject() and + // use this.ctx.storage directly. +} + +// The pool requires a default export so it can spin up a worker. +// We don't route any traffic through it. +export default { + async fetch(): Promise { + return new Response("dofs test worker", { status: 200 }); + }, +} satisfies ExportedHandler; diff --git a/spikes/349-dofs/vendor/dofs/tests/wrangler.jsonc b/spikes/349-dofs/vendor/dofs/tests/wrangler.jsonc new file mode 100644 index 00000000..f330d766 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/tests/wrangler.jsonc @@ -0,0 +1,20 @@ +{ + // Test-only worker. Not deployed. + "name": "dofs-tests", + "main": "./worker.ts", + "compatibility_date": "2026-05-26", + "durable_objects": { + "bindings": [ + { + "name": "TestStorage", + "class_name": "TestStorage" + } + ] + }, + "migrations": [ + { + "tag": "v1", + "new_sqlite_classes": ["TestStorage"] + } + ] +} diff --git a/spikes/349-dofs/vendor/dofs/tsconfig.build.json b/spikes/349-dofs/vendor/dofs/tsconfig.build.json new file mode 100644 index 00000000..e7086b47 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/tsconfig.build.json @@ -0,0 +1,22 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": false, + "noEmit": false, + "outDir": "dist", + "rootDir": "src", + "types": [ + "@cloudflare/workers-types", + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/**/*.test.ts", + "src/fs/with-db.workers.ts", + "src/bench/**" + ] +} diff --git a/spikes/349-dofs/vendor/dofs/tsconfig.json b/spikes/349-dofs/vendor/dofs/tsconfig.json new file mode 100644 index 00000000..39e35871 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "lib": ["ES2023", "ESNext.Disposable", "WebWorker"], + "strict": true, + "skipLibCheck": true, + "noEmit": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "isolatedModules": true, + "types": [ + "@cloudflare/workers-types", + "vitest/globals", + "node", + "@cloudflare/vitest-pool-workers/types" + ] + }, + "include": ["src/**/*.ts", "tests/**/*.ts", "*.ts"] +} diff --git a/spikes/349-dofs/vendor/dofs/vitest.config.bench.ts b/spikes/349-dofs/vendor/dofs/vitest.config.bench.ts new file mode 100644 index 00000000..b9e25298 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/vitest.config.bench.ts @@ -0,0 +1,24 @@ +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; +import { defineConfig } from "vitest/config"; + +// Benchmark runner. Reuses the workerd-backed pool (same wrangler +// config as the workers test project) so the harness drives a REAL +// Durable Object SqlStorage — NOT the node SQLiteTestStorage fixture, +// which caches prepared statements and would understate per-statement +// cost. Scoped to the *.bench.ts glob so it never runs during +// `npm test`; invoke explicitly via `npm run bench`. +export default defineConfig({ + plugins: [ + cloudflareTest({ + wrangler: { configPath: "./tests/wrangler.jsonc" }, + }), + ], + test: { + globals: true, + include: ["src/bench/**/*.bench.ts"], + // The harness builds large trees and loops tens of thousands of + // synchronous ops; the default 5s timeout is far too tight. + testTimeout: 600_000, + hookTimeout: 600_000, + }, +}); diff --git a/spikes/349-dofs/vendor/dofs/vitest.config.ts b/spikes/349-dofs/vendor/dofs/vitest.config.ts new file mode 100644 index 00000000..e0512789 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + globals: true, + include: ["src/**/*.test.ts", "test/**/*.test.ts"], + }, +}); diff --git a/spikes/349-dofs/vendor/dofs/vitest.config.workers.ts b/spikes/349-dofs/vendor/dofs/vitest.config.workers.ts new file mode 100644 index 00000000..d912d613 --- /dev/null +++ b/spikes/349-dofs/vendor/dofs/vitest.config.workers.ts @@ -0,0 +1,35 @@ +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; +import { defineConfig } from "vitest/config"; + +// Workerd-backed runner. Same .test.ts files as the node project, but +// withDB resolves to the Durable Object-backed implementation via the +// alias below. +export default defineConfig({ + plugins: [ + cloudflareTest({ + wrangler: { configPath: "./tests/wrangler.jsonc" }, + }), + ], + resolve: { + // Match any relative import that lands on src/fs/with-db.js so + // tests under src/fs/, src/, and src/sync/ all resolve to the + // workerd-backed implementation regardless of their depth. + alias: [ + { + find: /^.*\/with-db\.js$/, + replacement: new URL("./src/fs/with-db.workers.ts", import.meta.url).pathname, + }, + ], + }, + test: { + globals: true, + include: ["src/**/*.test.ts"], + // testing.test.ts exercises SQLiteTestStorage directly — the + // node:sqlite-backed fixture has no analogue under workerd, so + // the test is meaningful only against the real node runtime. + // All other tests run under both backends; provider/provider-fd + // use a withProvider helper that delegates to withDB, which the + // workers config aliases to a DO-backed implementation. + exclude: ["src/testing.test.ts"], + }, +});