diff --git a/README.md b/README.md index 1ab54e70..975eea3a 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,106 @@ For a complete scaffold with migrations + codegen, see the [examples](#examples). Or try it interactively at [typegres.com/play](https://typegres.com/play). +## Clients compose the queries + +The class surface is the contract. A client composes against `@expose`-marked +methods, the closure is serialized, and the server evaluates it under a +constrained interpreter — so a client can write any query it likes, and still +reach only what you exposed. + +```bash +npm install typegres better-sqlite3 zod +``` + +```typescript +import { typegres, expose, sql } from "typegres"; +import { doRpc, toRpc, newMessagePortRpcSession, type ShimStub } from "typegres/capnweb"; +import { SqliteDriver } from "typegres/drivers/sqlite"; +import { Integer, Text } from "typegres/sqlite"; +import z from "zod"; + +const db = typegres(); +db.connect(SqliteDriver.create()); + +await db.defaultConnection.execute(sql`CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + team_token TEXT NOT NULL +)`); +await db.defaultConnection.execute(sql`CREATE TABLE posts ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL, + body TEXT NOT NULL +)`); + +class Users extends db.Table("users") { + @expose() id = Integer.column({ nonNull: true, generated: true }); + @expose() name = Text.column({ nonNull: true }); + // No @expose: the server scopes on it, and no client query can select + // or filter by it. + team_token = Text.column({ nonNull: true }); +} + +class Posts extends db.Table("posts") { + @expose() id = Integer.column({ nonNull: true, generated: true }); + @expose() user_id = Integer.column({ nonNull: true }); + @expose() body = Text.column({ nonNull: true }); +} + +await Users.insert( + { name: "Alice", team_token: "t-acme" }, + { name: "Bob", team_token: "t-acme" }, + { name: "Carol", team_token: "t-other" }, // different team +).execute(); +await Posts.insert( + { user_id: 1, body: "one" }, + { user_id: 1, body: "two" }, + { user_id: 2, body: "three" }, + { user_id: 3, body: "not yours" }, +).execute(); + +// The capability root — the entire surface a client can reach. +class Api { + // Hands back a builder over one team's posts, already joined to authors. + // Everything the client writes is rooted here, so it can only narrow. + @expose(z.string()) + feedFor(teamToken: string) { + return Posts.from() + .join(Users, ({ posts, users }) => posts.user_id.eq(users.id)) + .where(({ users }) => users.team_token.eq(teamToken)); + } +} + +// Server and client, joined here by a MessagePort so this runs in one +// process. `examples/chat` is the same two lines over a WebSocket. +const { port1, port2 } = new MessageChannel(); +newMessagePortRpcSession(port1, toRpc(new Api())); +const api = newMessagePortRpcSession(port2) as unknown as ShimStub; + +// "Top posters" — written on the client, evaluated on the server. There is +// no endpoint for this: the client composed the group-by, the aggregate and +// the ordering itself. The team scoping is baked into the builder, so the +// refinement can only narrow it, and Carol's row never appears. +const rows = await doRpc(api, (a) => + a + .feedFor("t-acme") + .groupBy(({ users }) => [users.name]) + .select(({ users, posts }) => ({ author: users.name, posts: posts.id.count() })) + .orderBy(({ posts }) => [posts.id.count(), "desc"]) + .execute(), +); + +console.log(rows); + +port1.close(); +port2.close(); +await db.defaultConnection.close(); +``` + +Swap the MessagePort for `newWebSocketRpcSession` / `newWorkersRpcResponse` and +the same code runs browser-to-server, with capabilities, promise pipelining, +and live subscriptions — see [`examples/chat`](./examples/chat). + ## Backends `typegres()` is a synchronous schema handle — no top-level await, so table @@ -143,6 +243,18 @@ Deeper dive in [docs/ARCHITECTURE.md](./docs/ARCHITECTURE.md). - [x] Cap'n Web transport (`typegres/capnweb`) — capabilities, promises, and live subscriptions over a single WebSocket +> **Import Cap'n Web from `typegres/capnweb`, not from `capnweb`.** The +> transport needs a fork that isn't published yet (closure serialization, +> synchronous replay, `getLocalTarget` — see +> [cloudflare/capnweb#162](https://github.com/cloudflare/capnweb/pull/162)), +> so it ships bundled, and `typegres/capnweb` re-exports what you need: +> `RpcTarget`, `RpcStub`, `newWebSocketRpcSession`, `newWorkersRpcResponse`. +> Installing `capnweb` alongside it gives you a second copy whose +> `RpcTarget`/`RpcStub` fail `instanceof` against the bundled one — which +> surfaces as confusing RPC errors at the boundary rather than a clean +> failure. When #162 lands, capnweb becomes an ordinary dependency and these +> imports keep working unchanged. + ## Planned - [ ] `pg_notify`-driven live updates (Postgres currently uses a single shared polling loop, not per-subscription) diff --git a/examples/chat/package-lock.json b/examples/chat/package-lock.json index 507543ae..5a125ef5 100644 --- a/examples/chat/package-lock.json +++ b/examples/chat/package-lock.json @@ -5,8 +5,8 @@ "packages": { "": { "name": "typegres-chat-example", + "hasInstallScript": true, "dependencies": { - "capnweb": "file:../../packages/capnweb", "react": "^19.2.7", "react-dom": "^19.2.7", "typegres": "file:../..", @@ -30,16 +30,15 @@ } }, "../..": { - "name": "typegres", "version": "0.2.0", "dependencies": { - "camelcase": "^9.0.0", - "capnweb": "file:packages/capnweb" + "camelcase": "^9.0.0" }, "bin": { "tg": "dist/cli.mjs" }, "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.18.6", "@electric-sql/pglite": "^0.5.4", "@eslint/js": "^10.0.1", "@standard-schema/spec": "^1.1.0", @@ -53,6 +52,7 @@ "@typescript/native-preview": "7.0.0-dev.20260707.2", "acorn": "^8.17.0", "better-sqlite3": "^12.11.1", + "capnweb": "file:packages/capnweb", "eslint": "^10.7.0", "fast-check": "^4.9.0", "pg": "^8.22.0", @@ -86,6 +86,7 @@ }, "../../packages/capnweb": { "version": "0.6.1", + "extraneous": true, "license": "MIT", "devDependencies": { "@changesets/changelog-github": "^0.5.2", @@ -2854,10 +2855,6 @@ ], "license": "CC-BY-4.0" }, - "node_modules/capnweb": { - "resolved": "../../packages/capnweb", - "link": true - }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", diff --git a/examples/chat/package.json b/examples/chat/package.json index 2cce8356..512d8328 100644 --- a/examples/chat/package.json +++ b/examples/chat/package.json @@ -14,7 +14,6 @@ "reset:local": "rm -rf .wrangler && echo 'local DO storage cleared — next dev/test re-runs migrations'" }, "dependencies": { - "capnweb": "file:../../packages/capnweb", "react": "^19.2.7", "react-dom": "^19.2.7", "typegres": "file:../..", diff --git a/examples/chat/src/rpc.ts b/examples/chat/src/rpc.ts index 41e8b7f3..782b89e9 100644 --- a/examples/chat/src/rpc.ts +++ b/examples/chat/src/rpc.ts @@ -1,5 +1,4 @@ -import { newWebSocketRpcSession } from "capnweb"; -import { doRpc, type ShimStub, type Stubbed } from "typegres/capnweb"; +import { newWebSocketRpcSession, doRpc, type ShimStub, type Stubbed } from "typegres/capnweb"; import type { Chat, Users, Rooms } from "../worker/api"; import { wireLog } from "./wire-log"; diff --git a/examples/chat/tests/capabilities.test.ts b/examples/chat/tests/capabilities.test.ts index c8e1bb6a..b5b7a98b 100644 --- a/examples/chat/tests/capabilities.test.ts +++ b/examples/chat/tests/capabilities.test.ts @@ -9,8 +9,7 @@ import { describe, test, expect, expectTypeOf, vi } from "vitest"; import { SELF } from "cloudflare:test"; -import { newWebSocketRpcSession } from "capnweb"; -import { byRef, doRpc, type ShimStub } from "typegres/capnweb"; +import { newWebSocketRpcSession, byRef, doRpc, type ShimStub } from "typegres/capnweb"; import type { Chat, Users, Rooms, Memberships } from "../worker/api"; type Principal = InstanceType>; diff --git a/examples/chat/tests/facet-spike.test.ts b/examples/chat/tests/facet-spike.test.ts index 2466f992..31a40b96 100644 --- a/examples/chat/tests/facet-spike.test.ts +++ b/examples/chat/tests/facet-spike.test.ts @@ -12,8 +12,7 @@ import { test, expect } from "vitest"; import { env, runInDurableObject } from "cloudflare:test"; -import { newWebSocketRpcSession, type RpcTarget } from "capnweb"; -import { doRpc, toRpc, type ShimStub } from "typegres/capnweb"; +import { newWebSocketRpcSession, doRpc, toRpc, type RpcTarget, type ShimStub } from "typegres/capnweb"; import { Chat, Users, Memberships } from "../worker/api"; import type { ChatDo } from "../worker/chat-do"; diff --git a/examples/chat/worker/chat-do.ts b/examples/chat/worker/chat-do.ts index b67f0ce2..5baa8247 100644 --- a/examples/chat/worker/chat-do.ts +++ b/examples/chat/worker/chat-do.ts @@ -1,6 +1,5 @@ import { DurableObject } from "cloudflare:workers"; -import { newWorkersRpcResponse, type RpcTarget } from "capnweb"; -import { toRpc } from "typegres/capnweb"; +import { newWorkersRpcResponse, toRpc, type RpcTarget } from "typegres/capnweb"; import { DoSqliteDriver } from "typegres/drivers/do"; import type { Connection } from "typegres"; import { db, Chat } from "./api"; diff --git a/package-lock.json b/package-lock.json index 12f5cf4c..b4472968 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,8 +8,7 @@ "name": "typegres", "version": "0.2.0", "dependencies": { - "camelcase": "^9.0.0", - "capnweb": "file:packages/capnweb" + "camelcase": "^9.0.0" }, "bin": { "tg": "dist/cli.mjs" @@ -29,6 +28,7 @@ "@typescript/native-preview": "7.0.0-dev.20260707.2", "acorn": "^8.17.0", "better-sqlite3": "^12.11.1", + "capnweb": "file:packages/capnweb", "eslint": "^10.7.0", "fast-check": "^4.9.0", "pg": "^8.22.0", @@ -10964,6 +10964,7 @@ }, "packages/capnweb": { "version": "0.6.1", + "dev": true, "license": "MIT", "devDependencies": { "@changesets/changelog-github": "^0.5.2", diff --git a/package.json b/package.json index 961829ff..f77094d1 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,11 @@ "types": "./dist/exoeval/index.d.mts" }, "./capnweb": { - "import": "./dist/capnweb/shim.mjs", - "types": "./dist/capnweb/shim.d.mts" + "types": "./dist/capnweb/shim.d.mts", + "import": { + "workerd": "./dist/capnweb/shim-workers.mjs", + "default": "./dist/capnweb/shim.mjs" + } }, "./drivers/do": { "import": "./dist/drivers/do.mjs", @@ -57,8 +60,8 @@ "dist" ], "peerDependencies": { - "@electric-sql/pglite": "^0.4.4", - "better-sqlite3": "^12.11.1", + "@electric-sql/pglite": "^0.4.4 || ^0.5.0", + "better-sqlite3": "^12.11.1 || ^13.0.0", "pg": "^8.20.0" }, "peerDependenciesMeta": { @@ -109,12 +112,12 @@ "secure-json-parse": "^4.1.0", "tsdown": "^0.22.7", "typescript": "^6.0.3", + "capnweb": "file:packages/capnweb", "unplugin-swc": "^1.5.9", "vitest": "^4.1.10", "zod": "^4.4.3" }, "dependencies": { - "camelcase": "^9.0.0", - "capnweb": "file:packages/capnweb" + "camelcase": "^9.0.0" } } diff --git a/src/capnweb/shim.ts b/src/capnweb/shim.ts index f25094bd..23dbb717 100644 --- a/src/capnweb/shim.ts +++ b/src/capnweb/shim.ts @@ -2,6 +2,24 @@ import { getLocalTarget, RpcStub, RpcTarget } from "capnweb"; import { getTool } from "../exoeval/tool"; import { isPlainObject, isThenable } from "../util"; +// capnweb is bundled into this entry point rather than resolved from the +// consumer's node_modules: typegres needs a fork (closure serialization, +// synchronous replay, getLocalTarget) that isn't published. Re-exporting the +// surface a caller needs means they never `import ... from "capnweb"` +// themselves, so exactly one copy is ever in play — which matters because +// RpcTarget/RpcStub identity is compared across the boundary, and two copies +// would fail `instanceof` in ways that surface as baffling RPC errors. +// +// When cloudflare/capnweb#162 lands, capnweb becomes an ordinary dependency +// and these re-exports keep working unchanged. +export { + RpcTarget, + RpcStub, + newWebSocketRpcSession, + newWorkersRpcResponse, + newMessagePortRpcSession, +} from "capnweb"; + // --- capnweb shim for @expose classes --- // // Adapts typegres's @expose capability vocabulary to capnweb RPC. A class instance crossing diff --git a/src/readme.test.ts b/src/readme.test.ts index 33621267..04dee6a8 100644 --- a/src/readme.test.ts +++ b/src/readme.test.ts @@ -10,10 +10,10 @@ // `npm install`, paste, run. // // Two install modes: -// - working-tree (default): `npm install file:`, which packs the -// local repo internally and honors the package.json `files` -// manifest. Tests what the README *will* be when this code -// publishes — catches drift in PRs. Requires dist/ to be built. +// - working-tree (default): `npm pack` the repo and install the tarball. +// Tests what the README *will* be when this code publishes — both that +// the snippet runs and that the published artifact resolves. Requires +// dist/ to be built. // - registry (TYPEGRES_README_TEST_REGISTRY=1): install `typegres` // from npm. Tests what the README *currently is* for someone // running it against the latest published version. Useful @@ -38,25 +38,57 @@ const README_PATH = path.join(REPO_ROOT, "README.md"); type InstallMode = "working-tree" | "registry"; -const runReadmeUsage = async (mode: InstallMode): Promise => { +// Working-tree mode installs a real tarball, not `file:${REPO_ROOT}`. +// +// That distinction is load-bearing. npm resolves a `file:` directory dep by +// symlinking, and Node resolves through the symlink's real path — so the +// consumer transitively sees the *repo's own* node_modules, and a dependency +// the published package can't actually resolve still works. That is exactly +// how `typegres/capnweb` shipped importable-but-unloadable while this suite +// stayed green. Only a tarball install reproduces what a registry consumer +// gets. +// +// Packed once and shared: `npm pack` costs a second or two, and every +// section installs the same artifact. +let packed: Promise | undefined; +const packTypegres = (): Promise => { + packed ??= (async () => { + if (!fs.existsSync(path.join(REPO_ROOT, "dist", "index.mjs"))) { + throw new Error("working-tree mode needs dist/ — run `npm run build` first"); + } + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "typegres-pack-")); + const { stdout } = await execFileP( + "npm", + ["pack", "--pack-destination", dir, "--silent"], + { cwd: REPO_ROOT }, + ); + return path.join(dir, stdout.trim().split("\n").pop()!); + })(); + return packed; +}; + +// Each runnable README section owns an install line and a program, so a +// section is testable in isolation and the install stays honest (the RPC +// section needs zod; Usage doesn't). +const runReadmeSection = async ( + heading: string, + mode: InstallMode, + expected: string[], + unexpected: string[] = [], +): Promise => { const readme = fs.readFileSync(README_PATH, "utf8"); - // Scope to the Usage section so we don't pick up code blocks from - // other sections (Backends, Development, etc.). - const usageSection = /## Usage[\s\S]*?(?=\n## |$)/.exec(readme)?.[0] ?? ""; - const bashSnippet = /```bash\n([\s\S]*?)```/.exec(usageSection)?.[1]?.trim(); - const tsSnippet = /```typescript\n([\s\S]*?)```/.exec(usageSection)?.[1]; + // Scope to one section so we don't pick up code blocks from the others + // (Backends, Development, ...). + const section = + new RegExp(`## ${heading}[\\s\\S]*?(?=\\n## |$)`).exec(readme)?.[0] ?? ""; + const bashSnippet = /```bash\n([\s\S]*?)```/.exec(section)?.[1]?.trim(); + const tsSnippet = /```typescript\n([\s\S]*?)```/.exec(section)?.[1]; if (!bashSnippet || !tsSnippet) { throw new Error( - "README: couldn't find both ```bash``` and ```typescript``` blocks under ## Usage", + `README: couldn't find both \`\`\`bash\`\`\` and \`\`\`typescript\`\`\` blocks under ## ${heading}`, ); } - // working-tree mode installs from the repo via a file: reference, - // which honors `files: ["dist"]` — so dist/ must be built. - if (mode === "working-tree" && !fs.existsSync(path.join(REPO_ROOT, "dist", "index.mjs"))) { - throw new Error("working-tree mode needs dist/ — run `npm run build` first"); - } - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `typegres-readme-${mode}-`)); fs.writeFileSync( @@ -64,22 +96,22 @@ const runReadmeUsage = async (mode: InstallMode): Promise => { JSON.stringify({ name: "readme-test", type: "module", private: true }), ); - // working-tree: file: reference to the repo. npm packs the local - // directory using the `files` manifest internally — same effect as - // `npm pack && npm install `, no tarball management. - // registry: install verbatim from npm. + // working-tree: swap the `typegres` package name for the packed tarball, + // leaving the rest of the README's install line intact (better-sqlite3, + // zod, ...). registry: install verbatim from npm. // - // Flags shave ~3-5s off install: skip the security audit (we're a - // disposable tmp dir), skip funding messages, prefer the offline - // cache before hitting the registry. + // --no-audit/--no-fund shave a few seconds off a disposable tmp dir. + // + // Deliberately NOT --prefer-offline: it makes npm trust a cached + // packument, so a developer whose cache predates a peer's current + // versions gets ETARGET on a range that resolves fine against the + // registry. That's a failure about the machine, not the change under + // test, and it costs more in confusion than the flag saves in seconds. const installCmd = ( mode === "working-tree" - ? bashSnippet.replace(/\btypegres\b/, JSON.stringify(`file:${REPO_ROOT}`)) + ? bashSnippet.replace(/\btypegres\b/, JSON.stringify(await packTypegres())) : bashSnippet - ).replace( - /\bnpm install\b/, - `npm install --no-audit --no-fund ${mode === "working-tree" ? "--prefer-offline" : ""}`, - ); + ).replace(/\bnpm install\b/, "npm install --no-audit --no-fund"); await execFileP("sh", ["-c", installCmd], { cwd: tmpDir }); // Compile the snippet via swc — handles stage-3 decorators that @@ -98,8 +130,12 @@ const runReadmeUsage = async (mode: InstallMode): Promise => { const { stdout } = await execFileP("node", ["main.mjs"], { cwd: tmpDir }); - expect(stdout).toContain("Alice Smith"); - expect(stdout).toContain("Bob Jones"); + for (const want of expected) { + expect(stdout).toContain(want); + } + for (const avoid of unexpected) { + expect(stdout).not.toContain(avoid); + } // Only delete the temp dir if everything succeeded — leave it behind // for debugging on failure. @@ -107,16 +143,37 @@ const runReadmeUsage = async (mode: InstallMode): Promise => { }; test( - "README.md Usage snippet — working tree (file:)", - () => runReadmeUsage("working-tree"), + "README.md Usage snippet — working tree (packed tarball)", + () => runReadmeSection("Usage", "working-tree", ["Alice Smith", "Bob Jones"]), 60_000, // typical: ~5s; generous for better-sqlite3 prebuilt download on cache misses. ); +// The RPC section demonstrates the project's actual claim — a client +// composing a query that reaches only the @expose surface — so it's held to +// the same "it runs" bar as Usage. The negative assertions are what make it +// meaningful: `Carol` absent proves feedFor's team scoping survived a +// client-authored group-by (she has a post, on another team), and +// `team_token` absent proves the un-@expose'd column never crossed the wire +// even though the server filtered on it. +test( + "README.md RPC snippet — working tree (packed tarball)", + () => + runReadmeSection( + "Clients compose the queries", + "working-tree", + // "posts: 2" pins the aggregate itself — without it the test would + // pass on any query that merely returned both names. + ["Alice", "Bob", "posts: 2"], + ["Carol", "t-acme", "team_token"], + ), + 60_000, +); + // Registry mode: opt-in via env var. Tests the currently-published // `typegres` against the README — useful post-release. Skipped by // default so PR CI doesn't fail on registry hiccups or version drift. test.runIf(process.env["TYPEGRES_README_TEST_REGISTRY"] === "1")( "README.md Usage snippet — registry (npm install typegres)", - () => runReadmeUsage("registry"), + () => runReadmeSection("Usage", "registry", ["Alice Smith", "Bob Jones"]), 120_000, ); diff --git a/tsdown.config.ts b/tsdown.config.ts index 5e6b5105..0a4d13e6 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -23,7 +23,31 @@ export default defineConfig([ entry: ["src/index.ts", "src/config.ts", "src/builder/sql.ts", "src/types/postgres/index.ts", "src/types/sqlite/index.ts", "src/cli.ts", "src/exoeval/index.ts", "src/capnweb/shim.ts", "src/drivers/do.ts", "src/drivers/pg.ts", "src/drivers/pglite.ts", "src/drivers/sqlite.ts"], format: ["esm"], clean: true, - deps: { neverBundle: ["pg", "@electric-sql/pglite", "better-sqlite3", "capnweb"] }, + // capnweb is force-bundled: typegres needs a fork that isn't on npm, so it + // ships inlined and the shim re-exports the surface consumers need (see + // src/capnweb/shim.ts). + deps: { + neverBundle: ["pg", "@electric-sql/pglite", "better-sqlite3"], + alwaysBundle: ["capnweb"], + }, + plugins: [swcPlugin()], + }, + // Same shim, resolved through capnweb's `workerd` export condition. That + // build imports `inject-workers-module` first, which stashes + // `cloudflare:workers` on globalThis so capnweb interoperates with the + // runtime's built-in RPC. Bundling only the default build would silently + // drop that on Workers/Durable Objects; `package.json` routes the workerd + // condition here. `cloudflare:workers` stays external — it's runtime-provided. + { + entry: { "capnweb/shim-workers": "src/capnweb/shim.ts" }, + format: ["esm"], + dts: false, + clean: false, + inputOptions: { resolve: { conditionNames: ["workerd", "import", "default"] } }, + deps: { + neverBundle: ["pg", "@electric-sql/pglite", "better-sqlite3", "cloudflare:workers"], + alwaysBundle: ["capnweb"], + }, plugins: [swcPlugin()], }, // Playground single-file bundle for the site's Monaco + esbuild-wasm