From 64e8694d5b7b189e4983ea485f9c066cdb05d252 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Thu, 30 Jul 2026 16:00:19 -0400 Subject: [PATCH 1/7] fix collection query to show private collections --- src/api/collections.ts | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/api/collections.ts b/src/api/collections.ts index 8a61444..05ce930 100644 --- a/src/api/collections.ts +++ b/src/api/collections.ts @@ -15,12 +15,15 @@ import { type AuthEnv } from './auth.server.js' import { requireAuth } from './auth.server.js' const app = new Hono() - // Browse public collections + // Browse collections — public by default, or the caller's own with ?mine=true .get( '/collections', openApi({ tags: ['Collections'], - summary: 'Browse public collections', + summary: 'Browse collections', + description: + 'Lists public collections. Pass `mine=true` (authenticated) to list collections belonging ' + + 'to the organizations the caller is a member of instead, including private ones.', responses: { 200: z.any() }, }), async (c) => { @@ -28,10 +31,30 @@ const app = new Hono() const owner = c.req.query('owner') const tag = c.req.query('tag') const sort = c.req.query('sort') + const mine = c.req.query('mine') === 'true' const take = Math.min(parseInt(c.req.query('limit') ?? '50', 10), 100) const skip = parseInt(c.req.query('offset') ?? '0', 10) - const conditions = [eq(schema.collections.public, true)] + // Visibility scope. Public collections by default; with ?mine=true, every + // collection owned by an org the caller belongs to — private ones included, + // since org membership is what grants access elsewhere (hasOrgAccess). + if (mine && !c.get('userId')) { + return c.json( + { error: 'Unauthorized — mine=true requires a session', statusCode: 401 }, + 401, + ) + } + const visibility = mine + ? inArray( + schema.collections.organizationId, + db + .select({ id: schema.member.organizationId }) + .from(schema.member) + .where(eq(schema.member.userId, c.get('userId')!)), + ) + : eq(schema.collections.public, true) + + const conditions = [visibility] if (q) { conditions.push(ilike(schema.collections.name, `%${q}%`)) } @@ -44,6 +67,7 @@ const app = new Hono() id: schema.collections.id, slug: schema.collections.slug, name: schema.collections.name, + public: schema.collections.public, ownerSlug: schema.organization.slug, ownerName: schema.organization.name, createdAt: schema.collections.createdAt, @@ -129,7 +153,9 @@ const app = new Hono() .map(([name, count]) => ({ name, count })) .sort((a, b) => b.count - a.count) - const facetConditions = [eq(schema.collections.public, true)] + // Facets must describe the same scope as the results, or ?mine=true would + // show owner counts for collections the caller can't see. + const facetConditions = [visibility] if (q) { facetConditions.push(ilike(schema.collections.name, `%${q}%`)) } From 4ec8f511de0b48e39c7c6a0d18403e05c51a692d Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Fri, 31 Jul 2026 16:06:13 -0400 Subject: [PATCH 2/7] First steps of scaling work. --- docker-compose.yml | 25 +- src/api/ark.ts | 7 +- src/api/collections.ts | 70 +- src/api/negotiate.ts | 31 +- src/api/versions.ts | 635 +++-- src/db/migrations/0007_lucky_rhino.sql | 39 + src/db/migrations/meta/0007_snapshot.json | 2703 +++++++++++++++++++++ src/db/migrations/meta/_journal.json | 7 + src/db/schema.ts | 19 + src/db/seed.ts | 11 +- src/db/seedKfCollections.ts | 9 +- src/lib/mirror-sync.ts | 19 +- src/routes/docs/api/versions.tsx | 122 +- 13 files changed, 3386 insertions(+), 311 deletions(-) create mode 100644 src/db/migrations/0007_lucky_rhino.sql create mode 100644 src/db/migrations/meta/0007_snapshot.json diff --git a/docker-compose.yml b/docker-compose.yml index 425b131..15da65f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,20 @@ services: - .env environment: NODE_ENV: production - NODE_OPTIONS: '--max-old-space-size=448' + # Push commits accumulate per-record structures in memory, so the heap has + # to scale with the largest collection pushed, not with steady-state + # traffic. 448 MB could not commit much past ~100k records. + # + # Measured: a 500k-record push costs ~900 MB of heap above baseline + # (~1.8 KB/record), so 2 GB carries a ~1M-record push with headroom. + # Raise APP_HEAP_MB (and APP_MEMORY_LIMIT with it) for a one-off larger + # ingest rather than making it the standing default — dev and prod share + # one 16 GB box, so the defaults here are paid four times over. + # + # Do not read this as "3.11M records will fit if we go high enough": at + # that size the push path needs the streaming commit, not a bigger number. + # See planning/local/demos/arxiv-ingest-measurements-v1.md. + NODE_OPTIONS: '--max-old-space-size=${APP_HEAP_MB:-2048}' PORT: ${PORT:-3000} UNDERLAY_MODE: ${UNDERLAY_MODE:-origin} UNDERLAY_UPSTREAM: ${UNDERLAY_UPSTREAM:-} @@ -20,7 +33,15 @@ services: replicas: ${APP_REPLICAS:-2} resources: limits: - memory: 640m + # Must exceed APP_HEAP_MB — the V8 heap is only part of RSS. + # + # Sizing on the shared 16 GB box: dev + prod × 2 replicas = 4 app + # containers. They idle at a few hundred MB each and only one climbs + # at a time (a push is one request on one replica of one stack), so + # the realistic peak is 3× idle + 1× limit, not 4× limit. Reservations + # stay low deliberately — they are what Swarm schedules against, and + # over-reserving would strand memory the other stack needs. + memory: ${APP_MEMORY_LIMIT:-2560m} cpus: '1.0' reservations: memory: 384m diff --git a/src/api/ark.ts b/src/api/ark.ts index 68f43eb..0579a54 100644 --- a/src/api/ark.ts +++ b/src/api/ark.ts @@ -173,10 +173,13 @@ export async function resolve(c: Context) { eq(schema.versionRecords.recordHash, schema.recordObjects.hash), ) .where( + // Seeks (version_id, type, record_id) directly — before these columns + // were denormalized this scanned every version_records row for the + // version just to resolve one ARK. and( eq(schema.versionRecords.versionId, versionRow.id), - eq(schema.recordObjects.recordId, recordId), - eq(schema.recordObjects.type, recordType), + eq(schema.versionRecords.recordId, recordId), + eq(schema.versionRecords.type, recordType), ), ) .limit(1) diff --git a/src/api/collections.ts b/src/api/collections.ts index 05ce930..06752cb 100644 --- a/src/api/collections.ts +++ b/src/api/collections.ts @@ -369,21 +369,25 @@ const app = new Hono() // Get latest version info const latestVersion = await getLatestReadyVersion(result.id) - // Get per-type record counts for latest version + // Per-type record counts for the latest version. Stored on the version row + // at commit — this used to be a COUNT(*) GROUP BY over every + // version_records row on each page view. Versions written before that + // column existed fall back to the aggregate, now index-only. let typeCounts: { type: string; count: number }[] = [] - if (latestVersion) { + if (latestVersion?.typeCounts) { + typeCounts = Object.entries(latestVersion.typeCounts).map(([type, count]) => ({ + type, + count, + })) + } else if (latestVersion) { const rows = await db .select({ - type: schema.recordObjects.type, + type: schema.versionRecords.type, count: sql`count(*)::int`, }) .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) .where(eq(schema.versionRecords.versionId, latestVersion.id)) - .groupBy(schema.recordObjects.type) + .groupBy(schema.versionRecords.type) typeCounts = rows.map((r) => ({ type: r.type, count: r.count })) } @@ -866,12 +870,8 @@ const app = new Hono() // Stream records per-type into tar — avoids loading all records at once const types = await db - .selectDistinct({ type: schema.recordObjects.type }) + .selectDistinct({ type: schema.versionRecords.type }) .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) .where(eq(schema.versionRecords.versionId, version.id)) for (const { type } of types) { @@ -879,19 +879,20 @@ const app = new Hono() let batchCursor: string | null = null let batchHasMore = true while (batchHasMore) { + // Walk the (version_id, type, record_id) index; record_objects is + // joined only to pick up the body for the rows on this page. const conditions = [ eq(schema.versionRecords.versionId, version.id), - eq(schema.recordObjects.type, type), + eq(schema.versionRecords.type, type), ] if (batchCursor) { - conditions.push(sql`${schema.recordObjects.hash} > ${batchCursor}`) + conditions.push(sql`${schema.versionRecords.recordId} > ${batchCursor}`) } const batch = await db .select({ - recordId: schema.recordObjects.recordId, - type: schema.recordObjects.type, + recordId: schema.versionRecords.recordId, + type: schema.versionRecords.type, data: schema.recordObjects.data, - hash: schema.recordObjects.hash, }) .from(schema.versionRecords) .innerJoin( @@ -899,12 +900,12 @@ const app = new Hono() eq(schema.versionRecords.recordHash, schema.recordObjects.hash), ) .where(and(...conditions)) - .orderBy(schema.recordObjects.hash) + .orderBy(schema.versionRecords.recordId) .limit(5001) batchHasMore = batch.length > 5000 const page = batchHasMore ? batch.slice(0, 5000) : batch - if (page.length > 0) batchCursor = page[page.length - 1]!.hash + if (page.length > 0) batchCursor = page[page.length - 1]!.recordId for (const r of page) { lines.push(JSON.stringify({ id: r.recordId, type: r.type, data: r.data })) } @@ -1073,35 +1074,26 @@ const app = new Hono() appId: 'fork', recordCount: latestVersion.recordCount, fileCount: latestVersion.fileCount, + typeCounts: latestVersion.typeCounts, totalBytes: latestVersion.totalBytes, }) .returning({ id: schema.versions.id }) - const sourceRecords = await tx - .select({ - recordHash: schema.versionRecords.recordHash, - publicRecordHash: schema.versionRecords.publicRecordHash, - }) - .from(schema.versionRecords) - .where(eq(schema.versionRecords.versionId, latestVersion.id)) - - const FORK_BATCH = 5000 - for (let i = 0; i < sourceRecords.length; i += FORK_BATCH) { - const batch = sourceRecords.slice(i, i + FORK_BATCH) - await tx.insert(schema.versionRecords).values( - batch.map((r) => ({ - versionId: newVersion!.id, - recordHash: r.recordHash, - publicRecordHash: r.publicRecordHash, - })), - ) - } + // Copy the record set server-side. A fork of a multi-million-record + // collection has no reason to round-trip every row through the app. + await tx.execute(sql` + INSERT INTO version_records (version_id, record_hash, public_record_hash, record_id, type) + SELECT ${newVersion!.id}, record_hash, public_record_hash, record_id, type + FROM version_records + WHERE version_id = ${latestVersion.id} + `) const sourceFiles = await tx .select({ fileHash: schema.versionFiles.fileHash }) .from(schema.versionFiles) .where(eq(schema.versionFiles.versionId, latestVersion.id)) + const FORK_BATCH = 5000 for (let i = 0; i < sourceFiles.length; i += FORK_BATCH) { const batch = sourceFiles.slice(i, i + FORK_BATCH) await tx diff --git a/src/api/negotiate.ts b/src/api/negotiate.ts index 5cbf9cf..cacf277 100644 --- a/src/api/negotiate.ts +++ b/src/api/negotiate.ts @@ -670,6 +670,10 @@ app.post( }[] = [] const validationErrors: { recordId: string; type: string; errors: string[] }[] = [] const extraFieldWarnings: { recordId: string; type: string; fields: string[] }[] = [] + // Per-type counts, stored on the version row. The commit already walks every + // record, so counting here is free and saves a COUNT(*) GROUP BY on every + // subsequent collection page view. + const typeCounts = new Map() let totalBytes = 0 const LOAD_BATCH = 1000 @@ -739,6 +743,7 @@ app.post( } finalRecordHashes.push(hash) + typeCounts.set(rec.type, (typeCounts.get(rec.type) ?? 0) + 1) totalBytes += size // Compute public record hash inline @@ -902,6 +907,7 @@ app.post( actorId: session.actorId, recordCount: finalRecordHashes.length, fileCount: session.fileHashes.length, + typeCounts: Object.fromEntries(typeCounts), totalBytes, status: 'creating', }) @@ -944,18 +950,27 @@ app.post( ) }) - // Batch-insert version_records outside the main transaction + // Batch-insert version_records outside the main transaction. + // record_id and type are read back out of record_objects rather than + // carried in process: the rows are guaranteed to exist by this point (they + // were either already stored or inserted by the transaction above), and at + // millions of records two more in-memory string arrays are exactly what the + // heap can't afford. try { const VR_BATCH = 5000 for (let i = 0; i < finalRecordHashes.length; i += VR_BATCH) { const batch = finalRecordHashes.slice(i, i + VR_BATCH) - await db.insert(schema.versionRecords).values( - batch.map((hash) => ({ - versionId: versionId!, - recordHash: hash, - publicRecordHash: publicHashByRecordHash.get(hash) ?? null, - })), - ) + const publicHashes = batch.map((hash) => publicHashByRecordHash.get(hash) ?? null) + // sql.param binds each list as one array parameter; interpolating a bare + // array would expand it to a parenthesized list of placeholders, which + // Postgres rejects past 1,664 entries. + await db.execute(sql` + INSERT INTO version_records (version_id, record_hash, public_record_hash, record_id, type) + SELECT ${versionId!}, ro.hash, t.public_hash, ro.record_id, ro.type + FROM unnest(${sql.param(batch)}::text[], ${sql.param(publicHashes)}::text[]) + AS t(hash, public_hash) + INNER JOIN record_objects ro ON ro.hash = t.hash + `) } } catch (err) { await db.delete(schema.versions).where(eq(schema.versions.id, versionId!)) diff --git a/src/api/versions.ts b/src/api/versions.ts index 7915988..b3c7f66 100644 --- a/src/api/versions.ts +++ b/src/api/versions.ts @@ -39,11 +39,136 @@ const MAX_RECORDS_OFFSET = 10_000 // clean 503 (with Retry-After) instead of hanging or returning an opaque 500. const RECORDS_STATEMENT_TIMEOUT_MS = 10_000 +// Delta and diff run three set operations per request, so they get more room +// than a single record page. +const DELTA_STATEMENT_TIMEOUT_MS = 30_000 + +// Enumeration caps. Fewer, larger pages is the single biggest lever on the +// wall-clock of a full-collection walk: an anonymous caller gets 60 requests a +// minute, an authenticated one 5,000, so a multi-million-record collection is +// bounded by request count long before it is bounded by bytes. +// +// Records carry bodies, so their cap is set by response size (2,000 × ~3 KB ≈ +// 6 MB); manifest entries are ~120 bytes, so 100,000 is ~12 MB. +const MAX_RECORDS_LIMIT = 2_000 +const MAX_MANIFEST_LIMIT = 100_000 +const MAX_DIFF_LIMIT = 5_000 + // Postgres SQLSTATE 57014 = query_canceled, raised when statement_timeout fires. const isStatementTimeout = (err: unknown): boolean => typeof err === 'object' && err !== null && 'code' in err && err.code === '57014' +/** + * Run `fn` with a scoped statement timeout. SET LOCAL is reset when the + * transaction ends, so this bounds one query rather than the connection. + */ +async function withStatementTimeout( + timeoutMs: number, + fn: (tx: Parameters[0]>[0]) => Promise, +): Promise { + return db.transaction(async (tx) => { + // The timeout value can't be a bind parameter, so it's inlined. + await tx.execute(sql`SET LOCAL statement_timeout = ${sql.raw(String(timeoutMs))}`) + return fn(tx) + }) +} + +/** + * Keyset position within one of the delta/diff result lists: + * null — not started, read from the beginning + * [id, hash] — resume strictly after this row + * DONE — this list is exhausted, skip its query entirely + * + * The three lists of a delta drain at different rates, so "exhausted" has to be + * distinguishable from "not started". Collapsing them would restart a finished + * list on the next page and loop forever. + * + * A version can legitimately hold two records with the same `record_id` and + * different bodies — the manifest is deduplicated by hash, not by id — so the + * key is (record_id, record_hash), which is unique per version and matches the + * (version_id, record_id) index order. + */ +const DONE = 'done' +type ListCursor = [recordId: string, recordHash: string] | null | typeof DONE + +/** Per-list cursors for the three delta lists, carried as one opaque token. */ +interface DeltaCursor { + added: ListCursor + updated: ListCursor + removed: ListCursor +} + +const EMPTY_DELTA_CURSOR: DeltaCursor = { added: null, updated: null, removed: null } + +const encodeDeltaCursor = (cursor: DeltaCursor): string => + Buffer.from(JSON.stringify(cursor), 'utf-8').toString('base64url') + +/** Malformed cursors restart from the beginning rather than erroring. */ +function decodeDeltaCursor(raw: string | undefined): DeltaCursor { + if (!raw) return EMPTY_DELTA_CURSOR + try { + const parsed = JSON.parse(Buffer.from(raw, 'base64url').toString('utf-8')) as DeltaCursor + const list = (v: unknown): ListCursor => { + if (v === DONE) return DONE + return Array.isArray(v) && + v.length === 2 && + typeof v[0] === 'string' && + typeof v[1] === 'string' + ? [v[0], v[1]] + : null + } + return { + added: list(parsed?.added), + updated: list(parsed?.updated), + removed: list(parsed?.removed), + } + } catch { + return EMPTY_DELTA_CURSOR + } +} + +/** `WHERE (record_id, record_hash) > (…)` against the aliased version_records row. */ +const afterCursor = (alias: string, cursor: ListCursor) => + Array.isArray(cursor) + ? sql`AND (${sql.raw(alias)}.record_id, ${sql.raw(alias)}.record_hash) > (${cursor[0]}, ${cursor[1]})` + : sql`` + +/** Split a limit+1 fetch into a page plus the cursor for the next one. */ +function paginate( + rows: T[], + limit: number, +): { page: Omit[]; next: ListCursor; hasMore: boolean } { + const hasMore = rows.length > limit + const page = hasMore ? rows.slice(0, limit) : rows + const last = page[page.length - 1] + return { + page: page.map(({ recordHash: _recordHash, ...rest }) => rest), + next: hasMore && last ? [last.id, last.recordHash] : DONE, + hasMore, + } +} + const app = new Hono() + // A query cancelled by its scoped statement_timeout is a load signal, not a + // server fault: answer 503 + Retry-After so a client backs off and retries + // rather than treating it as a permanent failure. + .use('*', async (c, next) => { + try { + await next() + } catch (err) { + if (!isStatementTimeout(err)) throw err + c.header('Retry-After', '5') + return c.json( + { + error: + 'Query timed out. Page large result sets with keyset pagination ' + + '(?after= on records, ?cursor= on manifest and diff).', + statusCode: 503, + }, + 503, + ) + } + }) // List versions .get( '/:owner/:slug/versions', @@ -224,12 +349,16 @@ const app = new Hono() if (!version) return c.json({ error: 'Version not found', statusCode: 404 }, 404) + // Filtering and ordering run entirely off version_records, which carries + // denormalized record_id + type and is indexed on + // (version_id, [type,] record_id). record_objects is joined only to + // fetch bodies for the page that survives the index scan. const conditions = [eq(schema.versionRecords.versionId, version.id)] - if (type) conditions.push(eq(schema.recordObjects.type, type)) + if (type) conditions.push(eq(schema.versionRecords.type, type)) // Cursor-based pagination: ?after=recordId (keyset pagination) if (after) { - conditions.push(sql`${schema.recordObjects.recordId} > ${after}`) + conditions.push(sql`${schema.versionRecords.recordId} > ${after}`) } // Determine visibility @@ -246,14 +375,14 @@ const app = new Hono() return c.json([]) // requesting a private type as non-owner } for (const pt of privateTypes) { - conditions.push(sql`${schema.recordObjects.type} != ${pt}`) + conditions.push(sql`${schema.versionRecords.type} != ${pt}`) } } // Exclude record-level private records conditions.push(eq(schema.recordObjects.private, false)) } - const pageLimit = Math.min(parseInt(limit ?? '100', 10), 1000) + const pageLimit = Math.min(parseInt(limit ?? '100', 10), MAX_RECORDS_LIMIT) // Resolve offset (ignored when a keyset cursor is supplied). Reject deep // offsets with a 400 rather than letting an O(offset) scan time out. @@ -282,8 +411,8 @@ const app = new Hono() ) return tx .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, + id: schema.versionRecords.recordId, + type: schema.versionRecords.type, data: schema.recordObjects.data, // Non-owners see the public content-address (hash of the // private-field-stripped record they receive) @@ -297,7 +426,7 @@ const app = new Hono() eq(schema.versionRecords.recordHash, schema.recordObjects.hash), ) .where(and(...conditions)) - .orderBy(schema.recordObjects.recordId) + .orderBy(schema.versionRecords.recordId) .limit(pageLimit + 1) .offset(after ? 0 : offsetValue) }) @@ -372,7 +501,7 @@ const app = new Hono() limit: pageLimit, hasMore, nextCursor, - total: version.recordCount, + total: await countVersionRecords(version, type, privateTypes), }, }) }, @@ -486,7 +615,7 @@ const app = new Hono() if (!version) return c.json({ error: 'Version not found', statusCode: 404 }, 404) - const limit = Math.min(parseInt(c.req.query('limit') ?? '10000', 10), 50000) + const limit = Math.min(parseInt(c.req.query('limit') ?? '10000', 10), MAX_MANIFEST_LIMIT) const cursor = c.req.query('cursor') const fileHashes = await db @@ -500,18 +629,23 @@ const app = new Hono() // Records whose type has private *fields* are listed under their public // content-address (hash of the filtered record), so readers can verify // what they actually receive. + // + // Type and record id come off version_records, so only the record-level + // `private` flag still needs record_objects — the owner path joins + // nothing at all. const ownerAccess = collection.ownerAccess const privateTypes = ownerAccess ? new Set() : getPrivateTypes(schemaEntries) - const privacyConditions = [] - if (!ownerAccess) { - privacyConditions.push(eq(schema.recordObjects.private, false)) - for (const pt of privateTypes) { - privacyConditions.push(sql`${schema.recordObjects.type} != ${pt}`) - } - } + const privacyJoin = ownerAccess + ? sql`` + : sql`INNER JOIN record_objects ro ON ro.hash = vr.record_hash` + const privacyWhere = ownerAccess + ? sql`` + : privateTypes.size > 0 + ? sql`AND ro.private = false AND vr.type NOT IN (${[...privateTypes]})` + : sql`AND ro.private = false` const servedHash = ownerAccess - ? sql`${schema.recordObjects.hash}` - : sql`coalesce(${schema.versionRecords.publicRecordHash}, ${schema.recordObjects.hash})` + ? sql`vr.record_hash` + : sql`coalesce(vr.public_record_hash, vr.record_hash)` const manifestHash = ownerAccess ? version.hash : (version.publicHash ?? version.hash) const schemasOut = ownerAccess ? Object.fromEntries(schemaEntries.map((e) => [e.slug, e.schemaHash])) @@ -521,7 +655,10 @@ const app = new Hono() .map((e) => [e.slug, hashSchema(filterTypeSchema(e.schema))]), ) - // Delta manifest via SQL set operations — no in-memory Maps + // Delta manifest. The three set operations run as anti-/semi-joins between + // two versions over (version_id, record_id) — no correlated lookup through + // record_objects, and each list is keyset-paginated so a delta of any size + // can be walked to completion. if (sinceParam) { const { semver: sinceSemver } = parseSemver(sinceParam) @@ -542,140 +679,117 @@ const app = new Hono() const targetId = version.id const sinceId = sinceVersion.id + const at = decodeDeltaCursor(cursor) - const added = await db - .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, - hash: servedHash, - }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where( - and( - eq(schema.versionRecords.versionId, targetId), - sql`NOT EXISTS ( - SELECT 1 FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${sinceId} - AND sro.record_id = ${schema.recordObjects.recordId} - )`, - ...privacyConditions, - ), - ) - .limit(limit) - - const removed = await db - .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, - hash: servedHash, - }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where( - and( - eq(schema.versionRecords.versionId, sinceId), - sql`NOT EXISTS ( - SELECT 1 FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${targetId} - AND sro.record_id = ${schema.recordObjects.recordId} - )`, - ...privacyConditions, - ), - ) - .limit(limit) + type DeltaRow = { id: string; type: string; hash: string; recordHash: string } + type UpdatedRow = DeltaRow & { previousHash: string | null } const previousServedHash = ownerAccess - ? sql`( - SELECT sro.hash FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${sinceId} - AND sro.record_id = ${schema.recordObjects.recordId} - )` - : sql`( - SELECT coalesce(svr.public_record_hash, sro.hash) FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${sinceId} - AND sro.record_id = ${schema.recordObjects.recordId} - )` - - const updated = await db - .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, - hash: servedHash, - previousHash: previousServedHash, - }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where( - and( - eq(schema.versionRecords.versionId, targetId), - sql`EXISTS ( - SELECT 1 FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${sinceId} - AND sro.record_id = ${schema.recordObjects.recordId} - AND sro.hash != ${schema.recordObjects.hash} - )`, - ...privacyConditions, - ), - ) - .limit(limit) + ? sql`(SELECT s.record_hash FROM version_records s + WHERE s.version_id = ${sinceId} AND s.record_id = vr.record_id LIMIT 1)` + : sql`(SELECT coalesce(s.public_record_hash, s.record_hash) FROM version_records s + WHERE s.version_id = ${sinceId} AND s.record_id = vr.record_id LIMIT 1)` + + /** One delta list. `presence` selects the anti- or semi-join. */ + const deltaQuery = ( + versionId: number, + otherId: number, + presence: 'absent' | 'changed', + cursor: ListCursor, + extraColumn = sql``, + ) => sql` + SELECT vr.record_id AS id, vr.type, ${servedHash} AS hash, + vr.record_hash AS "recordHash"${extraColumn} + FROM version_records vr ${privacyJoin} + WHERE vr.version_id = ${versionId} + ${ + presence === 'absent' + ? sql`AND NOT EXISTS ( + SELECT 1 FROM version_records s + WHERE s.version_id = ${otherId} AND s.record_id = vr.record_id + )` + : sql`AND EXISTS ( + SELECT 1 FROM version_records s + WHERE s.version_id = ${otherId} AND s.record_id = vr.record_id + AND s.record_hash <> vr.record_hash + )` + } + ${privacyWhere} ${afterCursor('vr', cursor)} + ORDER BY vr.record_id, vr.record_hash + LIMIT ${limit + 1} + ` + + const [addedRows, removedRows, updatedRows] = await withStatementTimeout( + DELTA_STATEMENT_TIMEOUT_MS, + async (tx) => { + // A list the caller has already drained is skipped, not re-run. + const run = (cursor: ListCursor, query: ReturnType) => + cursor === DONE + ? Promise.resolve([] as T[]) + : (tx.execute(query) as unknown as Promise) + return Promise.all([ + run(at.added, deltaQuery(targetId, sinceId, 'absent', at.added)), + run(at.removed, deltaQuery(sinceId, targetId, 'absent', at.removed)), + run( + at.updated, + deltaQuery( + targetId, + sinceId, + 'changed', + at.updated, + sql`, ${previousServedHash} AS "previousHash"`, + ), + ), + ]) + }, + ) + + const added = paginate(addedRows, limit) + const removed = paginate(removedRows, limit) + const updated = paginate(updatedRows, limit) - const truncated = - added.length === limit || updated.length === limit || removed.length === limit + const hasMore = added.hasMore || removed.hasMore || updated.hasMore + const nextCursor = hasMore + ? encodeDeltaCursor({ + added: at.added === DONE ? DONE : added.next, + updated: at.updated === DONE ? DONE : updated.next, + removed: at.removed === DONE ? DONE : removed.next, + }) + : null return c.json({ semver: version.semver, hash: manifestHash, since: sinceSemver, schemas: schemasOut, - delta: { added, updated, removed }, + delta: { added: added.page, updated: updated.page, removed: removed.page }, files: fileHashes.map((f) => f.hash), - truncated, + pagination: { limit, hasMore, nextCursor }, + // Retained for clients written against the pre-cursor response, which + // treat a capped delta as "give up and rebuild". They still can; a + // client that understands `pagination.nextCursor` should page instead. + truncated: hasMore, }) } - // Full manifest with cursor-based pagination (keyed on the served hash so - // public readers can resume with the hashes they were given) - const recordConditions = [ - eq(schema.versionRecords.versionId, version.id), - ...privacyConditions, - ] - if (cursor) { - recordConditions.push(sql`${servedHash} > ${cursor}`) - } - - const recordRows = await db - .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, - hash: servedHash, - }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where(and(...recordConditions)) - .orderBy(servedHash) - .limit(limit + 1) - - const hasMore = recordRows.length > limit - const page = hasMore ? recordRows.slice(0, limit) : recordRows - const nextCursor = hasMore ? page[page.length - 1]!.hash : null + // Full manifest, keyset-paginated on (record_id, record_hash) — the + // (version_id, record_id) index order. Previously ordered by the served + // hash, which for public readers is a coalesce() expression and therefore + // an unindexed sort of the whole version. + const at = decodeDeltaCursor(cursor) + const recordRows = (await withStatementTimeout(DELTA_STATEMENT_TIMEOUT_MS, async (tx) => + tx.execute(sql` + SELECT vr.record_id AS id, vr.type, ${servedHash} AS hash, + vr.record_hash AS "recordHash" + FROM version_records vr ${privacyJoin} + WHERE vr.version_id = ${version.id} + ${privacyWhere} ${afterCursor('vr', at.added)} + ORDER BY vr.record_id, vr.record_hash + LIMIT ${limit + 1} + `), + )) as unknown as { id: string; type: string; hash: string; recordHash: string }[] + + const { page, next, hasMore } = paginate(recordRows, limit) return c.json({ semver: version.semver, @@ -683,7 +797,11 @@ const app = new Hono() schemas: schemasOut, records: page, files: fileHashes.map((f) => f.hash), - pagination: { limit, hasMore, nextCursor }, + pagination: { + limit, + hasMore, + nextCursor: hasMore ? encodeDeltaCursor({ ...EMPTY_DELTA_CURSOR, added: next }) : null, + }, }) }, ) @@ -699,7 +817,8 @@ const app = new Hono() async (c) => { const { owner, slug, n } = c.req.valid('param') const from = c.req.query('from') - const diffLimit = Math.min(parseInt(c.req.query('limit') ?? '500', 10), 5000) + const diffLimit = Math.min(parseInt(c.req.query('limit') ?? '500', 10), MAX_DIFF_LIMIT) + const diffCursor = decodeDeltaCursor(c.req.query('cursor')) const collection = await resolveAccessibleCollection(owner, slug, c.get('userId')) if (!collection) return c.json({ error: 'Collection not found', statusCode: 404 }, 404) @@ -743,108 +862,79 @@ const app = new Hono() const fromId = fromVersion?.id - // Privacy filtering for non-owners: hide private types and private records + // Privacy filtering for non-owners: hide private types and private records. + // record_objects is joined for the body (and the record-level `private` + // flag); the set operations themselves run on version_records alone. const targetSchemas = await loadVersionSchemas(targetVersion.id) const ownerAccess = collection.ownerAccess const privateTypes = ownerAccess ? new Set() : getPrivateTypes(targetSchemas) - const privacyConditions = [] - if (!ownerAccess) { - privacyConditions.push(eq(schema.recordObjects.private, false)) - for (const pt of privateTypes) { - privacyConditions.push(sql`${schema.recordObjects.type} != ${pt}`) - } - } - - // SQL set operations — only fetch diff rows, not all records from both versions - const added = fromId - ? await db - .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, - data: schema.recordObjects.data, - }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where( - and( - eq(schema.versionRecords.versionId, targetId), - sql`NOT EXISTS ( - SELECT 1 FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${fromId} - AND sro.record_id = ${schema.recordObjects.recordId} - )`, - ...privacyConditions, - ), - ) - .limit(diffLimit) - : await db - .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, - data: schema.recordObjects.data, - }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where(and(eq(schema.versionRecords.versionId, targetId), ...privacyConditions)) - .limit(diffLimit) - - const removed = fromId - ? await db - .select({ id: schema.recordObjects.recordId }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where( - and( - eq(schema.versionRecords.versionId, fromId), - sql`NOT EXISTS ( - SELECT 1 FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${targetId} - AND sro.record_id = ${schema.recordObjects.recordId} - )`, - ...privacyConditions, - ), - ) - .limit(diffLimit) - : [] + const privacyWhere = ownerAccess + ? sql`` + : privateTypes.size > 0 + ? sql`AND ro.private = false AND vr.type NOT IN (${[...privateTypes]})` + : sql`AND ro.private = false` + + type DiffRow = { id: string; type: string; data: unknown; recordHash: string } + + /** One side of the diff: rows of `versionId` absent from / changed in `otherId`. */ + const diffQuery = ( + versionId: number, + otherId: number | undefined, + mode: 'absent' | 'changed' | 'all', + cursor: ListCursor, + ) => sql` + SELECT vr.record_id AS id, vr.type, ro.data, vr.record_hash AS "recordHash" + FROM version_records vr + INNER JOIN record_objects ro ON ro.hash = vr.record_hash + WHERE vr.version_id = ${versionId} + ${ + mode === 'absent' + ? sql`AND NOT EXISTS ( + SELECT 1 FROM version_records s + WHERE s.version_id = ${otherId!} AND s.record_id = vr.record_id + )` + : mode === 'changed' + ? sql`AND EXISTS ( + SELECT 1 FROM version_records s + WHERE s.version_id = ${otherId!} AND s.record_id = vr.record_id + AND s.record_hash <> vr.record_hash + )` + : sql`` + } + ${privacyWhere} ${afterCursor('vr', cursor)} + ORDER BY vr.record_id, vr.record_hash + LIMIT ${diffLimit + 1} + ` + + const [addedRows, removedRows, updatedRows] = await withStatementTimeout( + DELTA_STATEMENT_TIMEOUT_MS, + async (tx) => { + // Skip lists the caller has already drained, and — with no ?from= — + // the two that don't apply. + const run = (cursor: ListCursor, query: ReturnType | null) => + cursor === DONE || !query + ? Promise.resolve([] as DiffRow[]) + : (tx.execute(query) as unknown as Promise) + return Promise.all([ + run( + diffCursor.added, + diffQuery(targetId, fromId, fromId ? 'absent' : 'all', diffCursor.added), + ), + run( + diffCursor.removed, + fromId ? diffQuery(fromId, targetId, 'absent', diffCursor.removed) : null, + ), + run( + diffCursor.updated, + fromId ? diffQuery(targetId, fromId, 'changed', diffCursor.updated) : null, + ), + ]) + }, + ) - const updated = fromId - ? await db - .select({ - id: schema.recordObjects.recordId, - type: schema.recordObjects.type, - data: schema.recordObjects.data, - }) - .from(schema.versionRecords) - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where( - and( - eq(schema.versionRecords.versionId, targetId), - sql`EXISTS ( - SELECT 1 FROM version_records svr - INNER JOIN record_objects sro ON svr.record_hash = sro.hash - WHERE svr.version_id = ${fromId} - AND sro.record_id = ${schema.recordObjects.recordId} - AND sro.hash != ${schema.recordObjects.hash} - )`, - ...privacyConditions, - ), - ) - .limit(diffLimit) - : [] + const added = paginate(addedRows, diffLimit) + const removed = paginate(removedRows, diffLimit) + const updated = paginate(updatedRows, diffLimit) // Compare schema sets const fromSchemas = fromVersion ? await loadVersionSchemas(fromVersion.id) : [] @@ -896,12 +986,25 @@ const app = new Hono() } } + const hasMore = added.hasMore || removed.hasMore || updated.hasMore + return c.json({ from: fromVersion?.semver ?? null, to: targetVersion.semver, - added: added.map(stripPrivateFields), - updated: (updated as { id: string; type: string; data: unknown }[]).map(stripPrivateFields), - removed: (removed as { id: string }[]).map((r) => r.id), + added: added.page.map(stripPrivateFields), + updated: updated.page.map(stripPrivateFields), + removed: removed.page.map((r) => r.id), + pagination: { + limit: diffLimit, + hasMore, + nextCursor: hasMore + ? encodeDeltaCursor({ + added: diffCursor.added === DONE ? DONE : added.next, + updated: diffCursor.updated === DONE ? DONE : updated.next, + removed: diffCursor.removed === DONE ? DONE : removed.next, + }) + : null, + }, meta: { schemaChanged, metadataChanged, @@ -976,10 +1079,13 @@ const app = new Hono() .select({ hash: schema.versionRecords.recordHash, publicRecordHash: schema.versionRecords.publicRecordHash, - type: schema.recordObjects.type, + recordId: schema.versionRecords.recordId, + type: schema.versionRecords.type, private: schema.recordObjects.private, }) .from(schema.versionRecords) + // Only `private` still lives on record_objects; record_id and type are + // denormalized onto version_records. .innerJoin( schema.recordObjects, eq(schema.versionRecords.recordHash, schema.recordObjects.hash), @@ -1029,6 +1135,9 @@ const app = new Hono() pushedBy: userId ?? null, recordCount: latest.recordCount, fileCount: latest.fileCount, + // Same record set as the base version, so the per-type counts carry + // over unchanged. + typeCounts: latest.typeCounts, totalBytes: latest.totalBytes, }) .returning({ id: schema.versions.id }) @@ -1052,6 +1161,8 @@ const app = new Hono() versionId: version!.id, recordHash: r.hash, publicRecordHash: r.publicRecordHash, + recordId: r.recordId, + type: r.type, })), ) } @@ -1093,6 +1204,44 @@ const app = new Hono() }, ) +/** + * Total records in a version under the caller's visibility and an optional + * `?type=` filter. Reads the per-type counts stored on the version row at + * commit; falls back to a COUNT(*) over the (version_id, type, record_id) index + * for versions written before that column existed. + * + * Row-level private records are not represented in `type_counts`, so on a + * collection that uses them this is an upper bound for non-owners rather than + * an exact count. It was previously the whole version's `recordCount` + * regardless of the filter, which was simply wrong under `?type=`. + */ +async function countVersionRecords( + version: { id: number; recordCount: number; typeCounts: Record | null }, + type: string | undefined, + privateTypes: Set, +): Promise { + const counts = version.typeCounts + if (counts) { + if (type) return counts[type] ?? 0 + let total = 0 + for (const [slug, n] of Object.entries(counts)) { + if (!privateTypes.has(slug)) total += n + } + return total + } + + if (!type && privateTypes.size === 0) return version.recordCount + + const conditions = [eq(schema.versionRecords.versionId, version.id)] + if (type) conditions.push(eq(schema.versionRecords.type, type)) + for (const pt of privateTypes) conditions.push(sql`${schema.versionRecords.type} != ${pt}`) + const [row] = await db + .select({ n: sql`count(*)::int` }) + .from(schema.versionRecords) + .where(and(...conditions)) + return row?.n ?? 0 +} + /** ARK info is decorative on these endpoints — failures are logged, not fatal */ async function getCollectionArkInfo( collectionId: string, diff --git a/src/db/migrations/0007_lucky_rhino.sql b/src/db/migrations/0007_lucky_rhino.sql new file mode 100644 index 0000000..ad1a0b2 --- /dev/null +++ b/src/db/migrations/0007_lucky_rhino.sql @@ -0,0 +1,39 @@ +-- Denormalize record_id + type onto version_records, and store per-type counts +-- on versions. +-- +-- Hand-edited after drizzle-kit generate: the generated file added both columns +-- as NOT NULL in one statement, which fails on a non-empty table. Columns are +-- added nullable, backfilled, then constrained. +-- +-- The backfill rewrites every version_records row and holds an ACCESS EXCLUSIVE +-- lock for the duration. At the current scale (hundreds of thousands of rows) +-- that is seconds; on a much larger table, run the UPDATE in batches out of +-- band first so the in-migration UPDATE is a no-op. + +ALTER TABLE "version_records" ADD COLUMN "record_id" text;--> statement-breakpoint +ALTER TABLE "version_records" ADD COLUMN "type" text;--> statement-breakpoint +ALTER TABLE "versions" ADD COLUMN "type_counts" jsonb;--> statement-breakpoint + +UPDATE "version_records" vr +SET "record_id" = ro."record_id", "type" = ro."type" +FROM "record_objects" ro +WHERE ro."hash" = vr."record_hash" AND vr."record_id" IS NULL;--> statement-breakpoint + +ALTER TABLE "version_records" ALTER COLUMN "record_id" SET NOT NULL;--> statement-breakpoint +ALTER TABLE "version_records" ALTER COLUMN "type" SET NOT NULL;--> statement-breakpoint + +CREATE INDEX "version_records_version_record_idx" ON "version_records" USING btree ("version_id","record_id");--> statement-breakpoint +CREATE INDEX "version_records_version_type_record_idx" ON "version_records" USING btree ("version_id","type","record_id");--> statement-breakpoint + +UPDATE "versions" v +SET "type_counts" = c.counts +FROM ( + SELECT "version_id", jsonb_object_agg("type", n) AS counts + FROM ( + SELECT "version_id", "type", count(*) AS n + FROM "version_records" + GROUP BY "version_id", "type" + ) per_type + GROUP BY "version_id" +) c +WHERE c."version_id" = v."id"; diff --git a/src/db/migrations/meta/0007_snapshot.json b/src/db/migrations/meta/0007_snapshot.json new file mode 100644 index 0000000..af62439 --- /dev/null +++ b/src/db/migrations/meta/0007_snapshot.json @@ -0,0 +1,2703 @@ +{ + "id": "3dee55e9-4689-42bf-add6-d3b586d0ff2a", + "prevId": "6a188973-de2c-4826-af54-a3bfb08dd5d7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.apikey": { + "name": "apikey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_request": { + "name": "last_request", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + { + "expression": "config_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_collections": { + "name": "ark_collections", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "nullsNotDistinct": false, + "columns": ["ark_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_record_types": { + "name": "ark_record_types", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "name": "ark_record_types_collection_id_record_type_pk", + "columns": ["collection_id", "record_type"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_shoulders": { + "name": "ark_shoulders", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_shoulders_organization_id_unique": { + "name": "ark_shoulders_organization_id_unique", + "nullsNotDistinct": false, + "columns": ["organization_id"] + }, + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "nullsNotDistinct": false, + "columns": ["shoulder"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collection_webhooks": { + "name": "collection_webhooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bump_filter": { + "name": "bump_filter", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{major,minor,patch}'::text[]" + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "collection_webhooks_collection_id_idx": { + "name": "collection_webhooks_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collection_webhooks_created_by_user_id_fk": { + "name": "collection_webhooks_created_by_user_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collections": { + "name": "collections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "forked_from": { + "name": "forked_from", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "collections_organization_id_idx": { + "name": "collections_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collections_forked_from_collections_id_fk": { + "name": "collections_forked_from_collections_id_fk", + "tableFrom": "collections", + "tableTo": "collections", + "columnsFrom": ["forked_from"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "collections_organization_id_slug_unique": { + "name": "collections_organization_id_slug_unique", + "nullsNotDistinct": false, + "columns": ["organization_id", "slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.files": { + "name": "files", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instance_settings": { + "name": "instance_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_session_manifest": { + "name": "negotiate_session_manifest", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "needed": { + "name": "needed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + } + }, + "indexes": { + "nsm_session_needed_idx": { + "name": "nsm_session_needed_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "needed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "negotiate_session_manifest_session_id_negotiate_sessions_id_fk": { + "name": "negotiate_session_manifest_session_id_negotiate_sessions_id_fk", + "tableFrom": "negotiate_session_manifest", + "tableTo": "negotiate_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "negotiate_session_manifest_session_id_hash_pk": { + "name": "negotiate_session_manifest_session_id_hash_pk", + "columns": ["session_id", "hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_sessions": { + "name": "negotiate_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schemas": { + "name": "schemas", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_hashes": { + "name": "file_hashes", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "needed_files": { + "name": "needed_files", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "negotiate_sessions_collection_id_collections_id_fk": { + "name": "negotiate_sessions_collection_id_collections_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "negotiate_sessions_user_id_user_id_fk": { + "name": "negotiate_sessions_user_id_user_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": { + "organization_slug_uidx": { + "name": "organization_slug_uidx", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.page_comments": { + "name": "page_comments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor": { + "name": "anchor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "quote": { + "name": "quote", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "quote_context": { + "name": "quote_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "resolution_note": { + "name": "resolution_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "edited_at": { + "name": "edited_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "page_comments_page_anchor_idx": { + "name": "page_comments_page_anchor_idx", + "columns": [ + { + "expression": "page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "anchor", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "page_comments_user_id_idx": { + "name": "page_comments_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "page_comments_user_id_user_id_fk": { + "name": "page_comments_user_id_user_id_fk", + "tableFrom": "page_comments", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.record_objects": { + "name": "record_objects", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "record_objects_record_id_idx": { + "name": "record_objects_record_id_idx", + "columns": [ + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schema_labels": { + "name": "schema_labels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": [ + { + "expression": "label", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "schema_labels_schema_id_schemas_id_fk": { + "name": "schema_labels_schema_id_schemas_id_fk", + "tableFrom": "schema_labels", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schema_labels_schema_id_label_unique": { + "name": "schema_labels_schema_id_label_unique", + "nullsNotDistinct": false, + "columns": ["schema_id", "label"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schemas": { + "name": "schemas", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schemas_schema_hash_unique": { + "name": "schemas_schema_hash_unique", + "nullsNotDistinct": false, + "columns": ["schema_hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sync_runs": { + "name": "sync_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collections_synced": { + "name": "collections_synced", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_created": { + "name": "collections_created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_failed": { + "name": "collections_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "versions_pulled": { + "name": "versions_pulled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_downloaded": { + "name": "files_downloaded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_skipped": { + "name": "files_skipped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "logs": { + "name": "logs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_files": { + "name": "version_files", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_hash": { + "name": "file_hash", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_files_file_hash_idx": { + "name": "version_files_file_hash_idx", + "columns": [ + { + "expression": "file_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_files_version_id_versions_id_fk": { + "name": "version_files_version_id_versions_id_fk", + "tableFrom": "version_files", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_files_file_hash_files_hash_fk": { + "name": "version_files_file_hash_files_hash_fk", + "tableFrom": "version_files", + "tableTo": "files", + "columnsFrom": ["file_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_files_version_id_file_hash_pk": { + "name": "version_files_version_id_file_hash_pk", + "columns": ["version_id", "file_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_records": { + "name": "version_records", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "record_hash": { + "name": "record_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_record_hash": { + "name": "public_record_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_records_record_hash_idx": { + "name": "version_records_record_hash_idx", + "columns": [ + { + "expression": "record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_public_record_hash_idx": { + "name": "version_records_public_record_hash_idx", + "columns": [ + { + "expression": "public_record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "public_record_hash IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_record_idx": { + "name": "version_records_version_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_type_record_idx": { + "name": "version_records_version_type_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_records_version_id_versions_id_fk": { + "name": "version_records_version_id_versions_id_fk", + "tableFrom": "version_records", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_records_record_hash_record_objects_hash_fk": { + "name": "version_records_record_hash_record_objects_hash_fk", + "tableFrom": "version_records", + "tableTo": "record_objects", + "columnsFrom": ["record_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_records_version_id_record_hash_pk": { + "name": "version_records_version_id_record_hash_pk", + "columns": ["version_id", "record_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_schemas": { + "name": "version_schemas", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_schemas_schema_id_idx": { + "name": "version_schemas_schema_id_idx", + "columns": [ + { + "expression": "schema_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_schemas_version_id_versions_id_fk": { + "name": "version_schemas_version_id_versions_id_fk", + "tableFrom": "version_schemas", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_schemas_schema_id_schemas_id_fk": { + "name": "version_schemas_schema_id_schemas_id_fk", + "tableFrom": "version_schemas", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_schemas_version_id_slug_pk": { + "name": "version_schemas_version_id_slug_pk", + "columns": ["version_id", "slug"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.versions": { + "name": "versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_hash": { + "name": "public_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_counts": { + "name": "type_counts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "versions_ordering_idx": { + "name": "versions_ordering_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "major", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "minor", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "patch", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "versions_pushed_by_user_id_fk": { + "name": "versions_pushed_by_user_id_fk", + "tableFrom": "versions", + "tableTo": "user", + "columnsFrom": ["pushed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "versions_collection_id_semver_unique": { + "name": "versions_collection_id_semver_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "semver"] + }, + "versions_collection_id_hash_unique": { + "name": "versions_collection_id_hash_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "webhook_id": { + "name": "webhook_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "webhook_deliveries_webhook_id_idx": { + "name": "webhook_deliveries_webhook_id_idx", + "columns": [ + { + "expression": "webhook_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_collection_id_idx": { + "name": "webhook_deliveries_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_created_at_idx": { + "name": "webhook_deliveries_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_sweep_idx": { + "name": "webhook_deliveries_sweep_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_collection_id_collections_id_fk": { + "name": "webhook_deliveries_collection_id_collections_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_version_id_versions_id_fk": { + "name": "webhook_deliveries_version_id_versions_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/src/db/migrations/meta/_journal.json b/src/db/migrations/meta/_journal.json index 20dc478..a131c5f 100644 --- a/src/db/migrations/meta/_journal.json +++ b/src/db/migrations/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1784050001911, "tag": "0006_moaning_mauler", "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1785525852093, + "tag": "0007_lucky_rhino", + "breakpoints": true } ] } diff --git a/src/db/schema.ts b/src/db/schema.ts index 5308a5a..372c7ef 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -238,6 +238,12 @@ export const versions = pgTable( signature: text('signature'), recordCount: integer('record_count').notNull(), fileCount: integer('file_count').notNull(), + // { [type]: count } for this version's record set, computed once at commit. + // Avoids a COUNT(*) GROUP BY over version_records on every collection page + // view and gives `?type=` listings an accurate pagination.total. NULL on + // versions written before this column existed — callers fall back to a + // count query. + typeCounts: jsonb('type_counts').$type>(), totalBytes: bigint('total_bytes', { mode: 'number' }).notNull(), status: text('status').notNull().default('ready'), createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), @@ -279,6 +285,14 @@ export const versionRecords = pgTable( // equals record_hash (i.e. the type has no private fields), which is the // common case — only private-field bindings pay the storage cost. publicRecordHash: text('public_record_hash'), + // Denormalized from record_objects. Records are immutable and + // content-addressed, so these can never drift from the row they were copied + // from. Carrying them here is what lets record listing, `?type=` filtering + // and the diff/delta set operations run as index-only scans over + // version_records instead of joining every candidate row through + // record_objects just to order or filter it. + recordId: text('record_id').notNull(), + type: text('type').notNull(), }, (t) => [ primaryKey({ columns: [t.versionId, t.recordHash] }), @@ -286,6 +300,11 @@ export const versionRecords = pgTable( index('version_records_public_record_hash_idx') .on(t.publicRecordHash) .where(sql`public_record_hash IS NOT NULL`), + // Drives keyset record listing and the added/removed/updated anti- and + // semi-joins between two versions. + index('version_records_version_record_idx').on(t.versionId, t.recordId), + // Same, for `?type=`-filtered listings. + index('version_records_version_type_record_idx').on(t.versionId, t.type, t.recordId), ], ) diff --git a/src/db/seed.ts b/src/db/seed.ts index d4e9dd4..2f89002 100644 --- a/src/db/seed.ts +++ b/src/db/seed.ts @@ -65,9 +65,14 @@ async function insertRecords( } }) await db.insert(schema.recordObjects).values(objectRows).onConflictDoNothing() - await db - .insert(schema.versionRecords) - .values(objectRows.map((r) => ({ versionId, recordHash: r.hash }))) + await db.insert(schema.versionRecords).values( + objectRows.map((r) => ({ + versionId, + recordHash: r.hash, + recordId: r.recordId, + type: r.type, + })), + ) } /** Insert schemas into global table, returning schema IDs. Deduplicates by hash. */ diff --git a/src/db/seedKfCollections.ts b/src/db/seedKfCollections.ts index d47829b..f43135c 100644 --- a/src/db/seedKfCollections.ts +++ b/src/db/seedKfCollections.ts @@ -86,7 +86,14 @@ async function insertRecords( await db.insert(schema.recordObjects).values(objectRows).onConflictDoNothing() await db .insert(schema.versionRecords) - .values(objectRows.map((r) => ({ versionId, recordHash: r.hash }))) + .values( + objectRows.map((r) => ({ + versionId, + recordHash: r.hash, + recordId: r.recordId, + type: r.type, + })), + ) .onConflictDoNothing() } diff --git a/src/lib/mirror-sync.ts b/src/lib/mirror-sync.ts index 288e572..5d614b6 100644 --- a/src/lib/mirror-sync.ts +++ b/src/lib/mirror-sync.ts @@ -687,6 +687,10 @@ async function pullVersion( actorId: uv.actorId, recordCount: manifest.records.length, fileCount: manifest.files.length, + typeCounts: manifest.records.reduce>((acc, r) => { + acc[r.type] = (acc[r.type] ?? 0) + 1 + return acc + }, {}), totalBytes: uv.totalBytes, status: 'creating', }) @@ -702,11 +706,16 @@ async function pullVersion( } } - for (let i = 0; i < manifestHashes.length; i += BATCH_SIZE) { - const batch = manifestHashes.slice(i, i + BATCH_SIZE) - await db - .insert(schema.versionRecords) - .values(batch.map((hash) => ({ versionId, recordHash: hash }))) + for (let i = 0; i < manifest.records.length; i += BATCH_SIZE) { + const batch = manifest.records.slice(i, i + BATCH_SIZE) + await db.insert(schema.versionRecords).values( + batch.map((r) => ({ + versionId, + recordHash: r.hash, + recordId: r.id, + type: r.type, + })), + ) } const fileHashList = manifest.files.filter((h) => availableFileHashes.has(h)) diff --git a/src/routes/docs/api/versions.tsx b/src/routes/docs/api/versions.tsx index a13464e..220c20e 100644 --- a/src/routes/docs/api/versions.tsx +++ b/src/routes/docs/api/versions.tsx @@ -86,7 +86,32 @@ const manifestRes = `{ {"id": "pub-001", "type": "Publication", "hash": "sha256:def456..."}, {"id": "pub-002", "type": "Publication", "hash": "sha256:789abc..."} ], - "files": ["sha256:a1b2c3...", "sha256:d4e5f6..."] + "files": ["sha256:a1b2c3...", "sha256:d4e5f6..."], + "pagination": { + "limit": 10000, + "hasMore": true, + "nextCursor": "eyJhZGRlZCI6WyJwdWItMDAyIiwiZGVmNDU2Il0..." + } +}` + +const manifestDeltaRes = `{ + "semver": "v1.1.0", + "hash": "a1b2c3d4...", + "since": "v1.0.0", + "schemas": {"Publication": "sha256:abc123..."}, + "delta": { + "added": [{"id": "pub-003", "type": "Publication", "hash": "sha256:..."}], + "updated": [{"id": "pub-001", "type": "Publication", "hash": "sha256:...", + "previousHash": "sha256:..."}], + "removed": [{"id": "pub-old", "type": "Publication", "hash": "sha256:..."}] + }, + "files": ["sha256:a1b2c3..."], + "pagination": { + "limit": 10000, + "hasMore": false, + "nextCursor": null + }, + "truncated": false }` const diffRes = `{ @@ -98,7 +123,18 @@ const diffRes = `{ "updated": [ {"id": "pub-001", "type": "Publication", "data": {...}} ], - "removed": ["pub-old"] + "removed": ["pub-old"], + "pagination": { + "limit": 500, + "hasMore": false, + "nextCursor": null + }, + "meta": { + "schemaChanged": false, + "metadataChanged": false, + "filesAdded": 0, + "filesRemoved": 0 + } }` export default function DocsApiVersions() { @@ -406,7 +442,7 @@ export default function DocsApiVersions() { limit - Max results (default 100, max 1000) + Max results (default 100, max 2000) @@ -428,6 +464,11 @@ export default function DocsApiVersions() { +

+ Walking a whole collection is bounded by request count, not bytes: 60 requests/minute + anonymous, 5,000 authenticated. Ask for the largest page you can handle — a + 3-million-record collection is 6,200 requests at 500/page and 1,550 at 2,000/page. +

Response 200

@@ -440,9 +481,10 @@ export default function DocsApiVersions() { collections, always paginate with after rather than offset.

- Note: pagination.total is the whole-version record count and is not adjusted - when type is set — use hasMore to detect the end of a filtered - result set. + pagination.total respects the type filter and excludes private + types. On collections that mark individual records private it is an upper bound for + anonymous callers, since those records are hidden but still counted — use{' '} + hasMore if you need an exact end-of-set signal.

@@ -452,15 +494,63 @@ export default function DocsApiVersions() {

GET /api/collections/:owner/:slug/versions/:n/manifest

No auth for public collections

- Get the manifest: a lightweight summary of what's in a version without the full record - data. + Get the manifest: every record's id, type and content hash, without the bodies. This is + the cheapest way to learn what a version contains — at roughly 120 bytes per entry, a + million records is one order of magnitude smaller than fetching them.

+

Query parameters

+ + + + + + + + + + + + + + + +
+ limit + Entries per page (default 10000, max 100000)
+ cursor + + Opaque keyset cursor from pagination.nextCursor. Do not construct or + parse it — pass back exactly what you were given. +
+ since + + Return a delta against this semver instead of the full manifest: which records were + added, updated and removed between the two versions. +

Response 200 

           {manifestRes}
+

+ Response with ?since= 200 +

+ 
+          {manifestDeltaRes}
+
+

+ A delta of any size can be walked to completion: keep re-requesting with{' '} + cursor=pagination.nextCursor until hasMore is false. The three + lists drain independently and the cursor tracks each one, so a page late in the walk may + contain only updated entries. +

+

+ truncated is retained for older clients, which treated a capped delta as + "give up and rebuild from the full manifest". It now simply mirrors{' '} + pagination.hasMore. Clients that understand the cursor should page instead of + rebuilding. +

@@ -483,6 +573,22 @@ export default function DocsApiVersions() { Semver to diff from (e.g. v1.0.0). Default: previous version. + + + limit + + Entries per list per page (default 500, max 5000) + + + + cursor + + + Opaque keyset cursor from pagination.nextCursor, as on the manifest + endpoint. Diff returns full record bodies, so pages are much larger than manifest + pages — prefer manifest?since= when you only need the hashes. + +

From c05e59abeed397242c49f381c22932cd0372d7a7 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Fri, 31 Jul 2026 16:43:48 -0400 Subject: [PATCH 3/7] Streaming part 1 --- src/api/negotiate.ts | 360 ++- src/db/migrations/0008_mixed_moonstone.sql | 13 + src/db/migrations/meta/0008_snapshot.json | 2736 ++++++++++++++++++++ src/db/migrations/meta/_journal.json | 7 + src/db/schema.ts | 15 + src/lib/core/index.ts | 7 +- src/lib/core/version-hash.test.ts | 90 +- src/lib/core/version-hash.ts | 66 + src/lib/version-helpers.server.ts | 1 + 9 files changed, 3176 insertions(+), 119 deletions(-) create mode 100644 src/db/migrations/0008_mixed_moonstone.sql create mode 100644 src/db/migrations/meta/0008_snapshot.json diff --git a/src/api/negotiate.ts b/src/api/negotiate.ts index cacf277..ee3d3b4 100644 --- a/src/api/negotiate.ts +++ b/src/api/negotiate.ts @@ -8,7 +8,6 @@ import { ajv, canonicalize, checkSchemaBounds, - computeVersionHash, deriveSemver, filterRecordData, filterTypeSchema, @@ -24,6 +23,7 @@ import { resolveCollection, type SchemaEntry, stripToSchema, + VersionHashStream, } from '../lib/version-helpers.server.js' import { bumpTypeFromChanges, @@ -565,15 +565,6 @@ app.post( ) } - const manifestEntries = await db - .select({ - hash: schema.negotiateSessionManifest.hash, - needed: schema.negotiateSessionManifest.needed, - private: schema.negotiateSessionManifest.private, - }) - .from(schema.negotiateSessionManifest) - .where(eq(schema.negotiateSessionManifest.sessionId, sessionId)) - // --- Schema resolution --- const newSchemaSet: { slug: string @@ -654,61 +645,100 @@ app.post( if (fields.size > 0) privateFieldsByType.set(entry.slug, fields) } - const manifestHashes = manifestEntries.map((r) => r.hash) - const manifestPrivateMap = new Map(manifestEntries.map((r) => [r.hash, r.private])) - const finalRecordHashes: string[] = [] - const publicRecordHashes: string[] = [] - // record hash → public record hash, only where they differ (private-field types) - const publicHashByRecordHash = new Map() - const strippedRecordObjects: { - hash: string - recordId: string - type: string - data: unknown - private: boolean - size: number - }[] = [] + // Walk the manifest in keyset batches, writing each record's outcome back to + // the session manifest row instead of accumulating it in process. Nothing + // here grows with collection size: at 3.11M records the old arrays alone + // (final hashes, public hashes, the manifest itself) ran to several GB. const validationErrors: { recordId: string; type: string; errors: string[] }[] = [] const extraFieldWarnings: { recordId: string; type: string; fields: string[] }[] = [] + // Errors are reported, not accumulated: a schema change that invalidates + // every record would otherwise build a multi-million-entry response. + const MAX_REPORTED_ERRORS = 100 + let validationErrorCount = 0 + let extraFieldCount = 0 + let recordCount = 0 + let totalBytes = 0 // Per-type counts, stored on the version row. The commit already walks every // record, so counting here is free and saves a COUNT(*) GROUP BY on every // subsequent collection page view. const typeCounts = new Map() - let totalBytes = 0 - const LOAD_BATCH = 1000 - for (let i = 0; i < manifestHashes.length; i += LOAD_BATCH) { - const batchHashes = manifestHashes.slice(i, i + LOAD_BATCH) - const rows = await db + // Each batch costs a read plus a write-back, so the round-trip count is what + // dominates commit wall-clock. 5,000 records of bodies is ~10 MB in flight — + // bounded, and constant regardless of how large the collection is. + const LOAD_BATCH = 5000 + let cursor: string | null = null + for (;;) { + const batch: { + hash: string + manifestPrivate: boolean + recordId: string + type: string + data: unknown + private: boolean + size: number + }[] = await db .select({ - hash: schema.recordObjects.hash, + hash: schema.negotiateSessionManifest.hash, + manifestPrivate: schema.negotiateSessionManifest.private, recordId: schema.recordObjects.recordId, type: schema.recordObjects.type, data: schema.recordObjects.data, private: schema.recordObjects.private, size: schema.recordObjects.size, }) - .from(schema.recordObjects) - .where(inArray(schema.recordObjects.hash, batchHashes)) - - for (const rec of rows) { + .from(schema.negotiateSessionManifest) + .innerJoin( + schema.recordObjects, + eq(schema.negotiateSessionManifest.hash, schema.recordObjects.hash), + ) + .where( + and( + eq(schema.negotiateSessionManifest.sessionId, sessionId), + ...(cursor ? [sql`${schema.negotiateSessionManifest.hash} > ${cursor}`] : []), + ), + ) + .orderBy(schema.negotiateSessionManifest.hash) + .limit(LOAD_BATCH) + + if (batch.length === 0) break + cursor = batch[batch.length - 1]!.hash + + // Per-batch outcomes, flushed to Postgres before the next batch is read. + const stripped: { + hash: string + recordId: string + type: string + data: unknown + private: boolean + size: number + }[] = [] + const outcomes: { hash: string; finalHash: string; publicHash: string | null }[] = [] + + for (const rec of batch) { const validate = validators.get(rec.type) if (!validate) { - validationErrors.push({ - recordId: rec.recordId, - type: rec.type, - errors: [`No schema defined for record type "${rec.type}"`], - }) + validationErrorCount++ + if (validationErrors.length < MAX_REPORTED_ERRORS) { + validationErrors.push({ + recordId: rec.recordId, + type: rec.type, + errors: [`No schema defined for record type "${rec.type}"`], + }) + } continue } if (!validate(rec.data)) { - validationErrors.push({ - recordId: rec.recordId, - type: rec.type, - errors: (validate.errors ?? []).map( - (e) => `${e.instancePath || '/'} ${e.message ?? 'validation failed'}`, - ), - }) + validationErrorCount++ + if (validationErrors.length < MAX_REPORTED_ERRORS) { + validationErrors.push({ + recordId: rec.recordId, + type: rec.type, + errors: (validate.errors ?? []).map( + (e) => `${e.instancePath || '/'} ${e.message ?? 'validation failed'}`, + ), + }) + } continue } @@ -722,14 +752,17 @@ app.post( const extra = Object.keys(data).filter((k) => !(k in typeSchema.properties!)) if (extra.length > 0) { if (!session.stripUnknownFields) { - extraFieldWarnings.push({ recordId: rec.recordId, type: rec.type, fields: extra }) + extraFieldCount++ + if (extraFieldWarnings.length < MAX_REPORTED_ERRORS) { + extraFieldWarnings.push({ recordId: rec.recordId, type: rec.type, fields: extra }) + } } else { data = stripToSchema(data as Record, typeSchema.properties) const result = hashRecord({ id: rec.recordId, type: rec.type, data }) if (hash !== result.hash) { hash = result.hash size = Buffer.byteLength(result.canonical, 'utf-8') - strippedRecordObjects.push({ + stripped.push({ hash, recordId: rec.recordId, type: rec.type, @@ -742,35 +775,81 @@ app.post( } } - finalRecordHashes.push(hash) + recordCount++ typeCounts.set(rec.type, (typeCounts.get(rec.type) ?? 0) + 1) totalBytes += size - // Compute public record hash inline - const isPrivate = rec.private || manifestPrivateMap.get(rec.hash) || false + // Compute the public record hash inline + const isPrivate = rec.private || rec.manifestPrivate + let publicHash: string | null = null if (!isPrivate && !privateTypes.has(rec.type)) { const privateFields = privateFieldsByType.get(rec.type) const publicData = privateFields && privateFields.size > 0 ? filterRecordData(data, privateFields) : data - const publicHash = hashRecord({ id: rec.recordId, type: rec.type, data: publicData }).hash - publicRecordHashes.push(publicHash) - // Persist the public address only when filtering changed the content - if (publicHash !== hash) publicHashByRecordHash.set(hash, publicHash) + publicHash = hashRecord({ id: rec.recordId, type: rec.type, data: publicData }).hash } + outcomes.push({ hash: rec.hash, finalHash: hash, publicHash }) + } + + // Stop as soon as the push is known to be rejected — there is no point + // walking millions more records to grow an error list we already capped. + if (validationErrorCount > 0 || extraFieldCount > 0) break + + if (stripped.length > 0) { + // Record objects are global, immutable and content-addressed, so writing + // them before the version exists is safe; a failed commit leaves them + // orphaned exactly as a failed record submission already does. Conflicts + // are expected and mean the identical body is already stored — stripping + // the same records twice is a normal repeat push, not an error. + await db + .insert(schema.recordObjects) + .values( + stripped.map((r) => ({ + hash: r.hash, + recordId: r.recordId, + type: r.type, + data: r.data as any, + private: r.private, + size: r.size, + })), + ) + .onConflictDoNothing() + } + + if (outcomes.length > 0) { + await db.execute(sql` + UPDATE negotiate_session_manifest m + SET final_hash = o.final_hash, public_hash = o.public_hash + FROM unnest( + ${sql.param(outcomes.map((o) => o.hash))}::text[], + ${sql.param(outcomes.map((o) => o.finalHash))}::text[], + ${sql.param(outcomes.map((o) => o.publicHash))}::text[] + ) AS o(hash, final_hash, public_hash) + WHERE m.session_id = ${sessionId} AND m.hash = o.hash + `) } } - if (validationErrors.length > 0) { + if (validationErrorCount > 0) { await expireSession(sessionId) - return c.json({ error: 'Schema validation failed', validationErrors, statusCode: 422 }, 422) + return c.json( + { + error: 'Schema validation failed', + validationErrors, + totalErrors: validationErrorCount, + statusCode: 422, + }, + 422, + ) } - if (extraFieldWarnings.length > 0) { + if (extraFieldCount > 0) { await expireSession(sessionId) return c.json( { error: 'Records contain fields not defined in schema', extraFields: extraFieldWarnings, + totalRecords: extraFieldCount, hint: 'Set strip_unknown_fields: true in the negotiate request to strip these fields.', statusCode: 422, }, @@ -815,16 +894,26 @@ app.post( } } - // Determine if records changed vs previous version + // Determine if records changed vs previous version. Set comparison done in + // Postgres: loading the previous version's hashes was a second full-size + // array on top of everything else. let recordsChanged = true if (latest) { - const prevHashes = await db - .select({ hash: schema.versionRecords.recordHash }) - .from(schema.versionRecords) - .where(eq(schema.versionRecords.versionId, latest.id)) - const prevSet = new Set(prevHashes.map((r) => r.hash)) - const newSet = new Set(finalRecordHashes) - recordsChanged = prevSet.size !== newSet.size || [...newSet].some((h) => !prevSet.has(h)) + const [cmp] = (await db.execute(sql` + SELECT + (SELECT count(DISTINCT final_hash) FROM negotiate_session_manifest + WHERE session_id = ${sessionId}) AS new_count, + (SELECT count(*) FROM version_records WHERE version_id = ${latest.id}) AS old_count, + EXISTS ( + SELECT 1 FROM negotiate_session_manifest m + WHERE m.session_id = ${sessionId} + AND NOT EXISTS ( + SELECT 1 FROM version_records vr + WHERE vr.version_id = ${latest.id} AND vr.record_hash = m.final_hash + ) + ) AS has_new + `)) as unknown as { new_count: string; old_count: string; has_new: boolean }[] + recordsChanged = Number(cmp!.new_count) !== Number(cmp!.old_count) || cmp!.has_new } const prevMetadata = (latest?.metadata as Record) ?? null @@ -835,27 +924,64 @@ app.post( JSON.stringify(metadataValue ? canonicalize(metadataValue) : null) !== JSON.stringify(prevMetadata ? canonicalize(prevMetadata) : null) - const schemaSetForHash = newSchemaSet.map((e) => ({ slug: e.slug, schemaHash: e.schemaHash })) - const versionHash = computeVersionHash( - schemaSetForHash, - finalRecordHashes, - session.fileHashes, - metadataValue, - ) - - // Compute public hash from pre-accumulated public record hashes const publicSchemaSet: { slug: string; schemaHash: string }[] = [] for (const entry of schemaEntriesForPublicHash) { if (privateTypes.has(entry.slug)) continue const filtered = filterTypeSchema(entry.schema) publicSchemaSet.push({ slug: entry.slug, schemaHash: hashSchema(filtered) }) } - const publicHash = computeVersionHash( + + // Both version hashes are folded incrementally over hashes streamed out of + // Postgres in sorted order, rather than sorting N hashes in memory and + // stringifying them into one ~200 MB document. VersionHashStream is + // byte-compatible with computeVersionHash — see its test. + // + // COLLATE "C" is required, not cosmetic: the digest must see the hashes in + // the same order Array.prototype.sort() would produce, which is byte order, + // not the database's locale collation. + const versionHashStream = new VersionHashStream( + newSchemaSet.map((e) => ({ slug: e.slug, schemaHash: e.schemaHash })), + session.fileHashes, + metadataValue, + ) + const publicHashStream = new VersionHashStream( publicSchemaSet, - publicRecordHashes, session.fileHashes, metadataValue, - ).replace('private:', 'public:') + ) + + const HASH_PAGE = 50_000 + for (const [stream, column] of [ + [versionHashStream, sql`final_hash`], + [publicHashStream, sql`public_hash`], + ] as const) { + // Two records that differ only in private fields share a public hash, so + // the value alone is not a unique cursor — the tiebreak is the manifest's + // own primary key, or duplicates straddling a page boundary get dropped + // and the digest silently changes. + let at: { value: string; tiebreak: string } | null = null + for (;;) { + const rows = (await db.execute(sql` + SELECT ${column} AS h, hash AS tiebreak FROM negotiate_session_manifest + WHERE session_id = ${sessionId} AND ${column} IS NOT NULL + ${ + at + ? sql`AND (${column} COLLATE "C", hash COLLATE "C") + > (${at.value} COLLATE "C", ${at.tiebreak} COLLATE "C")` + : sql`` + } + ORDER BY ${column} COLLATE "C", hash COLLATE "C" + LIMIT ${HASH_PAGE} + `)) as unknown as { h: string; tiebreak: string }[] + if (rows.length === 0) break + for (const row of rows) stream.push(row.h) + if (rows.length < HASH_PAGE) break + const last = rows[rows.length - 1]! + at = { value: last.h, tiebreak: last.tiebreak } + } + } + const versionHash = versionHashStream.digest() + const publicHash = publicHashStream.digest().replace('private:', 'public:') const sv = deriveSemver(latest?.semver ?? null, schemaChanged, recordsChanged, metadataChanged) @@ -905,7 +1031,7 @@ app.post( pushedBy: c.get('userId') ?? null, appId: session.appId, actorId: session.actorId, - recordCount: finalRecordHashes.length, + recordCount, fileCount: session.fileHashes.length, typeCounts: Object.fromEntries(typeCounts), totalBytes, @@ -915,26 +1041,6 @@ app.post( versionId = version!.id - if (strippedRecordObjects.length > 0) { - const BATCH = 1000 - for (let i = 0; i < strippedRecordObjects.length; i += BATCH) { - const batch = strippedRecordObjects.slice(i, i + BATCH) - await tx - .insert(schema.recordObjects) - .values( - batch.map((r) => ({ - hash: r.hash, - recordId: r.recordId, - type: r.type, - data: r.data as any, - private: r.private, - size: r.size, - })), - ) - .onConflictDoNothing() - } - } - if (session.fileHashes.length > 0) { await tx .insert(schema.versionFiles) @@ -950,27 +1056,47 @@ app.post( ) }) - // Batch-insert version_records outside the main transaction. - // record_id and type are read back out of record_objects rather than - // carried in process: the rows are guaranteed to exist by this point (they - // were either already stored or inserted by the transaction above), and at - // millions of records two more in-memory string arrays are exactly what the - // heap can't afford. + // Populate version_records outside the main transaction, server-side, in + // keyset batches over the session manifest. Nothing about the record set + // passes through the app: record_id and type come from record_objects, the + // hashes and public addresses from the manifest rows the validation pass + // already wrote. public_record_hash is stored only where it differs from + // the record hash, which is the column's existing contract. try { const VR_BATCH = 5000 - for (let i = 0; i < finalRecordHashes.length; i += VR_BATCH) { - const batch = finalRecordHashes.slice(i, i + VR_BATCH) - const publicHashes = batch.map((hash) => publicHashByRecordHash.get(hash) ?? null) - // sql.param binds each list as one array parameter; interpolating a bare - // array would expand it to a parenthesized list of placeholders, which - // Postgres rejects past 1,664 entries. + let vrCursor: string | null = null + for (;;) { + // Resolve the page's upper bound first. Doing it in one statement with + // RETURNING doesn't work: ON CONFLICT DO NOTHING makes the number of + // returned rows a count of insertions, not of manifest rows read, so it + // can't drive the cursor. + const [bound] = (await db.execute(sql` + SELECT max(hash) AS hi, count(*)::int AS n FROM ( + SELECT hash FROM negotiate_session_manifest + WHERE session_id = ${sessionId} + ${vrCursor ? sql`AND hash > ${vrCursor}` : sql``} + ORDER BY hash + LIMIT ${VR_BATCH} + ) page + `)) as unknown as { hi: string | null; n: number }[] + + if (!bound || bound.n === 0 || bound.hi === null) break + const hi = bound.hi + await db.execute(sql` INSERT INTO version_records (version_id, record_hash, public_record_hash, record_id, type) - SELECT ${versionId!}, ro.hash, t.public_hash, ro.record_id, ro.type - FROM unnest(${sql.param(batch)}::text[], ${sql.param(publicHashes)}::text[]) - AS t(hash, public_hash) - INNER JOIN record_objects ro ON ro.hash = t.hash + SELECT ${versionId!}, ro.hash, nullif(m.public_hash, m.final_hash), + ro.record_id, ro.type + FROM negotiate_session_manifest m + INNER JOIN record_objects ro ON ro.hash = m.final_hash + WHERE m.session_id = ${sessionId} + ${vrCursor ? sql`AND m.hash > ${vrCursor}` : sql``} + AND m.hash <= ${hi} + ON CONFLICT DO NOTHING `) + + vrCursor = hi + if (bound.n < VR_BATCH) break } } catch (err) { await db.delete(schema.versions).where(eq(schema.versions.id, versionId!)) @@ -1005,7 +1131,7 @@ app.post( major: sv.major, minor: sv.minor, patch: sv.patch, - recordCount: finalRecordHashes.length, + recordCount, fileCount: session.fileHashes.length, }, bumpTypeFromChanges(schemaChanged, recordsChanged), @@ -1020,7 +1146,7 @@ app.post( { semver: sv.semver, hash: versionHash, - recordCount: finalRecordHashes.length, + recordCount, fileCount: session.fileHashes.length, }, 201, diff --git a/src/db/migrations/0008_mixed_moonstone.sql b/src/db/migrations/0008_mixed_moonstone.sql new file mode 100644 index 0000000..720a580 --- /dev/null +++ b/src/db/migrations/0008_mixed_moonstone.sql @@ -0,0 +1,13 @@ +-- Commit scratch columns on the negotiate session manifest. +-- +-- The commit's validation pass writes each record's post-strip hash and public +-- content-address back here instead of accumulating them in app memory; the +-- version hash and the version_records insert are then streamed out of Postgres +-- in sorted order. +-- +-- Nullable with no backfill: they are only ever written during a commit, and +-- sessions expire after 10 minutes, so nothing pre-existing needs a value. + +ALTER TABLE "negotiate_session_manifest" ADD COLUMN "final_hash" text;--> statement-breakpoint +ALTER TABLE "negotiate_session_manifest" ADD COLUMN "public_hash" text;--> statement-breakpoint +CREATE INDEX "nsm_session_final_hash_idx" ON "negotiate_session_manifest" USING btree ("session_id","final_hash"); \ No newline at end of file diff --git a/src/db/migrations/meta/0008_snapshot.json b/src/db/migrations/meta/0008_snapshot.json new file mode 100644 index 0000000..deac199 --- /dev/null +++ b/src/db/migrations/meta/0008_snapshot.json @@ -0,0 +1,2736 @@ +{ + "id": "5321bb12-5ad4-4a57-a77a-c949f94c965d", + "prevId": "3dee55e9-4689-42bf-add6-d3b586d0ff2a", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.apikey": { + "name": "apikey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_request": { + "name": "last_request", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + { + "expression": "config_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_collections": { + "name": "ark_collections", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "nullsNotDistinct": false, + "columns": ["ark_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_record_types": { + "name": "ark_record_types", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "name": "ark_record_types_collection_id_record_type_pk", + "columns": ["collection_id", "record_type"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_shoulders": { + "name": "ark_shoulders", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_shoulders_organization_id_unique": { + "name": "ark_shoulders_organization_id_unique", + "nullsNotDistinct": false, + "columns": ["organization_id"] + }, + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "nullsNotDistinct": false, + "columns": ["shoulder"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collection_webhooks": { + "name": "collection_webhooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bump_filter": { + "name": "bump_filter", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{major,minor,patch}'::text[]" + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "collection_webhooks_collection_id_idx": { + "name": "collection_webhooks_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collection_webhooks_created_by_user_id_fk": { + "name": "collection_webhooks_created_by_user_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collections": { + "name": "collections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "forked_from": { + "name": "forked_from", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "collections_organization_id_idx": { + "name": "collections_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collections_forked_from_collections_id_fk": { + "name": "collections_forked_from_collections_id_fk", + "tableFrom": "collections", + "tableTo": "collections", + "columnsFrom": ["forked_from"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "collections_organization_id_slug_unique": { + "name": "collections_organization_id_slug_unique", + "nullsNotDistinct": false, + "columns": ["organization_id", "slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.files": { + "name": "files", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instance_settings": { + "name": "instance_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_session_manifest": { + "name": "negotiate_session_manifest", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "needed": { + "name": "needed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "final_hash": { + "name": "final_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_hash": { + "name": "public_hash", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "nsm_session_needed_idx": { + "name": "nsm_session_needed_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "needed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "nsm_session_final_hash_idx": { + "name": "nsm_session_final_hash_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "final_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "negotiate_session_manifest_session_id_negotiate_sessions_id_fk": { + "name": "negotiate_session_manifest_session_id_negotiate_sessions_id_fk", + "tableFrom": "negotiate_session_manifest", + "tableTo": "negotiate_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "negotiate_session_manifest_session_id_hash_pk": { + "name": "negotiate_session_manifest_session_id_hash_pk", + "columns": ["session_id", "hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_sessions": { + "name": "negotiate_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schemas": { + "name": "schemas", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_hashes": { + "name": "file_hashes", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "needed_files": { + "name": "needed_files", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "negotiate_sessions_collection_id_collections_id_fk": { + "name": "negotiate_sessions_collection_id_collections_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "negotiate_sessions_user_id_user_id_fk": { + "name": "negotiate_sessions_user_id_user_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": { + "organization_slug_uidx": { + "name": "organization_slug_uidx", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.page_comments": { + "name": "page_comments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor": { + "name": "anchor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "quote": { + "name": "quote", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "quote_context": { + "name": "quote_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "resolution_note": { + "name": "resolution_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "edited_at": { + "name": "edited_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "page_comments_page_anchor_idx": { + "name": "page_comments_page_anchor_idx", + "columns": [ + { + "expression": "page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "anchor", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "page_comments_user_id_idx": { + "name": "page_comments_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "page_comments_user_id_user_id_fk": { + "name": "page_comments_user_id_user_id_fk", + "tableFrom": "page_comments", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.record_objects": { + "name": "record_objects", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "record_objects_record_id_idx": { + "name": "record_objects_record_id_idx", + "columns": [ + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schema_labels": { + "name": "schema_labels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": [ + { + "expression": "label", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "schema_labels_schema_id_schemas_id_fk": { + "name": "schema_labels_schema_id_schemas_id_fk", + "tableFrom": "schema_labels", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schema_labels_schema_id_label_unique": { + "name": "schema_labels_schema_id_label_unique", + "nullsNotDistinct": false, + "columns": ["schema_id", "label"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schemas": { + "name": "schemas", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schemas_schema_hash_unique": { + "name": "schemas_schema_hash_unique", + "nullsNotDistinct": false, + "columns": ["schema_hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sync_runs": { + "name": "sync_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collections_synced": { + "name": "collections_synced", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_created": { + "name": "collections_created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_failed": { + "name": "collections_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "versions_pulled": { + "name": "versions_pulled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_downloaded": { + "name": "files_downloaded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_skipped": { + "name": "files_skipped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "logs": { + "name": "logs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_files": { + "name": "version_files", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_hash": { + "name": "file_hash", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_files_file_hash_idx": { + "name": "version_files_file_hash_idx", + "columns": [ + { + "expression": "file_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_files_version_id_versions_id_fk": { + "name": "version_files_version_id_versions_id_fk", + "tableFrom": "version_files", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_files_file_hash_files_hash_fk": { + "name": "version_files_file_hash_files_hash_fk", + "tableFrom": "version_files", + "tableTo": "files", + "columnsFrom": ["file_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_files_version_id_file_hash_pk": { + "name": "version_files_version_id_file_hash_pk", + "columns": ["version_id", "file_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_records": { + "name": "version_records", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "record_hash": { + "name": "record_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_record_hash": { + "name": "public_record_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_records_record_hash_idx": { + "name": "version_records_record_hash_idx", + "columns": [ + { + "expression": "record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_public_record_hash_idx": { + "name": "version_records_public_record_hash_idx", + "columns": [ + { + "expression": "public_record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "public_record_hash IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_record_idx": { + "name": "version_records_version_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_type_record_idx": { + "name": "version_records_version_type_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_records_version_id_versions_id_fk": { + "name": "version_records_version_id_versions_id_fk", + "tableFrom": "version_records", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_records_record_hash_record_objects_hash_fk": { + "name": "version_records_record_hash_record_objects_hash_fk", + "tableFrom": "version_records", + "tableTo": "record_objects", + "columnsFrom": ["record_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_records_version_id_record_hash_pk": { + "name": "version_records_version_id_record_hash_pk", + "columns": ["version_id", "record_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_schemas": { + "name": "version_schemas", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_schemas_schema_id_idx": { + "name": "version_schemas_schema_id_idx", + "columns": [ + { + "expression": "schema_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_schemas_version_id_versions_id_fk": { + "name": "version_schemas_version_id_versions_id_fk", + "tableFrom": "version_schemas", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_schemas_schema_id_schemas_id_fk": { + "name": "version_schemas_schema_id_schemas_id_fk", + "tableFrom": "version_schemas", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_schemas_version_id_slug_pk": { + "name": "version_schemas_version_id_slug_pk", + "columns": ["version_id", "slug"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.versions": { + "name": "versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_hash": { + "name": "public_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_counts": { + "name": "type_counts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "versions_ordering_idx": { + "name": "versions_ordering_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "major", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "minor", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "patch", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "versions_pushed_by_user_id_fk": { + "name": "versions_pushed_by_user_id_fk", + "tableFrom": "versions", + "tableTo": "user", + "columnsFrom": ["pushed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "versions_collection_id_semver_unique": { + "name": "versions_collection_id_semver_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "semver"] + }, + "versions_collection_id_hash_unique": { + "name": "versions_collection_id_hash_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "webhook_id": { + "name": "webhook_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "webhook_deliveries_webhook_id_idx": { + "name": "webhook_deliveries_webhook_id_idx", + "columns": [ + { + "expression": "webhook_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_collection_id_idx": { + "name": "webhook_deliveries_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_created_at_idx": { + "name": "webhook_deliveries_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_sweep_idx": { + "name": "webhook_deliveries_sweep_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_collection_id_collections_id_fk": { + "name": "webhook_deliveries_collection_id_collections_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_version_id_versions_id_fk": { + "name": "webhook_deliveries_version_id_versions_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/src/db/migrations/meta/_journal.json b/src/db/migrations/meta/_journal.json index a131c5f..5bcc370 100644 --- a/src/db/migrations/meta/_journal.json +++ b/src/db/migrations/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1785525852093, "tag": "0007_lucky_rhino", "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1785528674026, + "tag": "0008_mixed_moonstone", + "breakpoints": true } ] } diff --git a/src/db/schema.ts b/src/db/schema.ts index 372c7ef..789d85a 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -412,10 +412,25 @@ export const negotiateSessionManifest = pgTable( hash: text('hash').notNull(), private: boolean('private').notNull().default(false), needed: boolean('needed').notNull().default(true), + // Commit scratch space. The commit walks every record once to validate and + // hash it; rather than accumulating the results in arrays that grow with + // collection size, it writes them back here and then streams them out of + // Postgres in sorted order. This table already exists per push and is + // already the size of the manifest, so it costs nothing new. + // + // finalHash is the record's hash after strip_unknown_fields rewrote it, or + // the submitted hash when nothing was stripped. publicHash is the + // content-address of the privacy-filtered record, NULL when the record does + // not appear in the public view at all. + finalHash: text('final_hash'), + publicHash: text('public_hash'), }, (t) => [ primaryKey({ columns: [t.sessionId, t.hash] }), index('nsm_session_needed_idx').on(t.sessionId, t.needed), + // Drives the sorted streams that feed the version hash and the + // version_records insert. + index('nsm_session_final_hash_idx').on(t.sessionId, t.finalHash), ], ) diff --git a/src/lib/core/index.ts b/src/lib/core/index.ts index 52538b4..fd17f06 100644 --- a/src/lib/core/index.ts +++ b/src/lib/core/index.ts @@ -8,5 +8,10 @@ export { stripToSchema, type ExtraFieldWarning, } from './validate.js' -export { computePublicHash, computeVersionHash, filterSchemasForPublic } from './version-hash.js' +export { + computePublicHash, + computeVersionHash, + filterSchemasForPublic, + VersionHashStream, +} from './version-hash.js' export type { SchemaEntry } from './types.js' diff --git a/src/lib/core/version-hash.test.ts b/src/lib/core/version-hash.test.ts index 858b1d7..581e309 100644 --- a/src/lib/core/version-hash.test.ts +++ b/src/lib/core/version-hash.test.ts @@ -3,7 +3,12 @@ import { describe, expect, it } from 'vitest' import { hashRecord, hashSchema } from './hash.js' import { filterRecordData } from './privacy.js' import type { SchemaEntry } from './types.js' -import { computePublicHash, computeVersionHash, filterSchemasForPublic } from './version-hash.js' +import { + computePublicHash, + computeVersionHash, + filterSchemasForPublic, + VersionHashStream, +} from './version-hash.js' const entry = (slug: string, schema: Record): SchemaEntry => ({ slug, @@ -155,3 +160,86 @@ describe('computePublicHash', () => { expect(publicHash).toBe(privateEquivalent.replace('private:', 'public:')) }) }) + +describe('VersionHashStream', () => { + // The streaming writer exists only so a multi-million-record commit doesn't + // have to hold the canonical document in memory. It is worth nothing unless it + // is byte-identical to computeVersionHash, so the whole test is: does it agree, + // on everything. If these fail, server and CLI version hashes have diverged. + const agree = ( + schemaSet: { slug: string; schemaHash: string }[], + recordHashes: string[], + fileHashes: string[], + metadata: Record | null, + ) => { + const stream = new VersionHashStream(schemaSet, fileHashes, metadata) + for (const h of [...recordHashes].sort()) stream.push(h) + expect(stream.digest()).toBe(computeVersionHash(schemaSet, recordHashes, fileHashes, metadata)) + } + + it('matches the golden hash', () => { + const stream = new VersionHashStream([{ slug: 'Author', schemaHash: 'aaa' }], [], null) + for (const h of ['h1', 'h2']) stream.push(h) + expect(stream.digest()).toBe( + 'private:6a382212927aee2474d30565a55f30fe5c128610998190756a41d629422b6dba', + ) + }) + + it('agrees on an empty record set', () => { + agree([{ slug: 'A', schemaHash: 'a' }], [], ['f1'], { title: 'x' }) + }) + + it('agrees with no schemas, files or metadata', () => { + agree([], ['h1'], [], null) + }) + + it('agrees on realistic sha256 digests', () => { + const hashes = Array.from( + { length: 500 }, + (_, i) => hashRecord({ id: `rec-${i}`, type: 'T', data: { i } }).hash, + ) + agree( + [ + { slug: 'Zeta', schemaHash: hashSchema({ type: 'object' }) }, + { slug: 'Alpha', schemaHash: hashSchema({ type: 'string' }) }, + ], + hashes, + ['f2', 'f1'], + { description: 'Ünïcödé "quoted" \\ and \n newline', nested: { b: 2, a: [1, 2] } }, + ) + }) + + it('agrees when the record set contains duplicates', () => { + // Two records differing only in private fields share a public hash, so the + // public-hash stream really can see the same value twice. + agree([{ slug: 'A', schemaHash: 'a' }], ['h1', 'h1', 'h2'], [], null) + }) + + it('agrees on hashes needing JSON escaping', () => { + agree([{ slug: 'A', schemaHash: 'a' }], ['a"b', 'a\\b', 'ab'], [], null) + }) + + it('agrees across randomized inputs', () => { + let seed = 12345 + const rand = () => (seed = (seed * 1103515245 + 12345) & 0x7fffffff) / 0x7fffffff + for (let round = 0; round < 200; round++) { + const hashes = Array.from({ length: Math.floor(rand() * 40) }, () => + rand().toString(16).slice(2), + ) + const files = Array.from({ length: Math.floor(rand() * 5) }, () => rand().toString(16)) + const schemas = Array.from({ length: Math.floor(rand() * 4) }, (_, i) => ({ + slug: `S${Math.floor(rand() * 100)}-${i}`, + schemaHash: rand().toString(16), + })) + agree(schemas, hashes, files, rand() > 0.5 ? { k: rand() } : null) + } + }) + + it('refuses reuse after digest()', () => { + const stream = new VersionHashStream([], [], null) + stream.push('h1') + stream.digest() + expect(() => stream.push('h2')).toThrow() + expect(() => stream.digest()).toThrow() + }) +}) diff --git a/src/lib/core/version-hash.ts b/src/lib/core/version-hash.ts index e3057f3..32d4014 100644 --- a/src/lib/core/version-hash.ts +++ b/src/lib/core/version-hash.ts @@ -27,6 +27,72 @@ export function computeVersionHash( return 'private:' + createHash('sha256').update(canonical).digest('hex') } +/** + * Streaming form of {@link computeVersionHash}, for versions whose record set is + * too large to materialize. + * + * `computeVersionHash` builds the whole canonical document in memory: a sorted + * copy of every record hash, then a JSON string containing all of them. At a few + * million records that is a sorted array plus a ~200 MB string, which is one of + * the reasons a large push cannot commit. + * + * This produces **byte-identical** output by emitting the same document + * incrementally through the same digest. The caller feeds record hashes in + * ascending order — which Postgres can supply straight off an index, so nothing + * is ever fully in memory — and the writer emits the surrounding JSON around + * them. `version-hash.test.ts` locks the two implementations together against + * randomized inputs; they must never diverge, or server and CLI version hashes + * do. + * + * Ordering must match `Array.prototype.sort()`, i.e. UTF-16 code-unit order. For + * the lowercase hex digests this is used with, that is plain byte order — so the + * SQL side must sort with `COLLATE "C"` and not a locale collation. + */ +export class VersionHashStream { + #digest = createHash('sha256') + #wroteRecord = false + #closed = false + + constructor( + schemaSet: { slug: string; schemaHash: string }[], + private readonly fileHashes: string[], + private readonly metadata: Record | null, + ) { + // JSON.stringify emits object keys in insertion order, so the prefix here + // has to match computeVersionHash's literal field order exactly. + const schemas = JSON.stringify( + Object.fromEntries( + [...schemaSet] + .sort((a, b) => a.slug.localeCompare(b.slug)) + .map((s) => [s.slug, s.schemaHash]), + ), + ) + this.#digest.update(`{"schemas":${schemas},"records":[`) + } + + /** Feed the next record hash. Hashes must arrive in ascending sort order. */ + push(recordHash: string): void { + if (this.#closed) throw new Error('VersionHashStream: push after digest()') + if (this.#wroteRecord) this.#digest.update(',') + this.#wroteRecord = true + this.#digest.update(JSON.stringify(recordHash)) + } + + /** Close the document and return the `private:`-prefixed version hash. */ + digest(): string { + if (this.#closed) throw new Error('VersionHashStream: digest() called twice') + this.#closed = true + const tail = JSON.stringify({ + files: [...this.fileHashes].sort(), + metadata: this.metadata ? canonicalize(this.metadata) : null, + }) + // Splice the tail object's fields onto the open document: drop its leading + // `{` and reuse the rest, which already carries the closing brace. + this.#digest.update(`],${tail.slice(1)}`) + return 'private:' + this.#digest.digest('hex') + } +} + /** Build a public-facing schemas map (excluding private types, stripping private fields) */ export function filterSchemasForPublic(schemaEntries: SchemaEntry[]): Record { const result: Record = {} diff --git a/src/lib/version-helpers.server.ts b/src/lib/version-helpers.server.ts index 043bb34..feee1d6 100644 --- a/src/lib/version-helpers.server.ts +++ b/src/lib/version-helpers.server.ts @@ -22,6 +22,7 @@ export { type SchemaEntry, type SemverComponents, stripToSchema, + VersionHashStream, } from './core/index.js' import { type SchemaEntry } from './core/index.js' From 353aba52ec78045f511a542620b513af5edae46c Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Fri, 31 Jul 2026 17:28:05 -0400 Subject: [PATCH 4/7] streaming push part 2 --- docker-compose.local.yml | 3 + docker-compose.yml | 7 + src/api/negotiate.ts | 1188 +++---- src/db/migrations/0009_boring_tiger_shark.sql | 34 + src/db/migrations/meta/0009_snapshot.json | 2740 +++++++++++++++++ src/db/migrations/meta/_journal.json | 7 + src/db/schema.ts | 47 +- src/routes/docs/api/versions.tsx | 48 + tools/cleanupSessions.ts | 63 +- 9 files changed, 3606 insertions(+), 531 deletions(-) create mode 100644 src/db/migrations/0009_boring_tiger_shark.sql create mode 100644 src/db/migrations/meta/0009_snapshot.json diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 8c71bb5..6e5ff81 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -18,6 +18,9 @@ services: -c work_mem=8MB -c maintenance_work_mem=64MB -c max_connections=50 + # Matches the production stack: Docker's 64 MB /dev/shm default is too small + # for Postgres parallel query workers once tables get large. + shm_size: 1gb ports: - '5433:5432' volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 15da65f..4a7da9d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -130,6 +130,13 @@ services: -c random_page_cost=1.1 volumes: - pgdata:/var/lib/postgresql/data + # Postgres allocates dynamic shared memory in /dev/shm for parallel query + # workers. Docker's 64 MB default is enough for small scans and not enough + # for a parallel aggregate over a few million rows, which fails with + # "could not resize shared memory segment ... No space left on device" — + # observed on a 500k-record commit. + tmpfs: + - /dev/shm:size=1g ports: - '${DB_PORT:-5432}:5432' networks: diff --git a/src/api/negotiate.ts b/src/api/negotiate.ts index ee3d3b4..5cfbec2 100644 --- a/src/api/negotiate.ts +++ b/src/api/negotiate.ts @@ -1,6 +1,7 @@ import { and, eq, inArray, sql } from 'drizzle-orm' import { Hono } from 'hono' import { openApi } from 'hono-zod-openapi' +import type { ContentfulStatusCode } from 'hono/utils/http-status' import { z } from 'zod' import { db, schema } from '../db/client.server.js' @@ -54,6 +55,9 @@ const NegotiateBody = z.object({ strip_unknown_fields: z.boolean().optional(), }) +/** Mirrors `c.json(body, status)` so the finalize body reads unchanged. */ +const reply = (body: unknown, status: ContentfulStatusCode = 200) => ({ status, body }) + async function expireSession(sessionId: string) { await db .update(schema.negotiateSessions) @@ -236,6 +240,11 @@ app.get( needed_files: session.neededFiles, expires_at: session.expiresAt, created_at: session.createdAt, + // Populated for an async commit: `result` once status is 'committed', + // `error` once it is 'failed'. Both null while status is 'committing'. + finalize_started_at: session.finalizeStartedAt, + result: session.result ?? null, + error: session.error ?? null, }) }, ) @@ -443,11 +452,13 @@ app.post( } } - // Mark received hashes as no longer needed (single-row updates, not JSONB rewrite) + // Mark received hashes as no longer needed (single-row updates, not JSONB + // rewrite). `submitted` records this record was validated here, against this + // session's schemas, so commit can skip revalidating it. if (receivedHashes.size > 0) { await db .update(schema.negotiateSessionManifest) - .set({ needed: false }) + .set({ needed: false, submitted: true }) .where( and( eq(schema.negotiateSessionManifest.sessionId, sessionId), @@ -489,6 +500,17 @@ app.post( async (c) => { const { sessionId } = c.req.valid('param') const userId = c.get('userId') + + // Opt-in async finalize. Accepted as a query param or a JSON body field so a + // client can use it without sending a body at all; a commit has no other + // payload. + const asyncQuery = c.req.query('async') + const asyncBody = await c.req.json().catch(() => null) + const wantsAsync = + asyncQuery === 'true' || + asyncQuery === '1' || + (asyncBody !== null && typeof asyncBody === 'object' && asyncBody.async === true) + const [sessionRow] = await db .select() .from(schema.negotiateSessions) @@ -533,19 +555,25 @@ app.post( stripUnknownFields: sessionRow.stripUnknownFields, } - // Check if any needed records remain - const [neededCount] = await db - .select({ count: sql`count(*)::int` }) - .from(schema.negotiateSessionManifest) - .where( - and( - eq(schema.negotiateSessionManifest.sessionId, sessionId), - eq(schema.negotiateSessionManifest.needed, true), - ), - ) - if ((neededCount?.count ?? 0) > 0) { - const neededRows = await db - .select({ hash: schema.negotiateSessionManifest.hash }) + // Everything past this point is the expensive half of the commit: it walks + // the record set, folds two digests over it and writes version_records. At a + // few million records that is minutes, which is too long to hold an HTTP + // request open — so it is expressed as a value-returning function that the + // caller either awaits (the default, unchanged behaviour) or runs in the + // background after answering 202. + // A rejection inside finalize expires the session synchronously, but in + // async mode the terminal status is 'failed' and the caller writes it + // alongside the error body. Expiring here first would briefly publish + // 'expired', which a poller waiting on committed/failed would either miss or + // misread as a terminal state of its own. + const abandonSession = async () => { + if (!wantsAsync) await expireSession(sessionId) + } + + const finalize = async (): Promise<{ status: ContentfulStatusCode; body: unknown }> => { + // Check if any needed records remain + const [neededCount] = await db + .select({ count: sql`count(*)::int` }) .from(schema.negotiateSessionManifest) .where( and( @@ -553,355 +581,409 @@ app.post( eq(schema.negotiateSessionManifest.needed, true), ), ) - .limit(100) - return c.json( - { - error: 'Missing records', - missing_hashes: neededRows.map((r) => r.hash), - message: `${neededCount!.count} needed record(s) have not been submitted. Use POST .../negotiate/${sessionId}/records first.`, - statusCode: 400, - }, - 400, - ) - } - - // --- Schema resolution --- - const newSchemaSet: { - slug: string - schemaId: string - schemaHash: string - schema: Record - }[] = [] - for (const [typeSlug, typeSchema] of Object.entries(session.schemas)) { - const hash = hashSchema(typeSchema) - const [existing] = await db - .select({ id: schema.schemas.id }) - .from(schema.schemas) - .where(eq(schema.schemas.schemaHash, hash)) - .limit(1) - - let schemaId: string - if (existing) { - schemaId = existing.id - } else { - const [inserted] = await db - .insert(schema.schemas) - .values({ schema: typeSchema as any, schemaHash: hash }) - .returning({ id: schema.schemas.id }) - schemaId = inserted!.id - } - newSchemaSet.push({ - slug: typeSlug, - schemaId, - schemaHash: hash, - schema: typeSchema as Record, - }) - } - - // Check files exist - if (session.fileHashes.length > 0) { - const existingFiles = await db - .select({ hash: schema.files.hash }) - .from(schema.files) - .where(inArray(schema.files.hash, session.fileHashes)) - const existingFileSet = new Set(existingFiles.map((f) => f.hash)) - const missingFiles = session.fileHashes.filter((h) => !existingFileSet.has(h)) - if (missingFiles.length > 0) { - return c.json( + if ((neededCount?.count ?? 0) > 0) { + const neededRows = await db + .select({ hash: schema.negotiateSessionManifest.hash }) + .from(schema.negotiateSessionManifest) + .where( + and( + eq(schema.negotiateSessionManifest.sessionId, sessionId), + eq(schema.negotiateSessionManifest.needed, true), + ), + ) + .limit(100) + return reply( { - error: 'Missing files', - filesNeeded: missingFiles.map((h) => `sha256:${h}`), - statusCode: 422, + error: 'Missing records', + missing_hashes: neededRows.map((r) => r.hash), + message: `${neededCount!.count} needed record(s) have not been submitted. Use POST .../negotiate/${sessionId}/records first.`, + statusCode: 400, }, - 422, + 400, ) } - } - // --- Streaming validation + hash accumulation --- - // Process records in batches instead of loading all into memory at once. - // Newly submitted records were validated during submitRecords(); existing - // records are validated here against the current schemas. - const validators = new Map>() - for (const entry of newSchemaSet) { - validators.set(entry.slug, ajv.compile(entry.schema as object)) - } + // --- Schema resolution --- + const newSchemaSet: { + slug: string + schemaId: string + schemaHash: string + schema: Record + }[] = [] + for (const [typeSlug, typeSchema] of Object.entries(session.schemas)) { + const hash = hashSchema(typeSchema) + const [existing] = await db + .select({ id: schema.schemas.id }) + .from(schema.schemas) + .where(eq(schema.schemas.schemaHash, hash)) + .limit(1) + + let schemaId: string + if (existing) { + schemaId = existing.id + } else { + const [inserted] = await db + .insert(schema.schemas) + .values({ schema: typeSchema as any, schemaHash: hash }) + .returning({ id: schema.schemas.id }) + schemaId = inserted!.id + } + newSchemaSet.push({ + slug: typeSlug, + schemaId, + schemaHash: hash, + schema: typeSchema as Record, + }) + } - const schemasForCheck: Record }> = {} - for (const entry of newSchemaSet) { - schemasForCheck[entry.slug] = entry.schema as { properties?: Record } - } + // Check files exist + if (session.fileHashes.length > 0) { + const existingFiles = await db + .select({ hash: schema.files.hash }) + .from(schema.files) + .where(inArray(schema.files.hash, session.fileHashes)) + const existingFileSet = new Set(existingFiles.map((f) => f.hash)) + const missingFiles = session.fileHashes.filter((h) => !existingFileSet.has(h)) + if (missingFiles.length > 0) { + return reply( + { + error: 'Missing files', + filesNeeded: missingFiles.map((h) => `sha256:${h}`), + statusCode: 422, + }, + 422, + ) + } + } - const schemaEntriesForPublicHash: SchemaEntry[] = newSchemaSet.map((e) => ({ - slug: e.slug, - schemaId: e.schemaId, - schema: e.schema, - schemaHash: e.schemaHash, - })) - const privateTypes = getPrivateTypes(schemaEntriesForPublicHash) - const privateFieldsByType = new Map>() - for (const entry of schemaEntriesForPublicHash) { - const fields = getPrivateFields(entry.schema) - if (fields.size > 0) privateFieldsByType.set(entry.slug, fields) - } + // --- Streaming validation + hash accumulation --- + // Process records in batches instead of loading all into memory at once. + // Newly submitted records were validated during submitRecords(); existing + // records are validated here against the current schemas. + const validators = new Map>() + for (const entry of newSchemaSet) { + validators.set(entry.slug, ajv.compile(entry.schema as object)) + } - // Walk the manifest in keyset batches, writing each record's outcome back to - // the session manifest row instead of accumulating it in process. Nothing - // here grows with collection size: at 3.11M records the old arrays alone - // (final hashes, public hashes, the manifest itself) ran to several GB. - const validationErrors: { recordId: string; type: string; errors: string[] }[] = [] - const extraFieldWarnings: { recordId: string; type: string; fields: string[] }[] = [] - // Errors are reported, not accumulated: a schema change that invalidates - // every record would otherwise build a multi-million-entry response. - const MAX_REPORTED_ERRORS = 100 - let validationErrorCount = 0 - let extraFieldCount = 0 - let recordCount = 0 - let totalBytes = 0 - // Per-type counts, stored on the version row. The commit already walks every - // record, so counting here is free and saves a COUNT(*) GROUP BY on every - // subsequent collection page view. - const typeCounts = new Map() - - // Each batch costs a read plus a write-back, so the round-trip count is what - // dominates commit wall-clock. 5,000 records of bodies is ~10 MB in flight — - // bounded, and constant regardless of how large the collection is. - const LOAD_BATCH = 5000 - let cursor: string | null = null - for (;;) { - const batch: { - hash: string - manifestPrivate: boolean - recordId: string - type: string - data: unknown - private: boolean - size: number - }[] = await db - .select({ - hash: schema.negotiateSessionManifest.hash, - manifestPrivate: schema.negotiateSessionManifest.private, - recordId: schema.recordObjects.recordId, - type: schema.recordObjects.type, - data: schema.recordObjects.data, - private: schema.recordObjects.private, - size: schema.recordObjects.size, - }) - .from(schema.negotiateSessionManifest) - .innerJoin( - schema.recordObjects, - eq(schema.negotiateSessionManifest.hash, schema.recordObjects.hash), - ) - .where( - and( - eq(schema.negotiateSessionManifest.sessionId, sessionId), - ...(cursor ? [sql`${schema.negotiateSessionManifest.hash} > ${cursor}`] : []), - ), - ) - .orderBy(schema.negotiateSessionManifest.hash) - .limit(LOAD_BATCH) - - if (batch.length === 0) break - cursor = batch[batch.length - 1]!.hash - - // Per-batch outcomes, flushed to Postgres before the next batch is read. - const stripped: { - hash: string - recordId: string - type: string - data: unknown - private: boolean - size: number - }[] = [] - const outcomes: { hash: string; finalHash: string; publicHash: string | null }[] = [] - - for (const rec of batch) { - const validate = validators.get(rec.type) - if (!validate) { - validationErrorCount++ - if (validationErrors.length < MAX_REPORTED_ERRORS) { - validationErrors.push({ - recordId: rec.recordId, - type: rec.type, - errors: [`No schema defined for record type "${rec.type}"`], - }) - } - continue + const schemasForCheck: Record }> = {} + for (const entry of newSchemaSet) { + schemasForCheck[entry.slug] = entry.schema as { properties?: Record } + } + + const schemaEntriesForPublicHash: SchemaEntry[] = newSchemaSet.map((e) => ({ + slug: e.slug, + schemaId: e.schemaId, + schema: e.schema, + schemaHash: e.schemaHash, + })) + const privateTypes = getPrivateTypes(schemaEntriesForPublicHash) + const privateFieldsByType = new Map>() + for (const entry of schemaEntriesForPublicHash) { + const fields = getPrivateFields(entry.schema) + if (fields.size > 0) privateFieldsByType.set(entry.slug, fields) + } + + // The base version, needed early: whether a record still has to be validated + // depends on whether it was already in the base under an unchanged schema. + const latest = await getLatestReadyVersion(session.collectionId) + + const currentSemver = latest?.semver ?? null + if (session.baseSemver !== null && session.baseSemver !== currentSemver) { + const normalized = session.baseSemver ? parseSemver(session.baseSemver).semver : null + if (normalized !== currentSemver) { + await abandonSession() + return reply( + { error: 'Version conflict', currentVersion: currentSemver, statusCode: 409 }, + 409, + ) } - if (!validate(rec.data)) { - validationErrorCount++ - if (validationErrors.length < MAX_REPORTED_ERRORS) { - validationErrors.push({ - recordId: rec.recordId, - type: rec.type, - errors: (validate.errors ?? []).map( - (e) => `${e.instancePath || '/'} ${e.message ?? 'validation failed'}`, - ), - }) + } + + const prevSchemaEntries = latest ? await loadVersionSchemas(latest.id) : [] + const prevSchemaMap = new Map(prevSchemaEntries.map((e) => [e.slug, e.schemaHash])) + const newSchemaMap = new Map(newSchemaSet.map((e) => [e.slug, e.schemaHash])) + let schemaChanged = prevSchemaMap.size !== newSchemaMap.size + if (!schemaChanged) { + for (const [s, hash] of newSchemaMap) { + if (prevSchemaMap.get(s) !== hash) { + schemaChanged = true + break } - continue } + } - let data = rec.data - let hash = rec.hash - let size = rec.size + // Which records appear in the public view. Every input — the manifest's + // private flag, the record's own flag, the type — is already in Postgres, so + // this is a predicate rather than a materialized column: writing it to the + // manifest would mean a full UPDATE pass over every row of every push to + // record something derivable on the spot. + // + // `<> ALL` over an empty array is TRUE, so the type clause needs no special + // case when nothing is private — which lets the same predicate be written + // through drizzle and through the raw driver used for the digest cursors. + const privateTypeList = [...privateTypes] + const publicRows = sql`NOT m.private AND NOT ro.private + AND ro.type <> ALL(${sql.param(privateTypeList)}::text[])` + + const validationErrors: { recordId: string; type: string; errors: string[] }[] = [] + const extraFieldWarnings: { recordId: string; type: string; fields: string[] }[] = [] + // Errors are reported, not accumulated: a schema change that invalidates + // every record would otherwise build a multi-million-entry response. + const MAX_REPORTED_ERRORS = 100 + let validationErrorCount = 0 + let extraFieldCount = 0 + + // Which records still need validating. + // + // Records submitted during this session were validated on arrival against + // these exact schemas, so re-running AJV over them is pure waste — and on a + // first push that is *every* record. Records inherited from the base version + // were validated when that version was pushed, so they only need rechecking + // when the schema set changed. What remains is the genuinely unchecked set: + // records that were already in record_objects (globally, possibly from + // another collection) and are new to this collection. + const revalidateAll = schemaChanged || !latest + const needsValidation = revalidateAll + ? sql`NOT m.submitted` + : sql`NOT m.submitted AND NOT EXISTS ( + SELECT 1 FROM version_records vr + WHERE vr.version_id = ${latest.id} AND vr.record_hash = m.hash + )` + + const WALK_BATCH = 5000 + let cursor: string | null = null + for (;;) { + const batch = (await db.execute(sql` + SELECT m.hash, ro.record_id AS "recordId", ro.type, ro.data, ro.private, ro.size + FROM negotiate_session_manifest m + INNER JOIN record_objects ro ON ro.hash = m.hash + WHERE m.session_id = ${sessionId} AND (${needsValidation}) + ${cursor ? sql`AND m.hash > ${cursor}` : sql``} + ORDER BY m.hash + LIMIT ${WALK_BATCH} + `)) as unknown as { + hash: string + recordId: string + type: string + data: unknown + private: boolean + size: number + }[] + + if (batch.length === 0) break + cursor = batch[batch.length - 1]!.hash + + // Per-batch outcomes, flushed before the next batch is read. Only records + // whose hash actually changed are written back — NULL means unchanged, so + // a push with no stripping performs no UPDATEs. + const stripped: { + hash: string + recordId: string + type: string + data: unknown + private: boolean + size: number + }[] = [] + const rehashed: { hash: string; finalHash: string }[] = [] + + for (const rec of batch) { + const validate = validators.get(rec.type) + if (!validate) { + validationErrorCount++ + if (validationErrors.length < MAX_REPORTED_ERRORS) { + validationErrors.push({ + recordId: rec.recordId, + type: rec.type, + errors: [`No schema defined for record type "${rec.type}"`], + }) + } + continue + } + if (!validate(rec.data)) { + validationErrorCount++ + if (validationErrors.length < MAX_REPORTED_ERRORS) { + validationErrors.push({ + recordId: rec.recordId, + type: rec.type, + errors: (validate.errors ?? []).map( + (e) => `${e.instancePath || '/'} ${e.message ?? 'validation failed'}`, + ), + }) + } + continue + } - // Check for extra fields - const typeSchema = schemasForCheck[rec.type] - if (typeSchema?.properties && typeof data === 'object' && data !== null) { - const extra = Object.keys(data).filter((k) => !(k in typeSchema.properties!)) - if (extra.length > 0) { + // Check for extra fields + const typeSchema = schemasForCheck[rec.type] + if (typeSchema?.properties && typeof rec.data === 'object' && rec.data !== null) { + const extra = Object.keys(rec.data).filter((k) => !(k in typeSchema.properties!)) + if (extra.length === 0) continue if (!session.stripUnknownFields) { extraFieldCount++ if (extraFieldWarnings.length < MAX_REPORTED_ERRORS) { extraFieldWarnings.push({ recordId: rec.recordId, type: rec.type, fields: extra }) } - } else { - data = stripToSchema(data as Record, typeSchema.properties) - const result = hashRecord({ id: rec.recordId, type: rec.type, data }) - if (hash !== result.hash) { - hash = result.hash - size = Buffer.byteLength(result.canonical, 'utf-8') - stripped.push({ - hash, - recordId: rec.recordId, - type: rec.type, - data, - private: rec.private, - size, - }) - } + continue + } + const data = stripToSchema(rec.data as Record, typeSchema.properties) + const result = hashRecord({ id: rec.recordId, type: rec.type, data }) + if (result.hash !== rec.hash) { + stripped.push({ + hash: result.hash, + recordId: rec.recordId, + type: rec.type, + data, + private: rec.private, + size: Buffer.byteLength(result.canonical, 'utf-8'), + }) + rehashed.push({ hash: rec.hash, finalHash: result.hash }) } } } - recordCount++ - typeCounts.set(rec.type, (typeCounts.get(rec.type) ?? 0) + 1) - totalBytes += size - - // Compute the public record hash inline - const isPrivate = rec.private || rec.manifestPrivate - let publicHash: string | null = null - if (!isPrivate && !privateTypes.has(rec.type)) { - const privateFields = privateFieldsByType.get(rec.type) - const publicData = - privateFields && privateFields.size > 0 ? filterRecordData(data, privateFields) : data - publicHash = hashRecord({ id: rec.recordId, type: rec.type, data: publicData }).hash + // Stop as soon as the push is known to be rejected — there is no point + // walking millions more records to grow an error list we already capped. + if (validationErrorCount > 0 || extraFieldCount > 0) break + + if (stripped.length > 0) { + // Record objects are global, immutable and content-addressed, so writing + // them before the version exists is safe; a failed commit leaves them + // orphaned exactly as a failed record submission already does. Conflicts + // are expected and mean the identical body is already stored — stripping + // the same records twice is a normal repeat push, not an error. + await db + .insert(schema.recordObjects) + .values( + stripped.map((r) => ({ + hash: r.hash, + recordId: r.recordId, + type: r.type, + data: r.data as any, + private: r.private, + size: r.size, + })), + ) + .onConflictDoNothing() + + await db.execute(sql` + UPDATE negotiate_session_manifest m SET final_hash = o.final_hash + FROM unnest( + ${sql.param(rehashed.map((o) => o.hash))}::text[], + ${sql.param(rehashed.map((o) => o.finalHash))}::text[] + ) AS o(hash, final_hash) + WHERE m.session_id = ${sessionId} AND m.hash = o.hash + `) } - outcomes.push({ hash: rec.hash, finalHash: hash, publicHash }) } - // Stop as soon as the push is known to be rejected — there is no point - // walking millions more records to grow an error list we already capped. - if (validationErrorCount > 0 || extraFieldCount > 0) break + if (validationErrorCount > 0) { + await abandonSession() + return reply( + { + error: 'Schema validation failed', + validationErrors, + totalErrors: validationErrorCount, + statusCode: 422, + }, + 422, + ) + } - if (stripped.length > 0) { - // Record objects are global, immutable and content-addressed, so writing - // them before the version exists is safe; a failed commit leaves them - // orphaned exactly as a failed record submission already does. Conflicts - // are expected and mean the identical body is already stored — stripping - // the same records twice is a normal repeat push, not an error. - await db - .insert(schema.recordObjects) - .values( - stripped.map((r) => ({ - hash: r.hash, - recordId: r.recordId, - type: r.type, - data: r.data as any, - private: r.private, - size: r.size, - })), - ) - .onConflictDoNothing() + if (extraFieldCount > 0) { + await abandonSession() + return reply( + { + error: 'Records contain fields not defined in schema', + extraFields: extraFieldWarnings, + totalRecords: extraFieldCount, + hint: 'Set strip_unknown_fields: true in the negotiate request to strip these fields.', + statusCode: 422, + }, + 422, + ) } - if (outcomes.length > 0) { - await db.execute(sql` - UPDATE negotiate_session_manifest m - SET final_hash = o.final_hash, public_hash = o.public_hash + // Public record addresses. Only types that declare private *fields* can have + // a public address that differs from the record hash, so this pass reads + // bodies for those types alone — for a collection with no private fields, + // which is the normal case, it does nothing at all. + if (privateFieldsByType.size > 0) { + const filteredTypes = [...privateFieldsByType.keys()] + let pubCursor: string | null = null + for (;;) { + const batch = (await db.execute(sql` + SELECT m.hash, ro.record_id AS "recordId", ro.type, ro.data + FROM negotiate_session_manifest m + INNER JOIN record_objects ro ON ro.hash = coalesce(m.final_hash, m.hash) + WHERE m.session_id = ${sessionId} AND (${publicRows}) + AND ro.type = ANY(${sql.param(filteredTypes)}::text[]) + ${pubCursor ? sql`AND m.hash > ${pubCursor}` : sql``} + ORDER BY m.hash + LIMIT 5000 + `)) as unknown as { hash: string; recordId: string; type: string; data: unknown }[] + + if (batch.length === 0) break + pubCursor = batch[batch.length - 1]!.hash + + const updates: { hash: string; publicHash: string }[] = [] + for (const rec of batch) { + const privateFields = privateFieldsByType.get(rec.type)! + const publicData = filterRecordData(rec.data, privateFields) + updates.push({ + hash: rec.hash, + publicHash: hashRecord({ id: rec.recordId, type: rec.type, data: publicData }).hash, + }) + } + + await db.execute(sql` + UPDATE negotiate_session_manifest m SET public_hash = o.public_hash FROM unnest( - ${sql.param(outcomes.map((o) => o.hash))}::text[], - ${sql.param(outcomes.map((o) => o.finalHash))}::text[], - ${sql.param(outcomes.map((o) => o.publicHash))}::text[] - ) AS o(hash, final_hash, public_hash) + ${sql.param(updates.map((u) => u.hash))}::text[], + ${sql.param(updates.map((u) => u.publicHash))}::text[] + ) AS o(hash, public_hash) WHERE m.session_id = ${sessionId} AND m.hash = o.hash `) + } } - } - - if (validationErrorCount > 0) { - await expireSession(sessionId) - return c.json( - { - error: 'Schema validation failed', - validationErrors, - totalErrors: validationErrorCount, - statusCode: 422, - }, - 422, - ) - } - - if (extraFieldCount > 0) { - await expireSession(sessionId) - return c.json( - { - error: 'Records contain fields not defined in schema', - extraFields: extraFieldWarnings, - totalRecords: extraFieldCount, - hint: 'Set strip_unknown_fields: true in the negotiate request to strip these fields.', - statusCode: 422, - }, - 422, - ) - } - - // Add file sizes - if (session.fileHashes.length > 0) { - const [fileSizeSum] = await db - .select({ total: sql`coalesce(sum(${schema.files.size}), 0)` }) - .from(schema.files) - .where(inArray(schema.files.hash, session.fileHashes)) - totalBytes += Number(fileSizeSum?.total ?? 0) - } - // Determine semver - const latest = await getLatestReadyVersion(session.collectionId) - - const currentSemver = latest?.semver ?? null - if (session.baseSemver !== null && session.baseSemver !== currentSemver) { - const normalized = session.baseSemver ? parseSemver(session.baseSemver).semver : null - if (normalized !== currentSemver) { - await expireSession(sessionId) - return c.json( - { error: 'Version conflict', currentVersion: currentSemver, statusCode: 409 }, - 409, - ) + // Record count, per-type counts and byte total, aggregated in Postgres over + // the final hashes. These used to be tallied in the app during the walk, + // which only worked because the walk visited every record — it no longer + // does. + const typeRows = (await db.execute(sql` + SELECT ro.type, count(*)::int AS n, sum(ro.size)::bigint AS bytes + FROM negotiate_session_manifest m + INNER JOIN record_objects ro ON ro.hash = coalesce(m.final_hash, m.hash) + WHERE m.session_id = ${sessionId} + GROUP BY ro.type + `)) as unknown as { type: string; n: number; bytes: string }[] + + const typeCounts = new Map() + let recordCount = 0 + let totalBytes = 0 + for (const row of typeRows) { + typeCounts.set(row.type, row.n) + recordCount += row.n + totalBytes += Number(row.bytes) } - } - const prevSchemaEntries = latest ? await loadVersionSchemas(latest.id) : [] - const prevSchemaMap = new Map(prevSchemaEntries.map((e) => [e.slug, e.schemaHash])) - const newSchemaMap = new Map(newSchemaSet.map((e) => [e.slug, e.schemaHash])) - let schemaChanged = prevSchemaMap.size !== newSchemaMap.size - if (!schemaChanged) { - for (const [s, hash] of newSchemaMap) { - if (prevSchemaMap.get(s) !== hash) { - schemaChanged = true - break - } + // Add file sizes + if (session.fileHashes.length > 0) { + const [fileSizeSum] = await db + .select({ total: sql`coalesce(sum(${schema.files.size}), 0)` }) + .from(schema.files) + .where(inArray(schema.files.hash, session.fileHashes)) + totalBytes += Number(fileSizeSum?.total ?? 0) } - } - // Determine if records changed vs previous version. Set comparison done in - // Postgres: loading the previous version's hashes was a second full-size - // array on top of everything else. - let recordsChanged = true - if (latest) { - const [cmp] = (await db.execute(sql` + // Determine if records changed vs previous version. Set comparison done in + // Postgres: loading the previous version's hashes was a second full-size + // array on top of everything else. + let recordsChanged = true + if (latest) { + const [cmp] = (await db.execute(sql` SELECT - (SELECT count(DISTINCT final_hash) FROM negotiate_session_manifest + (SELECT count(DISTINCT coalesce(final_hash, hash)) FROM negotiate_session_manifest WHERE session_id = ${sessionId}) AS new_count, (SELECT count(*) FROM version_records WHERE version_id = ${latest.id}) AS old_count, EXISTS ( @@ -909,168 +991,173 @@ app.post( WHERE m.session_id = ${sessionId} AND NOT EXISTS ( SELECT 1 FROM version_records vr - WHERE vr.version_id = ${latest.id} AND vr.record_hash = m.final_hash + WHERE vr.version_id = ${latest.id} + AND vr.record_hash = coalesce(m.final_hash, m.hash) ) ) AS has_new `)) as unknown as { new_count: string; old_count: string; has_new: boolean }[] - recordsChanged = Number(cmp!.new_count) !== Number(cmp!.old_count) || cmp!.has_new - } - - const prevMetadata = (latest?.metadata as Record) ?? null - const metadataValue = session.metadata - ? { ...prevMetadata, ...(session.metadata as Record) } - : prevMetadata - const metadataChanged = - JSON.stringify(metadataValue ? canonicalize(metadataValue) : null) !== - JSON.stringify(prevMetadata ? canonicalize(prevMetadata) : null) - - const publicSchemaSet: { slug: string; schemaHash: string }[] = [] - for (const entry of schemaEntriesForPublicHash) { - if (privateTypes.has(entry.slug)) continue - const filtered = filterTypeSchema(entry.schema) - publicSchemaSet.push({ slug: entry.slug, schemaHash: hashSchema(filtered) }) - } - - // Both version hashes are folded incrementally over hashes streamed out of - // Postgres in sorted order, rather than sorting N hashes in memory and - // stringifying them into one ~200 MB document. VersionHashStream is - // byte-compatible with computeVersionHash — see its test. - // - // COLLATE "C" is required, not cosmetic: the digest must see the hashes in - // the same order Array.prototype.sort() would produce, which is byte order, - // not the database's locale collation. - const versionHashStream = new VersionHashStream( - newSchemaSet.map((e) => ({ slug: e.slug, schemaHash: e.schemaHash })), - session.fileHashes, - metadataValue, - ) - const publicHashStream = new VersionHashStream( - publicSchemaSet, - session.fileHashes, - metadataValue, - ) - - const HASH_PAGE = 50_000 - for (const [stream, column] of [ - [versionHashStream, sql`final_hash`], - [publicHashStream, sql`public_hash`], - ] as const) { - // Two records that differ only in private fields share a public hash, so - // the value alone is not a unique cursor — the tiebreak is the manifest's - // own primary key, or duplicates straddling a page boundary get dropped - // and the digest silently changes. - let at: { value: string; tiebreak: string } | null = null - for (;;) { - const rows = (await db.execute(sql` - SELECT ${column} AS h, hash AS tiebreak FROM negotiate_session_manifest - WHERE session_id = ${sessionId} AND ${column} IS NOT NULL - ${ - at - ? sql`AND (${column} COLLATE "C", hash COLLATE "C") - > (${at.value} COLLATE "C", ${at.tiebreak} COLLATE "C")` - : sql`` - } - ORDER BY ${column} COLLATE "C", hash COLLATE "C" - LIMIT ${HASH_PAGE} - `)) as unknown as { h: string; tiebreak: string }[] - if (rows.length === 0) break - for (const row of rows) stream.push(row.h) - if (rows.length < HASH_PAGE) break - const last = rows[rows.length - 1]! - at = { value: last.h, tiebreak: last.tiebreak } + recordsChanged = Number(cmp!.new_count) !== Number(cmp!.old_count) || cmp!.has_new } - } - const versionHash = versionHashStream.digest() - const publicHash = publicHashStream.digest().replace('private:', 'public:') - const sv = deriveSemver(latest?.semver ?? null, schemaChanged, recordsChanged, metadataChanged) + const prevMetadata = (latest?.metadata as Record) ?? null + const metadataValue = session.metadata + ? { ...prevMetadata, ...(session.metadata as Record) } + : prevMetadata + const metadataChanged = + JSON.stringify(metadataValue ? canonicalize(metadataValue) : null) !== + JSON.stringify(prevMetadata ? canonicalize(prevMetadata) : null) + + const publicSchemaSet: { slug: string; schemaHash: string }[] = [] + for (const entry of schemaEntriesForPublicHash) { + if (privateTypes.has(entry.slug)) continue + const filtered = filterTypeSchema(entry.schema) + publicSchemaSet.push({ slug: entry.slug, schemaHash: hashSchema(filtered) }) + } - // Check for duplicate - const [existingHash] = await db - .select({ semver: schema.versions.semver }) - .from(schema.versions) - .where( - and( - eq(schema.versions.collectionId, session.collectionId), - eq(schema.versions.hash, versionHash), - eq(schema.versions.status, 'ready'), - ), + // Both version hashes are folded incrementally over hashes streamed out of + // Postgres in sorted order, rather than sorting N hashes in memory and + // stringifying them into one ~200 MB document. VersionHashStream is + // byte-compatible with computeVersionHash — see its test. + // + // COLLATE "C" is required, not cosmetic: the digest must see the hashes in + // the same order Array.prototype.sort() would produce, which is byte order, + // not the database's locale collation. + const versionHashStream = new VersionHashStream( + newSchemaSet.map((e) => ({ slug: e.slug, schemaHash: e.schemaHash })), + session.fileHashes, + metadataValue, ) - .limit(1) - if (existingHash) { - await expireSession(sessionId) - return c.json( - { - error: 'No changes detected', - message: `Version ${existingHash.semver} already has identical content.`, - existingVersion: existingHash.semver, - }, - 409, + const publicHashStream = new VersionHashStream( + publicSchemaSet, + session.fileHashes, + metadataValue, ) - } - // Insert version row + small join tables in a transaction (status = 'creating') - // Then batch-insert version_records outside the transaction to avoid long locks. - // Finally mark the version as 'ready'. - let versionId: number + // A server-side cursor: one sorted scan per digest, delivered in chunks, with + // no keyset arithmetic to get wrong. The earlier keyset version had to + // tiebreak on the primary key because two records differing only in private + // fields share a public hash, and duplicates straddling a page boundary + // would silently change the digest. A cursor sidesteps that entirely. + const client = db.$client + const CURSOR_CHUNK = 10_000 + + await client` + SELECT coalesce(final_hash, hash) AS h + FROM negotiate_session_manifest + WHERE session_id = ${sessionId} + ORDER BY coalesce(final_hash, hash) COLLATE "C" + `.cursor(CURSOR_CHUNK, (rows) => { + for (const row of rows) versionHashStream.push(row['h'] as string) + }) - await db.transaction(async (tx) => { - const [version] = await tx - .insert(schema.versions) - .values({ - collectionId: session.collectionId, - semver: sv.semver, - major: sv.major, - minor: sv.minor, - patch: sv.patch, - hash: versionHash, - publicHash, - baseSemver: session.baseSemver, - message: session.message, - metadata: metadataValue, - pushedBy: c.get('userId') ?? null, - appId: session.appId, - actorId: session.actorId, - recordCount, - fileCount: session.fileHashes.length, - typeCounts: Object.fromEntries(typeCounts), - totalBytes, - status: 'creating', - }) - .returning() + await client` + SELECT coalesce(m.public_hash, m.final_hash, m.hash) AS h + FROM negotiate_session_manifest m + INNER JOIN record_objects ro ON ro.hash = coalesce(m.final_hash, m.hash) + WHERE m.session_id = ${sessionId} + AND NOT m.private AND NOT ro.private + AND ro.type <> ALL(${privateTypeList}::text[]) + ORDER BY coalesce(m.public_hash, m.final_hash, m.hash) COLLATE "C" + `.cursor(CURSOR_CHUNK, (rows) => { + for (const row of rows) publicHashStream.push(row['h'] as string) + }) - versionId = version!.id + const versionHash = versionHashStream.digest() + const publicHash = publicHashStream.digest().replace('private:', 'public:') - if (session.fileHashes.length > 0) { - await tx - .insert(schema.versionFiles) - .values(session.fileHashes.map((hash) => ({ versionId: versionId, fileHash: hash }))) + const sv = deriveSemver( + latest?.semver ?? null, + schemaChanged, + recordsChanged, + metadataChanged, + ) + + // Check for duplicate + const [existingHash] = await db + .select({ semver: schema.versions.semver }) + .from(schema.versions) + .where( + and( + eq(schema.versions.collectionId, session.collectionId), + eq(schema.versions.hash, versionHash), + eq(schema.versions.status, 'ready'), + ), + ) + .limit(1) + if (existingHash) { + await abandonSession() + return reply( + { + error: 'No changes detected', + message: `Version ${existingHash.semver} already has identical content.`, + existingVersion: existingHash.semver, + }, + 409, + ) } - await tx.insert(schema.versionSchemas).values( - newSchemaSet.map((entry) => ({ - versionId: versionId, - slug: entry.slug, - schemaId: entry.schemaId, - })), - ) - }) + // Insert version row + small join tables in a transaction (status = 'creating') + // Then batch-insert version_records outside the transaction to avoid long locks. + // Finally mark the version as 'ready'. + let versionId: number + + await db.transaction(async (tx) => { + const [version] = await tx + .insert(schema.versions) + .values({ + collectionId: session.collectionId, + semver: sv.semver, + major: sv.major, + minor: sv.minor, + patch: sv.patch, + hash: versionHash, + publicHash, + baseSemver: session.baseSemver, + message: session.message, + metadata: metadataValue, + pushedBy: userId ?? null, + appId: session.appId, + actorId: session.actorId, + recordCount, + fileCount: session.fileHashes.length, + typeCounts: Object.fromEntries(typeCounts), + totalBytes, + status: 'creating', + }) + .returning() + + versionId = version!.id + + if (session.fileHashes.length > 0) { + await tx + .insert(schema.versionFiles) + .values(session.fileHashes.map((hash) => ({ versionId: versionId, fileHash: hash }))) + } - // Populate version_records outside the main transaction, server-side, in - // keyset batches over the session manifest. Nothing about the record set - // passes through the app: record_id and type come from record_objects, the - // hashes and public addresses from the manifest rows the validation pass - // already wrote. public_record_hash is stored only where it differs from - // the record hash, which is the column's existing contract. - try { - const VR_BATCH = 5000 - let vrCursor: string | null = null - for (;;) { - // Resolve the page's upper bound first. Doing it in one statement with - // RETURNING doesn't work: ON CONFLICT DO NOTHING makes the number of - // returned rows a count of insertions, not of manifest rows read, so it - // can't drive the cursor. - const [bound] = (await db.execute(sql` + await tx.insert(schema.versionSchemas).values( + newSchemaSet.map((entry) => ({ + versionId: versionId, + slug: entry.slug, + schemaId: entry.schemaId, + })), + ) + }) + + // Populate version_records outside the main transaction, server-side, in + // keyset batches over the session manifest. Nothing about the record set + // passes through the app: record_id and type come from record_objects, the + // hashes and public addresses from the manifest rows the validation pass + // already wrote. public_record_hash is stored only where it differs from + // the record hash, which is the column's existing contract. + try { + const VR_BATCH = 5000 + let vrCursor: string | null = null + for (;;) { + // Resolve the page's upper bound first. Doing it in one statement with + // RETURNING doesn't work: ON CONFLICT DO NOTHING makes the number of + // returned rows a count of insertions, not of manifest rows read, so it + // can't drive the cursor. + const [bound] = (await db.execute(sql` SELECT max(hash) AS hi, count(*)::int AS n FROM ( SELECT hash FROM negotiate_session_manifest WHERE session_id = ${sessionId} @@ -1080,76 +1167,143 @@ app.post( ) page `)) as unknown as { hi: string | null; n: number }[] - if (!bound || bound.n === 0 || bound.hi === null) break - const hi = bound.hi + if (!bound || bound.n === 0 || bound.hi === null) break + const hi = bound.hi - await db.execute(sql` + await db.execute(sql` INSERT INTO version_records (version_id, record_hash, public_record_hash, record_id, type) - SELECT ${versionId!}, ro.hash, nullif(m.public_hash, m.final_hash), + SELECT ${versionId!}, ro.hash, + nullif(coalesce(m.public_hash, m.final_hash, m.hash), + coalesce(m.final_hash, m.hash)), ro.record_id, ro.type FROM negotiate_session_manifest m - INNER JOIN record_objects ro ON ro.hash = m.final_hash + INNER JOIN record_objects ro ON ro.hash = coalesce(m.final_hash, m.hash) WHERE m.session_id = ${sessionId} ${vrCursor ? sql`AND m.hash > ${vrCursor}` : sql``} AND m.hash <= ${hi} ON CONFLICT DO NOTHING `) - vrCursor = hi - if (bound.n < VR_BATCH) break + vrCursor = hi + if (bound.n < VR_BATCH) break + } + } catch (err) { + await db.delete(schema.versions).where(eq(schema.versions.id, versionId!)) + throw err } - } catch (err) { - await db.delete(schema.versions).where(eq(schema.versions.id, versionId!)) - throw err - } - // Mark version as ready and update collection timestamp - await db.transaction(async (tx) => { - await tx - .update(schema.versions) - .set({ status: 'ready' }) - .where(eq(schema.versions.id, versionId!)) - - await tx - .update(schema.collections) - .set({ updatedAt: new Date() }) - .where(eq(schema.collections.id, session.collectionId)) - }) + // Mark version as ready and update collection timestamp + await db.transaction(async (tx) => { + await tx + .update(schema.versions) + .set({ status: 'ready' }) + .where(eq(schema.versions.id, versionId!)) - await db - .update(schema.negotiateSessions) - .set({ status: 'committed' }) - .where(eq(schema.negotiateSessions.id, sessionId)) + await tx + .update(schema.collections) + .set({ updatedAt: new Date() }) + .where(eq(schema.collections.id, session.collectionId)) + }) + + // In async mode the caller owns the terminal status, so that `status` and + // `result` land in the same write. Flipping to 'committed' here as well + // would briefly publish a committed session with a null result, and a + // client polling on that boundary would read success with nothing in it. + if (!wantsAsync) { + await db + .update(schema.negotiateSessions) + .set({ status: 'committed' }) + .where(eq(schema.negotiateSessions.id, sessionId)) + } + + // Fire webhooks for the new version — best-effort, never blocks/denies the 201. + try { + const deliveryIds = await enqueueWebhookDeliveries( + { + id: versionId!, + semver: sv.semver, + hash: versionHash, + major: sv.major, + minor: sv.minor, + patch: sv.patch, + recordCount, + fileCount: session.fileHashes.length, + }, + bumpTypeFromChanges(schemaChanged, recordsChanged), + session.collectionId, + ) + dispatchDeliveries(deliveryIds) + } catch (err) { + console.error(`[webhooks] failed to enqueue for ${sv.semver}:`, err) + } - // Fire webhooks for the new version — best-effort, never blocks/denies the 201. - try { - const deliveryIds = await enqueueWebhookDeliveries( + return reply( { - id: versionId!, semver: sv.semver, hash: versionHash, - major: sv.major, - minor: sv.minor, - patch: sv.patch, recordCount, fileCount: session.fileHashes.length, }, - bumpTypeFromChanges(schemaChanged, recordsChanged), - session.collectionId, + 201, ) - dispatchDeliveries(deliveryIds) - } catch (err) { - console.error(`[webhooks] failed to enqueue for ${sv.semver}:`, err) } + // Synchronous by default: the CLI, mirror-sync and every existing client + // expect the version in the response, and for ordinary collections the whole + // thing takes well under a second. + if (!wantsAsync) { + const { status, body } = await finalize() + return c.json(body as object, status) + } + + await db + .update(schema.negotiateSessions) + .set({ status: 'committing', finalizeStartedAt: new Date() }) + .where(eq(schema.negotiateSessions.id, sessionId)) + + // Deliberately not awaited: the response goes out now and the outcome is + // recorded on the session for the client to poll. A crash mid-finalize + // leaves the session in 'committing' and its version in 'creating', which + // the cleanup job sweeps. + void (async () => { + const startedAt = Date.now() + try { + const { status, body } = await finalize() + const ok = status >= 200 && status < 300 + await db + .update(schema.negotiateSessions) + .set( + ok + ? { status: 'committed', result: body as never } + : { status: 'failed', error: body as never }, + ) + .where(eq(schema.negotiateSessions.id, sessionId)) + console.log( + `[negotiate] async finalize ${sessionId} ${ok ? 'committed' : `failed (${status})`} in ${Math.round((Date.now() - startedAt) / 1000)}s`, + ) + } catch (err) { + console.error(`[negotiate] async finalize ${sessionId} threw:`, err) + await db + .update(schema.negotiateSessions) + .set({ + status: 'failed', + error: { statusCode: 500, error: err instanceof Error ? err.message : String(err) }, + }) + .where(eq(schema.negotiateSessions.id, sessionId)) + .catch(() => {}) + } + })() + return c.json( { - semver: sv.semver, - hash: versionHash, - recordCount, - fileCount: session.fileHashes.length, + session_id: sessionId, + status: 'committing', + message: + 'Commit accepted. Poll GET .../versions/negotiate/' + + sessionId + + ' until status is "committed" or "failed".', }, - 201, + 202, ) }, ) diff --git a/src/db/migrations/0009_boring_tiger_shark.sql b/src/db/migrations/0009_boring_tiger_shark.sql new file mode 100644 index 0000000..37f4608 --- /dev/null +++ b/src/db/migrations/0009_boring_tiger_shark.sql @@ -0,0 +1,34 @@ +-- Corrects 0008 and adds async finalize. +-- +-- 1. The index 0008 created is dropped. It was meant to make the version-hash +-- streams index-ordered, but those must read in byte order and so sort with +-- COLLATE "C", which cannot use an index built in the database's default +-- collation — the planner sorted anyway. Measured at 209 MB of pure write +-- amplification: maintained by every commit write-back, usable by no read, +-- and blocking HOT updates on the column it covered. Dropping it is most of +-- the reason a 500k commit went from 151 s back to ~60 s. +-- +-- 2. `submitted` is set when a record arrives through the records endpoint, +-- which validates it against the session's schemas. Commit uses it to skip +-- re-running AJV over records validated minutes earlier; on a first push +-- that is every record. +-- +-- 3. `result` / `error` / `finalize_started_at` back the async finalize: a +-- commit can now return 202 and build the version in the background, so its +-- outcome has to live somewhere the client can poll. `result` holds what the +-- synchronous path would have returned inline, `error` the rejection body it +-- would have returned instead, and `finalize_started_at` tells a finalize +-- still running apart from one killed by a restart — which is what +-- tool:cleanupSessions sweeps on. +-- +-- The `status` column gains 'committing' and 'failed'. It is plain text with the +-- allowed values enforced in the application, so no constraint change here. +-- +-- IF EXISTS on the DROP: 0008 is unreleased, so a database that never ran it is +-- a normal state rather than an error. + +DROP INDEX IF EXISTS "nsm_session_final_hash_idx";--> statement-breakpoint +ALTER TABLE "negotiate_session_manifest" ADD COLUMN "submitted" boolean DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE "negotiate_sessions" ADD COLUMN "result" jsonb;--> statement-breakpoint +ALTER TABLE "negotiate_sessions" ADD COLUMN "error" jsonb;--> statement-breakpoint +ALTER TABLE "negotiate_sessions" ADD COLUMN "finalize_started_at" timestamp with time zone; \ No newline at end of file diff --git a/src/db/migrations/meta/0009_snapshot.json b/src/db/migrations/meta/0009_snapshot.json new file mode 100644 index 0000000..af64675 --- /dev/null +++ b/src/db/migrations/meta/0009_snapshot.json @@ -0,0 +1,2740 @@ +{ + "id": "615a0df1-89c2-41fd-be93-554d8c495b86", + "prevId": "5321bb12-5ad4-4a57-a77a-c949f94c965d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.apikey": { + "name": "apikey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_request": { + "name": "last_request", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + { + "expression": "config_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_collections": { + "name": "ark_collections", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "nullsNotDistinct": false, + "columns": ["ark_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_record_types": { + "name": "ark_record_types", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "name": "ark_record_types_collection_id_record_type_pk", + "columns": ["collection_id", "record_type"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_shoulders": { + "name": "ark_shoulders", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_shoulders_organization_id_unique": { + "name": "ark_shoulders_organization_id_unique", + "nullsNotDistinct": false, + "columns": ["organization_id"] + }, + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "nullsNotDistinct": false, + "columns": ["shoulder"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collection_webhooks": { + "name": "collection_webhooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bump_filter": { + "name": "bump_filter", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{major,minor,patch}'::text[]" + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "collection_webhooks_collection_id_idx": { + "name": "collection_webhooks_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collection_webhooks_created_by_user_id_fk": { + "name": "collection_webhooks_created_by_user_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collections": { + "name": "collections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "forked_from": { + "name": "forked_from", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "collections_organization_id_idx": { + "name": "collections_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collections_forked_from_collections_id_fk": { + "name": "collections_forked_from_collections_id_fk", + "tableFrom": "collections", + "tableTo": "collections", + "columnsFrom": ["forked_from"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "collections_organization_id_slug_unique": { + "name": "collections_organization_id_slug_unique", + "nullsNotDistinct": false, + "columns": ["organization_id", "slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.files": { + "name": "files", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instance_settings": { + "name": "instance_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_session_manifest": { + "name": "negotiate_session_manifest", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "needed": { + "name": "needed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "submitted": { + "name": "submitted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "final_hash": { + "name": "final_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_hash": { + "name": "public_hash", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "nsm_session_needed_idx": { + "name": "nsm_session_needed_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "needed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "negotiate_session_manifest_session_id_negotiate_sessions_id_fk": { + "name": "negotiate_session_manifest_session_id_negotiate_sessions_id_fk", + "tableFrom": "negotiate_session_manifest", + "tableTo": "negotiate_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "negotiate_session_manifest_session_id_hash_pk": { + "name": "negotiate_session_manifest_session_id_hash_pk", + "columns": ["session_id", "hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_sessions": { + "name": "negotiate_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schemas": { + "name": "schemas", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_hashes": { + "name": "file_hashes", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "needed_files": { + "name": "needed_files", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "negotiate_sessions_collection_id_collections_id_fk": { + "name": "negotiate_sessions_collection_id_collections_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "negotiate_sessions_user_id_user_id_fk": { + "name": "negotiate_sessions_user_id_user_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": { + "organization_slug_uidx": { + "name": "organization_slug_uidx", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.page_comments": { + "name": "page_comments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor": { + "name": "anchor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "quote": { + "name": "quote", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "quote_context": { + "name": "quote_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "resolution_note": { + "name": "resolution_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "edited_at": { + "name": "edited_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "page_comments_page_anchor_idx": { + "name": "page_comments_page_anchor_idx", + "columns": [ + { + "expression": "page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "anchor", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "page_comments_user_id_idx": { + "name": "page_comments_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "page_comments_user_id_user_id_fk": { + "name": "page_comments_user_id_user_id_fk", + "tableFrom": "page_comments", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.record_objects": { + "name": "record_objects", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "record_objects_record_id_idx": { + "name": "record_objects_record_id_idx", + "columns": [ + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schema_labels": { + "name": "schema_labels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": [ + { + "expression": "label", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "schema_labels_schema_id_schemas_id_fk": { + "name": "schema_labels_schema_id_schemas_id_fk", + "tableFrom": "schema_labels", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schema_labels_schema_id_label_unique": { + "name": "schema_labels_schema_id_label_unique", + "nullsNotDistinct": false, + "columns": ["schema_id", "label"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schemas": { + "name": "schemas", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schemas_schema_hash_unique": { + "name": "schemas_schema_hash_unique", + "nullsNotDistinct": false, + "columns": ["schema_hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sync_runs": { + "name": "sync_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collections_synced": { + "name": "collections_synced", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_created": { + "name": "collections_created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_failed": { + "name": "collections_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "versions_pulled": { + "name": "versions_pulled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_downloaded": { + "name": "files_downloaded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_skipped": { + "name": "files_skipped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "logs": { + "name": "logs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_files": { + "name": "version_files", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_hash": { + "name": "file_hash", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_files_file_hash_idx": { + "name": "version_files_file_hash_idx", + "columns": [ + { + "expression": "file_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_files_version_id_versions_id_fk": { + "name": "version_files_version_id_versions_id_fk", + "tableFrom": "version_files", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_files_file_hash_files_hash_fk": { + "name": "version_files_file_hash_files_hash_fk", + "tableFrom": "version_files", + "tableTo": "files", + "columnsFrom": ["file_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_files_version_id_file_hash_pk": { + "name": "version_files_version_id_file_hash_pk", + "columns": ["version_id", "file_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_records": { + "name": "version_records", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "record_hash": { + "name": "record_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_record_hash": { + "name": "public_record_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_records_record_hash_idx": { + "name": "version_records_record_hash_idx", + "columns": [ + { + "expression": "record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_public_record_hash_idx": { + "name": "version_records_public_record_hash_idx", + "columns": [ + { + "expression": "public_record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "public_record_hash IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_record_idx": { + "name": "version_records_version_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_type_record_idx": { + "name": "version_records_version_type_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_records_version_id_versions_id_fk": { + "name": "version_records_version_id_versions_id_fk", + "tableFrom": "version_records", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_records_record_hash_record_objects_hash_fk": { + "name": "version_records_record_hash_record_objects_hash_fk", + "tableFrom": "version_records", + "tableTo": "record_objects", + "columnsFrom": ["record_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_records_version_id_record_hash_pk": { + "name": "version_records_version_id_record_hash_pk", + "columns": ["version_id", "record_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_schemas": { + "name": "version_schemas", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_schemas_schema_id_idx": { + "name": "version_schemas_schema_id_idx", + "columns": [ + { + "expression": "schema_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_schemas_version_id_versions_id_fk": { + "name": "version_schemas_version_id_versions_id_fk", + "tableFrom": "version_schemas", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_schemas_schema_id_schemas_id_fk": { + "name": "version_schemas_schema_id_schemas_id_fk", + "tableFrom": "version_schemas", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_schemas_version_id_slug_pk": { + "name": "version_schemas_version_id_slug_pk", + "columns": ["version_id", "slug"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.versions": { + "name": "versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_hash": { + "name": "public_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_counts": { + "name": "type_counts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "versions_ordering_idx": { + "name": "versions_ordering_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "major", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "minor", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "patch", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "versions_pushed_by_user_id_fk": { + "name": "versions_pushed_by_user_id_fk", + "tableFrom": "versions", + "tableTo": "user", + "columnsFrom": ["pushed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "versions_collection_id_semver_unique": { + "name": "versions_collection_id_semver_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "semver"] + }, + "versions_collection_id_hash_unique": { + "name": "versions_collection_id_hash_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "webhook_id": { + "name": "webhook_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "webhook_deliveries_webhook_id_idx": { + "name": "webhook_deliveries_webhook_id_idx", + "columns": [ + { + "expression": "webhook_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_collection_id_idx": { + "name": "webhook_deliveries_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_created_at_idx": { + "name": "webhook_deliveries_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_sweep_idx": { + "name": "webhook_deliveries_sweep_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_collection_id_collections_id_fk": { + "name": "webhook_deliveries_collection_id_collections_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_version_id_versions_id_fk": { + "name": "webhook_deliveries_version_id_versions_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/src/db/migrations/meta/_journal.json b/src/db/migrations/meta/_journal.json index 5bcc370..de58452 100644 --- a/src/db/migrations/meta/_journal.json +++ b/src/db/migrations/meta/_journal.json @@ -64,6 +64,13 @@ "when": 1785528674026, "tag": "0008_mixed_moonstone", "breakpoints": true + }, + { + "idx": 9, + "version": "7", + "when": 1785532930491, + "tag": "0009_boring_tiger_shark", + "breakpoints": true } ] } diff --git a/src/db/schema.ts b/src/db/schema.ts index 789d85a..211edd5 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -394,9 +394,26 @@ export const negotiateSessions = pgTable('negotiate_sessions', { appId: text('app_id'), actorId: text('actor_id'), stripUnknownFields: boolean('strip_unknown_fields').notNull().default(false), - status: text('status', { enum: ['open', 'committed', 'expired'] }) + // 'committing' is the async-finalize state: the request has returned 202 and + // a background task is building the version. It ends at 'committed' or + // 'failed', both of which are reported through the session-status endpoint. + status: text('status', { + enum: ['open', 'committing', 'committed', 'failed', 'expired'], + }) .notNull() .default('open'), + // Outcome of an async finalize, so a client that polls after the fact gets the + // same answer the synchronous path would have returned inline. + result: jsonb('result').$type<{ + semver: string + hash: string + recordCount: number + fileCount: number + }>(), + error: jsonb('error').$type<{ statusCode: number; error: string; [k: string]: unknown }>(), + // When the background finalize started, so a task killed by a restart can be + // told apart from one still running. + finalizeStartedAt: timestamp('finalize_started_at', { withTimezone: true }), createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), expiresAt: timestamp('expires_at', { withTimezone: true }).notNull(), }) @@ -412,25 +429,29 @@ export const negotiateSessionManifest = pgTable( hash: text('hash').notNull(), private: boolean('private').notNull().default(false), needed: boolean('needed').notNull().default(true), - // Commit scratch space. The commit walks every record once to validate and - // hash it; rather than accumulating the results in arrays that grow with - // collection size, it writes them back here and then streams them out of - // Postgres in sorted order. This table already exists per push and is - // already the size of the manifest, so it costs nothing new. + // Set when the record arrives through the records endpoint, which validates + // it against this session's schemas. Commit uses this to skip re-running AJV + // over records that were validated minutes ago — for a first push that is + // every record, and revalidation was the single largest cost in the commit. + submitted: boolean('submitted').notNull().default(false), + // Commit scratch space, written back per record instead of accumulated in + // arrays that grow with collection size. // - // finalHash is the record's hash after strip_unknown_fields rewrote it, or - // the submitted hash when nothing was stripped. publicHash is the - // content-address of the privacy-filtered record, NULL when the record does - // not appear in the public view at all. + // Both hash columns mean "unchanged" when NULL, which is the overwhelmingly + // common case: finalHash is only set when strip_unknown_fields rewrote the + // record, and publicHash only when privacy filtering changed its content. A + // push with no private fields and no stripping — the normal case, and the + // arXiv case — writes neither, so the walk performs no UPDATEs at all. + // + // Whether a record appears in the public view is deliberately *not* stored: + // it is derivable from the private flags and the type, so materializing it + // would cost a full UPDATE pass over every row of every push. finalHash: text('final_hash'), publicHash: text('public_hash'), }, (t) => [ primaryKey({ columns: [t.sessionId, t.hash] }), index('nsm_session_needed_idx').on(t.sessionId, t.needed), - // Drives the sorted streams that feed the version hash and the - // version_records insert. - index('nsm_session_final_hash_idx').on(t.sessionId, t.finalHash), ], ) diff --git a/src/routes/docs/api/versions.tsx b/src/routes/docs/api/versions.tsx index 220c20e..fb85c98 100644 --- a/src/routes/docs/api/versions.tsx +++ b/src/routes/docs/api/versions.tsx @@ -45,6 +45,27 @@ const commitRes = `{ "fileCount": 1 }` +const asyncCommitRes = `{ + "session_id": "uuid", + "status": "committing", + "message": "Commit accepted. Poll GET .../versions/negotiate/uuid until status is \\"committed\\" or \\"failed\\"." +}` + +const sessionPollRes = `{ + "session_id": "uuid", + "status": "committed", + "total_records": 3110000, + "needed_records": 0, + "finalize_started_at": "2026-07-31T12:00:00.000Z", + "result": { + "semver": "v1.1.0", + "hash": "private:a1b2c3d4...", + "recordCount": 3110000, + "fileCount": 0 + }, + "error": null +}` + const listRes = `[ { "semver": "v1.1.0", @@ -279,6 +300,33 @@ export default function DocsApiVersions() { {commitRes} +

Large pushes: async finalize

+

+ Commit work is proportional to the size of the collection, so on a very large one it can + run for minutes — longer than a proxy or client will hold a request open. Pass{' '} + ?async=true (or {'{"async": true}'} in the body) and the server + answers 202 immediately and builds the version in the background: +

+ 
+          {asyncCommitRes}
+
+

+ Then poll GET .../versions/negotiate/:sessionId until status is{' '} + committed or failed. On success result holds + exactly what the synchronous 201 would have returned; on failure{' '} + error holds the rejection body it would have returned instead, so the two + paths are interchangeable apart from timing. +

+ 
+          {sessionPollRes}
+
+

+ The version is not visible to readers until the finalize completes — it is created in a{' '} + creating state and only published at the end, so there is no window where a + half-built version can be read. A finalize whose process dies is swept and marked{' '} + failed, and its partial version removed. +

+

Schema privacy

You can add "private": true at two levels in the schema: diff --git a/tools/cleanupSessions.ts b/tools/cleanupSessions.ts index 614f1fe..0c1919e 100644 --- a/tools/cleanupSessions.ts +++ b/tools/cleanupSessions.ts @@ -6,14 +6,75 @@ * Removes sessions whose expiry is older than the grace period regardless of * status — committed and expired sessions have no further use, and an "open" * session past expiry can never be committed. + * + * Also fails out stranded async finalizes. A background finalize that dies with + * its process (deploy, OOM, crash) leaves its session in 'committing' and its + * version in 'creating' forever; nothing else will ever move them, and the + * half-built version is invisible to readers but still holds version_records + * rows. Anything still 'committing' well past the point a finalize could + * plausibly still be running is treated as dead. */ -import { lt, sql } from 'drizzle-orm' +import { and, eq, lt, sql } from 'drizzle-orm' import { db, schema } from '../src/db/client.server.js' const GRACE_MS = 24 * 60 * 60 * 1000 // keep recent sessions for a day for debugging +// Generous: a multi-million-record finalize legitimately runs for many minutes, +// and failing a live one would be worse than leaving a dead one an hour longer. +const FINALIZE_TIMEOUT_MS = 2 * 60 * 60 * 1000 + +async function sweepStrandedFinalizes() { + const cutoff = new Date(Date.now() - FINALIZE_TIMEOUT_MS) + const stranded = await db + .select({ + id: schema.negotiateSessions.id, + collectionId: schema.negotiateSessions.collectionId, + }) + .from(schema.negotiateSessions) + .where( + and( + eq(schema.negotiateSessions.status, 'committing'), + lt(schema.negotiateSessions.finalizeStartedAt, cutoff), + ), + ) + + if (stranded.length === 0) return + + for (const session of stranded) { + // Drop the half-built version. It was never flipped to 'ready', so no reader + // has seen it; version_records cascade with it. + const removed = await db + .delete(schema.versions) + .where( + and( + eq(schema.versions.collectionId, session.collectionId), + eq(schema.versions.status, 'creating'), + ), + ) + .returning({ semver: schema.versions.semver }) + + await db + .update(schema.negotiateSessions) + .set({ + status: 'failed', + error: { + statusCode: 500, + error: 'Finalize did not complete — the process handling it went away.', + }, + }) + .where(eq(schema.negotiateSessions.id, session.id)) + + console.log( + `[cleanup-sessions] Failed stranded finalize ${session.id}` + + (removed.length > 0 ? `, removed partial version(s) ${removed.map((r) => r.semver)}` : ''), + ) + } +} + async function main() { + await sweepStrandedFinalizes() + const cutoff = new Date(Date.now() - GRACE_MS) const deleted = await db .delete(schema.negotiateSessions) From 73cc77411a21f274b185147a2c4d63b879dc4436 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Fri, 31 Jul 2026 19:15:19 -0400 Subject: [PATCH 5/7] docs and alignment --- README.md | 33 +- public/llms.txt | 162 +- src/api/agent.ts | 9 +- src/api/negotiate.ts | 330 ++- src/db/migrations/0010_unique_nemesis.sql | 15 + src/db/migrations/meta/0010_snapshot.json | 2746 +++++++++++++++++++++ src/db/migrations/meta/_journal.json | 7 + src/db/schema.ts | 6 + src/routes/docs/api/versions.tsx | 86 +- src/routes/docs/integration.tsx | 28 +- src/routes/protocol.tsx | 53 +- 11 files changed, 3398 insertions(+), 77 deletions(-) create mode 100644 src/db/migrations/0010_unique_nemesis.sql create mode 100644 src/db/migrations/meta/0010_snapshot.json diff --git a/README.md b/README.md index 47de1d6..d310fad 100644 --- a/README.md +++ b/README.md @@ -215,20 +215,25 @@ The protocol and the platform are documented together: ### Key API endpoints -All pushes use the negotiate protocol — a three-step flow similar to git's pack negotiation: - -| Endpoint | Purpose | -| ------------------------------------------------ | ----------------------------------------------------------- | -| `POST .../versions/negotiate` | Start a push session (server returns which hashes it needs) | -| `POST .../versions/negotiate/:sessionId/records` | Send only the needed records (NDJSON) | -| `POST .../versions/negotiate/:sessionId/commit` | Validate, hash, and create the immutable version | -| `GET .../versions/:semver/manifest` | Version manifest (add `?since=` for delta) | -| `GET .../versions/:semver/records` | Paginated records | -| `GET .../versions/:semver/diff?from=...` | Diff between two versions | -| `POST /api/records/batch` | Fetch records by hash (JSONL stream) | -| `GET /api/records/:hash/provenance` | Find all collections containing a record | -| `POST .../fork` | Fork a collection (copies manifest, not data) | -| `GET /api/schemas` | Search schemas across all collections | +All pushes use the negotiate protocol — a three-step flow similar to git's pack negotiation. Two of +those steps have a chunked form for collections that don't fit in a single request: the manifest can +upload in pieces, and the commit can run in the background. A chunked, asynchronous push produces +the same version hash as the simple one. + +| Endpoint | Purpose | +| ------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `POST .../versions/negotiate` | Start a push session (server returns which hashes it needs) | +| `POST .../versions/negotiate/:sessionId/manifest` | Upload the manifest in NDJSON chunks, when it is too large to send in one body | +| `POST .../versions/negotiate/:sessionId/records` | Send only the needed records (NDJSON) | +| `POST .../versions/negotiate/:sessionId/commit` | Validate, hash, and create the immutable version. `?async=true` returns 202 and finalizes in the background | +| `GET .../versions/negotiate/:sessionId` | Session status, and the result or error of an async commit | +| `GET .../versions/:semver/manifest` | Version manifest (add `?since=` for delta; both keyset-paginated) | +| `GET .../versions/:semver/records` | Paginated records | +| `GET .../versions/:semver/diff?from=...` | Diff between two versions | +| `POST /api/records/batch` | Fetch records by hash (JSONL stream) | +| `GET /api/records/:hash/provenance` | Find all collections containing a record | +| `POST .../fork` | Fork a collection (copies manifest, not data) | +| `GET /api/schemas` | Search schemas across all collections | ## Privacy diff --git a/public/llms.txt b/public/llms.txt index 5fd1acc..92575a0 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -84,6 +84,9 @@ POST /api/records/batch → fetch records by ha ### Diff GET /api/collections/:owner/:slug/versions/:semver/diff?from=:semver → diff between two versions (added, updated, removed records) + Returns full record bodies, keyset-paginated: limit defaults to 500, maxes at 5000, and + ?cursor= walks the rest. When you only need hashes, prefer + .../manifest?since= — its entries are ~120 bytes instead of whole records. ### Export GET /api/collections/:owner/:slug/export → download .tar.gz archive (manifest.json + records/*.ndjson + files/*) @@ -109,7 +112,7 @@ Response includes { semver, hash, recordCount, fileCount }. If 404, no versions exist yet — your first push should use base_version: null. ### Step 2: Fetch the manifest (optional, for diffing) -GET /api/collections/:owner/:slug/versions/:semver/manifest +GET /api/collections/:owner/:slug/versions/:semver/manifest?limit=10000 Response: { @@ -117,12 +120,37 @@ Response: "hash": "abc123...", "schemas": {"Article": "schema-hash...", ...}, "records": [{"id": "rec-1", "type": "Article", "hash": "record-hash..."}, ...], - "files": ["deadbeef...", ...] + "files": ["deadbeef...", ...], + "pagination": {"limit": 10000, "hasMore": true, "nextCursor": "eyJhZGRlZCI6..."} } +The manifest is paginated: limit defaults to 10000 and maxes at 100000. Pass +?cursor= to continue; repeat until hasMore is false. The cursor is +opaque — pass back exactly what you were given, don't construct or parse it. + +This is by far the cheapest way to learn what a version contains: manifest entries are ~120 +bytes each, so a million records is one request-order-of-magnitude smaller than fetching the +records themselves. Prefer it over walking /records when you only need hashes. + For delta manifests, add ?since= to get only the changes between two versions: GET /api/collections/:owner/:slug/versions/:semver/manifest?since=:semver -Response includes "delta": { "added": [...], "updated": [...], "removed": [...] } +Response: +{ + ..., + "delta": { + "added": [{"id": "rec-9", "type": "Article", "hash": "..."}], + "updated": [{"id": "rec-1", "type": "Article", "hash": "...", "previousHash": "..."}], + "removed": [{"id": "rec-4", "type": "Article", "hash": "..."}] + }, + "pagination": {"limit": 10000, "hasMore": false, "nextCursor": null}, + "truncated": false +} + +Deltas are keyset-paginated the same way, so a delta of any size can be walked to completion +with ?cursor=. The three lists drain independently, so a page late in the walk may contain +only "updated" entries. "truncated" is legacy: it now just mirrors pagination.hasMore, and +older clients treated it as "give up and rebuild from the full manifest". If you understand +the cursor, page instead of rebuilding. Compare this against your local data to determine what changed. @@ -139,6 +167,8 @@ Check existence first with HEAD if you want to skip uploads. ### Step 4: Push the version (negotiate protocol) All pushes use the negotiate protocol — a three-step flow similar to git's pack negotiation. +For very large collections two of those steps can be broken up (the manifest uploads in chunks, +the commit runs in the background) — see 4a-chunked and 4c-async below; the shape is the same. The client sends a manifest of record hashes; the server says which it needs; the client sends only those records; then commits. @@ -187,7 +217,8 @@ hashing, or the server will reject the records. Field reference: - base_version: the semver string of the version you diffed against (e.g. "v1.2.0"). null for first push. Used for optimistic locking. - schemas: per-type JSON Schema map. Required on every push. -- manifest: array of {id, type, hash} for every record in the new version. +- manifest: array of {id, type, hash} for every record in the new version. Capped at 500,000 entries — above that, upload it in chunks instead (see 4a-chunked below). Omit when using manifest_expected. +- manifest_expected: number of distinct record hashes you will upload in chunks. Mutually exclusive with manifest; sending both returns 400. - files: array of file hashes (SHA-256 hex strings) referenced by records. - metadata: optional JSON object for version metadata (description, readme, license, etc.). Merged with previous version's metadata. - message: human-readable commit message (optional). @@ -206,6 +237,56 @@ Response: "already_have_files": 1 } +#### Step 4a-chunked: Large collections — upload the manifest in chunks + +The manifest above is one JSON body. That is fine up to 500,000 entries; past that it would be +hundreds of megabytes parsed in one go, so upload it in chunks instead. Omit "manifest" and +declare the count: + +POST /api/collections/:owner/:slug/versions/negotiate +{ + "base_version": null, + "schemas": {...}, + "manifest_expected": 3110000, + "message": "arXiv metadata" +} + +Response — nothing here is proportional to the collection: +{ + "session_id": "uuid", + "manifest_expected": 3110000, + "manifest_received": 0, + "needed_files": [], + "total_files": 0, + "already_have_files": 0, + "next": "POST .../versions/negotiate/uuid/manifest" +} + +Then send the manifest as JSONL, up to 50,000 entries per request: + +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/manifest +Content-Type: application/x-ndjson +Authorization: Bearer ul_ + +{"id":"article-42","type":"Article","hash":"abc123..."} +{"id":"article-10","type":"Article","hash":"def456..."} + +Response: +{ + "received": 50000, + "needed_records": ["def456..."], + "manifest_received": 150000, + "manifest_expected": 3110000 +} + +Each response tells you which records from THAT chunk the server needs, so you can start +sending record bodies (step 4b) before the whole manifest is uploaded. + +Chunks are idempotent: entries are keyed by hash, so re-sending a chunk after a timeout is +safe and manifest_received will not move. Commit refuses to build a version until +manifest_received equals manifest_expected, so a client that dies partway through cannot +silently produce a version that dropped records. + #### Step 4b: Send needed records POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/records Content-Type: application/x-ndjson @@ -232,6 +313,40 @@ version hashes, and creates the new immutable version. Response (201): { "semver": "v1.3.0", "hash": "def456...", "recordCount": 2, "fileCount": 1 } +#### Step 4c-async: Large collections — commit in the background + +Commit work is proportional to collection size, so on a very large collection it can run for +minutes — longer than a proxy or client will hold a request open. Add ?async=true (or send +{"async": true}) and the server accepts the commit and builds the version in the background: + +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/commit?async=true + +Response (202): +{ + "session_id": "uuid", + "status": "committing", + "message": "Commit accepted. Poll GET .../versions/negotiate/uuid until status is \"committed\" or \"failed\"." +} + +Then poll GET .../versions/negotiate/:sessionId until status is "committed" or "failed": + +{ + "session_id": "uuid", + "status": "committed", + "finalize_started_at": "2026-07-31T12:00:00.000Z", + "result": {"semver": "v1.3.0", "hash": "private:def456...", "recordCount": 3110000, "fileCount": 0}, + "error": null +} + +On success "result" holds exactly what the synchronous 201 would have returned. On failure +"status" is "failed" and "error" holds the rejection body the synchronous path would have +returned (same statusCode and shape), so the two paths are interchangeable apart from timing. + +The version is invisible to readers until the finalize completes — there is no window in which +a half-built version can be read. The finalize is server-side work and does not depend on your +connection staying open: a client that disconnects while polling can reconnect and read the +result. A finalize whose server process dies is swept and marked "failed". + ### Step 5: Handle errors Conflict (409 — someone pushed while you were diffing): @@ -247,15 +362,34 @@ Missing files (422 — records reference files not yet uploaded): → Upload the listed files, then retry commit. Extra fields (422 — records contain fields not in the schema): -{ "error": "Records contain fields not defined in schema", "extraFields": [...], "statusCode": 422 } +{ "error": "Records contain fields not defined in schema", "extraFields": [...], "totalRecords": 12, "statusCode": 422 } → Either fix the records, or re-negotiate with "strip_unknown_fields": true. + extraFields lists at most the first 100; totalRecords is how many were affected. + Schema validation failures (422) are reported the same way, with "totalErrors". + +Manifest incomplete (400 — chunked upload, commit called before every chunk arrived): +{ "error": "Manifest incomplete", "manifest_expected": 3110000, "manifest_received": 3050000, "statusCode": 400 } +→ Upload the remaining chunks, then retry commit. -Sessions expire after 10 minutes. If the session expires, re-negotiate. +Manifest too large (413 — inline manifest over 500,000 entries): +{ "error": "Inline manifests are limited to 500000 entries...", "statusCode": 413 } +→ Re-negotiate with manifest_expected and upload the manifest in chunks. + +Sessions expire after 10 minutes of INACTIVITY. Every manifest chunk and record batch pushes +the expiry back, so a push that legitimately runs for an hour will not expire underneath you. +If a session does expire, re-negotiate. ### Session management GET /api/collections/:owner/:slug/versions/negotiate/:sessionId → check session status DELETE /api/collections/:owner/:slug/versions/negotiate/:sessionId → cancel session (204) +Session status is one of: +- open — accepting manifest chunks and records +- committing — async commit accepted, finalize running in the background +- committed — done; "result" holds the version +- failed — finalize rejected or died; "error" holds why +- expired — timed out or cancelled + ### First push (no existing versions) Set base_version to null. Include all records in the manifest. Include schemas for all types. The first version will be v1.0.0. @@ -280,7 +414,7 @@ Response: } Parameters: -- limit: max records per page (default 100, max 1000) +- limit: max records per page (default 100, max 2000) - after: keyset cursor (record ID) — return records with IDs lexicographically after this value. This is the canonical, scalable method: it is an index seek and stays fast at any depth. `cursor` is accepted as an alias for `after`. @@ -289,17 +423,23 @@ Parameters: - type: filter by record type Notes: -- `pagination.total` is the whole-version record count. When `type` is set it is NOT - the filtered count — use `hasMore` to detect the end of a filtered result set. +- `pagination.total` respects the `type` filter and excludes private types. On collections + that mark individual records private it is an upper bound for anonymous callers, since + those records are hidden but still counted — use `hasMore` for an exact end-of-set signal. - A query that exceeds the server's statement timeout returns 503 with a Retry-After header. If you hit this on `offset`, switch to `after`. To paginate through all records (works at any collection size): -1. First request: GET .../records?limit=1000 +1. First request: GET .../records?limit=2000 2. If pagination.hasMore is true, use pagination.nextCursor for the next request: - GET .../records?limit=1000&after= + GET .../records?limit=2000&after= 3. Repeat until hasMore is false. +Ask for the largest page you can handle. Walking a whole collection is bounded by request +count, not bytes — 60 requests/minute unauthenticated, 5,000 authenticated — so a +3-million-record collection is 6,200 requests at 500/page and 1,550 at 2,000/page. +Authenticate for any full-collection walk. + Do NOT paginate large collections with ?offset=; it is capped at 10000 and will 400 beyond that. Use ?after= keyset pagination instead. diff --git a/src/api/agent.ts b/src/api/agent.ts index c4dd5da..2707c22 100644 --- a/src/api/agent.ts +++ b/src/api/agent.ts @@ -228,7 +228,7 @@ Content-Type: application/json - +
base_versionThe semver of the version you’re building on (e.g. ${latest ? escapeHtml(latest.semver) : 'null'}). Use null for the first push.
schemasRequired. A map of type name → JSON Schema for every type in this version.
manifestRequired. Array of {id, type, hash} for every record in the new version. The hash is SHA-256 of the canonical JSON (see llms.txt for the exact algorithm).
manifestRequired (unless uploading it in chunks, see below). Array of {id, type, hash} for every record in the new version. The hash is SHA-256 of the canonical JSON (see llms.txt for the exact algorithm). Capped at 500,000 entries.
filesArray of file hashes referenced by records. Empty array if none.
messageOptional commit message describing this update.
@@ -253,6 +253,13 @@ Authorization: Bearer ${escapeHtml(token)}

Response (201)

${escapeHtml(JSON.stringify({ semver: 'v1.1.0', hash: '', recordCount: 1, fileCount: 0 }, null, 2))}
+

Pushing more than ~500,000 records

+

Two of the steps above assume the collection fits in one request. At a few million records the manifest would be hundreds of megabytes and the commit would run for minutes, so both have a chunked form. The push is otherwise identical and produces the same version hash.

+
    +
  • Chunked manifest. Omit manifest and send manifest_expected: <count> instead. Then POST the manifest as NDJSON to …/negotiate/<session_id>/manifest, up to 50,000 entries per request. Each response reports which records from that chunk are needed, so you can start sending record bodies before the manifest finishes. Commit refuses to build a version until the declared count has arrived.
    • +
  • Async commit. Add ?async=true to the commit. The server returns 202 and finalizes in the background; poll GET …/negotiate/<session_id> until status is committed (with result) or failed (with error). The finalize does not depend on your connection staying open.
    • +
+

References

diff --git a/src/api/negotiate.ts b/src/api/negotiate.ts index 5cfbec2..ebd42c4 100644 --- a/src/api/negotiate.ts +++ b/src/api/negotiate.ts @@ -33,20 +33,41 @@ import { } from '../lib/webhooks.server.js' import { requireAuth, type AuthEnv } from './auth.server.js' +// Idle timeout, not a total-duration budget: every manifest chunk and record +// batch pushes it back. A multi-million-record push legitimately runs for tens +// of minutes, and it would be perverse to expire a session that is actively +// receiving data. const SESSION_TTL_MS = 10 * 60 * 1000 const MAX_BATCH_RECORDS = 10_000 +// Entries per manifest chunk. At ~120 bytes each this is ~6 MB per request, +// which is the point: the whole reason chunked upload exists is that the body +// stops scaling with the collection. +const MAX_MANIFEST_CHUNK = 50_000 + +// Inline manifests are parsed whole — the JSON body, then zod's validated copy. +// 500k entries is ~58 MB of body and was measured at ~730 MB of heap; past that +// a push should use the chunked flow rather than gambling on the heap. +const MAX_INLINE_MANIFEST = 500_000 + +const ManifestEntry = z.object({ + id: z.string(), + type: z.string(), + hash: z.string().regex(/^[0-9a-f]{64}$/, 'must be a lowercase hex sha256'), + private: z.boolean().optional(), +}) + const NegotiateBody = z.object({ base_version: z.string().nullable().optional(), schemas: z.record(z.string(), z.record(z.string(), z.unknown())), - manifest: z.array( - z.object({ - id: z.string(), - type: z.string(), - hash: z.string().regex(/^[0-9a-f]{64}$/, 'must be a lowercase hex sha256'), - private: z.boolean().optional(), - }), - ), + // Omit (or send empty) together with `manifest_expected` to upload the + // manifest in chunks instead. + manifest: z.array(ManifestEntry).optional(), + // Declares how many distinct record hashes will be uploaded via + // POST .../manifest. Commit refuses to build a version until exactly that many + // have arrived, so a client that dies halfway through can never silently + // produce a truncated version. + manifest_expected: z.number().int().nonnegative().optional(), files: z.array(z.string().regex(/^[0-9a-f]{64}$/)).optional(), message: z.string().optional(), metadata: z.record(z.string(), z.unknown()).optional(), @@ -55,6 +76,62 @@ const NegotiateBody = z.object({ strip_unknown_fields: z.boolean().optional(), }) +/** Push the session's idle timeout back; called whenever it receives data. */ +async function touchSession(sessionId: string) { + await db + .update(schema.negotiateSessions) + .set({ expiresAt: new Date(Date.now() + SESSION_TTL_MS) }) + .where(eq(schema.negotiateSessions.id, sessionId)) +} + +/** + * Insert manifest entries and report which of them the server doesn't already + * hold. Idempotent by (session_id, hash), so a client that retries a chunk after + * a timeout gets the same answer rather than a conflict. + */ +async function ingestManifestEntries( + sessionId: string, + entries: z.infer[], +): Promise { + const seen = new Set() + const deduped = entries.filter((r) => { + if (seen.has(r.hash)) return false + seen.add(r.hash) + return true + }) + + const HASH_CHECK_BATCH = 5000 + const existingRecordSet = new Set() + for (let i = 0; i < deduped.length; i += HASH_CHECK_BATCH) { + const chunk = deduped.slice(i, i + HASH_CHECK_BATCH).map((r) => r.hash) + const existing = await db + .select({ hash: schema.recordObjects.hash }) + .from(schema.recordObjects) + .where(inArray(schema.recordObjects.hash, chunk)) + for (const r of existing) existingRecordSet.add(r.hash) + } + + const MANIFEST_BATCH = 1000 + for (let i = 0; i < deduped.length; i += MANIFEST_BATCH) { + const batch = deduped.slice(i, i + MANIFEST_BATCH) + await db + .insert(schema.negotiateSessionManifest) + .values( + batch.map((r) => ({ + sessionId, + recordId: r.id, + type: r.type, + hash: r.hash, + private: r.private ?? false, + needed: !existingRecordSet.has(r.hash), + })), + ) + .onConflictDoNothing() + } + + return deduped.filter((r) => !existingRecordSet.has(r.hash)).map((r) => r.hash) +} + /** Mirrors `c.json(body, status)` so the finalize body reads unchanged. */ const reply = (body: unknown, status: ContentfulStatusCode = 200) => ({ status, body }) @@ -114,29 +191,35 @@ app.post( } } - // Deduplicate manifest entries by hash (PK is sessionId+hash) - const seenHashes = new Set() - const dedupedManifest = body.manifest.filter((r) => { - if (seenHashes.has(r.hash)) return false - seenHashes.add(r.hash) - return true - }) + const inlineManifest = body.manifest ?? [] + const chunked = body.manifest_expected !== undefined - // Check which record hashes already exist in record_objects (batched for large manifests) - const manifestHashes = dedupedManifest.map((r) => r.hash) - const existingRecordSet = new Set() - const HASH_CHECK_BATCH = 5000 - for (let i = 0; i < manifestHashes.length; i += HASH_CHECK_BATCH) { - const chunk = manifestHashes.slice(i, i + HASH_CHECK_BATCH) - const existing = await db - .select({ hash: schema.recordObjects.hash }) - .from(schema.recordObjects) - .where(inArray(schema.recordObjects.hash, chunk)) - for (const r of existing) existingRecordSet.add(r.hash) + if (chunked && inlineManifest.length > 0) { + return c.json( + { + error: + 'Send either an inline `manifest` or `manifest_expected` with chunked upload, not both.', + statusCode: 400, + }, + 400, + ) + } + + if (inlineManifest.length > MAX_INLINE_MANIFEST) { + return c.json( + { + error: + `Inline manifests are limited to ${MAX_INLINE_MANIFEST} entries. Set ` + + '`manifest_expected` to the number of records and upload the manifest in chunks ' + + 'via POST .../versions/negotiate/:sessionId/manifest instead.', + statusCode: 413, + }, + 413, + ) } - const neededRecords = manifestHashes.filter((h) => !existingRecordSet.has(h)) // Check which file hashes already exist (batched) + const HASH_CHECK_BATCH = 5000 const fileHashes = body.files ?? [] const existingFileSet = new Set() for (let i = 0; i < fileHashes.length; i += HASH_CHECK_BATCH) { @@ -163,39 +246,154 @@ app.post( appId: body.app_id ?? null, actorId: body.actor_id ?? null, stripUnknownFields: body.strip_unknown_fields ?? false, + manifestExpected: body.manifest_expected ?? null, expiresAt: new Date(Date.now() + SESSION_TTL_MS), }) .returning({ id: schema.negotiateSessions.id }) - // Insert manifest entries into edge table - const neededSet = new Set(neededRecords) - const MANIFEST_BATCH = 1000 - for (let i = 0; i < dedupedManifest.length; i += MANIFEST_BATCH) { - const batch = dedupedManifest.slice(i, i + MANIFEST_BATCH) - await db.insert(schema.negotiateSessionManifest).values( - batch.map((r) => ({ - sessionId: session!.id, - recordId: r.id, - type: r.type, - hash: r.hash, - private: r.private ?? false, - needed: neededSet.has(r.hash), - })), - ) + if (chunked) { + return c.json({ + session_id: session!.id, + // No needed_records yet: they are reported per chunk, as the manifest + // arrives. Nothing here is proportional to the collection. + manifest_expected: body.manifest_expected, + manifest_received: 0, + needed_files: neededFiles, + total_files: fileHashes.length, + already_have_files: fileHashes.length - neededFiles.length, + next: `POST .../versions/negotiate/${session!.id}/manifest`, + }) } + const neededRecords = await ingestManifestEntries(session!.id, inlineManifest) + const [counts] = await db + .select({ total: sql`count(*)::int` }) + .from(schema.negotiateSessionManifest) + .where(eq(schema.negotiateSessionManifest.sessionId, session!.id)) + const totalRecords = counts?.total ?? 0 + return c.json({ session_id: session!.id, needed_records: neededRecords, needed_files: neededFiles, - total_records: manifestHashes.length, + total_records: totalRecords, total_files: fileHashes.length, - already_have_records: manifestHashes.length - neededRecords.length, + already_have_records: totalRecords - neededRecords.length, already_have_files: fileHashes.length - neededFiles.length, }) }, ) +// POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/manifest +// +// Upload one chunk of the manifest as JSONL. This exists because the inline +// manifest is a single JSON body: ~58 MB at 500k records and ~360 MB at 3.11M, +// parsed whole and then copied again by validation. Chunked, the request body +// stops scaling with the collection entirely. +app.post( + '/:owner/:slug/versions/negotiate/:sessionId/manifest', + requireAuth('write'), + openApi({ + tags: ['Negotiate'], + summary: 'Upload a chunk of the manifest for a negotiate session', + description: + 'For collections too large to send the manifest as one JSON body. Open the session with ' + + '`manifest_expected` instead of `manifest`, then POST the entries here as JSONL ' + + '(`Content-Type: application/x-ndjson`), one `{id, type, hash, private?}` object per line, ' + + `up to ${MAX_MANIFEST_CHUNK} per request. Each response reports which records from that ` + + 'chunk the server still needs, so record bodies can be sent before the manifest is ' + + 'complete. Chunks are idempotent: entries are keyed by hash, so re-sending one after a ' + + 'timeout is safe. Commit refuses to build a version until `manifest_received` equals ' + + '`manifest_expected`.', + request: { + param: z.object({ owner: z.string(), slug: z.string(), sessionId: z.string() }), + }, + responses: { 200: z.any() }, + }), + async (c) => { + const { sessionId } = c.req.valid('param') + + const [sessionRow] = await db + .select() + .from(schema.negotiateSessions) + .where(eq(schema.negotiateSessions.id, sessionId)) + .limit(1) + + if (!sessionRow || sessionRow.status !== 'open' || sessionRow.expiresAt < new Date()) { + if (sessionRow?.status === 'open') await expireSession(sessionId) + return c.json({ error: 'Session expired or not found', statusCode: 404 }, 404) + } + if (sessionRow.userId !== c.get('userId')) { + return c.json({ error: 'Not authorized', statusCode: 403 }, 403) + } + if (sessionRow.manifestExpected === null) { + return c.json( + { + error: + 'This session was opened with an inline manifest. Pass `manifest_expected` at ' + + 'negotiate time to upload the manifest in chunks.', + statusCode: 400, + }, + 400, + ) + } + + const lines = (await c.req.text()) + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.length > 0) + + if (lines.length > MAX_MANIFEST_CHUNK) { + return c.json( + { + error: `Chunk too large. Maximum ${MAX_MANIFEST_CHUNK} manifest entries per request.`, + statusCode: 400, + }, + 400, + ) + } + + const entries: z.infer[] = [] + for (const line of lines) { + let parsed: unknown + try { + parsed = JSON.parse(line) + } catch { + return c.json({ error: `Invalid JSONL line: ${line.slice(0, 100)}`, statusCode: 400 }, 400) + } + const result = ManifestEntry.safeParse(parsed) + if (!result.success) { + return c.json( + { + error: `Invalid manifest entry: ${line.slice(0, 100)}`, + details: result.error.issues.map((i) => `${i.path.join('.')} ${i.message}`), + statusCode: 400, + }, + 400, + ) + } + entries.push(result.data) + } + + const needed = await ingestManifestEntries(sessionId, entries) + await touchSession(sessionId) + + // Counted from the table rather than accumulated, so a retried chunk — which + // conflicts away to nothing — doesn't inflate the total. + const [counts] = await db + .select({ total: sql`count(*)::int` }) + .from(schema.negotiateSessionManifest) + .where(eq(schema.negotiateSessionManifest.sessionId, sessionId)) + + return c.json({ + received: entries.length, + needed_records: needed, + manifest_received: counts?.total ?? 0, + manifest_expected: sessionRow.manifestExpected, + }) + }, +) + // GET /api/collections/:owner/:slug/versions/negotiate/:sessionId app.get( '/:owner/:slug/versions/negotiate/:sessionId', @@ -203,6 +401,10 @@ app.get( openApi({ tags: ['Negotiate'], summary: 'Get a negotiate session', + description: + 'Session status and progress. `status` is one of `open`, `committing`, `committed`, ' + + '`failed` or `expired`. After an async commit, `result` holds the created version once ' + + 'status is `committed`, and `error` the rejection body once it is `failed`.', request: { param: z.object({ owner: z.string(), slug: z.string(), sessionId: z.string() }), }, @@ -467,6 +669,8 @@ app.post( ) } + await touchSession(sessionId) + const [remainingRow] = await db .select({ count: sql`count(*)::int` }) .from(schema.negotiateSessionManifest) @@ -492,10 +696,19 @@ app.post( openApi({ tags: ['Negotiate'], summary: 'Commit a negotiate session', + description: + 'Validates every record against the schemas, computes the version hashes and creates the ' + + 'new immutable version. Synchronous by default, returning 201 with the version. Pass ' + + '`?async=true` (or `{"async": true}`) on a large collection, where this work can run for ' + + 'minutes: the server returns 202 immediately and finalizes in the background. Poll ' + + 'GET .../negotiate/:sessionId until `status` is `committed` (with `result`) or `failed` ' + + '(with `error`); `result` holds exactly what the 201 would have returned. The version is ' + + 'not visible to readers until the finalize completes.', request: { param: z.object({ owner: z.string(), slug: z.string(), sessionId: z.string() }), + query: z.object({ async: z.enum(['true', '1']).optional() }), }, - responses: { 201: z.any() }, + responses: { 201: z.any(), 202: z.any() }, }), async (c) => { const { sessionId } = c.req.valid('param') @@ -571,6 +784,33 @@ app.post( } const finalize = async (): Promise<{ status: ContentfulStatusCode; body: unknown }> => { + // A chunked manifest has no natural end-of-stream, so the client's + // declared total is what tells a complete upload from one whose client + // died halfway. Without this a truncated manifest would commit happily as + // a version that silently dropped records. + if (sessionRow.manifestExpected !== null) { + const [received] = await db + .select({ total: sql`count(*)::int` }) + .from(schema.negotiateSessionManifest) + .where(eq(schema.negotiateSessionManifest.sessionId, sessionId)) + const total = received?.total ?? 0 + if (total !== sessionRow.manifestExpected) { + return reply( + { + error: 'Manifest incomplete', + message: + `Expected ${sessionRow.manifestExpected} manifest entries but received ${total}. ` + + 'Upload the remaining chunks, or restart the session if the difference is ' + + 'because the manifest contained duplicate record hashes.', + manifest_expected: sessionRow.manifestExpected, + manifest_received: total, + statusCode: 400, + }, + 400, + ) + } + } + // Check if any needed records remain const [neededCount] = await db .select({ count: sql`count(*)::int` }) diff --git a/src/db/migrations/0010_unique_nemesis.sql b/src/db/migrations/0010_unique_nemesis.sql new file mode 100644 index 0000000..0e326a6 --- /dev/null +++ b/src/db/migrations/0010_unique_nemesis.sql @@ -0,0 +1,15 @@ +-- Chunked manifest upload. +-- +-- The inline manifest is a single JSON body — ~58 MB at 500k records and ~360 MB +-- at 3.11M, parsed whole and then copied again by validation. A client can now +-- declare how many record hashes it will send and upload them in chunks through +-- POST .../versions/negotiate/:sessionId/manifest, so the request body stops +-- scaling with the collection. +-- +-- manifest_expected is that declared count, and it is load-bearing: a chunked +-- upload has no natural end-of-stream, so commit compares it against the rows +-- actually received and refuses to build a version if they differ. Without it a +-- client that died halfway through would produce a version that silently dropped +-- records. NULL for the inline path, where the manifest arrives atomically. + +ALTER TABLE "negotiate_sessions" ADD COLUMN "manifest_expected" integer; diff --git a/src/db/migrations/meta/0010_snapshot.json b/src/db/migrations/meta/0010_snapshot.json new file mode 100644 index 0000000..b54e4d4 --- /dev/null +++ b/src/db/migrations/meta/0010_snapshot.json @@ -0,0 +1,2746 @@ +{ + "id": "e6efbada-0ca0-4c16-9234-2dc933096a1e", + "prevId": "615a0df1-89c2-41fd-be93-554d8c495b86", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.apikey": { + "name": "apikey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_request": { + "name": "last_request", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + { + "expression": "config_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_collections": { + "name": "ark_collections", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "nullsNotDistinct": false, + "columns": ["ark_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_record_types": { + "name": "ark_record_types", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "name": "ark_record_types_collection_id_record_type_pk", + "columns": ["collection_id", "record_type"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ark_shoulders": { + "name": "ark_shoulders", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ark_shoulders_organization_id_unique": { + "name": "ark_shoulders_organization_id_unique", + "nullsNotDistinct": false, + "columns": ["organization_id"] + }, + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "nullsNotDistinct": false, + "columns": ["shoulder"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collection_webhooks": { + "name": "collection_webhooks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bump_filter": { + "name": "bump_filter", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{major,minor,patch}'::text[]" + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "collection_webhooks_collection_id_idx": { + "name": "collection_webhooks_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collection_webhooks_created_by_user_id_fk": { + "name": "collection_webhooks_created_by_user_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "user", + "columnsFrom": ["created_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collections": { + "name": "collections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "forked_from": { + "name": "forked_from", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "collections_organization_id_idx": { + "name": "collections_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "collections_forked_from_collections_id_fk": { + "name": "collections_forked_from_collections_id_fk", + "tableFrom": "collections", + "tableTo": "collections", + "columnsFrom": ["forked_from"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "collections_organization_id_slug_unique": { + "name": "collections_organization_id_slug_unique", + "nullsNotDistinct": false, + "columns": ["organization_id", "slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.files": { + "name": "files", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "size": { + "name": "size", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.instance_settings": { + "name": "instance_settings", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_session_manifest": { + "name": "negotiate_session_manifest", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "needed": { + "name": "needed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "submitted": { + "name": "submitted", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "final_hash": { + "name": "final_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "public_hash": { + "name": "public_hash", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "nsm_session_needed_idx": { + "name": "nsm_session_needed_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "needed", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "negotiate_session_manifest_session_id_negotiate_sessions_id_fk": { + "name": "negotiate_session_manifest_session_id_negotiate_sessions_id_fk", + "tableFrom": "negotiate_session_manifest", + "tableTo": "negotiate_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "negotiate_session_manifest_session_id_hash_pk": { + "name": "negotiate_session_manifest_session_id_hash_pk", + "columns": ["session_id", "hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negotiate_sessions": { + "name": "negotiate_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schemas": { + "name": "schemas", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "file_hashes": { + "name": "file_hashes", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "needed_files": { + "name": "needed_files", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "manifest_expected": { + "name": "manifest_expected", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "negotiate_sessions_collection_id_collections_id_fk": { + "name": "negotiate_sessions_collection_id_collections_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "negotiate_sessions_user_id_user_id_fk": { + "name": "negotiate_sessions_user_id_user_id_fk", + "tableFrom": "negotiate_sessions", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": { + "organization_slug_uidx": { + "name": "organization_slug_uidx", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "nullsNotDistinct": false, + "columns": ["slug"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.page_comments": { + "name": "page_comments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "anchor": { + "name": "anchor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "quote": { + "name": "quote", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "quote_context": { + "name": "quote_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "approved_by": { + "name": "approved_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "resolution_note": { + "name": "resolution_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "edited_at": { + "name": "edited_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "page_comments_page_anchor_idx": { + "name": "page_comments_page_anchor_idx", + "columns": [ + { + "expression": "page", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "anchor", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "page_comments_user_id_idx": { + "name": "page_comments_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "page_comments_user_id_user_id_fk": { + "name": "page_comments_user_id_user_id_fk", + "tableFrom": "page_comments", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.record_objects": { + "name": "record_objects", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "record_objects_record_id_idx": { + "name": "record_objects_record_id_idx", + "columns": [ + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schema_labels": { + "name": "schema_labels", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": [ + { + "expression": "label", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "schema_labels_schema_id_schemas_id_fk": { + "name": "schema_labels_schema_id_schemas_id_fk", + "tableFrom": "schema_labels", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schema_labels_schema_id_label_unique": { + "name": "schema_labels_schema_id_label_unique", + "nullsNotDistinct": false, + "columns": ["schema_id", "label"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.schemas": { + "name": "schemas", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "schema": { + "name": "schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "schemas_schema_hash_unique": { + "name": "schemas_schema_hash_unique", + "nullsNotDistinct": false, + "columns": ["schema_hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": ["token"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sync_runs": { + "name": "sync_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "collections_synced": { + "name": "collections_synced", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_created": { + "name": "collections_created", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "collections_failed": { + "name": "collections_failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "versions_pulled": { + "name": "versions_pulled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_downloaded": { + "name": "files_downloaded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "files_skipped": { + "name": "files_skipped", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "errors": { + "name": "errors", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "logs": { + "name": "logs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_files": { + "name": "version_files", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "file_hash": { + "name": "file_hash", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_files_file_hash_idx": { + "name": "version_files_file_hash_idx", + "columns": [ + { + "expression": "file_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_files_version_id_versions_id_fk": { + "name": "version_files_version_id_versions_id_fk", + "tableFrom": "version_files", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_files_file_hash_files_hash_fk": { + "name": "version_files_file_hash_files_hash_fk", + "tableFrom": "version_files", + "tableTo": "files", + "columnsFrom": ["file_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_files_version_id_file_hash_pk": { + "name": "version_files_version_id_file_hash_pk", + "columns": ["version_id", "file_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_records": { + "name": "version_records", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "record_hash": { + "name": "record_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_record_hash": { + "name": "public_record_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_id": { + "name": "record_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_records_record_hash_idx": { + "name": "version_records_record_hash_idx", + "columns": [ + { + "expression": "record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_public_record_hash_idx": { + "name": "version_records_public_record_hash_idx", + "columns": [ + { + "expression": "public_record_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "public_record_hash IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_record_idx": { + "name": "version_records_version_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "version_records_version_type_record_idx": { + "name": "version_records_version_type_record_idx", + "columns": [ + { + "expression": "version_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "record_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_records_version_id_versions_id_fk": { + "name": "version_records_version_id_versions_id_fk", + "tableFrom": "version_records", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_records_record_hash_record_objects_hash_fk": { + "name": "version_records_record_hash_record_objects_hash_fk", + "tableFrom": "version_records", + "tableTo": "record_objects", + "columnsFrom": ["record_hash"], + "columnsTo": ["hash"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_records_version_id_record_hash_pk": { + "name": "version_records_version_id_record_hash_pk", + "columns": ["version_id", "record_hash"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.version_schemas": { + "name": "version_schemas", + "schema": "", + "columns": { + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema_id": { + "name": "schema_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "version_schemas_schema_id_idx": { + "name": "version_schemas_schema_id_idx", + "columns": [ + { + "expression": "schema_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "version_schemas_version_id_versions_id_fk": { + "name": "version_schemas_version_id_versions_id_fk", + "tableFrom": "version_schemas", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "version_schemas_schema_id_schemas_id_fk": { + "name": "version_schemas_schema_id_schemas_id_fk", + "tableFrom": "version_schemas", + "tableTo": "schemas", + "columnsFrom": ["schema_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "version_schemas_version_id_slug_pk": { + "name": "version_schemas_version_id_slug_pk", + "columns": ["version_id", "slug"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.versions": { + "name": "versions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_hash": { + "name": "public_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "type_counts": { + "name": "type_counts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'ready'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "versions_ordering_idx": { + "name": "versions_ordering_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "major", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "minor", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "patch", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "versions_pushed_by_user_id_fk": { + "name": "versions_pushed_by_user_id_fk", + "tableFrom": "versions", + "tableTo": "user", + "columnsFrom": ["pushed_by"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "versions_collection_id_semver_unique": { + "name": "versions_collection_id_semver_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "semver"] + }, + "versions_collection_id_hash_unique": { + "name": "versions_collection_id_hash_unique", + "nullsNotDistinct": false, + "columns": ["collection_id", "hash"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "webhook_id": { + "name": "webhook_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "webhook_deliveries_webhook_id_idx": { + "name": "webhook_deliveries_webhook_id_idx", + "columns": [ + { + "expression": "webhook_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_collection_id_idx": { + "name": "webhook_deliveries_collection_id_idx", + "columns": [ + { + "expression": "collection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_created_at_idx": { + "name": "webhook_deliveries_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "webhook_deliveries_sweep_idx": { + "name": "webhook_deliveries_sweep_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_collection_id_collections_id_fk": { + "name": "webhook_deliveries_collection_id_collections_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "webhook_deliveries_version_id_versions_id_fk": { + "name": "webhook_deliveries_version_id_versions_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "versions", + "columnsFrom": ["version_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/src/db/migrations/meta/_journal.json b/src/db/migrations/meta/_journal.json index de58452..417ed07 100644 --- a/src/db/migrations/meta/_journal.json +++ b/src/db/migrations/meta/_journal.json @@ -71,6 +71,13 @@ "when": 1785532930491, "tag": "0009_boring_tiger_shark", "breakpoints": true + }, + { + "idx": 10, + "version": "7", + "when": 1785533520514, + "tag": "0010_unique_nemesis", + "breakpoints": true } ] } diff --git a/src/db/schema.ts b/src/db/schema.ts index 211edd5..2c0c4e0 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -394,6 +394,12 @@ export const negotiateSessions = pgTable('negotiate_sessions', { appId: text('app_id'), actorId: text('actor_id'), stripUnknownFields: boolean('strip_unknown_fields').notNull().default(false), + // Set when the manifest is uploaded in chunks instead of inline: how many + // distinct record hashes the client says it will send. Commit refuses to build + // a version until exactly that many have arrived, so a client that dies + // partway through the upload cannot silently produce a truncated version. + // NULL for the inline path, where the manifest arrives atomically. + manifestExpected: integer('manifest_expected'), // 'committing' is the async-finalize state: the request has returned 202 and // a background task is building the version. It ends at 'committed' or // 'failed', both of which are reported through the session-status endpoint. diff --git a/src/routes/docs/api/versions.tsx b/src/routes/docs/api/versions.tsx index fb85c98..3442b3f 100644 --- a/src/routes/docs/api/versions.tsx +++ b/src/routes/docs/api/versions.tsx @@ -45,6 +45,30 @@ const commitRes = `{ "fileCount": 1 }` +const chunkedStartReq = `{ + "base_version": null, + "schemas": { "Preprint": { "type": "object", "properties": {...} } }, + "manifest_expected": 3110000, + "message": "arXiv metadata" +}` + +const chunkedStartRes = `{ + "session_id": "uuid", + "manifest_expected": 3110000, + "manifest_received": 0, + "needed_files": [], + "total_files": 0, + "already_have_files": 0, + "next": "POST .../versions/negotiate/uuid/manifest" +}` + +const manifestChunkRes = `{ + "received": 50000, + "needed_records": ["def456...", "789abc..."], + "manifest_received": 150000, + "manifest_expected": 3110000 +}` + const asyncCommitRes = `{ "session_id": "uuid", "status": "committing", @@ -215,9 +239,19 @@ export default function DocsApiVersions() { manifest - Required. Array of {'{"id", "type", "hash"}'} objects. - Each hash is the SHA-256 of the canonical JSON{' '} - {'{"id":...,"type":...,"data":...}'}. + Array of {'{"id", "type", "hash"}'} objects. Each hash is + the SHA-256 of the canonical JSON {'{"id":...,"type":...,"data":...}'}. + Required unless you upload the manifest in chunks — see below. Capped at 500,000 + entries. + + + + + manifest_expected + + + Number of distinct record hashes you will upload in chunks. Mutually exclusive with{' '} + manifest. See "uploading the manifest in chunks" below. @@ -259,6 +293,47 @@ export default function DocsApiVersions() { {negotiateRes} +

Large collections: uploading the manifest in chunks

+

+ The manifest above is a single JSON body, which is fine up to{' '} + 500,000 entries (beyond that the endpoint returns 413). At a + few million records it would be hundreds of megabytes, parsed whole. Instead, omit{' '} + manifest and declare how many records you will send: +

+ 
+          {chunkedStartReq}
+
+

+ The server opens the session without asking for any records yet — nothing in this response + is proportional to the collection: +

+ 
+          {chunkedStartRes}
+
+

+ Then POST .../versions/negotiate/:sessionId/manifest with up to{' '} + 50,000 JSONL entries per request ( + Content-Type: application/x-ndjson), each line a{' '} + {'{"id", "type", "hash"}'} object. Each response tells you which records from{' '} + that chunk the server still needs, so you can start sending bodies before the + whole manifest is uploaded: +

+ 
+          {manifestChunkRes}
+
+

+ Chunks are idempotent: entries are keyed by hash, so re-sending a chunk after a timeout is + safe and manifest_received will not move. Commit refuses to build a version + until manifest_received equals manifest_expected, so a client + that dies partway through the upload cannot silently produce a version that dropped + records. +

+

+ The session's 10-minute expiry is an idle timeout — every manifest chunk and + record batch pushes it back — so a push that legitimately runs for an hour will not expire + underneath you. +

+

Step 2: POST .../negotiate/:sessionId/records

Auth: write scope

@@ -395,7 +470,10 @@ export default function DocsApiVersions() { 404 - Session expired or not found. Sessions expire after 10 minutes. + + Session expired or not found. Sessions expire after 10 minutes of inactivity — every + manifest chunk and record batch pushes the expiry back. + diff --git a/src/routes/docs/integration.tsx b/src/routes/docs/integration.tsx index 7a4e144..9632753 100644 --- a/src/routes/docs/integration.tsx +++ b/src/routes/docs/integration.tsx @@ -176,6 +176,15 @@ export default function DocsIntegration() { 

         {diffPush}
+

+ Above roughly half a million records, two of those steps stop fitting in one request: send + the manifest in chunks rather than as a single body, and commit asynchronously rather than + holding the connection open. See{' '} + + the versions API + {' '} + for both. The push is otherwise identical, and produces the same version hash. +

Record Hashing

@@ -314,6 +323,14 @@ export default function DocsIntegration() { Start a push (hash negotiation) + + + POST .../negotiate/:id/manifest + + + Upload the manifest in JSONL chunks, for collections too large to send it in one body + + POST .../negotiate/:id/records @@ -324,7 +341,16 @@ export default function DocsIntegration() { POST .../negotiate/:id/commit - Finalize and create the version + + Finalize and create the version. Add ?async=true to get a{' '} + 202 and poll instead of holding the request open + + + + + GET .../negotiate/:id + + Session status, and the result or error of an async commit diff --git a/src/routes/protocol.tsx b/src/routes/protocol.tsx index ca81b7d..fccc4a1 100644 --- a/src/routes/protocol.tsx +++ b/src/routes/protocol.tsx @@ -51,6 +51,28 @@ Content-Type: application/x-ndjson POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/commit # -> { "semver": "v1.2.0", "hash": "...", "recordCount": 2, "fileCount": 1 }` +const scaleExample = `# Manifests above 500k entries upload in chunks instead of one body. +# Declare the count; the server opens the session without asking for anything yet. +POST /api/collections/:owner/:slug/versions/negotiate +{ "base_version": null, "schemas": {...}, "manifest_expected": 3110000 } +# -> { "session_id": "...", "manifest_expected": 3110000, "manifest_received": 0 } + +# Send the manifest as JSONL, <= 50,000 entries per request. Each response says +# which records from THAT chunk are needed, so bodies can start flowing early. +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/manifest +Content-Type: application/x-ndjson + +{"id":"pub-001","type":"Publication","hash":"abc123..."} +# -> { "received": 50000, "needed_records": [...], "manifest_received": 150000 } + +# Commit in the background rather than holding a request open for minutes. +POST /api/collections/:owner/:slug/versions/negotiate/:sessionId/commit?async=true +# -> 202 { "session_id": "...", "status": "committing" } + +# Poll until the version lands. The finalize does not depend on your connection. +GET /api/collections/:owner/:slug/versions/negotiate/:sessionId +# -> { "status": "committed", "result": { "semver": "v1.2.0", "hash": "...", ... } }` + const pullExample = `# Full manifest GET /api/collections/:owner/:slug/versions/v2.0.0/manifest @@ -421,7 +443,29 @@ export default function Protocol() { For large pushes, the /records endpoint can be called multiple times (up to 10,000 records per batch). The server tracks which records have been received. Once all needed records are submitted, commit to finalize the version. Sessions expire - after 10 minutes. + after 10 minutes of inactivity — every manifest chunk and record batch pushes the + expiry back, so a push that runs for an hour will not expire underneath you. +

+

Pushes larger than one request

+

+ Two steps of the flow assume the collection fits comfortably in one request: the + manifest arrives as a single JSON body, and commit holds the connection open while it + validates and hashes everything. At a few million records neither holds — the manifest + would be hundreds of megabytes and the commit would run for minutes. Both have a + chunked form, and they change the shape of the exchange rather than its meaning: the + version hash a chunked, asynchronous push produces is identical to the one the simple + flow produces from the same content. +

+ 
+              {scaleExample}
+
+

+ A chunked manifest has no natural end-of-stream, so manifest_expected is + part of the contract rather than a hint: commit compares it against what actually + arrived and refuses to build a version if they differ. Chunks are keyed by hash and + therefore idempotent — re-sending one after a timeout is safe. An asynchronous commit + publishes nothing until it finishes, so there is no window in which a partially built + version can be read.

@@ -434,6 +478,13 @@ export default function Protocol() { 
               {pullExample}
+

+ Both are keyset-paginated: pass pagination.nextCursor back as{' '} + ?cursor= until hasMore is false. A delta of any size can be + walked to completion, and the three lists drain independently, so a page late in the + walk may hold only updated entries. The cursor is opaque — pass back what + you were given rather than constructing one. +

From ff3091fd787e2b225b4d38b754af33cd297197a5 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 1 Aug 2026 22:15:35 -0400 Subject: [PATCH 6/7] Small local compose tweak --- docker-compose.local.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 6e5ff81..42caeee 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -11,12 +11,18 @@ services: POSTGRES_USER: underlay POSTGRES_PASSWORD: underlay POSTGRES_DB: underlay + # Sized for bulk ingest as well as everyday dev. A multi-million-record push + # sorts the whole record set twice to fold the version digests, and builds + # indexes over it — at 8MB work_mem that spills to disk in many small merge + # passes. max_connections is low, so the worst case here is bounded. command: > postgres - -c shared_buffers=256MB - -c effective_cache_size=512MB - -c work_mem=8MB - -c maintenance_work_mem=64MB + -c shared_buffers=1GB + -c effective_cache_size=3GB + -c work_mem=64MB + -c maintenance_work_mem=512MB + -c max_wal_size=4GB + -c checkpoint_completion_target=0.9 -c max_connections=50 # Matches the production stack: Docker's 64 MB /dev/shm default is too small # for Postgres parallel query workers once tables get large. From 962b3c36ab530d0b00bff698793592581eee57f7 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Sat, 1 Aug 2026 23:06:13 -0400 Subject: [PATCH 7/7] Documentation updates --- public/llms.txt | 10 +++++ src/api/collections.ts | 27 ++++++++++++++ src/api/query.ts | 46 ++++++++++++++++++++++- src/api/versions.ts | 85 ++++++++++++++++++++++-------------------- 4 files changed, 126 insertions(+), 42 deletions(-) diff --git a/public/llms.txt b/public/llms.txt index 92575a0..d184487 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -92,6 +92,16 @@ GET /api/collections/:owner/:slug/versions/:semver/diff?from=:semver → diff be GET /api/collections/:owner/:slug/export → download .tar.gz archive (manifest.json + records/*.ndjson + files/*) GET /api/collections/:owner/:slug/export?version=v2.0.0 → export a specific version +Export assembles the whole archive in memory, so it is only offered on collections below +250,000 records. Above that it returns 413 with the record count and the limit. To read a +large collection, page the records endpoint with ?after= (see Pagination below), or fetch +the manifest if you only need id/type/hash — both work at any size. + +The same 250,000-record limit applies to the SQL explorer (/api/query/...), which builds an +in-memory SQLite copy of a version. It is a UI feature rather than a documented API; on a +large collection use the records endpoint, or Hot, which hydrates a collection into a +queryable database built for the purpose. + ### Fork POST /api/collections/:owner/:slug/fork → fork collection into caller's org (requires write auth) Body: { "targetOrg": "my-org", "slug": "optional-new-slug" } diff --git a/src/api/collections.ts b/src/api/collections.ts index 06752cb..7312ea1 100644 --- a/src/api/collections.ts +++ b/src/api/collections.ts @@ -14,6 +14,11 @@ import { getLatestReadyVersion, getOrgRole, hasOrgAccess } from '../lib/version- import { type AuthEnv } from './auth.server.js' import { requireAuth } from './auth.server.js' +// Export builds the whole archive in memory (see the guard in the export route), +// so it is offered only below this. Matches the SQL explorer's limit — both are +// whole-collection-in-memory features and should draw the line in the same place. +const MAX_EXPORT_RECORDS = 250_000 + const app = new Hono() // Browse collections — public by default, or the caller's own with ?mine=true .get( @@ -816,6 +821,28 @@ const app = new Hono() return c.json({ error: 'No versions found', statusCode: 404 }, 404) } + // The archive is assembled in memory: every record of a type is collected + // into a string[] and joined before it becomes a tar entry. That is fine + // for the collections this was built for and fatal on a multi-million + // record one — the join alone would exceed V8's maximum string length, + // and the array would exhaust the heap first, from an endpoint any + // visitor can reach. Refuse above the threshold rather than fall over. + if (version.recordCount > MAX_EXPORT_RECORDS) { + return c.json( + { + error: + `This collection has ${version.recordCount.toLocaleString()} records; export is ` + + `available below ${MAX_EXPORT_RECORDS.toLocaleString()}. Read it through the ` + + `records API instead (GET .../versions/:n/records?after=…), which pages at any ` + + `depth, or the manifest endpoint if you only need hashes.`, + recordCount: version.recordCount, + maxRecords: MAX_EXPORT_RECORDS, + statusCode: 413, + }, + 413, + ) + } + const versionFiles = await db .select({ hash: schema.versionFiles.fileHash, diff --git a/src/api/query.ts b/src/api/query.ts index 7ca74a8..6e16ee6 100644 --- a/src/api/query.ts +++ b/src/api/query.ts @@ -28,6 +28,11 @@ const sqliteCache = new Map< const CACHE_TTL_MS = 30 * 60 * 1000 // 30 minutes const CACHE_MAX_ENTRIES = 10 +// The SQLite build is proportional to collection size and holds the whole thing +// in memory, so it is offered only below this. Above it, the records API pages +// at any depth and Hot provides a SQL editor over a hydrated copy. +const MAX_QUERY_RECORDS = 250_000 + // In-memory rate limit for the LLM endpoint (public path, spends CF AI credits) const RATE_LIMIT_WINDOW_MS = 60_000 const RATE_LIMIT_MAX = 10 // requests per key per window @@ -96,7 +101,11 @@ async function getOrBuildSqlite( // Resolve version const [version] = await db - .select({ id: schema.versions.id, semver: schema.versions.semver }) + .select({ + id: schema.versions.id, + semver: schema.versions.semver, + recordCount: schema.versions.recordCount, + }) .from(schema.versions) .where( and( @@ -109,6 +118,15 @@ async function getOrBuildSqlite( if (!version) return null + // Building the SQLite artifact materializes every record body in memory. That + // is fine for the collections this feature was built for and fatal on a + // multi-million-record one: a 3.1M-record collection is several GB of jsonb + // that would OOM the container — from an endpoint any visitor can reach. + // Refuse above the threshold instead, and point at the paths that do scale. + if (version.recordCount > MAX_QUERY_RECORDS) { + return { tooLarge: true as const, recordCount: version.recordCount } + } + const cacheKey = `${collection.id}:${version.semver}:${ownerAccess ? 'full' : 'public'}` // Check cache (re-insert to move to end for LRU ordering) @@ -207,6 +225,28 @@ async function getOrBuildSqlite( return entry } +/** + * Shared 413 for collections above MAX_QUERY_RECORDS. Names the size, the limit + * and what to use instead — a bare "too large" leaves the caller guessing + * whether to retry. + */ +function tooLargeResponse(c: Context, recordCount: number) { + return c.json( + { + error: + `This collection has ${recordCount.toLocaleString()} records; the SQL explorer is ` + + `available below ${MAX_QUERY_RECORDS.toLocaleString()}. It builds an in-memory SQLite ` + + `copy of the whole version, which does not scale past that. Use the records API ` + + `(GET /api/collections/:owner/:slug/versions/:n/records?after=…), which pages at any ` + + `depth, or Hot for a SQL editor over a hydrated copy.`, + recordCount, + maxRecords: MAX_QUERY_RECORDS, + statusCode: 413, + }, + 413, + ) +} + // GET /query/sqlite/:owner/:slug/:version — Download SQLite file for a version export async function sqlite(c: Context) { const owner = c.req.param('owner')! @@ -216,6 +256,7 @@ export async function sqlite(c: Context) { const result = await getOrBuildSqlite(owner, slug, versionSemver, c.get('userId')) if (!result) return c.json({ error: 'Collection or version not found', statusCode: 404 }, 404) + if ('tooLarge' in result) return tooLargeResponse(c, result.recordCount) return new Response(new Uint8Array(result.buffer), { status: 200, @@ -235,6 +276,7 @@ export async function ddl(c: Context) { const result = await getOrBuildSqlite(owner, slug, versionSemver, c.get('userId')) if (!result) return c.json({ error: 'Collection or version not found', statusCode: 404 }, 404) + if ('tooLarge' in result) return tooLargeResponse(c, result.recordCount) return c.json({ ddl: result.ddl }) } @@ -282,6 +324,7 @@ export async function generateSql(c: Context) { { error: `Collection ${ref.owner}/${ref.slug} v${ref.version} not found`, statusCode: 404 }, 404, ) + if ('tooLarge' in result) return tooLargeResponse(c, result.recordCount) combinedDdl = result.ddlWithSamples // Count records from cache (approximation from the version table already captured) } else { @@ -293,6 +336,7 @@ export async function generateSql(c: Context) { { error: `Collection ${ref.owner}/${ref.slug} v${ref.version} not found` }, 404, ) + if ('tooLarge' in result) return tooLargeResponse(c, result.recordCount) const prefix = ref.slug.replace(/-/g, '_') // Prefix table names and add _source column to DDL const ddlPrefixed = result.ddlWithSamples diff --git a/src/api/versions.ts b/src/api/versions.ts index b3c7f66..8073745 100644 --- a/src/api/versions.ts +++ b/src/api/versions.ts @@ -7,7 +7,6 @@ import { db, schema } from '../db/client.server.js' import { buildArkUrl, DEFAULT_NAAN } from '../lib/ark.js' import { canonicalize, - computeVersionHash, deriveSemver, filterRecordData, filterSchemasForPublic, @@ -22,6 +21,7 @@ import { resolveAccessibleCollection, resolveCollection, type SchemaEntry, + VersionHashStream, } from '../lib/version-helpers.server.js' import { dispatchDeliveries, enqueueWebhookDeliveries } from '../lib/webhooks.server.js' import { type AuthEnv, requireAuth } from './auth.server.js' @@ -1075,23 +1075,6 @@ const app = new Hono() const schemaSet = schemaEntries.map((e) => ({ slug: e.slug, schemaHash: e.schemaHash })) // Hash-only load: public hashes were computed and stored at commit time, // so a metadata-only version never needs the record bodies - const recordRows = await db - .select({ - hash: schema.versionRecords.recordHash, - publicRecordHash: schema.versionRecords.publicRecordHash, - recordId: schema.versionRecords.recordId, - type: schema.versionRecords.type, - private: schema.recordObjects.private, - }) - .from(schema.versionRecords) - // Only `private` still lives on record_objects; record_id and type are - // denormalized onto version_records. - .innerJoin( - schema.recordObjects, - eq(schema.versionRecords.recordHash, schema.recordObjects.hash), - ) - .where(eq(schema.versionRecords.versionId, latest.id)) - const recordHashes = recordRows.map((r) => r.hash) const fileHashes = ( await db .select({ hash: schema.versionFiles.fileHash }) @@ -1099,21 +1082,45 @@ const app = new Hono() .where(eq(schema.versionFiles.versionId, latest.id)) ).map((f) => f.hash) - const versionHash = computeVersionHash(schemaSet, recordHashes, fileHashes, newMetadata) - const privateTypes = getPrivateTypes(schemaEntries) const publicSchemaSet = schemaEntries .filter((e) => !privateTypes.has(e.slug)) .map((e) => ({ slug: e.slug, schemaHash: hashSchema(filterTypeSchema(e.schema)) })) - const publicRecordHashes = recordRows - .filter((r) => !r.private && !privateTypes.has(r.type)) - .map((r) => r.publicRecordHash ?? r.hash) - const publicHash = computeVersionHash( - publicSchemaSet, - publicRecordHashes, - fileHashes, - newMetadata, - ).replace('private:', 'public:') + + // Both digests are folded over hashes streamed from Postgres in sorted + // order, exactly as the commit path does. Loading every row to build two + // in-memory arrays made a metadata edit cost as much as a full push — on a + // multi-million-record collection, several hundred MB of JS objects to + // change a description. + // + // COLLATE "C" is required: the digest must see byte order, which is what + // Array.prototype.sort() produces, not the database's locale collation. + const client = db.$client + const CURSOR_CHUNK = 10_000 + + const versionHashStream = new VersionHashStream(schemaSet, fileHashes, newMetadata) + await client` + SELECT record_hash AS h FROM version_records + WHERE version_id = ${latest.id} + ORDER BY record_hash COLLATE "C" + `.cursor(CURSOR_CHUNK, (rows) => { + for (const row of rows) versionHashStream.push(row['h'] as string) + }) + const versionHash = versionHashStream.digest() + + const publicHashStream = new VersionHashStream(publicSchemaSet, fileHashes, newMetadata) + await client` + SELECT coalesce(vr.public_record_hash, vr.record_hash) AS h + FROM version_records vr + INNER JOIN record_objects ro ON ro.hash = vr.record_hash + WHERE vr.version_id = ${latest.id} + AND NOT ro.private + AND vr.type <> ALL(${[...privateTypes]}::text[]) + ORDER BY coalesce(vr.public_record_hash, vr.record_hash) COLLATE "C" + `.cursor(CURSOR_CHUNK, (rows) => { + for (const row of rows) publicHashStream.push(row['h'] as string) + }) + const publicHash = publicHashStream.digest().replace('private:', 'public:') const sv = deriveSemver(latest.semver, false, false, true) @@ -1154,18 +1161,14 @@ const app = new Hono() ) } - if (recordRows.length > 0) { - // Same schema set as the previous version, so public hashes carry over - await tx.insert(schema.versionRecords).values( - recordRows.map((r) => ({ - versionId: version!.id, - recordHash: r.hash, - publicRecordHash: r.publicRecordHash, - recordId: r.recordId, - type: r.type, - })), - ) - } + // Copy the record set server-side. The schema set is unchanged, so every + // column — including the public content-addresses — carries over as-is. + await tx.execute(sql` + INSERT INTO version_records (version_id, record_hash, public_record_hash, record_id, type) + SELECT ${version!.id}, record_hash, public_record_hash, record_id, type + FROM version_records + WHERE version_id = ${latest.id} + `) if (fileHashes.length > 0) { await tx