From a965f3abe1f8e72d58ad042054fb98e28c4f4a19 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 8 Aug 2026 06:51:10 +0000 Subject: [PATCH 1/2] feat(storage): route uploads through managed files rows and tenant-resolved buckets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both upload transports now resolve the tenant's logical bucket in the database and create an authoritative files row, and return the projection document whose id is what storage GC counts before collecting an object. - upload field identity (schema/table/column) is reported to the resolver - file_ref_field registry lookup supplies storage module + bucket intent - omitted bucketKey resolves the reserved default tag via function_resolution.resolve_default_bucket instead of an env bucket - multipart uploads stage, hash, promote to the content key, dedup, and insert a files row; image/upload columns keep url/filename/mime - removes the global BUCKET_NAME upload lane Refs constructive-planning#1476 (item 3 of §6) --- .../__tests__/managed-upload.test.ts | 484 ++++++++++++++++++ .../src/default-bucket.ts | 85 +++ .../src/file-ref-registry.ts | 163 ++++++ .../src/index.ts | 16 +- .../src/managed-upload.ts | 363 +++++++++++++ .../src/physical-bucket.ts | 142 +++++ .../src/plugin.ts | 184 +++---- .../src/s3-signer.ts | 31 ++ .../src/types.ts | 37 +- .../__tests__/upload-resolver.test.ts | 348 ++++++++++--- .../src/presigned-url-resolver.ts | 7 +- .../graphile-settings/src/upload-resolver.ts | 274 +++++++--- graphile/graphile-upload-plugin/src/index.ts | 1 + graphile/graphile-upload-plugin/src/plugin.ts | 24 +- graphile/graphile-upload-plugin/src/types.ts | 23 + uploads/s3-streamer/src/streamer.ts | 8 +- 16 files changed, 1921 insertions(+), 269 deletions(-) create mode 100644 graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts create mode 100644 graphile/graphile-presigned-url-plugin/src/default-bucket.ts create mode 100644 graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts create mode 100644 graphile/graphile-presigned-url-plugin/src/managed-upload.ts create mode 100644 graphile/graphile-presigned-url-plugin/src/physical-bucket.ts diff --git a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts new file mode 100644 index 0000000000..aed9002b95 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts @@ -0,0 +1,484 @@ +/** + * The managed upload lifecycle: bucket resolution, bucket rules, and the + * staged-object → files row + projection promotion. + * + * The database is faked at the query boundary (each test answers only the + * statements its path issues) so these cover the decisions this module makes — + * which bucket, whether to dedup, what the document ends up containing — + * without a server or S3. + */ + +import { clearFileRefFieldCache } from '../src/file-ref-registry'; +import { clearBucketCache, clearStorageModuleCache } from '../src/storage-module-cache'; +import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from '../src/types'; + +const DATABASE_ID = '00000000-0000-0000-0000-0000000000db'; +const APP_MODULE_ID = '11111111-1111-1111-1111-111111111111'; +const OTHER_MODULE_ID = '22222222-2222-2222-2222-222222222222'; +const BUCKET_ID = '33333333-3333-3333-3333-333333333333'; +const FILE_ID = '44444444-4444-4444-4444-444444444444'; +const FIELD = { schemaName: 'app_public', tableName: 'posts', columnName: 'image' }; + +interface QueryHandler { + match: RegExp; + rows: (values: unknown[]) => unknown[]; +} + +interface FakeDb { + withPgClient: any; + queries: Array<{ text: string; values?: unknown[] }>; +} + +/** + * A withPgClient whose statements are answered by the first matching handler. + * + * Unmatched statements throw rather than returning zero rows: a resolver that + * silently proceeds on an unanswered query is exactly the bug these tests exist + * to catch. + */ +function fakeDb(handlers: QueryHandler[]): FakeDb { + const queries: Array<{ text: string; values?: unknown[] }> = []; + const client = { + async query(opts: { text: string; values?: unknown[] }) { + queries.push(opts); + const handler = handlers.find((h) => h.match.test(opts.text)); + if (!handler) throw new Error(`unexpected query: ${opts.text}`); + return { rows: handler.rows(opts.values ?? []) }; + }, + withTransaction: (cb: any) => cb(client), + }; + return { + withPgClient: (_settings: any, cb: any) => cb(client), + queries, + }; +} + +const SET_CONFIG: QueryHandler = { match: /set_config/, rows: () => [] }; + +function storageModuleRow(overrides: Record = {}): Record { + return { + id: APP_MODULE_ID, + scope: 'app', + entity_table_id: null, + buckets_schema: 'storage_public', + buckets_table: 'app_buckets', + files_schema: 'storage_public', + files_table: 'app_files', + endpoint: null, + public_url_prefix: 'https://cdn.example.com', + provider: 'minio', + allowed_origins: null, + upload_url_expiry_seconds: null, + download_url_expiry_seconds: null, + default_max_file_size: 1000, + max_filename_length: null, + cache_ttl_seconds: null, + max_bulk_files: null, + max_bulk_total_size: null, + has_path_shares: false, + entity_schema: null, + entity_table: null, + ...overrides, + }; +} + +function bucketRow(overrides: Record = {}): Record { + return { + id: BUCKET_ID, + key: 'default-public', + type: 'public', + is_public: true, + owner_id: null, + allowed_mime_types: null, + max_file_size: null, + allow_custom_keys: false, + physical_name: 'myapp-default-public-db', + ...overrides, + }; +} + +const STORAGE_MODULES: QueryHandler = { + match: /FROM metaschema_modules_public\.storage_module/, + rows: () => [storageModuleRow()], +}; + +const NO_REGISTRY_ROW: QueryHandler = { + match: /file_ref_field/, + rows: () => [], +}; + +function options(): PresignedUrlPluginOptions { + return { + s3: { + client: { send: jest.fn() } as any, + bucket: 'connection-default', + region: 'us-east-1', + publicUrlPrefix: 'https://cdn.example.com', + }, + resolveBucketName: (databaseId: string, bucketKey: string) => `myapp-${bucketKey}-${databaseId}`, + }; +} + +function storageConfig(): StorageModuleConfig { + return { + id: APP_MODULE_ID, + scope: 'app', + bucketsQualifiedName: 'storage_public.app_buckets', + filesQualifiedName: 'storage_public.app_files', + defaultMaxFileSize: 1000, + hasPathShares: false, + allowedOrigins: null, + } as unknown as StorageModuleConfig; +} + +// Every lookup on this path is cached for the life of the process; each test +// states its own database, so the caches start empty. +beforeEach(() => { + clearStorageModuleCache(); + clearBucketCache(); + clearFileRefFieldCache(); +}); + +describe('buildFileProjection', () => { + it('names the files row in `id` so a document reference is countable', async () => { + const { buildFileProjection } = await import('../src/managed-upload'); + + const projection = buildFileProjection( + { id: FILE_ID, key: 'abc123', bucketId: BUCKET_ID, mime: 'image/png', size: 42, filename: 'hero.png' }, + { is_public: true }, + { publicUrlPrefix: 'https://cdn.example.com/' } as S3Config, + ); + + expect(projection).toEqual({ + id: FILE_ID, + key: 'abc123', + bucket_id: BUCKET_ID, + mime: 'image/png', + size: 42, + filename: 'hero.png', + url: 'https://cdn.example.com/abc123', + }); + }); + + it('omits `url` for a private bucket rather than storing an expiring one', async () => { + const { buildFileProjection } = await import('../src/managed-upload'); + + const projection = buildFileProjection( + { id: FILE_ID, key: 'abc123', bucketId: BUCKET_ID, mime: 'image/png', size: 42 }, + { is_public: false }, + { publicUrlPrefix: 'https://cdn.example.com' } as S3Config, + ); + + expect(projection.url).toBeUndefined(); + expect(projection.id).toBe(FILE_ID); + }); +}); + +describe('resolveManagedUploadTarget', () => { + it('resolves the default tag for an unregistered column, never an env bucket', async () => { + const { resolveManagedUploadTarget } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + NO_REGISTRY_ROW, + STORAGE_MODULES, + { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'default-public', bucket_type: 'public', physical_name: 'myapp-default-public-db' }] }, + { match: /FROM storage_public\.app_buckets/, rows: () => [bucketRow()] }, + ]); + + const target = await resolveManagedUploadTarget({ + options: options(), + withPgClient: db.withPgClient, + pgSettings: { 'jwt.claims.database_id': DATABASE_ID }, + databaseId: DATABASE_ID, + field: FIELD, + defaultPublicAccess: true, + }); + + expect(target.binding).toBeNull(); + expect(target.physicalName).toBe('myapp-default-public-db'); + expect(target.s3.bucket).toBe('myapp-default-public-db'); + expect(target.s3.bucket).not.toBe('connection-default'); + + const resolveCall = db.queries.find((q) => /resolve_default_bucket/.test(q.text)); + // scope, entity, public_access, and no explicit key: the reserved default tag. + expect(resolveCall?.values).toEqual([DATABASE_ID, 'app', null, true, null]); + }); + + it('passes a registered field\'s declared bucket key and publicness through', async () => { + const { resolveManagedUploadTarget } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + { + match: /file_ref_field/, + rows: () => [{ + id: 'ref-1', + storage_module_id: APP_MODULE_ID, + bucket_key: 'avatars', + bucket_tags: null, + is_public: false, + enforce_fk: true, + }], + }, + STORAGE_MODULES, + { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'avatars', bucket_type: 'private', physical_name: 'myapp-avatars-db' }] }, + { match: /FROM storage_public\.app_buckets/, rows: () => [bucketRow({ key: 'avatars', type: 'private', is_public: false, physical_name: 'myapp-avatars-db' })] }, + ]); + + const target = await resolveManagedUploadTarget({ + options: options(), + withPgClient: db.withPgClient, + pgSettings: null, + databaseId: DATABASE_ID, + field: FIELD, + defaultPublicAccess: true, + }); + + const resolveCall = db.queries.find((q) => /resolve_default_bucket/.test(q.text)); + expect(resolveCall?.values).toEqual([DATABASE_ID, 'app', null, false, 'avatars']); + expect(target.bucket.key).toBe('avatars'); + }); + + it('records the physical name on first provision instead of re-minting it', async () => { + const { resolveManagedUploadTarget } = await import('../src/managed-upload'); + const ensureBucketProvisioned = jest.fn().mockResolvedValue(undefined); + const db = fakeDb([ + SET_CONFIG, + NO_REGISTRY_ROW, + STORAGE_MODULES, + { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'default-public', bucket_type: 'public', physical_name: null }] }, + { match: /SELECT id, key, type/, rows: () => [bucketRow({ physical_name: null })] }, + { match: /UPDATE storage_public\.app_buckets/, rows: () => [] }, + ]); + + const target = await resolveManagedUploadTarget({ + options: { ...options(), ensureBucketProvisioned }, + withPgClient: db.withPgClient, + pgSettings: null, + databaseId: DATABASE_ID, + field: FIELD, + defaultPublicAccess: true, + }); + + expect(target.physicalName).toBe(`myapp-default-public-${DATABASE_ID}`); + expect(ensureBucketProvisioned).toHaveBeenCalledWith( + `myapp-default-public-${DATABASE_ID}`, 'public', DATABASE_ID, null, + ); + const update = db.queries.find((q) => /UPDATE/.test(q.text)); + expect(update?.values).toEqual([`myapp-default-public-${DATABASE_ID}`, BUCKET_ID]); + }); + + it('raises when the database has no storage module to default to', async () => { + const { resolveManagedUploadTarget } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + NO_REGISTRY_ROW, + { match: /FROM metaschema_modules_public\.storage_module/, rows: () => [] }, + ]); + + await expect( + resolveManagedUploadTarget({ + options: options(), + withPgClient: db.withPgClient, + pgSettings: null, + databaseId: DATABASE_ID, + field: FIELD, + defaultPublicAccess: true, + }), + ).rejects.toThrow('STORAGE_MODULE_NOT_FOUND'); + }); + + it('raises when the registry names a module the database does not have', async () => { + const { resolveManagedUploadTarget } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + { + match: /file_ref_field/, + rows: () => [{ id: 'ref-1', storage_module_id: OTHER_MODULE_ID, bucket_key: null, bucket_tags: null, is_public: null, enforce_fk: false }], + }, + STORAGE_MODULES, + ]); + + await expect( + resolveManagedUploadTarget({ + options: options(), + withPgClient: db.withPgClient, + pgSettings: null, + databaseId: DATABASE_ID, + field: FIELD, + defaultPublicAccess: true, + }), + ).rejects.toThrow(OTHER_MODULE_ID); + }); + + it('refuses an entity-scoped module, whose bucket depends on an owner row', async () => { + const { resolveManagedUploadTarget } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + { + match: /file_ref_field/, + rows: () => [{ id: 'ref-1', storage_module_id: OTHER_MODULE_ID, bucket_key: null, bucket_tags: null, is_public: null, enforce_fk: false }], + }, + { + match: /FROM metaschema_modules_public\.storage_module/, + rows: () => [storageModuleRow({ + id: OTHER_MODULE_ID, + scope: 'data_room', + entity_table_id: 'et-1', + entity_schema: 'app_public', + entity_table: 'data_rooms', + buckets_table: 'data_room_buckets', + files_table: 'data_room_files', + })], + }, + ]); + + await expect( + resolveManagedUploadTarget({ + options: options(), + withPgClient: db.withPgClient, + pgSettings: null, + databaseId: DATABASE_ID, + field: FIELD, + defaultPublicAccess: true, + }), + ).rejects.toThrow('STORAGE_SCOPE_UNSUPPORTED'); + }); +}); + +describe('assertUploadAllowedByBucket', () => { + function target(bucket: Partial): any { + return { bucket: { ...bucketRow(), ...bucket }, storageConfig: storageConfig() }; + } + + it('enforces the bucket mime allowlist on the streaming transport too', async () => { + const { assertUploadAllowedByBucket } = await import('../src/managed-upload'); + expect(() => + assertUploadAllowedByBucket(target({ allowed_mime_types: ['image/*'] }), 'application/pdf', 10), + ).toThrow('CONTENT_TYPE_NOT_ALLOWED'); + expect(() => + assertUploadAllowedByBucket(target({ allowed_mime_types: ['image/*'] }), 'image/png', 10), + ).not.toThrow(); + }); + + it('enforces the size cap, falling back to the module default', async () => { + const { assertUploadAllowedByBucket } = await import('../src/managed-upload'); + expect(() => assertUploadAllowedByBucket(target({ max_file_size: 5 }), 'image/png', 6)).toThrow('FILE_TOO_LARGE'); + expect(() => assertUploadAllowedByBucket(target({ max_file_size: null }), 'image/png', 1001)).toThrow('FILE_TOO_LARGE'); + expect(() => assertUploadAllowedByBucket(target({}), 'image/png', 0)).toThrow('INVALID_FILE_SIZE'); + }); +}); + +describe('finalizeStagedUpload', () => { + const s3 = { + client: { send: jest.fn().mockResolvedValue({}) }, + bucket: 'myapp-default-public-db', + region: 'us-east-1', + publicUrlPrefix: 'https://cdn.example.com', + } as unknown as S3Config; + + const target: any = { + databaseId: DATABASE_ID, + storageConfig: storageConfig(), + bucket: bucketRow(), + physicalName: 'myapp-default-public-db', + s3, + binding: null, + }; + + const staged = { + stagingKey: '.staging/tmp-1', + contentHash: 'a'.repeat(64), + contentType: 'image/png', + size: 16, + filename: 'hero.png', + }; + + beforeEach(() => { + (s3.client.send as jest.Mock).mockClear(); + }); + + it('promotes the staged object to its content key and inserts the files row', async () => { + const { finalizeStagedUpload } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + { match: /SELECT id, key, mime_type/, rows: () => [] }, + { match: /INSERT INTO storage_public\.app_files/, rows: () => [{ id: FILE_ID }] }, + ]); + + const { projection, deduplicated } = await finalizeStagedUpload({ + target, withPgClient: db.withPgClient, pgSettings: null, staged, + }); + + expect(deduplicated).toBe(false); + expect(projection).toEqual({ + id: FILE_ID, + key: staged.contentHash, + bucket_id: BUCKET_ID, + mime: 'image/png', + size: 16, + filename: 'hero.png', + url: `https://cdn.example.com/${staged.contentHash}`, + }); + + const insert = db.queries.find((q) => /INSERT/.test(q.text)); + expect(insert?.values).toEqual([BUCKET_ID, staged.contentHash, staged.contentHash, 'image/png', 16, 'hero.png', true]); + // Copy to the content key, then drop the staged object. + expect(s3.client.send).toHaveBeenCalledTimes(2); + }); + + it('reuses the existing row and drops the staged bytes on a hash collision', async () => { + const { finalizeStagedUpload } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + { + match: /SELECT id, key, mime_type/, + rows: () => [{ id: FILE_ID, key: staged.contentHash, mime_type: 'image/png', size: 16, filename: 'original.png' }], + }, + ]); + + const { projection, deduplicated } = await finalizeStagedUpload({ + target, withPgClient: db.withPgClient, pgSettings: null, staged, + }); + + expect(deduplicated).toBe(true); + expect(projection.id).toBe(FILE_ID); + expect(projection.filename).toBe('original.png'); + expect(db.queries.some((q) => /INSERT/.test(q.text))).toBe(false); + // Only the staged object is deleted; nothing is copied. + expect(s3.client.send).toHaveBeenCalledTimes(1); + }); + + it('abandons both keys when the files row cannot be inserted', async () => { + const { finalizeStagedUpload } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + { match: /SELECT id, key, mime_type/, rows: () => [] }, + { match: /INSERT INTO storage_public\.app_files/, rows: () => { throw new Error('insert boom'); } }, + ]); + + await expect( + finalizeStagedUpload({ target, withPgClient: db.withPgClient, pgSettings: null, staged }), + ).rejects.toThrow('insert boom'); + + // Copy + delete promoted + delete staged: bytes no row names are bytes GC + // can never reach, so they do not outlive the failed call. + expect(s3.client.send).toHaveBeenCalledTimes(3); + }); + + it('rejects bytes the bucket does not allow before writing anything', async () => { + const { finalizeStagedUpload } = await import('../src/managed-upload'); + const db = fakeDb([SET_CONFIG]); + + await expect( + finalizeStagedUpload({ + target: { ...target, bucket: { ...bucketRow(), allowed_mime_types: ['image/jpeg'] } }, + withPgClient: db.withPgClient, + pgSettings: null, + staged, + }), + ).rejects.toThrow('CONTENT_TYPE_NOT_ALLOWED'); + + expect(db.queries).toHaveLength(0); + expect(s3.client.send).not.toHaveBeenCalled(); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/src/default-bucket.ts b/graphile/graphile-presigned-url-plugin/src/default-bucket.ts new file mode 100644 index 0000000000..eb7a791a98 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/default-bucket.ts @@ -0,0 +1,85 @@ +/** + * Server-side bucket resolution. + * + * Which bucket a write lands in belongs to the database, never to the client and + * never to the server's environment: a client-chosen key means a different + * bucket per tenant, and an env-level bucket name means storage that belongs to + * no tenant at all. `function_resolution.resolve_default_bucket` is the one + * place that answers it — a logical key when the field declares one, otherwise + * the reserved default tag for the requested access ('default' / 'default-public'). + * + * Zero matches and several matches both raise inside SQL, so there is nothing to + * guess here: this module only carries the question in and the coordinate out. + */ + +import { Logger } from '@pgpmjs/logger'; + +const log = new Logger('graphile-presigned-url:default-bucket'); + +/** + * The resolved bucket coordinate. + * + * `physicalName` is the recorded S3 bucket name, or null when the logical + * bucket has never been provisioned — the caller mints and records it then. + */ +export interface ResolvedBucketCoordinate { + bucketId: string; + resolvedKey: string; + bucketType: 'public' | 'private' | 'temp'; + physicalName: string | null; +} + +const RESOLVE_DEFAULT_BUCKET_QUERY = ` + SELECT bucket_id, resolved_key, bucket_type, physical_name + FROM function_resolution.resolve_default_bucket($1, $2, $3, $4, $5) +`; + +/** + * Resolve the bucket a write should land in. + * + * @param scope - The storage module's scope ('app' for database-wide storage) + * @param entityId - The owning entity row for an entity-scoped module, else null + * @param publicAccess - Which reserved default tag to use when no key is named, + * and an assertion on the named bucket's type when one is + * @param bucketKey - The field's declared logical key, or null for the default + */ +export async function resolveDefaultBucket( + pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, + databaseId: string, + scope: string, + entityId: string | null, + publicAccess: boolean, + bucketKey: string | null, +): Promise { + const result = await pgClient.query({ + text: RESOLVE_DEFAULT_BUCKET_QUERY, + values: [databaseId, scope, entityId, publicAccess, bucketKey], + }); + + const row = result.rows[0] as { + bucket_id: string; + resolved_key: string; + bucket_type: string; + physical_name: string | null; + } | undefined; + + if (!row) { + // resolve_default_bucket raises on zero and on several matches, so an empty + // result means the function did not run as declared rather than "no bucket". + throw new Error( + `STORAGE_DEFAULT_BUCKET_NO_ROW: resolve_default_bucket returned no row for ` + + `database=${databaseId} scope=${scope} public=${publicAccess} key=${bucketKey ?? ''}`, + ); + } + + log.debug( + `Resolved bucket ${row.resolved_key} (${row.bucket_type}) for database=${databaseId} scope=${scope}`, + ); + + return { + bucketId: row.bucket_id, + resolvedKey: row.resolved_key, + bucketType: row.bucket_type as ResolvedBucketCoordinate['bucketType'], + physicalName: row.physical_name, + }; +} diff --git a/graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts b/graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts new file mode 100644 index 0000000000..692c1cb281 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/file-ref-registry.ts @@ -0,0 +1,163 @@ +/** + * The `file_ref_field` registry: which storage module and bucket a managed + * document column writes into. + * + * An `image`/`upload` column is a projection of a files row, and the decision of + * *where* those bytes live is a property of the field declaration, not of the + * request. The registry records that intent per (table, column) — a storage + * module plus either a logical bucket key, a tag selector, or nothing at all + * (meaning the reserved default tag for the declared publicness). + * + * This module answers one question — "what does a write to this column bind + * to?" — and answers it loudly: an unregistered column raises rather than + * falling back to a server-global bucket, because a silent fallback is how the + * unmanaged lane produced objects no tenant owned. + */ + +import { Logger } from '@pgpmjs/logger'; +import { LRUCache } from 'lru-cache'; + +const log = new Logger('graphile-presigned-url:file-ref-registry'); + +const FIVE_MINUTES_MS = 1000 * 60 * 5; +const ONE_HOUR_MS = 1000 * 60 * 60; + +/** + * A field's recorded storage intent. + * + * `bucketKey` and `bucketTags` are mutually exclusive by table constraint, and + * both may be absent — resolution then uses the reserved default tag for + * `isPublic`. Nothing here is a physical bucket name or id: the concrete bucket + * is resolved per written row, inside the tenant. + */ +export interface FileRefFieldBinding { + id: string; + storageModuleId: string; + bucketKey: string | null; + bucketTags: string[] | null; + isPublic: boolean | null; + enforceFk: boolean; +} + +/** + * Resolve the registry row for a document column. + * + * Joined through metaschema rather than keyed by name, because the registry + * records field *ids*: the physical (schema, table, column) triple is what the + * GraphQL layer knows, and metaschema is the only thing that maps one to the + * other. + */ +const FILE_REF_FIELD_QUERY = ` + SELECT + frf.id, + frf.storage_module_id, + frf.bucket_key, + frf.bucket_tags::text[] AS bucket_tags, + frf.is_public, + frf.enforce_fk + FROM metaschema_modules_public.file_ref_field frf + JOIN metaschema_public.field f ON f.id = frf.field_id + JOIN metaschema_public.table t ON t.id = frf.table_id + JOIN metaschema_public.schema s ON s.id = t.schema_id + WHERE frf.database_id = $1 + AND s.schema_name = $2 + AND t.name = $3 + AND f.name = $4 + LIMIT 1 +`; + +interface FileRefFieldRow { + id: string; + storage_module_id: string; + bucket_key: string | null; + bucket_tags: string[] | null; + is_public: boolean | null; + enforce_fk: boolean; +} + +/** + * LRU cache of field bindings. + * + * A binding is schema, not data: it changes only when a database is + * re-provisioned, so it caches on the same terms as the storage module config + * next to it. Misses are never cached — an unregistered column is a hard error + * every time it is written, not a remembered "no". + */ +const bindingCache = new LRUCache({ + max: 500, + ttl: process.env.NODE_ENV === 'development' ? FIVE_MINUTES_MS : ONE_HOUR_MS, + updateAgeOnGet: true, +}); + +export class FileRefFieldNotRegisteredError extends Error { + constructor( + public readonly databaseId: string, + public readonly schemaName: string, + public readonly tableName: string, + public readonly columnName: string, + ) { + super( + `FILE_REF_FIELD_NOT_REGISTERED: ${schemaName}.${tableName}.${columnName} ` + + `is not a registered file-reference field in database ${databaseId}. ` + + 'A managed upload needs the declared storage module and bucket intent; ' + + 'there is no server-global bucket to fall back to.', + ); + this.name = 'FileRefFieldNotRegisteredError'; + } +} + +/** + * Look up the storage binding for a document column, or throw. + * + * The read runs on whichever client the caller passes. The registry is schema + * metadata rather than tenant rows, so callers resolve it in the system lane — + * the RLS that matters is on the files table the upload eventually writes. + */ +export async function getFileRefFieldBinding( + pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, + databaseId: string, + field: { schemaName: string; tableName: string; columnName: string }, +): Promise { + const cacheKey = `file-ref:${databaseId}:${field.schemaName}.${field.tableName}.${field.columnName}`; + const cached = bindingCache.get(cacheKey); + if (cached) return cached; + + const result = await pgClient.query({ + text: FILE_REF_FIELD_QUERY, + values: [databaseId, field.schemaName, field.tableName, field.columnName], + }); + + if (result.rows.length === 0) { + throw new FileRefFieldNotRegisteredError( + databaseId, + field.schemaName, + field.tableName, + field.columnName, + ); + } + + const row = result.rows[0] as FileRefFieldRow; + const binding: FileRefFieldBinding = { + id: row.id, + storageModuleId: row.storage_module_id, + bucketKey: row.bucket_key, + bucketTags: row.bucket_tags, + isPublic: row.is_public, + enforceFk: row.enforce_fk, + }; + + bindingCache.set(cacheKey, binding); + log.debug( + `Bound ${field.schemaName}.${field.tableName}.${field.columnName} to storage module ` + + `${binding.storageModuleId} (bucket_key=${binding.bucketKey ?? ''})`, + ); + + return binding; +} + +/** + * Drop cached bindings. Used by tests and after a re-provision. + */ +export function clearFileRefFieldCache(): void { + bindingCache.clear(); +} diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index 3d8ee4fe44..dbaf92d28c 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -27,10 +27,24 @@ * ``` */ +export type { ResolvedBucketCoordinate } from './default-bucket'; +export { resolveDefaultBucket } from './default-bucket'; export { createDownloadUrlPlugin } from './download-url-field'; +export type { FileRefFieldBinding } from './file-ref-registry'; +export { clearFileRefFieldCache, FileRefFieldNotRegisteredError, getFileRefFieldBinding } from './file-ref-registry'; +export { + assertUploadAllowedByBucket, + buildFileProjection, + type FileProjection, + finalizeStagedUpload, + type ManagedUploadTarget, + resolveManagedUploadTarget, +} from './managed-upload'; +export { mintPhysicalBucketName, provisionAndRecordPhysicalBucket, resolveS3, resolveS3ForDatabase } from './physical-bucket'; export { createPresignedUrlPlugin,PresignedUrlPlugin } from './plugin'; export { PresignedUrlPreset } from './preset'; -export { deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject } from './s3-signer'; +export { type WithPgClient, withRequestPgClient } from './request-pg-client'; +export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject } from './s3-signer'; export { clearBucketCache, clearStorageModuleCache, getBucketConfig, getStorageModuleConfig, getStorageModuleConfigForOwner, isS3BucketProvisioned, loadAllStorageModules, markS3BucketProvisioned,resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache'; export type { BucketConfig, diff --git a/graphile/graphile-presigned-url-plugin/src/managed-upload.ts b/graphile/graphile-presigned-url-plugin/src/managed-upload.ts new file mode 100644 index 0000000000..db864e9b70 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/managed-upload.ts @@ -0,0 +1,363 @@ +/** + * The managed upload lifecycle, shared by both transports. + * + * Multipart-through-GraphQL and presigned two-step are transports for one + * lifecycle, not two data models: either way an object gets a files row, a + * server-chosen key, a tenant-resolved bucket, and a projection document that + * carries the files row's id. The only difference is who moves the bytes. + * + * This module owns the parts that are the same: + * * `resolveManagedUploadTarget` — from a document column to a concrete + * (storage module, bucket, physical bucket, S3 config). + * * `finalizeStagedUpload` — from bytes already in S3 under a staging key to a + * files row and a projection document, deduplicating on content hash. + * * `buildFileProjection` — the document shape the column stores. + * + * Nothing here bakes a presigned URL into a row: `url` is populated only for a + * public bucket, where it is a stable CDN address rather than a credential with + * an expiry. + */ + +import { Logger } from '@pgpmjs/logger'; + +import { resolveDefaultBucket } from './default-bucket'; +import { type FileRefFieldBinding, getFileRefFieldBinding } from './file-ref-registry'; +import { provisionAndRecordPhysicalBucket, resolveS3ForDatabase } from './physical-bucket'; +import { type WithPgClient, withRequestPgClient } from './request-pg-client'; +import { copyS3Object, deleteS3Object } from './s3-signer'; +import { getBucketConfig, loadAllStorageModules } from './storage-module-cache'; +import type { BucketConfig, FileProjection, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; + +const log = new Logger('graphile-presigned-url:managed-upload'); + +/** + * The document a managed `image`/`upload` column stores. + * + * `id` is the files row — the load-bearing field: it is what makes the column a + * projection rather than a second, unmanaged copy of the truth, and it is what + * storage GC counts before collecting an object. + * + * `url` is retained for existing readers of the pre-managed shape and is set + * only for public buckets. Prefer `id` plus the files row's `downloadUrl`, which + * is late-bound and works for private buckets too. + */ +export type { FileProjection } from './types'; + +/** + * Build the projection document for a files row. + * + * A public bucket has a stable address, so `url` is a real, durable value there. + * A private bucket has no such address — only presigned, expiring ones — so the + * field is omitted rather than filled with a URL that dies in an hour. + */ +export function buildFileProjection( + file: { id: string; key: string; bucketId: string; mime: string; size: number; filename?: string | null }, + bucket: { is_public: boolean }, + s3: S3Config, +): FileProjection { + const projection: FileProjection = { + id: file.id, + key: file.key, + bucket_id: file.bucketId, + mime: file.mime, + size: file.size, + }; + if (file.filename) projection.filename = file.filename; + + if (bucket.is_public && s3.publicUrlPrefix) { + projection.url = `${s3.publicUrlPrefix.replace(/\/$/, '')}/${file.key}`; + } + + return projection; +} + +/** + * Everything a managed upload needs before bytes move. + */ +export interface ManagedUploadTarget { + databaseId: string; + storageConfig: StorageModuleConfig; + bucket: BucketConfig; + physicalName: string; + s3: S3Config; + /** The registry row, or null when the column predates registration. */ + binding: FileRefFieldBinding | null; +} + +/** + * Resolve where a write to a document column lands. + * + * Two routes, one rule — the bucket is always resolved inside the tenant: + * * a registered column names its storage module, and either a logical bucket + * key or the reserved default tag for its declared publicness; + * * an unregistered column (a bare `image`/`upload` on a database provisioned + * before the registry) falls back to the app-scope module and the same + * reserved default tag. That is a *tenant* default, not an environment one. + * + * A database with no storage module raises: there is nowhere tenant-owned to put + * the bytes, and the deployment's configured bucket is not an answer. + */ +export async function resolveManagedUploadTarget(args: { + options: PresignedUrlPluginOptions; + withPgClient: WithPgClient; + pgSettings: Record | null; + databaseId: string; + field: { schemaName: string; tableName: string; columnName: string }; + /** Publicness to use when the column is unregistered. */ + defaultPublicAccess: boolean; +}): Promise { + const { options, withPgClient, pgSettings, databaseId, field, defaultPublicAccess } = args; + + // Registry and module registration are schema metadata, not tenant rows: + // read them in the system lane, like every other config read here. + const binding = await withPgClient(null, async (pgClient: any): Promise => { + try { + return await getFileRefFieldBinding(pgClient, databaseId, field); + } catch (err: any) { + // An unregistered column is a legitimate state (it predates the registry) + // and falls back to the tenant's app-scope default below. Any other + // failure — a broken connection, a missing registry table — is not. + if (err?.name === 'FileRefFieldNotRegisteredError') return null; + throw err; + } + }); + + const allConfigs = await withPgClient(null, (pgClient: any) => + loadAllStorageModules(pgClient, databaseId), + ); + + const storageConfig = binding + ? allConfigs.find((c) => c.id === binding.storageModuleId) + : allConfigs.find((c) => c.scope === 'app'); + + if (!storageConfig) { + throw new Error( + binding + ? `STORAGE_MODULE_NOT_FOUND: file_ref_field ${binding.id} names storage module ` + + `${binding.storageModuleId}, which database ${databaseId} does not have` + : `STORAGE_MODULE_NOT_FOUND: ${field.schemaName}.${field.tableName}.${field.columnName} is an ` + + `unregistered upload column and database ${databaseId} has no app-scope storage module to ` + + 'default to; there is no environment bucket to fall back to', + ); + } + + if (storageConfig.scope !== 'app') { + // An entity-scoped module resolves its bucket per owning row, and a + // multipart column write does not carry one. Refuse rather than write a + // tenant's file into whichever bucket happened to resolve. + throw new Error( + `STORAGE_SCOPE_UNSUPPORTED: ${field.schemaName}.${field.tableName}.${field.columnName} binds to ` + + `'${storageConfig.scope}'-scoped storage, which resolves its bucket per owner row. ` + + 'Use the presigned upload mutation, which takes an ownerId.', + ); + } + + const publicAccess = binding?.isPublic ?? defaultPublicAccess; + + // Bucket resolution and the bucket read run under the request role: what the + // caller may store into is exactly what RLS lets them see. + const coordinate = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => + resolveDefaultBucket( + pgClient, + databaseId, + storageConfig.scope, + null, + publicAccess, + binding?.bucketKey ?? null, + ), + ); + + const bucket = await withRequestPgClient(withPgClient, pgSettings, (pgClient) => + getBucketConfig(pgClient, storageConfig, databaseId, coordinate.resolvedKey), + ); + if (!bucket) { + throw new Error( + `BUCKET_NOT_FOUND: bucket "${coordinate.resolvedKey}" resolved for ` + + `${field.schemaName}.${field.tableName}.${field.columnName} is not readable`, + ); + } + + const physicalName = bucket.physical_name === null + ? await provisionAndRecordPhysicalBucket( + options, withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins, + ) + : bucket.physical_name; + + return { + databaseId, + storageConfig, + bucket, + physicalName, + s3: resolveS3ForDatabase(options, storageConfig, physicalName), + binding, + }; +} + +/** + * Validate an upload against the resolved bucket's rules. + * + * The same rules the presigned lane enforces — a transport must not be a way + * around a bucket's mime allowlist or size cap. + */ +export function assertUploadAllowedByBucket( + target: ManagedUploadTarget, + contentType: string, + size: number, +): void { + const { bucket, storageConfig } = target; + + if (bucket.allowed_mime_types && bucket.allowed_mime_types.length > 0) { + const isAllowed = bucket.allowed_mime_types.some((pattern) => { + if (pattern === '*/*') return true; + if (pattern.endsWith('/*')) return contentType.startsWith(pattern.slice(0, -1)); + return contentType === pattern; + }); + if (!isAllowed) { + throw new Error(`CONTENT_TYPE_NOT_ALLOWED: ${contentType} not in bucket allowed types`); + } + } + + const maxSize = bucket.max_file_size ?? storageConfig.defaultMaxFileSize; + if (size > maxSize) { + throw new Error(`FILE_TOO_LARGE: ${size} bytes exceeds the ${maxSize} byte limit`); + } + if (size <= 0) { + throw new Error('INVALID_FILE_SIZE: an upload must carry at least one byte'); + } +} + +/** Delete an object we are abandoning, without masking the failure in progress. */ +async function bestEffortDelete(s3: S3Config, key: string): Promise { + try { + await deleteS3Object(s3, key); + } catch (err) { + log.warn(`Failed to clean up abandoned object ${key}: ${err}`); + } +} + +/** + * Turn bytes already staged in S3 into a files row and a projection document. + * + * The content hash is only known once the stream has been read, so a streaming + * transport writes to a staging key first and promotes here: + * + * * hash already present in this bucket → drop the staged object, reuse the + * existing files row. Dedup is a property of the object, so it holds no + * matter which transport wrote it first. + * * otherwise → server-side copy to the content-addressed key, drop the staged + * object, insert the files row. + * + * The row is inserted *after* the bytes land, so the confirm-upload job the + * insert trigger enqueues finds the object and completes the + * `requested → uploaded` transition without any extra wiring here. + * + * Every failure path leaves S3 as it found it. Bytes written by this call and + * not reachable through a files row would be invisible to storage GC, which + * collects objects by walking rows — so an object is only left behind once the + * row naming it exists. + */ +export async function finalizeStagedUpload(args: { + target: ManagedUploadTarget; + withPgClient: WithPgClient; + pgSettings: Record | null; + staged: { + stagingKey: string; + contentHash: string; + contentType: string; + size: number; + filename?: string | null; + }; +}): Promise<{ projection: FileProjection; deduplicated: boolean }> { + const { target, withPgClient, pgSettings, staged } = args; + const { storageConfig, bucket, s3 } = target; + + assertUploadAllowedByBucket(target, staged.contentType, staged.size); + + const finalKey = staged.contentHash; + + const existing = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { + const result = await pgClient.query({ + text: `SELECT id, key, mime_type, size, filename + FROM ${storageConfig.filesQualifiedName} + WHERE content_hash = $1 AND bucket_id = $2 + LIMIT 1`, + values: [staged.contentHash, bucket.id], + }); + return result.rows[0] as + | { id: string; key: string; mime_type: string; size: number; filename: string | null } + | undefined; + }); + + if (existing) { + log.info(`Dedup hit: file ${existing.id} already carries hash ${staged.contentHash}`); + await deleteS3Object(s3, staged.stagingKey); + + return { + projection: buildFileProjection( + { + id: existing.id, + key: existing.key, + bucketId: bucket.id, + mime: existing.mime_type, + size: Number(existing.size), + filename: existing.filename, + }, + bucket, + s3, + ), + deduplicated: true, + }; + } + + await copyS3Object(s3, staged.stagingKey, finalKey, staged.contentType); + + let fileId: string; + try { + fileId = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { + const result = await pgClient.query({ + text: `INSERT INTO ${storageConfig.filesQualifiedName} + (bucket_id, key, content_hash, mime_type, size, filename, is_public) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING id`, + values: [ + bucket.id, + finalKey, + staged.contentHash, + staged.contentType, + staged.size, + staged.filename ?? null, + bucket.is_public, + ], + }); + return (result.rows[0] as { id: string }).id; + }); + } catch (err) { + // No row names either key, so both are unreachable to GC. Dropping the + // promoted copy is safe precisely because the dedup probe above found no row + // on this hash: nothing else in this bucket is entitled to those bytes. + // Cleanup must never replace the failure that caused it. + await bestEffortDelete(s3, finalKey); + await bestEffortDelete(s3, staged.stagingKey); + throw err; + } + + await deleteS3Object(s3, staged.stagingKey); + + log.info(`Managed upload created file ${fileId} at ${bucket.key}/${finalKey}`); + + return { + projection: buildFileProjection( + { + id: fileId, + key: finalKey, + bucketId: bucket.id, + mime: staged.contentType, + size: staged.size, + filename: staged.filename, + }, + bucket, + s3, + ), + deduplicated: false, + }; +} diff --git a/graphile/graphile-presigned-url-plugin/src/physical-bucket.ts b/graphile/graphile-presigned-url-plugin/src/physical-bucket.ts new file mode 100644 index 0000000000..3b93f82251 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/physical-bucket.ts @@ -0,0 +1,142 @@ +/** + * Physical bucket coordinates: minting a name once, recording it, and building + * an S3 config against a *known* name. + * + * A logical bucket belongs to a tenant; a physical bucket is an S3 name. The + * mapping is recorded on the bucket row the first time it is provisioned, and + * from then on the recorded value is the only coordinate anything reads — no + * name is ever recomputed from a prefix convention, and there is no + * environment-level bucket standing in for a tenant's. + */ + +import { Logger } from '@pgpmjs/logger'; + +import { type WithPgClient, withRequestPgClient } from './request-pg-client'; +import { isS3BucketProvisioned, markS3BucketProvisioned } from './storage-module-cache'; +import type { BucketConfig, PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; + +const log = new Logger('graphile-presigned-url:physical-bucket'); + +/** + * Resolve the plugin's S3 connection (credentials, endpoint, region), memoizing + * a lazy getter on first use. + * + * `s3.bucket` on the result is the deployment's *default* physical bucket. It is + * a connection default only — never a tenant's bucket. Every upload path + * resolves its physical bucket from the tenant's bucket row. + */ +export function resolveS3(options: PresignedUrlPluginOptions): S3Config { + if (typeof options.s3 === 'function') { + const resolved = options.s3(); + options.s3 = resolved; + return resolved; + } + return options.s3; +} + +/** + * Mint the physical S3 bucket name for a logical bucket's first provision. + * + * This is a naming *policy*, consulted exactly once per bucket — before the + * physical bucket exists. Once provisioned, the recorded `physical_name` on the + * row is authoritative and this function must not be consulted again. + * + * There is no fallback to the configured `s3.bucket`: a deployment-wide bucket + * name is not a tenant's storage, and silently minting one is how objects ended + * up in a bucket no database owned. A deployment that wants per-tenant buckets + * must supply the policy. + */ +export function mintPhysicalBucketName( + options: PresignedUrlPluginOptions, + databaseId: string, + bucketKey: string, +): string { + if (!options.resolveBucketName) { + throw new Error( + 'STORAGE_BUCKET_NAME_POLICY_MISSING: no resolveBucketName was configured, so there is ' + + `no name to provision for bucket "${bucketKey}" of database ${databaseId}. ` + + 'Physical bucket naming is a deployment policy; the configured s3.bucket is a ' + + 'connection default and is never a tenant bucket.', + ); + } + return options.resolveBucketName(databaseId, bucketKey); +} + +/** + * Build the S3 config for a *known* physical bucket. `physicalName` is + * required — callers must resolve the coordinate (stored row value, or a + * freshly provisioned name) before getting here. No name is ever recomputed. + */ +export function resolveS3ForDatabase( + options: PresignedUrlPluginOptions, + storageConfig: StorageModuleConfig, + physicalName: string, +): S3Config { + const globalS3 = resolveS3(options); + const publicUrlPrefix = storageConfig.publicUrlPrefix != null + ? storageConfig.publicUrlPrefix + : globalS3.publicUrlPrefix; + + if (physicalName === globalS3.bucket && publicUrlPrefix === globalS3.publicUrlPrefix) { + return globalS3; + } + + return { + ...globalS3, + bucket: physicalName, + ...(publicUrlPrefix != null ? { publicUrlPrefix } : {}), + }; +} + +/** + * First provision of a logical bucket: mint a name, create the physical S3 + * bucket, and record the exact name on the source row. Returns the recorded + * physical name. + * + * Only called when the row has no `physical_name` yet. Afterwards the stored + * value is the durable coordinate: route resolution and every later read use + * it verbatim; nothing is recomputed. + * + * The record write runs in the system lane (privileged role, so it bypasses the + * RLS that stops request roles from UPDATE-ing bucket rows) — it is server + * bookkeeping, not request data. It still carries the tenant `database_id` + * claim, because the buckets table's catalog-sync trigger calls + * `jwt_private.current_database_id()` and would otherwise raise + * DATABASE_CLAIM_REQUIRED; `withRequestPgClient` applies that claim inside the + * write's transaction without switching off the privileged role. + * `bucket` (the cached config) is mutated in place so subsequent reads observe + * the recorded name without a DB round-trip. + */ +export async function provisionAndRecordPhysicalBucket( + options: PresignedUrlPluginOptions, + withPgClient: WithPgClient, + storageConfig: StorageModuleConfig, + databaseId: string, + bucket: BucketConfig, + allowedOrigins: string[] | null, +): Promise { + const s3BucketName = mintPhysicalBucketName(options, databaseId, bucket.key); + + if (options.ensureBucketProvisioned && !isS3BucketProvisioned(s3BucketName)) { + log.info(`Lazy-provisioning S3 bucket "${s3BucketName}" for database ${databaseId}`); + await options.ensureBucketProvisioned(s3BucketName, bucket.type, databaseId, allowedOrigins); + markS3BucketProvisioned(s3BucketName); + log.info(`Lazy-provisioned S3 bucket "${s3BucketName}" successfully`); + } + + // Record the physical coordinate on the source row. The `physical_name IS NULL` + // guard keeps this idempotent and race-safe across concurrent first uploads. + // The catalog-sync trigger on this UPDATE needs `jwt.claims.database_id`, so the + // write runs under the resolved database claim (privileged role preserved). + await withRequestPgClient(withPgClient, { 'jwt.claims.database_id': databaseId }, (client) => + client.query({ + text: `UPDATE ${storageConfig.bucketsQualifiedName} + SET physical_name = $1 + WHERE id = $2 AND physical_name IS NULL`, + values: [s3BucketName, bucket.id], + }), + ); + bucket.physical_name = s3BucketName; + log.info(`Recorded physical_name="${s3BucketName}" on bucket ${bucket.id}`); + return s3BucketName; +} diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index 6d27b29b19..7921175a2c 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -23,9 +23,12 @@ import { Logger } from '@pgpmjs/logger'; import { access, context as grafastContext, lambda, object } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; -import { type WithPgClient,withRequestPgClient } from './request-pg-client'; +import { resolveDefaultBucket } from './default-bucket'; +import { buildFileProjection, type FileProjection } from './managed-upload'; +import { provisionAndRecordPhysicalBucket, resolveS3ForDatabase } from './physical-bucket'; +import { withRequestPgClient } from './request-pg-client'; import { deleteS3Object,generatePresignedPutUrl } from './s3-signer'; -import { getBucketConfig, isS3BucketProvisioned, loadAllStorageModules, markS3BucketProvisioned,resolveStorageConfigFromCodec, storedPhysicalName } from './storage-module-cache'; +import { getBucketConfig, loadAllStorageModules, resolveStorageConfigFromCodec, storedPhysicalName } from './storage-module-cache'; import type { BucketConfig,PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; const log = new Logger('graphile-presigned-url:plugin'); @@ -81,111 +84,35 @@ async function resolveDatabaseId(pgClient: any): Promise { return result.rows[0]?.id ?? null; } -function resolveS3(options: PresignedUrlPluginOptions): S3Config { - if (typeof options.s3 === 'function') { - const resolved = options.s3(); - options.s3 = resolved; - return resolved; - } - return options.s3; -} - /** - * Mint the physical S3 bucket name for a logical bucket's first provision. + * Resolve the bucket an upload mutation writes into. * - * This is a naming *policy*, consulted exactly once per bucket — before the - * physical bucket exists. Once provisioned, the recorded `physical_name` on - * the row is authoritative and this function must not be consulted again. + * A named `bucketKey` is the caller's override and is read directly, as before. + * An omitted one asks the database for the tenant's reserved default tag for the + * requested access, so a missing or ambiguous default raises in SQL rather than + * falling back to a server-global bucket name here. */ -function mintPhysicalBucketName( - options: PresignedUrlPluginOptions, - databaseId: string, - bucketKey: string, -): string { - if (options.resolveBucketName) { - return options.resolveBucketName(databaseId, bucketKey); - } - // Single-bucket deployment: the globally configured bucket is the physical bucket. - return resolveS3(options).bucket; -} - -/** - * Build the S3 config for a *known* physical bucket. `physicalName` is - * required — callers must resolve the coordinate (stored row value, or a - * freshly provisioned name) before getting here. No name is ever recomputed. - */ -function resolveS3ForDatabase( - options: PresignedUrlPluginOptions, - storageConfig: StorageModuleConfig, - physicalName: string, -): S3Config { - const globalS3 = resolveS3(options); - const publicUrlPrefix = storageConfig.publicUrlPrefix != null - ? storageConfig.publicUrlPrefix - : globalS3.publicUrlPrefix; - - if (physicalName === globalS3.bucket && publicUrlPrefix === globalS3.publicUrlPrefix) { - return globalS3; - } - - return { - ...globalS3, - bucket: physicalName, - ...(publicUrlPrefix != null ? { publicUrlPrefix } : {}), - }; -} - -/** - * First provision of a logical bucket: mint a name, create the physical S3 - * bucket, and record the exact name on the source row. Returns the recorded - * physical name. - * - * Only called when the row has no `physical_name` yet. Afterwards the stored - * value is the durable coordinate: route resolution and every later read use - * it verbatim; nothing is recomputed. - * - * The record write runs in the system lane (privileged role, so it bypasses the - * RLS that stops request roles from UPDATE-ing bucket rows) — it is server - * bookkeeping, not request data. It still carries the tenant `database_id` - * claim, because the buckets table's catalog-sync trigger calls - * `jwt_private.current_database_id()` and would otherwise raise - * DATABASE_CLAIM_REQUIRED; `withRequestPgClient` applies that claim inside the - * write's transaction without switching off the privileged role. - * `bucket` (the cached config) is mutated in place so subsequent reads observe - * the recorded name without a DB round-trip. - */ -async function provisionAndRecordPhysicalBucket( - options: PresignedUrlPluginOptions, - withPgClient: WithPgClient, +async function resolveUploadBucket( + pgClient: any, storageConfig: StorageModuleConfig, databaseId: string, - bucket: BucketConfig, - allowedOrigins: string[] | null, -): Promise { - const s3BucketName = mintPhysicalBucketName(options, databaseId, bucket.key); - - if (options.ensureBucketProvisioned && !isS3BucketProvisioned(s3BucketName)) { - log.info(`Lazy-provisioning S3 bucket "${s3BucketName}" for database ${databaseId}`); - await options.ensureBucketProvisioned(s3BucketName, bucket.type, databaseId, allowedOrigins); - markS3BucketProvisioned(s3BucketName); - log.info(`Lazy-provisioned S3 bucket "${s3BucketName}" successfully`); + bucketKey: string | null, + ownerId: string | null, + isPublic: boolean, +): Promise { + if (bucketKey) { + return getBucketConfig(pgClient, storageConfig, databaseId, bucketKey, ownerId || undefined); } - // Record the physical coordinate on the source row. The `physical_name IS NULL` - // guard keeps this idempotent and race-safe across concurrent first uploads. - // The catalog-sync trigger on this UPDATE needs `jwt.claims.database_id`, so the - // write runs under the resolved database claim (privileged role preserved). - await withRequestPgClient(withPgClient, { 'jwt.claims.database_id': databaseId }, (client) => - client.query({ - text: `UPDATE ${storageConfig.bucketsQualifiedName} - SET physical_name = $1 - WHERE id = $2 AND physical_name IS NULL`, - values: [s3BucketName, bucket.id], - }), + const coordinate = await resolveDefaultBucket( + pgClient, + databaseId, + storageConfig.scope, + ownerId, + isPublic, + null, ); - bucket.physical_name = s3BucketName; - log.info(`Recorded physical_name="${s3BucketName}" on bucket ${bucket.id}`); - return s3BucketName; + return getBucketConfig(pgClient, storageConfig, databaseId, coordinate.resolvedKey, ownerId || undefined); } // --- Plugin factory --- @@ -225,6 +152,15 @@ export function createPresignedUrlPlugin( }, } = build; + // The projection document is jsonb-shaped. PostGraphile registers a JSON + // scalar whenever the schema has a jsonb column, which any storage-equipped + // database does; if it is absent the payload simply omits the field rather + // than failing schema build over a field nothing can have asked for yet. + const jsonType = build.getTypeByName('JSON') ?? null; + if (!jsonType) { + log.warn('No JSON scalar in this schema; upload payloads will omit the `file` projection'); + } + const bucketCodecs = Object.values((build.input as any).pgRegistry.pgCodecs).filter( (codec: any) => codec.attributes && (codec.extensions as any)?.tags?.storageBuckets, ); @@ -269,7 +205,8 @@ export function createPresignedUrlPlugin( const InputType = new GraphQLInputObjectType({ name: `Upload${filesTypeName}Input`, fields: { - bucketKey: { type: new GraphQLNonNull(GraphQLString), description: 'Bucket key (e.g., "public", "private")' }, + bucketKey: { type: GraphQLString, description: 'Bucket key (e.g., "public", "private"). Omit to use the database\'s default bucket for the requested access.' }, + isPublic: { type: GraphQLBoolean, description: 'Which default bucket to resolve when bucketKey is omitted: the public one (true) or the private one (default false). Ignored when bucketKey is given.' }, ...(hasOwnerId ? { ownerId: { type: new GraphQLNonNull(ownerIdGqlType || GraphQLString), description: 'Owner entity ID (required for entity-scoped buckets)' } } : {}), @@ -290,6 +227,17 @@ export function createPresignedUrlPlugin( deduplicated: { type: new GraphQLNonNull(GraphQLBoolean), description: 'Whether this file was deduplicated (content already exists)' }, expiresAt: { type: GraphQLString, description: 'Presigned URL expiry time (null if deduplicated)' }, previousVersionId: { type: GraphQLString, description: 'ID of the previous version (when using custom keys)' }, + ...(jsonType + ? { + file: { + type: jsonType, + description: + 'The projection document for the created file: {id, key, bucket_id, mime, size, filename, url?}. ' + + 'Store this verbatim in an image/upload column — its `id` is what keeps the object from being ' + + 'garbage collected while the column still references it.', + }, + } + : {}), }, }); @@ -308,6 +256,7 @@ export function createPresignedUrlPlugin( plan(_$mutation: any, fieldArgs: any) { const $input = fieldArgs.getRaw('input'); const $bucketKey = access($input, 'bucketKey'); + const $isPublic = access($input, 'isPublic'); const $contentHash = access($input, 'contentHash'); const $contentType = access($input, 'contentType'); const $size = access($input, 'size'); @@ -319,6 +268,7 @@ export function createPresignedUrlPlugin( const $combined = object({ bucketKey: $bucketKey, + isPublic: $isPublic, ownerId: $ownerId, contentHash: $contentHash, contentType: $contentType, @@ -346,9 +296,12 @@ export function createPresignedUrlPlugin( const storageConfig = resolveStorageConfigFromCodec(capturedFilesCodec, allConfigs); if (!storageConfig) throw new Error('STORAGE_MODULE_NOT_FOUND'); - // Bucket config read under the request role (RLS-gated visibility). + // Bucket resolution + read under the request role (RLS-gated visibility). const bucket = await withRequestPgClient(vals.withPgClient, vals.pgSettings, (pgClient) => - getBucketConfig(pgClient, storageConfig, databaseId, vals.bucketKey, vals.ownerId || undefined), + resolveUploadBucket( + pgClient, storageConfig, databaseId, + vals.bucketKey ?? null, vals.ownerId ?? null, vals.isPublic === true, + ), ); if (!bucket) throw new Error('BUCKET_NOT_FOUND'); @@ -395,13 +348,15 @@ export function createPresignedUrlPlugin( deduplicated: { type: new GraphQLNonNull(GraphQLBoolean) }, expiresAt: { type: GraphQLString }, previousVersionId: { type: GraphQLString }, + ...(jsonType ? { file: { type: jsonType, description: 'The projection document for the created file.' } } : {}), }, }); const BulkInputType = new GraphQLInputObjectType({ name: `Upload${filesTypeName}BulkInput`, fields: { - bucketKey: { type: new GraphQLNonNull(GraphQLString), description: 'Bucket key (e.g., "public", "private")' }, + bucketKey: { type: GraphQLString, description: 'Bucket key (e.g., "public", "private"). Omit to use the database\'s default bucket for the requested access.' }, + isPublic: { type: GraphQLBoolean, description: 'Which default bucket to resolve when bucketKey is omitted. Ignored when bucketKey is given.' }, ...(hasOwnerId ? { ownerId: { type: new GraphQLNonNull(ownerIdGqlType || GraphQLString), description: 'Owner entity ID (required for entity-scoped buckets)' } } : {}), @@ -430,6 +385,7 @@ export function createPresignedUrlPlugin( plan(_$mutation: any, fieldArgs: any) { const $input = fieldArgs.getRaw('input'); const $bucketKey = access($input, 'bucketKey'); + const $isPublic = access($input, 'isPublic'); const $ownerId = hasOwnerId ? access($input, 'ownerId') : lambda(null, (): null => null); const $files = access($input, 'files'); const $withPgClient = (grafastContext() as any).get('withPgClient'); @@ -437,6 +393,7 @@ export function createPresignedUrlPlugin( const $combined = object({ bucketKey: $bucketKey, + isPublic: $isPublic, ownerId: $ownerId, files: $files, withPgClient: $withPgClient, @@ -460,9 +417,12 @@ export function createPresignedUrlPlugin( const storageConfig = resolveStorageConfigFromCodec(capturedFilesCodec, allConfigs); if (!storageConfig) throw new Error('STORAGE_MODULE_NOT_FOUND'); - // Bucket config read under the request role (RLS-gated visibility). + // Bucket resolution + read under the request role (RLS-gated visibility). const bucket = await withRequestPgClient(vals.withPgClient, vals.pgSettings, (pgClient) => - getBucketConfig(pgClient, storageConfig, databaseId, vals.bucketKey, vals.ownerId || undefined), + resolveUploadBucket( + pgClient, storageConfig, databaseId, + vals.bucketKey ?? null, vals.ownerId ?? null, vals.isPublic === true, + ), ); if (!bucket) throw new Error('BUCKET_NOT_FOUND'); @@ -714,6 +674,17 @@ async function processSingleFile( throw new Error(`FILE_TOO_LARGE: exceeds bucket max of ${bucket.max_file_size} bytes`); } + // The projection document the caller stores in an image/upload column. Built + // from the same values the files row carries, so the column and the row cannot + // disagree, and it names the files row by id — which is what stops GC from + // collecting an object a document still points at. + const projectFile = (fileId: string, key: string): FileProjection => + buildFileProjection( + { id: fileId, key, bucketId: bucket.id, mime: contentType, size, filename }, + bucket, + s3ForDb, + ); + // Determine S3 key let s3Key: string; let isCustomKey = false; @@ -756,6 +727,7 @@ async function processSingleFile( deduplicated: true, expiresAt: null as string | null, previousVersionId: null as string | null, + file: projectFile(existing.id as string, s3Key), }; } previousVersionId = existing.id; @@ -782,6 +754,7 @@ async function processSingleFile( deduplicated: true, expiresAt: null as string | null, previousVersionId: null as string | null, + file: projectFile(existingFile.id as string, s3Key), }; } } @@ -836,6 +809,7 @@ async function processSingleFile( deduplicated: false, expiresAt, previousVersionId, + file: projectFile(fileId, s3Key), }; } diff --git a/graphile/graphile-presigned-url-plugin/src/s3-signer.ts b/graphile/graphile-presigned-url-plugin/src/s3-signer.ts index c2e21d8608..6b26745b11 100644 --- a/graphile/graphile-presigned-url-plugin/src/s3-signer.ts +++ b/graphile/graphile-presigned-url-plugin/src/s3-signer.ts @@ -1,4 +1,5 @@ import { + CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, @@ -99,6 +100,36 @@ export async function deleteS3Object( log.debug(`Deleted S3 object: bucket=${s3Config.bucket}, key=${key}`); } +/** + * Copy an object within the same physical bucket, preserving its content type. + * + * Used to promote a staged upload to its content-addressed key: the bytes are + * hashed as they stream in, so the final key is only known once the stream ends. + * Server-side copy keeps that promotion off the application's wire. + * + * @param s3Config - S3 client and bucket configuration + * @param sourceKey - The staged key the bytes were written to + * @param destinationKey - The final key (the content hash) + * @param contentType - MIME type to record on the destination object + */ +export async function copyS3Object( + s3Config: S3Config, + sourceKey: string, + destinationKey: string, + contentType: string, +): Promise { + await s3Config.client.send( + new CopyObjectCommand({ + Bucket: s3Config.bucket, + Key: destinationKey, + CopySource: `${s3Config.bucket}/${sourceKey}`, + ContentType: contentType, + MetadataDirective: 'REPLACE', + }), + ); + log.debug(`Copied S3 object: bucket=${s3Config.bucket}, ${sourceKey} → ${destinationKey}`); +} + /** * Check if an object exists in S3 and optionally verify its content-type. * diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index 79fe686af1..0eaf03244c 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -85,8 +85,19 @@ export interface StorageModuleConfig { * Input for the requestUploadUrl mutation. */ export interface RequestUploadUrlInput { - /** Bucket key (e.g., "public", "private") */ - bucketKey: string; + /** + * Logical bucket key (e.g., "public", "private"). + * + * Optional: when omitted the database resolves its own default bucket for the + * requested access (see `isPublic`), so a client never has to know a tenant's + * bucket naming to upload. + */ + bucketKey?: string; + /** + * Which default bucket to resolve when `bucketKey` is omitted: the public one + * (true) or the private one (default false). Ignored when `bucketKey` is given. + */ + isPublic?: boolean; /** * Owner entity ID for entity-scoped uploads. * Omit for app-level (database-wide) storage. @@ -127,6 +138,28 @@ export interface RequestUploadUrlPayload { expiresAt: string | null; /** ID of the previous version (set when re-uploading to an existing custom key) */ previousVersionId: string | null; + /** + * The projection document to store in an `image`/`upload` column. + * + * Its `id` is the files row, which is what makes the column a reference the + * server can count — storage GC will not collect an object while a registered + * document column still names its file. + */ + file: FileProjection; +} + +/** + * The document a managed `image`/`upload` column stores. See `./managed-upload`. + */ +export interface FileProjection { + id: string; + key: string; + bucket_id: string; + mime: string; + size: number; + filename?: string; + /** @deprecated Read the files row's `downloadUrl` via `id` instead. */ + url?: string; } /** diff --git a/graphile/graphile-settings/__tests__/upload-resolver.test.ts b/graphile/graphile-settings/__tests__/upload-resolver.test.ts index 1b64f119d5..e3e1155e94 100644 --- a/graphile/graphile-settings/__tests__/upload-resolver.test.ts +++ b/graphile/graphile-settings/__tests__/upload-resolver.test.ts @@ -1,14 +1,121 @@ +/** + * The multipart upload lane, end to end minus the network. + * + * S3 is mocked at the streamer/client boundary and the database at the query + * boundary, so these assert what the lane decides: that the bytes go to the + * tenant's resolved bucket rather than an environment one, that a files row is + * created, and that the value stored in the column names that row while keeping + * the `url` its existing readers depend on. + */ + import { Readable } from 'stream'; -interface MockUploadResult { - upload: { Location: string }; - contentType: string; +const DATABASE_ID = '00000000-0000-0000-0000-0000000000db'; +const MODULE_ID = '11111111-1111-1111-1111-111111111111'; +const BUCKET_ID = '33333333-3333-3333-3333-333333333333'; +const FILE_ID = '44444444-4444-4444-4444-444444444444'; + +const FIELD = { schemaName: 'app_public', tableName: 'posts', columnName: 'image' }; + +interface QueryHandler { + match: RegExp; + rows: () => unknown[]; +} + +function storageModuleRow(): Record { + return { + id: MODULE_ID, + scope: 'app', + entity_table_id: null, + buckets_schema: 'storage_public', + buckets_table: 'app_buckets', + files_schema: 'storage_public', + files_table: 'app_files', + endpoint: null, + public_url_prefix: 'https://cdn.example.com', + provider: 'minio', + allowed_origins: null, + upload_url_expiry_seconds: null, + download_url_expiry_seconds: null, + default_max_file_size: 1048576, + max_filename_length: null, + cache_ttl_seconds: null, + max_bulk_files: null, + max_bulk_total_size: null, + has_path_shares: false, + entity_schema: null, + entity_table: null, + }; +} + +function bucketRow(isPublic: boolean): Record { + return { + id: BUCKET_ID, + key: isPublic ? 'default-public' : 'default', + type: isPublic ? 'public' : 'private', + is_public: isPublic, + owner_id: null, + allowed_mime_types: null, + max_file_size: null, + allow_custom_keys: false, + physical_name: 'myapp-default-public-db', + }; +} + +/** A withPgClient answering exactly the statements this lane issues. */ +function fakeContext(opts: { isPublic?: boolean; existingFile?: boolean; failInsert?: boolean } = {}) { + const isPublic = opts.isPublic ?? true; + const queries: Array<{ text: string; values?: unknown[] }> = []; + const handlers: QueryHandler[] = [ + { match: /set_config/, rows: () => [] }, + { match: /current_database_id/, rows: () => [{ id: DATABASE_ID }] }, + { match: /file_ref_field/, rows: () => [] }, + { match: /FROM metaschema_modules_public\.storage_module/, rows: () => [storageModuleRow()] }, + { + match: /resolve_default_bucket/, + rows: () => [{ + bucket_id: BUCKET_ID, + resolved_key: isPublic ? 'default-public' : 'default', + bucket_type: isPublic ? 'public' : 'private', + physical_name: 'myapp-default-public-db', + }], + }, + { match: /SELECT id, key, type/, rows: () => [bucketRow(isPublic)] }, + { + match: /SELECT id, key, mime_type/, + rows: () => opts.existingFile + ? [{ id: FILE_ID, key: 'existing-key', mime_type: 'image/png', size: 16, filename: 'first.png' }] + : [], + }, + { + match: /INSERT INTO storage_public\.app_files/, + rows: () => { + if (opts.failInsert) throw new Error('insert boom'); + return [{ id: FILE_ID }]; + }, + }, + ]; + + const client = { + async query(o: { text: string; values?: unknown[] }) { + queries.push(o); + const handler = handlers.find((h) => h.match.test(o.text)); + if (!handler) throw new Error(`unexpected query: ${o.text}`); + return { rows: handler.rows() }; + }, + withTransaction: (cb: any) => cb(client), + }; + + return { + context: { + withPgClient: (_settings: any, cb: any) => cb(client), + pgSettings: { 'jwt.claims.database_id': DATABASE_ID }, + }, + queries, + }; } -async function loadUploadResolverModule(opts: { - detectedContentType: string; - uploadResultContentType?: string; -}) { +async function loadUploadResolverModule(opts: { detectedContentType: string }) { jest.resetModules(); const mockDetectContentType = jest.fn().mockResolvedValue({ @@ -17,49 +124,57 @@ async function loadUploadResolverModule(opts: { contentType: opts.detectedContentType, }); - const mockUploadWithContentType = jest.fn().mockResolvedValue({ - upload: { Location: 'https://cdn.example.com/uploaded-file' }, - contentType: opts.uploadResultContentType ?? opts.detectedContentType, - } as MockUploadResult); - - const mockUpload = jest.fn().mockResolvedValue({ - upload: { Location: 'https://cdn.example.com/storage-upload' }, - contentType: 'application/octet-stream', - } as MockUploadResult); + const mockUploadWithContentType = jest.fn().mockImplementation(async ({ readStream }: any) => { + // Drain the stream so the hashing pass-through sees every byte, exactly as + // a real multipart upload to S3 would. + for await (const _chunk of readStream) { /* consumed */ } + return { + upload: { Location: 'https://cdn.example.com/uploaded-file' }, + contentType: opts.detectedContentType, + }; + }); jest.doMock('@constructive-io/graphql-env', () => ({ getEnvOptions: jest.fn(() => ({ cdn: { provider: 'minio', - bucketName: 'test-bucket', + bucketName: 'myapp', awsRegion: 'us-east-1', awsAccessKey: 'test', awsSecretKey: 'test', endpoint: 'http://localhost:9000', + publicUrlPrefix: 'https://cdn.example.com', }, })), })); - jest.doMock('@constructive-io/s3-streamer', () => { - const StreamerMock = jest.fn().mockImplementation(() => ({ - upload: mockUpload, + jest.doMock('@constructive-io/s3-streamer', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({ uploadWithContentType: mockUploadWithContentType, detectContentType: mockDetectContentType, - })); - return { - __esModule: true, - default: StreamerMock, - }; - }); + })), + })); + + const s3Send = jest.fn().mockResolvedValue({}); + jest.doMock('@constructive-io/s3-utils', () => ({ + createS3Client: jest.fn(() => ({ send: s3Send })), + })); const mod = await import('../src/upload-resolver'); + const { clearBucketCache, clearStorageModuleCache, clearFileRefFieldCache } = + await import('graphile-presigned-url-plugin'); + clearBucketCache(); + clearStorageModuleCache(); + clearFileRefFieldCache(); - return { - ...mod, - mockDetectContentType, - mockUploadWithContentType, - mockUpload, - }; + return { ...mod, mockDetectContentType, mockUploadWithContentType, s3Send }; +} + +function definitionFor(defs: any[], name: string) { + const def = defs.find((d) => 'name' in d && d.name === name); + if (!def) throw new Error(`Missing ${name} upload field definition`); + return def; } function makeFakeUpload(filename: string) { @@ -69,31 +184,18 @@ function makeFakeUpload(filename: string) { }; } -describe('uploadResolver MIME validation', () => { - it('rejects disallowed MIME before uploading to storage', async () => { - const { - constructiveUploadFieldDefinitions, - mockDetectContentType, - mockUploadWithContentType, - } = await loadUploadResolverModule({ - detectedContentType: 'application/pdf', - }); - - const imageDef = constructiveUploadFieldDefinitions.find( - (def) => 'name' in def && def.name === 'image', - ); - if (!imageDef) { - throw new Error('Missing image upload field definition'); - } - - const fakeUpload = makeFakeUpload('document.pdf'); +describe('multipart upload resolver', () => { + it('rejects a disallowed MIME before anything is written', async () => { + const { constructiveUploadFieldDefinitions, mockDetectContentType, mockUploadWithContentType } = + await loadUploadResolverModule({ detectedContentType: 'application/pdf' }); + const { context } = fakeContext(); await expect( - imageDef.resolve( - fakeUpload as any, + definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( + makeFakeUpload('document.pdf') as any, {}, - {}, - { uploadPlugin: { tags: {}, type: 'image' } }, + context, + { uploadPlugin: { tags: {}, type: 'image', field: FIELD } }, ), ).rejects.toThrow('UPLOAD_MIMETYPE'); @@ -101,43 +203,119 @@ describe('uploadResolver MIME validation', () => { expect(mockUploadWithContentType).not.toHaveBeenCalled(); }); - it('uploads and returns image metadata when MIME is allowed', async () => { - const { - constructiveUploadFieldDefinitions, - mockDetectContentType, - mockUploadWithContentType, - } = await loadUploadResolverModule({ - detectedContentType: 'image/png', - uploadResultContentType: 'image/png', - }); - - const imageDef = constructiveUploadFieldDefinitions.find( - (def) => 'name' in def && def.name === 'image', + it('stores a projection that names the files row, keeping url/filename/mime', async () => { + const { constructiveUploadFieldDefinitions, mockUploadWithContentType } = + await loadUploadResolverModule({ detectedContentType: 'image/png' }); + const { context, queries } = fakeContext(); + + const result: any = await definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( + makeFakeUpload('photo.png') as any, + {}, + context, + { uploadPlugin: { tags: {}, type: 'image', field: FIELD } }, ); - if (!imageDef) { - throw new Error('Missing image upload field definition'); - } - const fakeUpload = makeFakeUpload('photo.png'); + // The load-bearing new field: GC counts document references by files-row id. + expect(result.id).toBe(FILE_ID); + expect(result.bucket_id).toBe(BUCKET_ID); + expect(result.size).toBe(16); + // The content hash of 16 zero bytes — the key is derived from the bytes, not + // from a random string. + expect(result.key).toBe('374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb'); + // Retained for existing readers of the pre-managed shape. + expect(result.filename).toBe('photo.png'); + expect(result.mime).toBe('image/png'); + expect(result.url).toBe('https://cdn.example.com/374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb'); - const result = await imageDef.resolve( - fakeUpload as any, - {}, + // The bytes went to the tenant's bucket, and to a staging key first. + const call = mockUploadWithContentType.mock.calls[0][0]; + expect(call.bucket).toBe('myapp-default-public-db'); + expect(call.key).toMatch(/^\.staging\//); + + const insert = queries.find((q) => /INSERT INTO storage_public\.app_files/.test(q.text)); + expect(insert).toBeDefined(); + }); + + it('deduplicates against an existing row rather than inserting a second one', async () => { + const { constructiveUploadFieldDefinitions } = + await loadUploadResolverModule({ detectedContentType: 'image/png' }); + const { context, queries } = fakeContext({ existingFile: true }); + + const result: any = await definitionFor(constructiveUploadFieldDefinitions, 'upload').resolve( + makeFakeUpload('photo.png') as any, {}, - { uploadPlugin: { tags: {}, type: 'image' } }, + context, + { uploadPlugin: { tags: {}, type: 'upload', field: FIELD } }, ); - expect(result).toEqual({ - filename: 'photo.png', - mime: 'image/png', - url: 'https://cdn.example.com/uploaded-file', - }); - expect(mockDetectContentType).toHaveBeenCalledTimes(1); - expect(mockUploadWithContentType).toHaveBeenCalledTimes(1); - expect(mockUploadWithContentType).toHaveBeenCalledWith( - expect.objectContaining({ - contentType: 'image/png', - }), + expect(result.id).toBe(FILE_ID); + expect(result.key).toBe('existing-key'); + expect(queries.some((q) => /INSERT/.test(q.text))).toBe(false); + }); + + it('returns a bare URL for an attachment column, whose type cannot hold a document', async () => { + const { constructiveUploadFieldDefinitions } = + await loadUploadResolverModule({ detectedContentType: 'application/pdf' }); + const { context } = fakeContext(); + + const result = await definitionFor(constructiveUploadFieldDefinitions, 'attachment').resolve( + makeFakeUpload('doc.pdf') as any, + {}, + context, + { uploadPlugin: { tags: {}, type: 'attachment', field: { ...FIELD, columnName: 'doc' } } }, ); + + expect(typeof result).toBe('string'); + expect(result).toMatch(/^https:\/\/cdn\.example\.com\//); + }); + + it('refuses to put an expiring URL in an attachment column on a private bucket', async () => { + const { constructiveUploadFieldDefinitions } = + await loadUploadResolverModule({ detectedContentType: 'application/pdf' }); + const { context } = fakeContext({ isPublic: false }); + + await expect( + definitionFor(constructiveUploadFieldDefinitions, 'attachment').resolve( + makeFakeUpload('doc.pdf') as any, + {}, + context, + { uploadPlugin: { tags: {}, type: 'attachment', field: { ...FIELD, columnName: 'doc' } } }, + ), + ).rejects.toThrow('ATTACHMENT_BUCKET_NOT_PUBLIC'); + }); + + it('leaves nothing in S3 when the files row cannot be created', async () => { + const { constructiveUploadFieldDefinitions, s3Send } = + await loadUploadResolverModule({ detectedContentType: 'image/png' }); + // The insert is the last step, so it fails with the bytes already staged. + const { context } = fakeContext({ failInsert: true }); + + await expect( + definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( + makeFakeUpload('photo.png') as any, + {}, + context, + { uploadPlugin: { tags: {}, type: 'image', field: FIELD } }, + ), + ).rejects.toThrow('insert boom'); + + // Copy to the content key, then both the promoted and the staged object are + // removed: no files row names either, so GC could never reach them. + expect(s3Send).toHaveBeenCalledTimes(3); + }); + + it('refuses to upload when the plugin cannot say which column is being written', async () => { + const { constructiveUploadFieldDefinitions } = + await loadUploadResolverModule({ detectedContentType: 'image/png' }); + const { context } = fakeContext(); + + await expect( + definitionFor(constructiveUploadFieldDefinitions, 'image').resolve( + makeFakeUpload('photo.png') as any, + {}, + context, + { uploadPlugin: { tags: {}, type: 'image' } }, + ), + ).rejects.toThrow('UPLOAD_FIELD_UNKNOWN'); }); }); diff --git a/graphile/graphile-settings/src/presigned-url-resolver.ts b/graphile/graphile-settings/src/presigned-url-resolver.ts index 2326483751..850a6a6517 100644 --- a/graphile/graphile-settings/src/presigned-url-resolver.ts +++ b/graphile/graphile-settings/src/presigned-url-resolver.ts @@ -31,9 +31,10 @@ let s3Config: S3Config | null = null; * pgpmDefaults → config file → env vars), creates an S3Client, and caches * the result. Same CDN config as upload-resolver.ts. * - * NOTE: The `bucket` field here is the global fallback bucket name - * (from BUCKET_NAME env var). When `resolveBucketName` is provided, - * per-database bucket names take precedence for all S3 operations. + * NOTE: The `bucket` field here is only the connection's default and is never + * uploaded to. Every managed upload names its bucket explicitly, resolved from + * the tenant's logical bucket row via `resolveBucketName`; there is no + * environment-global upload bucket. */ export function getPresignedUrlS3Config(): S3Config { if (s3Config) return s3Config; diff --git a/graphile/graphile-settings/src/upload-resolver.ts b/graphile/graphile-settings/src/upload-resolver.ts index 09df772021..945b3efc33 100644 --- a/graphile/graphile-settings/src/upload-resolver.ts +++ b/graphile/graphile-settings/src/upload-resolver.ts @@ -1,15 +1,24 @@ /** - * Upload resolver for the Constructive upload plugin. + * Upload resolver for the Constructive upload plugin (multipart `Upload` scalar). * - * Reads CDN/S3/MinIO configuration from environment variables (via getEnvOptions) - * and streams uploaded files to the configured storage backend. + * This is the streaming transport into the *managed* storage lane: bytes arrive + * on the mutation, and the file they carry gets the same treatment a presigned + * upload gets — a bucket resolved inside the tenant, a content-addressed key, a + * files row, and a projection document naming that row. * - * Lazily initializes the S3 streamer on first upload to avoid requiring - * env vars at module load time. + * It used to be a second storage model: stream to `BUCKET_NAME` under a random + * key, hand back a URL, record nothing. Objects written that way belonged to no + * database, could not be deduplicated, listed, or access-controlled, and storage + * GC could not see that a document still pointed at them. There is no + * environment bucket in this path any more; `cdn.*` supplies S3 credentials and + * an endpoint only. * - * ENV VARS: + * Compatibility: `image`/`upload` columns still receive `url` alongside the new + * `id`/`key`/`bucket_id`/`size` fields, so existing readers of `photo.url` keep + * working while they migrate to `id` + the files row's late-bound `downloadUrl`. + * + * ENV VARS (S3 connection only): * BUCKET_PROVIDER - 'minio' | 's3' (default: 'minio') - * BUCKET_NAME - bucket name (default: 'test-bucket') * AWS_REGION - AWS region (default: 'us-east-1') * AWS_ACCESS_KEY - access key (default: 'minioadmin') * AWS_SECRET_KEY - secret key (default: 'minioadmin') @@ -18,128 +27,253 @@ import { getEnvOptions } from '@constructive-io/graphql-env'; import Streamer from '@constructive-io/s3-streamer'; -import uploadNames from '@constructive-io/upload-names'; import { Logger } from '@pgpmjs/logger'; -import { randomBytes } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; +import { + finalizeStagedUpload, + type PresignedUrlPluginOptions, + resolveManagedUploadTarget, + withRequestPgClient, +} from 'graphile-presigned-url-plugin'; import type { FileUpload, UploadFieldDefinition, + UploadFieldIdentity, UploadPluginInfo, } from 'graphile-upload-plugin'; +import { Transform } from 'stream'; + +import { + createBucketNameResolver, + createEnsureBucketProvisioned, + getPresignedUrlS3Config, +} from './presigned-url-resolver'; const log = new Logger('upload-resolver'); const DEFAULT_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/svg+xml']; let streamer: Streamer | null = null; -let bucketName: string; +/** + * The S3 streamer, built from the CDN connection settings. + * + * Deliberately constructed with no `defaultBucket`: every upload names the + * bucket it resolved, and a default here would be an environment-owned bucket + * standing in for a tenant's. + */ function getStreamer(): Streamer { if (streamer) return streamer; - const opts = getEnvOptions(); - const cdn = opts.cdn || {}; + const { cdn = {} } = getEnvOptions(); - const provider = cdn.provider || 'minio'; - bucketName = cdn.bucketName || 'test-bucket'; - const awsRegion = cdn.awsRegion || 'us-east-1'; - const awsAccessKey = cdn.awsAccessKey || 'minioadmin'; - const awsSecretKey = cdn.awsSecretKey || 'minioadmin'; - const endpoint = cdn.endpoint || 'http://localhost:9000'; - - if (process.env.NODE_ENV === 'production') { - if (!cdn.awsAccessKey || !cdn.awsSecretKey) { - log.warn('[upload-resolver] WARNING: Using default credentials in production.'); - } + if (process.env.NODE_ENV === 'production' && (!cdn.awsAccessKey || !cdn.awsSecretKey)) { + log.warn('[upload-resolver] WARNING: Using default credentials in production.'); } - log.info( - `[upload-resolver] Initializing: provider=${provider} bucket=${bucketName}`, - ); + const provider = cdn.provider || 'minio'; + log.info(`[upload-resolver] Initializing: provider=${provider}`); streamer = new Streamer({ - defaultBucket: bucketName, - awsRegion, - awsSecretKey, - awsAccessKey, - endpoint, provider, + awsRegion: cdn.awsRegion || 'us-east-1', + awsAccessKey: cdn.awsAccessKey || 'minioadmin', + awsSecretKey: cdn.awsSecretKey || 'minioadmin', + endpoint: cdn.endpoint || 'http://localhost:9000', }); return streamer; } /** - * Generates a randomized storage key from a filename. - * Format: {random10chars}-{sanitized-filename} + * The upload lane's view of the presigned plugin's options: the same S3 + * connection, physical-name policy, and provisioning hook the presigned lane + * uses, so both transports resolve identical coordinates for a bucket. + * + * Built on first upload rather than at import time — `createBucketNameResolver` + * throws on a missing name prefix, and that must surface as a failed upload, not + * as a server that will not boot. */ -function generateKey(filename: string): string { - const rand = randomBytes(12).toString('hex'); - return `${rand}-${uploadNames(filename)}`; +let managedOptions: PresignedUrlPluginOptions | null = null; + +function getManagedOptions(): PresignedUrlPluginOptions { + if (!managedOptions) { + managedOptions = { + s3: getPresignedUrlS3Config, + resolveBucketName: createBucketNameResolver(), + ensureBucketProvisioned: createEnsureBucketProvisioned(), + }; + } + return managedOptions; +} + +/** A staging key: transient, and never what the object ends up under. */ +function stagingKey(): string { + return `.staging/${randomUUID()}`; +} + +async function resolveDatabaseId(pgClient: any): Promise { + const result = await pgClient.query({ text: `SELECT jwt_private.current_database_id() AS id` }); + return result.rows[0]?.id ?? null; } /** - * Upload resolver that streams files to S3/MinIO. + * Which default bucket an *unregistered* column resolves to. * - * Returns different shapes based on the column's type hint: - * - 'image' / 'upload' → { filename, mime, url } (for jsonb domain columns) - * - 'attachment' / default → url string (for text domain columns) + * Columns written by the pre-managed resolver held a directly-embedded URL, so + * their readers assume a publicly addressable object; resolving them to the + * private default would break every page rendering one. A registered column + * states its own intent and this is not consulted. + */ +const LEGACY_DEFAULT_PUBLIC_ACCESS = true; + +/** The mime allowlist for a column, from its smart tags or its type. */ +function allowedMimeTypes(tags: Record | undefined, typ: string | undefined): string[] { + const VALID_MIME = /^[a-z]+\/[a-z0-9][a-z0-9!#$&\-.^_+]*$/i; + if (tags?.mime) { + return String(tags.mime) + .trim() + .split(',') + .map((a: string) => a.trim()) + .filter((m: string) => VALID_MIME.test(m)); + } + return typ === 'image' ? DEFAULT_IMAGE_MIME_TYPES : []; +} + +/** + * Hash and measure bytes as they stream past, without buffering them. * - * MIME validation happens before persistence: content type is detected from - * stream bytes, validated against smart-tag/type rules, and only then uploaded. + * The final key is the content hash, which is only known once the last byte has + * gone by — so the object is staged first and promoted after. Nothing is held in + * memory: a 2GB upload streams through this the same as a 2KB one. + */ +function hashingPassThrough(): Transform & { digest: () => string; bytes: () => number } { + const hash = createHash('sha256'); + let bytes = 0; + const stream = new Transform({ + transform(chunk, _encoding, callback) { + hash.update(chunk); + bytes += chunk.length; + callback(null, chunk); + }, + }); + return Object.assign(stream, { + digest: () => hash.digest('hex'), + bytes: () => bytes, + }); +} + +/** + * Stream an upload into managed storage and return the value the column stores. + * + * Shape by column type hint: + * * `image` / `upload` (jsonb domains) → the projection document, including a + * compatibility `url` for public buckets. + * * `attachment` (text domain) → the object's public URL. A text column cannot + * hold a projection, so the files row is still authoritative but the column + * itself carries no id; the row keeps the object alive. A private bucket + * raises rather than storing an expiring presigned URL in a column. */ async function uploadResolver( upload: FileUpload, _args: unknown, - _context: unknown, + context: any, info: { uploadPlugin: UploadPluginInfo }, ): Promise { - const { tags, type } = info.uploadPlugin; + const { tags, type, field } = info.uploadPlugin; + const typ = type || tags?.type; + + const withPgClient = context?.withPgClient; + const pgSettings = context?.pgSettings ?? null; + if (!withPgClient) { + throw new Error( + 'UPLOAD_NO_PG_CLIENT: a managed upload resolves its bucket in the database, so the ' + + 'GraphQL context must carry withPgClient', + ); + } + if (!field) { + throw new Error( + 'UPLOAD_FIELD_UNKNOWN: the upload plugin did not report which column is being written, ' + + 'so the storage module and bucket backing it cannot be resolved', + ); + } + + const databaseId = await withRequestPgClient(withPgClient, pgSettings, (pgClient: any) => + resolveDatabaseId(pgClient), + ); + if (!databaseId) throw new Error('DATABASE_NOT_FOUND'); + + const target = await resolveManagedUploadTarget({ + options: getManagedOptions(), + withPgClient, + pgSettings, + databaseId, + field: field as UploadFieldIdentity, + defaultPublicAccess: LEGACY_DEFAULT_PUBLIC_ACCESS, + }); + + if (typ === 'attachment' && !target.bucket.is_public) { + throw new Error( + 'ATTACHMENT_BUCKET_NOT_PUBLIC: an attachment column stores a plain URL, and the resolved ' + + `bucket "${target.bucket.key}" is private, whose only URLs expire. Use an upload column, ` + + 'which stores the file id and resolves a fresh download URL on read.', + ); + } + const s3 = getStreamer(); const { filename } = upload; - const key = generateKey(filename); - - // MIME type validation from smart tags - const typ = type || tags?.type; - const VALID_MIME = /^[a-z]+\/[a-z0-9][a-z0-9!#$&\-.^_+]*$/i; - const mim: string[] = tags?.mime - ? String(tags.mime) - .trim() - .split(',') - .map((a: string) => a.trim()) - .filter((m: string) => VALID_MIME.test(m)) - : typ === 'image' - ? DEFAULT_IMAGE_MIME_TYPES - : []; + // Validate before persisting: content type comes from the leading bytes, not + // from the client's claim about them. const detected = await s3.detectContentType({ readStream: upload.createReadStream(), filename, }); - const detectedContentType = detected.contentType; - - if (mim.length && !mim.includes(detectedContentType)) { + const allowed = allowedMimeTypes(tags, typ); + if (allowed.length && !allowed.includes(detected.contentType)) { detected.stream.destroy(); throw new Error('UPLOAD_MIMETYPE'); } - const result = await s3.uploadWithContentType({ - readStream: detected.stream, - contentType: detectedContentType, + const staged = stagingKey(); + const hashing = hashingPassThrough(); + const uploadResult = await s3.uploadWithContentType({ + readStream: detected.stream.pipe(hashing), + contentType: detected.contentType, magic: detected.magic, - key, - bucket: bucketName, + key: staged, + bucket: target.physicalName, }); - const url = result.upload.Location; - const { contentType } = result; + // Owns the staged key from here: it either promotes it into a files row or + // removes it, so a failed upload leaves nothing behind in S3. + const { projection } = await finalizeStagedUpload({ + target, + withPgClient, + pgSettings, + staged: { + stagingKey: staged, + contentHash: hashing.digest(), + contentType: uploadResult.contentType, + size: hashing.bytes(), + filename, + }, + }); switch (typ) { case 'image': case 'upload': - return { filename, mime: contentType, url }; + // `filename` and `mime` were in the pre-managed shape and stay in it; + // `url` is populated for public buckets and deprecated in favour of `id`. + return { ...projection, filename, mime: uploadResult.contentType }; case 'attachment': default: - return url; + if (!projection.url) { + throw new Error( + `ATTACHMENT_NO_PUBLIC_URL: bucket "${target.bucket.key}" has no public URL prefix ` + + 'configured, so there is no durable URL to store in a text column', + ); + } + return projection.url; } } diff --git a/graphile/graphile-upload-plugin/src/index.ts b/graphile/graphile-upload-plugin/src/index.ts index ef83dc060b..d555a29aba 100644 --- a/graphile/graphile-upload-plugin/src/index.ts +++ b/graphile/graphile-upload-plugin/src/index.ts @@ -32,6 +32,7 @@ export { UploadPreset } from './preset'; export type { FileUpload, UploadFieldDefinition, + UploadFieldIdentity, UploadPluginInfo, UploadPluginOptions, UploadResolver diff --git a/graphile/graphile-upload-plugin/src/plugin.ts b/graphile/graphile-upload-plugin/src/plugin.ts index a3c712d6d9..1a157f50e9 100644 --- a/graphile/graphile-upload-plugin/src/plugin.ts +++ b/graphile/graphile-upload-plugin/src/plugin.ts @@ -28,7 +28,24 @@ import type { GraphileConfig } from 'graphile-config'; import { isInputType } from 'graphql'; import { Readable, Transform } from 'stream'; -import type { UploadFieldDefinition, UploadPluginOptions } from './types'; +import type { UploadFieldDefinition, UploadFieldIdentity, UploadPluginOptions } from './types'; + +/** + * Read the mutated column's PG identity off the codec. + * + * Returns null when the codec is not table-backed (no `extensions.pg`), so a + * resolver that needs the identity can refuse rather than guess. + */ +function uploadFieldIdentity( + pgCodec: any, + attributeName: string, +): UploadFieldIdentity | null { + const pgExt = pgCodec?.extensions?.pg; + const schemaName = pgExt?.schemaName; + const tableName = pgExt?.name; + if (!schemaName || !tableName) return null; + return { schemaName, tableName, columnName: attributeName }; +} /** * Determines whether a codec attribute matches an upload field definition. @@ -264,6 +281,7 @@ export function createUploadPlugin( const tags: Record = {}; const types: Record = {}; const originals: Record = {}; + const identities: Record = {}; for (const [attributeName, attribute] of Object.entries( pgCodec.attributes as Record @@ -281,6 +299,7 @@ export function createUploadPlugin( tags[uploadFieldName] = attribute.extensions?.tags || {}; types[uploadFieldName] = matchedDef.type || ''; originals[uploadFieldName] = baseFieldName; + identities[uploadFieldName] = uploadFieldIdentity(pgCodec, attributeName); } // If no upload fields match this mutation's codec, skip wrapping @@ -316,7 +335,8 @@ export function createUploadPlugin( ...info, uploadPlugin: { tags: tags[key], - type: types[key] + type: types[key], + ...(identities[key] ? { field: identities[key] } : {}) } } ); diff --git a/graphile/graphile-upload-plugin/src/types.ts b/graphile/graphile-upload-plugin/src/types.ts index 5f337d66ec..b31f5b99d6 100644 --- a/graphile/graphile-upload-plugin/src/types.ts +++ b/graphile/graphile-upload-plugin/src/types.ts @@ -10,12 +10,35 @@ export interface FileUpload { createReadStream: () => Readable; } +/** + * Identifies the column an upload is being written into. + * + * A resolver that persists through managed storage needs the column's identity, + * not just its type: which bucket the bytes land in is a property of the field + * declaration, recorded per (table, column), so a resolver given only + * `{tags, type}` can do no better than a server-global bucket. + */ +export interface UploadFieldIdentity { + /** PostgreSQL schema of the table being mutated (e.g. 'app_public') */ + schemaName: string; + /** PostgreSQL table name being mutated (e.g. 'products') */ + tableName: string; + /** PostgreSQL column name receiving the upload (e.g. 'photo') */ + columnName: string; +} + /** * Additional metadata passed to the upload resolver via the info parameter. */ export interface UploadPluginInfo { tags: Record; type?: string; + /** + * The column being written. Absent only when the codec carries no PG + * identity, which a managed resolver must treat as an error rather than + * falling back to a default bucket. + */ + field?: UploadFieldIdentity; } /** diff --git a/uploads/s3-streamer/src/streamer.ts b/uploads/s3-streamer/src/streamer.ts index 3ce5059d78..6597b961e9 100644 --- a/uploads/s3-streamer/src/streamer.ts +++ b/uploads/s3-streamer/src/streamer.ts @@ -16,7 +16,13 @@ interface StreamerOptions { awsAccessKey: string; endpoint?: string; provider?: BucketProvider; - defaultBucket: string; + /** + * Bucket used when a call does not name one. Optional: a caller that resolves + * the bucket per upload (tenant-resolved storage) has no deployment-wide + * default to give, and passing a placeholder would let a missing `bucket` + * argument write somewhere plausible instead of failing. + */ + defaultBucket?: string; } interface UploadParams { From d2b54389cf7832a91509ca86b46d56f43ba148b0 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Sat, 8 Aug 2026 07:15:12 +0000 Subject: [PATCH 2/2] feat(storage): refuse multipart writes into path-keyed buckets A bucket with allow_custom_keys=true (e.g. a static site's) is addressed by publisher-chosen paths; the multipart lane only mints content-hash keys, which would pollute it with unreachable objects. Path-keyed uploads belong to the presigned lane, which accepts an explicit key. --- .../__tests__/managed-upload.test.ts | 23 +++++++++++++++++++ .../src/managed-upload.ts | 12 ++++++++++ 2 files changed, 35 insertions(+) diff --git a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts index aed9002b95..56d1c6fed6 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts @@ -238,6 +238,29 @@ describe('resolveManagedUploadTarget', () => { expect(target.bucket.key).toBe('avatars'); }); + it('refuses a path-keyed bucket, whose keys are chosen by its publisher', async () => { + const { resolveManagedUploadTarget } = await import('../src/managed-upload'); + // A static site's bucket: public, custom keys allowed, addressed by path. + const db = fakeDb([ + SET_CONFIG, + NO_REGISTRY_ROW, + STORAGE_MODULES, + { match: /resolve_default_bucket/, rows: () => [{ bucket_id: BUCKET_ID, resolved_key: 'site', bucket_type: 'public', physical_name: 'myapp-site-db' }] }, + { match: /FROM storage_public\.app_buckets/, rows: () => [bucketRow({ key: 'site', allow_custom_keys: true, physical_name: 'myapp-site-db' })] }, + ]); + + await expect( + resolveManagedUploadTarget({ + options: options(), + withPgClient: db.withPgClient, + pgSettings: null, + databaseId: DATABASE_ID, + field: FIELD, + defaultPublicAccess: true, + }), + ).rejects.toThrow('BUCKET_PATH_KEYED'); + }); + it('records the physical name on first provision instead of re-minting it', async () => { const { resolveManagedUploadTarget } = await import('../src/managed-upload'); const ensureBucketProvisioned = jest.fn().mockResolvedValue(undefined); diff --git a/graphile/graphile-presigned-url-plugin/src/managed-upload.ts b/graphile/graphile-presigned-url-plugin/src/managed-upload.ts index db864e9b70..26f730baeb 100644 --- a/graphile/graphile-presigned-url-plugin/src/managed-upload.ts +++ b/graphile/graphile-presigned-url-plugin/src/managed-upload.ts @@ -177,6 +177,18 @@ export async function resolveManagedUploadTarget(args: { ); } + if (bucket.allow_custom_keys) { + // A path-keyed bucket (e.g. a static site's) is addressed by the keys its + // publisher chose; this lane can only mint content-hash keys, which would + // pollute it with unreachable objects. Path-keyed uploads go through the + // presigned lane, which accepts an explicit `key`. + throw new Error( + `BUCKET_PATH_KEYED: bucket "${bucket.key}" allows custom keys and is addressed by path; ` + + 'the multipart upload lane only writes content-addressed keys. Use the presigned upload ' + + 'mutation with an explicit key.', + ); + } + const physicalName = bucket.physical_name === null ? await provisionAndRecordPhysicalBucket( options, withPgClient, storageConfig, databaseId, bucket, storageConfig.allowedOrigins,