diff --git a/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx b/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx index 64716f9f0f3..7233cb1bc70 100644 --- a/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx +++ b/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx @@ -237,6 +237,26 @@ AZURE_STORAGE_OG_IMAGES_CONTAINER_NAME=og-images AZURE_STORAGE_WORKSPACE_LOGOS_CONTAINER_NAME=workspace-logos ``` +Direct browser uploads require a Blob service CORS rule on the storage account. Allow your exact +Sim origin, `GET` and `PUT`, the `Content-Type` header, and the `x-ms-*` prefix used by signed blob +and metadata headers: + +```bash +az storage cors add \ + --services b \ + --methods GET PUT \ + --origins https://sim.yourdomain.com \ + --allowed-headers content-type 'x-ms-*' \ + --exposed-headers ETag \ + --max-age 3600 \ + --account-name mystorageaccount \ + --account-key '' +``` + +If you authenticate with a connection string, replace the last two options with +`--connection-string "$AZURE_CONNECTION_STRING"`. CORS is configured once for the account's Blob +service and applies to all of its containers. + A full Helm example lives at `helm/sim/examples/values-azure.yaml`. ## Set up Google Cloud Storage @@ -276,11 +296,13 @@ cat > /tmp/cors.json <<'EOF' "responseHeader": [ "Content-Type", "ETag", + "x-goog-meta-uploadid", "x-goog-meta-originalname", "x-goog-meta-uploadedat", "x-goog-meta-purpose", "x-goog-meta-userid", "x-goog-meta-workspaceid", + "x-goog-meta-knowledgebaseid", "x-goog-meta-folderid", "x-goog-meta-workflowid", "x-goog-meta-executionid" @@ -445,6 +467,27 @@ The same browser-reachability and CORS requirements apply. +## Configure temporary upload cleanup + +Sim stages every direct upload under the `upload-sessions/` prefix before promoting it to its final, +immutable object key. Apply the cleanup policy to **every** purpose-specific bucket or container +configured above: + +- On AWS S3 and Google Cloud Storage, expire objects under `upload-sessions/` after two days and + abort incomplete multipart uploads after two days. +- On Azure Blob, expire committed blobs under `upload-sessions/` after two days. Azure automatically + removes uncommitted blocks after seven days. +- For an S3-compatible provider, configure both rules when its lifecycle implementation supports + them. Check the provider's documentation because lifecycle feature support varies. + +The two-day window exceeds the 24-hour upload-token lifetime and leaves time to retry completion. +Do not apply this prefix rule to final objects outside `upload-sessions/`. + + + Configure both expiration and incomplete-multipart cleanup where available. Expiring staged + objects alone does not necessarily remove abandoned multipart parts. + + ## Verify it works After restarting with the new configuration: diff --git a/apps/docs/openapi-v2-files-audit.json b/apps/docs/openapi-v2-files-audit.json index c379903774c..8ad1d4363ce 100644 --- a/apps/docs/openapi-v2-files-audit.json +++ b/apps/docs/openapi-v2-files-audit.json @@ -169,56 +169,75 @@ } } }, - "x-removed-buffered-post": { - "operationId": "uploadFile", - "summary": "Upload File", - "description": "Upload a file to a workspace as `multipart/form-data` with a single `file` field. The workspace — and the optional target `folderId` — are supplied as query parameters (not form fields) so authorization runs before the request body is buffered. Maximum file size is 100MB. A name already taken in the destination folder is **not** an error: the name is auto-suffixed (`data.csv` -> `data (1).csv`), matching the in-app uploader, so a `201` can come back with a `name` different from the one you sent — always read `name` from the response rather than assuming it. `409` is returned only if a unique name cannot be allocated after several attempts. Use `PATCH /api/v2/files/{fileId}` if you need a specific name to be exact-or-fail. Returns `201 Created`.\n\nPresigned upload is not part of the public API: it debits the storage quota only in a separate register step, so a caller that never registers would leave unaccounted bytes in storage. This buffered path debits inside the upload transaction.", + "post": { + "operationId": "createFile", + "summary": "Create File", + "description": "Create an authored workspace file, either empty or with initial inline content. Use this endpoint for files whose bytes are already available as UTF-8 text or base64 and are at most 50 MiB after decoding. Use the upload-session endpoints for streamed or larger files. A live file with the same name in the same folder is rejected with `409`.", "tags": ["Files"], "x-codeSamples": [ { "id": "curl", "label": "cURL", "lang": "bash", - "source": "curl -X POST \\\n \"https://www.sim.ai/api/v2/files?workspaceId=YOUR_WORKSPACE_ID\" \\\n -H \"X-API-Key: YOUR_API_KEY\" \\\n -F \"file=@/path/to/file.csv\"" - } - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceIdQuery" - }, - { - "name": "folderId", - "in": "query", - "required": false, - "description": "Target file folder. Omit to upload to the workspace root. Supplied as a query parameter, like `workspaceId`, so authorization runs before the multipart body is buffered.", - "schema": { - "type": "string", - "example": "fold_9Kq2mZ7pR4tLxWc0Ye3Nu" - } + "source": "curl -X POST \\\n \"https://www.sim.ai/api/v2/files\" \\\n -H \"X-API-Key: YOUR_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"workspaceId\": \"YOUR_WORKSPACE_ID\", \"name\": \"notes.md\"}'" } ], "requestBody": { "required": true, - "description": "The file to upload, sent as multipart/form-data.", "content": { - "multipart/form-data": { + "application/json": { "schema": { "type": "object", - "required": ["file"], + "additionalProperties": false, + "required": ["workspaceId", "name"], "properties": { - "file": { + "workspaceId": { + "type": "string", + "minLength": 1, + "description": "Workspace in which to create the file." + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "File name, including its extension. Path separators and dot segments are rejected." + }, + "contentType": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "MIME type. When omitted, it is inferred from the file extension." + }, + "folderId": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "Destination folder. Omit to create the file at the workspace root." + }, + "content": { + "type": "string", + "maxLength": 70000000, + "default": "", + "description": "Initial file content. Omit or send an empty string to create a zero-byte file." + }, + "encoding": { "type": "string", - "format": "binary", - "description": "The file to upload. Maximum size is 100MB." + "enum": ["utf-8", "base64"], + "default": "utf-8", + "description": "Encoding of `content`." } } + }, + "example": { + "workspaceId": "a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64", + "name": "notes.md" } } } }, "responses": { "201": { - "description": "The file was uploaded successfully.", + "description": "The created file.", "headers": { "X-RateLimit-Limit": { "$ref": "#/components/headers/X-RateLimit-Limit" @@ -238,12 +257,12 @@ "example": { "data": { "id": "wf_V1StGXR8z5jdHi6BmyT91", - "name": "data.csv", - "size": 1024, - "type": "text/csv", - "key": "workspace/a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64/1709571234-xyz-data.csv", - "folderId": "fold_9Kq2mZ7pR4tLxWc0Ye3Nu", - "folderPath": "Reports/Q1", + "name": "notes.md", + "size": 0, + "type": "text/markdown", + "key": "workspace/a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64/1709571234-xyz-notes.md", + "folderId": null, + "folderPath": null, "uploadedBy": "user_abc123", "uploadedAt": "2026-01-15T10:30:00Z", "updatedAt": "2026-01-15T10:30:00Z" @@ -252,66 +271,14 @@ } } }, - "400": { - "description": "The request was malformed: an invalid `workspaceId` query parameter, a body that is not valid multipart form data, or a missing `file` form field.", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/V2Error" - }, - "example": { - "error": { - "code": "BAD_REQUEST", - "message": "file form field is required" - } - } - } - } - }, - "401": { - "$ref": "#/components/responses/Unauthorized" - }, - "403": { - "$ref": "#/components/responses/Forbidden" - }, - "409": { - "description": "A unique filename could not be allocated in the destination folder after several attempts. An ordinary name collision is auto-suffixed instead, not rejected.", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/V2Error" - }, - "example": { - "error": { - "code": "CONFLICT", - "message": "A file named \"data.csv\" already exists in this workspace" - } - } - } - } - }, - "413": { - "description": "The upload exceeds the 100MB file size limit, or the workspace storage limit would be exceeded.", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/V2Error" - }, - "example": { - "error": { - "code": "PAYLOAD_TOO_LARGE", - "message": "File size exceeds 100MB limit (142.30MB)" - } - } - } - } - }, - "429": { - "$ref": "#/components/responses/RateLimited" - }, - "500": { - "$ref": "#/components/responses/InternalError" - } + "400": { "$ref": "#/components/responses/BadRequest" }, + "401": { "$ref": "#/components/responses/Unauthorized" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "413": { "$ref": "#/components/responses/PayloadTooLarge" }, + "429": { "$ref": "#/components/responses/RateLimited" }, + "500": { "$ref": "#/components/responses/InternalError" } } } }, @@ -319,7 +286,7 @@ "post": { "operationId": "createFileUpload", "summary": "Create File Upload", - "description": "Create a stateless multipart upload session and signed upload token. Every file uses this flow; a small file is a single part. The maximum file size is 5 GB.", + "description": "Create an upload session and signed control token. Empty files and files up to and including 50 MiB receive a single signed PUT URL; larger files receive multipart transfer instructions. The maximum file size is 5 GB.", "tags": ["Files"], "requestBody": { "required": true, @@ -405,7 +372,7 @@ "post": { "operationId": "completeFileUpload", "summary": "Complete File Upload", - "description": "Verify every part, assemble the object, and atomically register the workspace file.", + "description": "Verify the single PUT or assemble every multipart part, then atomically register the workspace file.", "tags": ["Files"], "parameters": [ { @@ -1681,7 +1648,7 @@ "name": "upload-token", "in": "header", "required": true, - "description": "The signed token returned when the multipart upload was created.", + "description": "The signed control token returned when the upload session was created.", "schema": { "type": "string", "minLength": 1 } }, "FileIdPath": { diff --git a/apps/docs/openapi-v2-knowledge.json b/apps/docs/openapi-v2-knowledge.json index f9c45723d60..4a5924ad949 100644 --- a/apps/docs/openapi-v2-knowledge.json +++ b/apps/docs/openapi-v2-knowledge.json @@ -790,7 +790,7 @@ "post": { "operationId": "createKnowledgeDocumentUpload", "summary": "Create Document Upload", - "description": "Create a stateless multipart upload session for a knowledge document. Write access, billing, usage, file type, file size, and workspace storage are checked before provider storage is allocated. The signed upload token binds the caller, workspace, knowledge base, filename, content type, byte size, provider, and knowledge-document purpose. Files may be up to 100 MB.", + "description": "Create an upload session for a knowledge document. Files up to and including 50 MiB use a single signed PUT; larger files use multipart transfer. Write access, billing, usage, file type, file size, and workspace storage are checked before provider storage is allocated. The signed upload token binds the caller, workspace, knowledge base, filename, content type, byte size, provider, and knowledge-document purpose. Files may be up to 100 MB.", "tags": ["Knowledge Bases"], "x-codeSamples": [ { @@ -801,7 +801,7 @@ ], "requestBody": { "required": true, - "description": "Metadata for the document that will be uploaded through signed part URLs.", + "description": "Metadata for the document that will be uploaded through the returned transfer instructions.", "content": { "application/json": { "schema": { @@ -812,11 +812,11 @@ }, "responses": { "201": { - "description": "The multipart upload session and its signed control-plane token.", + "description": "The terminal-safe upload session, signed control-plane token, and PUT or multipart transfer instructions.", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DocumentUploadEnvelope" + "$ref": "#/components/schemas/CreateDocumentUploadEnvelope" } } } @@ -918,7 +918,7 @@ "post": { "operationId": "completeKnowledgeDocumentUpload", "summary": "Complete Document Upload", - "description": "Verify and assemble all parts, record knowledge-base storage ownership, create the knowledge document, and queue asynchronous processing. Repeating the same completion is idempotent and returns the same document. It never registers a general workspace file.", + "description": "Verify the single PUT or assemble all multipart parts, record knowledge-base storage ownership, create the knowledge document, and queue asynchronous processing. Repeating the same completion is idempotent and returns the same document. It never registers a general workspace file.", "tags": ["Knowledge Bases"], "parameters": [{ "$ref": "#/components/parameters/WorkspaceIdQuery" }], "requestBody": { @@ -1480,9 +1480,6 @@ "name", "contentType", "size", - "partSize", - "partCount", - "uploadToken", "expiresAt", "error", "document" @@ -1510,19 +1507,6 @@ "type": "integer", "minimum": 1 }, - "partSize": { - "type": "integer", - "minimum": 1 - }, - "partCount": { - "type": "integer", - "minimum": 1 - }, - "uploadToken": { - "type": "string", - "minLength": 1, - "description": "Signed token required for part URLs, completion, and abort." - }, "expiresAt": { "type": "string", "format": "date-time" @@ -1545,6 +1529,52 @@ } } }, + "PutUploadTransfer": { + "type": "object", + "additionalProperties": false, + "required": ["method", "url", "headers"], + "properties": { + "method": { "type": "string", "const": "put" }, + "url": { "type": "string", "format": "uri" }, + "headers": { + "type": "object", + "additionalProperties": { "type": "string" } + } + } + }, + "MultipartUploadTransfer": { + "type": "object", + "additionalProperties": false, + "required": ["method", "partSize", "partCount"], + "properties": { + "method": { "type": "string", "const": "multipart" }, + "partSize": { "type": "integer", "minimum": 1 }, + "partCount": { "type": "integer", "minimum": 1, "maximum": 640 } + } + }, + "UploadTransfer": { + "oneOf": [ + { "$ref": "#/components/schemas/PutUploadTransfer" }, + { "$ref": "#/components/schemas/MultipartUploadTransfer" } + ], + "discriminator": { "propertyName": "method" } + }, + "CreateDocumentUploadEnvelope": { + "type": "object", + "required": ["data"], + "properties": { + "data": { + "type": "object", + "additionalProperties": false, + "required": ["session", "uploadToken", "transfer"], + "properties": { + "session": { "$ref": "#/components/schemas/DocumentUpload" }, + "uploadToken": { "type": "string", "minLength": 1 }, + "transfer": { "$ref": "#/components/schemas/UploadTransfer" } + } + } + } + }, "CreatePartUrlsBody": { "type": "object", "additionalProperties": false, @@ -1605,31 +1635,39 @@ } }, "CompleteUploadBody": { - "type": "object", - "additionalProperties": false, - "required": ["parts"], - "properties": { - "parts": { - "type": "array", - "minItems": 1, - "maxItems": 640, - "items": { - "type": "object", - "additionalProperties": false, - "required": ["partNumber"], - "properties": { - "partNumber": { - "type": "integer", - "minimum": 1 - }, - "etag": { - "type": "string", - "minLength": 1 + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["parts"], + "properties": { + "parts": { + "type": "array", + "minItems": 1, + "maxItems": 640, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["partNumber"], + "properties": { + "partNumber": { + "type": "integer", + "minimum": 1 + }, + "etag": { + "type": "string", + "minLength": 1 + } + } } } } + }, + { + "type": "object", + "additionalProperties": false } - } + ] }, "DocumentSummary": { "type": "object", diff --git a/apps/docs/openapi-v2-resources.json b/apps/docs/openapi-v2-resources.json index f7a6117a2c9..3e4466ecbcc 100644 --- a/apps/docs/openapi-v2-resources.json +++ b/apps/docs/openapi-v2-resources.json @@ -2824,7 +2824,13 @@ "maxLength": 1024, "description": "Write-only. Client-credentials secret." }, - "orgId": { "type": "string", "minLength": 1, "maxLength": 255 } + "orgId": { "type": "string", "minLength": 1, "maxLength": 255 }, + "dataCenter": { + "type": "string", + "minLength": 1, + "maxLength": 32, + "description": "Optional provider region selector, such as a Zoho Desk data center." + } } }, "UpdateCredentialBody": { @@ -2860,7 +2866,13 @@ "maxLength": 1024, "description": "Write-only." }, - "orgId": { "type": "string", "minLength": 1, "maxLength": 255 } + "orgId": { "type": "string", "minLength": 1, "maxLength": 255 }, + "dataCenter": { + "type": "string", + "minLength": 1, + "maxLength": 32, + "description": "Optional provider region selector, such as a Zoho Desk data center." + } } } } diff --git a/apps/docs/openapi-v2-tables.json b/apps/docs/openapi-v2-tables.json index 2c82c47276a..a80ff6774b4 100644 --- a/apps/docs/openapi-v2-tables.json +++ b/apps/docs/openapi-v2-tables.json @@ -3500,7 +3500,7 @@ "post": { "operationId": "createTableImport", "summary": "Create Table Import", - "description": "Create a table import. Upload sources return a stateless multipart token; workspace-file sources start immediately and both use table jobs for processing state.", + "description": "Create a table import. Upload sources return a signed control token plus single-PUT or multipart transfer instructions; workspace-file sources start immediately. Both use table jobs for processing state.", "tags": ["Tables"], "requestBody": { "required": true, @@ -3697,7 +3697,7 @@ "post": { "operationId": "completeTableImportUpload", "summary": "Complete Table Import Upload", - "description": "Verify and assemble the uploaded CSV or TSV, then start processing with the same import id.", + "description": "Verify the single PUT or assemble the multipart CSV or TSV, then start processing with the same import id.", "tags": ["Tables"], "parameters": [ { diff --git a/apps/sim/app/api/files/multipart/route.test.ts b/apps/sim/app/api/files/multipart/route.test.ts deleted file mode 100644 index a1200ec18c9..00000000000 --- a/apps/sim/app/api/files/multipart/route.test.ts +++ /dev/null @@ -1,315 +0,0 @@ -/** - * @vitest-environment node - */ -import { authMockFns, permissionsMock, permissionsMockFns } from '@sim/testing' -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockIsUsingCloudStorage, - mockGetStorageProvider, - mockGetStorageConfig, - mockCompleteS3MultipartUpload, - mockCompleteBlobMultipartUpload, - mockDeriveBlobBlockId, - mockVerifyUploadToken, - mockSignUploadToken, -} = vi.hoisted(() => ({ - mockIsUsingCloudStorage: vi.fn(), - mockGetStorageProvider: vi.fn(), - mockGetStorageConfig: vi.fn(), - mockCompleteS3MultipartUpload: vi.fn(), - mockCompleteBlobMultipartUpload: vi.fn(), - mockDeriveBlobBlockId: vi.fn(), - mockVerifyUploadToken: vi.fn(), - mockSignUploadToken: vi.fn(), -})) - -vi.mock('@/lib/uploads', () => ({ - isUsingCloudStorage: mockIsUsingCloudStorage, - getStorageProvider: mockGetStorageProvider, - getStorageConfig: mockGetStorageConfig, -})) - -vi.mock('@/lib/uploads/core/upload-token', () => ({ - signUploadToken: mockSignUploadToken, - verifyUploadToken: mockVerifyUploadToken, -})) - -vi.mock('@/lib/uploads/providers/s3/client', () => ({ - completeS3MultipartUpload: mockCompleteS3MultipartUpload, - initiateS3MultipartUpload: mockInitiateS3MultipartUpload, - getS3MultipartPartUrls: vi.fn(), - abortS3MultipartUpload: vi.fn(), -})) - -vi.mock('@/lib/uploads/providers/blob/client', () => ({ - completeMultipartUpload: mockCompleteBlobMultipartUpload, - deriveBlobBlockId: mockDeriveBlobBlockId, - initiateMultipartUpload: vi.fn(), - getMultipartPartUrls: vi.fn(), - abortMultipartUpload: vi.fn(), -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) - -const { mockCheckStorageQuota, mockInitiateS3MultipartUpload, mockResolveStorageBillingContext } = - vi.hoisted(() => ({ - mockCheckStorageQuota: vi.fn(), - mockInitiateS3MultipartUpload: vi.fn(), - mockResolveStorageBillingContext: vi.fn(), - })) - -vi.mock('@/lib/billing/storage', () => ({ - checkStorageQuotaForBillingContext: mockCheckStorageQuota, - resolveStorageBillingContext: mockResolveStorageBillingContext, -})) - -import { POST } from '@/app/api/files/multipart/route' - -const STORAGE_CONTEXT = { - workspaceId: 'ws-1', - billedAccountUserId: 'workspace-owner', - billingEntity: { type: 'organization' as const, id: 'workspace-org' }, - plan: 'team_25000', - customStorageLimitGB: null, -} - -const tokenPayload = { - uploadId: 'upload-1', - key: 'workspace/ws-1/123-abc-file.bin', - userId: 'user-1', - workspaceId: 'ws-1', - context: 'workspace' as const, -} - -const makeRequest = (action: string, body: unknown) => - new NextRequest(`http://localhost/api/files/multipart?action=${action}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }) - -describe('POST /api/files/multipart action=complete', () => { - beforeEach(() => { - vi.clearAllMocks() - authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - mockIsUsingCloudStorage.mockReturnValue(true) - mockGetStorageConfig.mockReturnValue({ bucket: 'b', region: 'r' }) - mockVerifyUploadToken.mockReturnValue({ valid: true, payload: tokenPayload }) - mockSignUploadToken.mockReturnValue('signed-token') - mockCompleteS3MultipartUpload.mockResolvedValue({ - location: 'loc', - path: '/api/files/serve/...', - key: tokenPayload.key, - }) - mockCompleteBlobMultipartUpload.mockResolvedValue({ - location: 'loc', - path: '/api/files/serve/...', - key: tokenPayload.key, - }) - mockDeriveBlobBlockId.mockImplementation( - (n: number) => `block-${n.toString().padStart(6, '0')}` - ) - }) - - it('rejects parts without partNumber', async () => { - mockGetStorageProvider.mockReturnValue('s3') - const res = await POST( - makeRequest('complete', { - uploadToken: 'tok', - parts: [{ etag: 'abc' }], - }) - ) - expect(res.status).toBe(400) - expect(mockCompleteS3MultipartUpload).not.toHaveBeenCalled() - }) - - it('S3 path requires etag and forwards { ETag, PartNumber }', async () => { - mockGetStorageProvider.mockReturnValue('s3') - - const missingEtag = await POST( - makeRequest('complete', { - uploadToken: 'tok', - parts: [{ partNumber: 1 }], - }) - ) - expect(missingEtag.status).toBe(500) - - mockCompleteS3MultipartUpload.mockClear() - - const ok = await POST( - makeRequest('complete', { - uploadToken: 'tok', - parts: [ - { partNumber: 1, etag: 'aaa' }, - { partNumber: 2, etag: 'bbb' }, - ], - }) - ) - expect(ok.status).toBe(200) - expect(mockCompleteS3MultipartUpload).toHaveBeenCalledWith( - tokenPayload.key, - tokenPayload.uploadId, - [ - { ETag: 'aaa', PartNumber: 1 }, - { ETag: 'bbb', PartNumber: 2 }, - ], - expect.any(Object) - ) - }) - - it('Blob path derives blockId from partNumber and ignores etag', async () => { - mockGetStorageProvider.mockReturnValue('blob') - mockGetStorageConfig.mockReturnValue({ - containerName: 'c', - accountName: 'a', - accountKey: 'k', - }) - - const res = await POST( - makeRequest('complete', { - uploadToken: 'tok', - parts: [{ partNumber: 1, etag: 'irrelevant' }, { partNumber: 2 }], - }) - ) - - expect(res.status).toBe(200) - expect(mockDeriveBlobBlockId).toHaveBeenCalledWith(1) - expect(mockDeriveBlobBlockId).toHaveBeenCalledWith(2) - expect(mockCompleteBlobMultipartUpload).toHaveBeenCalledWith( - tokenPayload.key, - [ - { partNumber: 1, blockId: 'block-000001' }, - { partNumber: 2, blockId: 'block-000002' }, - ], - expect.objectContaining({ containerName: 'c' }) - ) - }) - - it('returns 403 when token is invalid', async () => { - mockGetStorageProvider.mockReturnValue('s3') - mockVerifyUploadToken.mockReturnValueOnce({ valid: false }) - const res = await POST( - makeRequest('complete', { - uploadToken: 'bad', - parts: [{ partNumber: 1, etag: 'a' }], - }) - ) - expect(res.status).toBe(403) - }) - - it('batch complete normalizes per upload', async () => { - mockGetStorageProvider.mockReturnValue('s3') - const res = await POST( - makeRequest('complete', { - uploads: [ - { - uploadToken: 'tok-a', - parts: [{ partNumber: 1, etag: 'aaa' }], - }, - { - uploadToken: 'tok-b', - parts: [{ partNumber: 1, etag: 'bbb' }], - }, - ], - }) - ) - expect(res.status).toBe(200) - expect(mockCompleteS3MultipartUpload).toHaveBeenCalledTimes(2) - }) -}) - -describe('POST /api/files/multipart action=initiate quota enforcement', () => { - const makeInitiateRequest = (body: unknown) => - new NextRequest('http://localhost/api/files/multipart?action=initiate', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }) - - beforeEach(() => { - vi.clearAllMocks() - authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - mockIsUsingCloudStorage.mockReturnValue(true) - mockGetStorageProvider.mockReturnValue('s3') - mockGetStorageConfig.mockReturnValue({ bucket: 'b', region: 'r' }) - mockSignUploadToken.mockReturnValue('signed-token') - mockResolveStorageBillingContext.mockResolvedValue(STORAGE_CONTEXT) - mockCheckStorageQuota.mockResolvedValue({ allowed: true }) - mockInitiateS3MultipartUpload.mockResolvedValue({ uploadId: 'up-1', key: 'k/file.bin' }) - }) - - it('blocks upload when fileSize: 0 exceeds quota', async () => { - mockCheckStorageQuota.mockResolvedValue({ allowed: false, error: 'Storage limit exceeded' }) - - const res = await makeInitiateRequest({ - fileName: 'file.bin', - contentType: 'application/octet-stream', - fileSize: 0, - workspaceId: 'ws-1', - context: 'knowledge-base', - }) - - const response = await POST(res) - expect(response.status).toBe(413) - const body = await response.json() - expect(body.error).toContain('Storage limit exceeded') - }) - - it('allows quota-enforced contexts that pass the quota check', async () => { - const res = await makeInitiateRequest({ - fileName: 'doc.pdf', - contentType: 'application/pdf', - fileSize: 99999, - workspaceId: 'ws-1', - context: 'knowledge-base', - }) - - const response = await POST(res) - expect(response.status).toBe(200) - expect(mockResolveStorageBillingContext).toHaveBeenCalledWith('ws-1') - expect(mockCheckStorageQuota).toHaveBeenCalledWith(STORAGE_CONTEXT, 99999) - expect(mockInitiateS3MultipartUpload).toHaveBeenCalled() - }) - - it('keeps mothership chat uploads outside workspace storage quotas', async () => { - mockCheckStorageQuota.mockResolvedValue({ allowed: false, error: 'Storage limit exceeded' }) - - const res = await makeInitiateRequest({ - fileName: 'conversation.bin', - contentType: 'application/octet-stream', - fileSize: 99999, - workspaceId: 'ws-1', - context: 'mothership', - }) - - const response = await POST(res) - expect(response.status).toBe(200) - expect(mockResolveStorageBillingContext).not.toHaveBeenCalled() - expect(mockCheckStorageQuota).not.toHaveBeenCalled() - expect(mockInitiateS3MultipartUpload).toHaveBeenCalled() - }) - - it.each(['og-images', 'profile-pictures', 'workspace-logos', 'logs'])( - 'rejects quota-exempt context %s — not allowed via the multipart endpoint', - async (context) => { - const res = await makeInitiateRequest({ - fileName: 'asset.png', - contentType: 'image/png', - fileSize: 100 * 1024 * 1024 * 1024, - workspaceId: 'ws-1', - context, - }) - - const response = await POST(res) - expect(response.status).toBe(400) - const body = await response.json() - expect(body.error).toMatch(/invalid storage context/i) - expect(mockCheckStorageQuota).not.toHaveBeenCalled() - expect(mockInitiateS3MultipartUpload).not.toHaveBeenCalled() - } - ) -}) diff --git a/apps/sim/app/api/files/multipart/route.ts b/apps/sim/app/api/files/multipart/route.ts deleted file mode 100644 index 07fbac67361..00000000000 --- a/apps/sim/app/api/files/multipart/route.ts +++ /dev/null @@ -1,533 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - abortMultipartUploadContract, - type CompleteMultipartBody, - completeMultipartUploadContract, - getMultipartPartUrlsContract, - initiateMultipartUploadContract, - multipartActionSchema, -} from '@/lib/api/contracts/storage-transfer' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - getStorageConfig, - getStorageProvider, - isUsingCloudStorage, - type StorageContext, -} from '@/lib/uploads' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { - signUploadToken, - type UploadTokenPayload, - verifyUploadToken, -} from '@/lib/uploads/core/upload-token' -import { recordKnowledgeBaseFileOwnership } from '@/lib/uploads/server/metadata' -import { QUOTA_EXEMPT_STORAGE_CONTEXTS, type StorageConfig } from '@/lib/uploads/shared/types' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' - -const logger = createLogger('MultipartUploadAPI') - -/** - * Contexts the multipart endpoint accepts. Small public assets and internal logs - * are excluded because they have no large-file flow. Mothership remains - * available for large chat attachments but is quota-exempt because chat uploads - * do not count as durable workspace-file storage. Every other accepted context - * is quota-enforced below. - */ -const ALLOWED_UPLOAD_CONTEXTS = new Set([ - 'knowledge-base', - 'chat', - 'copilot', - 'mothership', - 'execution', - 'workspace', -]) - -/** - * Unified part identity sent by the client when completing a multipart upload. - * `etag` is required for S3 and GCS (CompleteMultipartUpload). For Azure the - * server derives the block id from `partNumber` via {@link deriveBlobBlockId}. - */ -interface ClientCompletedPart { - partNumber: number - etag?: string -} - -const isClientCompletedParts = (value: unknown): value is ClientCompletedPart[] => - Array.isArray(value) && - value.every( - (p) => - p !== null && - typeof p === 'object' && - typeof (p as ClientCompletedPart).partNumber === 'number' && - ((p as ClientCompletedPart).etag === undefined || - typeof (p as ClientCompletedPart).etag === 'string') - ) - -const buildS3CustomConfig = (config: StorageConfig) => - config.bucket && config.region ? { bucket: config.bucket, region: config.region } : undefined - -const buildBlobCustomConfig = (config: StorageConfig) => ({ - containerName: config.containerName!, - accountName: config.accountName!, - accountKey: config.accountKey, - connectionString: config.connectionString, -}) - -const buildGcsCustomConfig = (config: StorageConfig) => - config.bucket ? { bucket: config.bucket } : undefined - -const verifyTokenForUser = (token: string | undefined, userId: string) => { - if (!token || typeof token !== 'string') { - return null - } - const result = verifyUploadToken(token) - if (!result.valid || result.payload.userId !== userId) { - return null - } - return result.payload -} - -/** - * Record a trusted storage-key -> workspace ownership binding for completed - * knowledge-base uploads. KB file authorization resolves the owning workspace - * from this binding, so every KB object must have one. No-op for other contexts. - */ -const recordKnowledgeBaseOwnership = async ( - payload: UploadTokenPayload, - key: string -): Promise => { - if (payload.context !== 'knowledge-base' || !payload.workspaceId) { - return - } - await recordKnowledgeBaseFileOwnership({ - key, - userId: payload.userId, - workspaceId: payload.workspaceId, - originalName: payload.fileName ?? key.split('/').pop() ?? key, - contentType: payload.contentType ?? 'application/octet-stream', - size: typeof payload.fileSize === 'number' ? payload.fileSize : 0, - }) -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = session.user.id - - const actionParam = request.nextUrl.searchParams.get('action') - const actionResult = multipartActionSchema.safeParse(actionParam) - const action = actionResult.success ? actionResult.data : null - - if (!isUsingCloudStorage()) { - return NextResponse.json( - { - error: - 'Multipart upload is only available with cloud storage (S3, Azure Blob, or Google Cloud Storage)', - }, - { status: 400 } - ) - } - - const storageProvider = getStorageProvider() - - switch (action) { - case 'initiate': { - const parsed = await parseRequest( - initiateMultipartUploadContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const data = parsed.data.body - const { fileName, contentType, fileSize, workspaceId, context = 'knowledge-base' } = data - - if (!workspaceId || typeof workspaceId !== 'string') { - return NextResponse.json({ error: 'workspaceId is required' }, { status: 400 }) - } - - if (!ALLOWED_UPLOAD_CONTEXTS.has(context as StorageContext)) { - return NextResponse.json({ error: 'Invalid storage context' }, { status: 400 }) - } - const storageContext = context as StorageContext - - const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } - - const config = getStorageConfig(storageContext) - - if (!QUOTA_EXEMPT_STORAGE_CONTEXTS.has(storageContext)) { - const { checkStorageQuotaForBillingContext, resolveStorageBillingContext } = await import( - '@/lib/billing/storage' - ) - const storageBillingContext = await resolveStorageBillingContext(workspaceId) - const quotaCheck = await checkStorageQuotaForBillingContext( - storageBillingContext, - fileSize ?? 0 - ) - if (!quotaCheck.allowed) { - return NextResponse.json( - { error: quotaCheck.error || 'Storage limit exceeded' }, - { status: 413 } - ) - } - } - - let customKey: string | undefined - if (context === 'workspace' || context === 'mothership') { - const { MAX_WORKSPACE_FILE_SIZE } = await import('@/lib/uploads/shared/types') - if (typeof fileSize === 'number' && fileSize > MAX_WORKSPACE_FILE_SIZE) { - return NextResponse.json( - { error: `File size exceeds maximum of ${MAX_WORKSPACE_FILE_SIZE} bytes` }, - { status: 413 } - ) - } - - const { generateWorkspaceFileKey } = await import( - '@/lib/uploads/contexts/workspace/workspace-file-manager' - ) - customKey = generateWorkspaceFileKey(workspaceId, fileName) - } else if (context === 'execution') { - const workflowId = (data as { workflowId?: unknown }).workflowId - const executionId = (data as { executionId?: unknown }).executionId - if (typeof workflowId !== 'string' || !workflowId.trim()) { - return NextResponse.json( - { error: 'workflowId is required for execution uploads' }, - { status: 400 } - ) - } - if (typeof executionId !== 'string' || !executionId.trim()) { - return NextResponse.json( - { error: 'executionId is required for execution uploads' }, - { status: 400 } - ) - } - const { generateExecutionFileKey } = await import( - '@/lib/uploads/contexts/execution/utils' - ) - customKey = generateExecutionFileKey({ workspaceId, workflowId, executionId }, fileName) - } - - let uploadId: string - let key: string - - if (storageProvider === 's3') { - const { initiateS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') - const result = await initiateS3MultipartUpload({ - fileName, - contentType, - fileSize, - customConfig: buildS3CustomConfig(config), - customKey, - purpose: context, - }) - uploadId = result.uploadId - key = result.key - } else if (storageProvider === 'blob') { - const { initiateMultipartUpload } = await import('@/lib/uploads/providers/blob/client') - const result = await initiateMultipartUpload({ - fileName, - contentType, - fileSize, - customConfig: buildBlobCustomConfig(config), - customKey, - }) - uploadId = result.uploadId - key = result.key - } else if (storageProvider === 'gcs') { - const { initiateGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') - const result = await initiateGcsMultipartUpload({ - fileName, - contentType, - fileSize, - customConfig: buildGcsCustomConfig(config), - customKey, - purpose: context, - }) - uploadId = result.uploadId - key = result.key - } else { - return NextResponse.json( - { error: `Unsupported storage provider: ${storageProvider}` }, - { status: 400 } - ) - } - - const uploadToken = signUploadToken({ - uploadId, - key, - userId, - workspaceId, - context: storageContext, - fileName, - contentType, - ...(typeof fileSize === 'number' ? { fileSize } : {}), - }) - - logger.info( - `Initiated ${storageProvider} multipart upload for ${fileName} (context: ${storageContext}, workspace: ${workspaceId}): ${uploadId}` - ) - - return NextResponse.json({ uploadId, key, uploadToken }) - } - - case 'get-part-urls': { - const parsed = await parseRequest( - getMultipartPartUrlsContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const data = parsed.data.body - const { partNumbers } = data - - const tokenPayload = verifyTokenForUser(data.uploadToken, userId) - if (!tokenPayload) { - return NextResponse.json({ error: 'Invalid or expired upload token' }, { status: 403 }) - } - - const { uploadId, key, context } = tokenPayload - const config = getStorageConfig(context) - - if (storageProvider === 's3') { - const { getS3MultipartPartUrls } = await import('@/lib/uploads/providers/s3/client') - const presignedUrls = await getS3MultipartPartUrls( - key, - uploadId, - partNumbers, - buildS3CustomConfig(config) - ) - return NextResponse.json({ presignedUrls }) - } - if (storageProvider === 'blob') { - const { getMultipartPartUrls } = await import('@/lib/uploads/providers/blob/client') - const presignedUrls = await getMultipartPartUrls( - key, - partNumbers, - buildBlobCustomConfig(config) - ) - return NextResponse.json({ presignedUrls }) - } - if (storageProvider === 'gcs') { - const { getGcsMultipartPartUrls } = await import('@/lib/uploads/providers/gcs/client') - const presignedUrls = await getGcsMultipartPartUrls( - key, - uploadId, - partNumbers, - buildGcsCustomConfig(config) - ) - return NextResponse.json({ presignedUrls }) - } - - return NextResponse.json( - { error: `Unsupported storage provider: ${storageProvider}` }, - { status: 400 } - ) - } - - case 'complete': { - const parsed = await parseRequest( - completeMultipartUploadContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const data: CompleteMultipartBody = parsed.data.body - - const s3Module = - storageProvider === 's3' ? await import('@/lib/uploads/providers/s3/client') : null - const blobModule = - storageProvider === 'blob' ? await import('@/lib/uploads/providers/blob/client') : null - const gcsModule = - storageProvider === 'gcs' ? await import('@/lib/uploads/providers/gcs/client') : null - - const completeOne = async (payload: UploadTokenPayload, parts: ClientCompletedPart[]) => { - const { uploadId, key, context } = payload - const config = getStorageConfig(context) - - let completed: { location: string; path: string; key: string } - if (storageProvider === 's3' && s3Module) { - const { completeS3MultipartUpload } = s3Module - const s3Parts = parts.map((p) => { - if (!p.etag) { - throw new Error(`Missing etag for S3 part ${p.partNumber}`) - } - return { ETag: p.etag, PartNumber: p.partNumber } - }) - completed = await completeS3MultipartUpload( - key, - uploadId, - s3Parts, - buildS3CustomConfig(config) - ) - } else if (storageProvider === 'blob' && blobModule) { - const { completeMultipartUpload, deriveBlobBlockId } = blobModule - const blobParts = parts.map((p) => ({ - partNumber: p.partNumber, - blockId: deriveBlobBlockId(p.partNumber), - })) - completed = await completeMultipartUpload(key, blobParts, buildBlobCustomConfig(config)) - } else if (storageProvider === 'gcs' && gcsModule) { - const { completeGcsMultipartUpload } = gcsModule - const gcsParts = parts.map((p) => { - if (!p.etag) { - throw new Error(`Missing etag for GCS part ${p.partNumber}`) - } - return { ETag: p.etag, PartNumber: p.partNumber } - }) - completed = await completeGcsMultipartUpload( - key, - uploadId, - gcsParts, - buildGcsCustomConfig(config) - ) - } else { - throw new Error(`Unsupported storage provider: ${storageProvider}`) - } - - try { - await recordKnowledgeBaseOwnership(payload, completed.key) - } catch (error) { - // The object is committed, but without an ownership binding a KB file - // is unreadable and undeletable via the KB paths. Remove the orphan - // best-effort and surface a retryable error so the client re-uploads. - if (payload.context === 'knowledge-base') { - await deleteFile({ key: completed.key, context: 'knowledge-base' }).catch(() => {}) - } - throw error - } - - return { - success: true as const, - location: completed.location, - path: completed.path, - key: completed.key, - } - } - - if ('uploads' in data && Array.isArray(data.uploads)) { - const verified: Array<{ payload: UploadTokenPayload; parts: ClientCompletedPart[] }> = [] - for (const upload of data.uploads) { - const payload = verifyTokenForUser(upload.uploadToken, userId) - if (!payload) { - return NextResponse.json( - { error: 'Invalid or expired upload token' }, - { status: 403 } - ) - } - if (!isClientCompletedParts(upload.parts)) { - return NextResponse.json( - { error: 'Invalid parts payload: expected [{ partNumber, etag? }]' }, - { status: 400 } - ) - } - verified.push({ payload, parts: upload.parts }) - } - - const results = await Promise.all( - verified.map(({ payload, parts }) => completeOne(payload, parts)) - ) - - logger.info(`Completed ${verified.length} multipart uploads`) - return NextResponse.json({ results }) - } - - const single = data - const tokenPayload = verifyTokenForUser(single.uploadToken, userId) - if (!tokenPayload) { - return NextResponse.json({ error: 'Invalid or expired upload token' }, { status: 403 }) - } - if (!isClientCompletedParts(single.parts)) { - return NextResponse.json( - { error: 'Invalid parts payload: expected [{ partNumber, etag? }]' }, - { status: 400 } - ) - } - - const result = await completeOne(tokenPayload, single.parts) - logger.info( - `Completed ${storageProvider} multipart upload for key ${tokenPayload.key} (context: ${tokenPayload.context})` - ) - return NextResponse.json(result) - } - - case 'abort': { - const parsed = await parseRequest( - abortMultipartUploadContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json({ error: getValidationErrorMessage(error) }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const data = parsed.data.body - const tokenPayload = verifyTokenForUser(data.uploadToken, userId) - if (!tokenPayload) { - return NextResponse.json({ error: 'Invalid or expired upload token' }, { status: 403 }) - } - - const { uploadId, key, context } = tokenPayload - const config = getStorageConfig(context) - - if (storageProvider === 's3') { - const { abortS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') - await abortS3MultipartUpload(key, uploadId, buildS3CustomConfig(config)) - logger.info(`Aborted S3 multipart upload for key ${key} (context: ${context})`) - } else if (storageProvider === 'blob') { - const { abortMultipartUpload } = await import('@/lib/uploads/providers/blob/client') - await abortMultipartUpload(key, buildBlobCustomConfig(config)) - logger.info(`Aborted Azure multipart upload for key ${key} (context: ${context})`) - } else if (storageProvider === 'gcs') { - const { abortGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') - await abortGcsMultipartUpload(key, uploadId, buildGcsCustomConfig(config)) - logger.info(`Aborted GCS multipart upload for key ${key} (context: ${context})`) - } else { - return NextResponse.json( - { error: `Unsupported storage provider: ${storageProvider}` }, - { status: 400 } - ) - } - - return NextResponse.json({ success: true }) - } - - default: - return NextResponse.json( - { error: 'Invalid action. Use: initiate, get-part-urls, complete, or abort' }, - { status: 400 } - ) - } - } catch (error) { - logger.error('Multipart upload error:', error) - return NextResponse.json( - { error: getErrorMessage(error, 'Multipart upload failed') }, - { status: 500 } - ) - } -}) diff --git a/apps/sim/app/api/files/presigned/batch/route.test.ts b/apps/sim/app/api/files/presigned/batch/route.test.ts deleted file mode 100644 index 988fae9cce4..00000000000 --- a/apps/sim/app/api/files/presigned/batch/route.test.ts +++ /dev/null @@ -1,189 +0,0 @@ -/** - * Tests for the batch presigned upload API route - * - * @vitest-environment node - */ - -import { authMockFns, storageServiceMock, storageServiceMockFns } from '@sim/testing' -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockValidateFileType, - mockGetUserEntityPermissions, - mockRecordKnowledgeBaseFileOwnershipMany, -} = vi.hoisted(() => ({ - mockValidateFileType: vi.fn().mockReturnValue(null), - mockGetUserEntityPermissions: vi.fn().mockResolvedValue('write'), - mockRecordKnowledgeBaseFileOwnershipMany: vi.fn().mockResolvedValue(undefined), -})) - -vi.mock('@/lib/uploads/config', () => ({ - getServeStoragePrefix: () => 's3', -})) - -vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) - -vi.mock('@/lib/uploads/utils/validation', () => ({ - validateFileType: mockValidateFileType, - SUPPORTED_ARCHIVE_EXTENSIONS: ['zip'] as const, -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - getUserEntityPermissions: mockGetUserEntityPermissions, -})) - -vi.mock('@/lib/uploads/server/metadata', () => ({ - recordKnowledgeBaseFileOwnershipMany: mockRecordKnowledgeBaseFileOwnershipMany, -})) - -import { POST } from '@/app/api/files/presigned/batch/route' - -const KB_QUERY = 'type=knowledge-base&workspaceId=ws-1' - -const buildRequest = (query: string, files?: unknown) => - new NextRequest(`http://localhost:3000/api/files/presigned/batch?${query}`, { - method: 'POST', - body: JSON.stringify({ - files: files ?? [{ fileName: 'doc.pdf', contentType: 'application/pdf', fileSize: 1024 }], - }), - }) - -describe('/api/files/presigned/batch', () => { - beforeEach(() => { - vi.clearAllMocks() - authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - mockValidateFileType.mockReturnValue(null) - mockGetUserEntityPermissions.mockResolvedValue('write') - mockRecordKnowledgeBaseFileOwnershipMany.mockResolvedValue(undefined) - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) - storageServiceMockFns.mockGenerateBatchPresignedUploadUrls.mockImplementation( - async (files: Array<{ fileName: string }>, context: string) => - files.map((file) => ({ - url: `https://example.com/${context}/${file.fileName}`, - key: `${context}/${file.fileName}`, - })) - ) - }) - - it('returns 401 when the caller has no session', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const response = await POST(buildRequest(KB_QUERY)) - - expect(response.status).toBe(401) - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).not.toHaveBeenCalled() - }) - - it.each([ - 'workspace-logos', - 'profile-pictures', - 'execution', - 'mothership', - 'chat', - 'copilot', - 'workspace', - ])('refuses to presign the %s context', async (type) => { - const response = await POST(buildRequest(`type=${type}&workspaceId=ws-1`)) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('Invalid type parameter') - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).not.toHaveBeenCalled() - }) - - it('returns 400 when type is missing', async () => { - const response = await POST(buildRequest('workspaceId=ws-1')) - - expect(response.status).toBe(400) - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).not.toHaveBeenCalled() - }) - - it('returns 400 when workspaceId is missing', async () => { - const response = await POST(buildRequest('type=knowledge-base')) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('workspaceId') - expect(mockGetUserEntityPermissions).not.toHaveBeenCalled() - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).not.toHaveBeenCalled() - }) - - it.each([['read'], [null]])( - 'returns 403 when the caller has %s access to the workspace', - async (permission) => { - mockGetUserEntityPermissions.mockResolvedValue(permission) - - const response = await POST(buildRequest(KB_QUERY)) - - expect(response.status).toBe(403) - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).not.toHaveBeenCalled() - } - ) - - it('authorizes the workspace before returning the local-storage fallback', async () => { - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(false) - mockGetUserEntityPermissions.mockResolvedValue('read') - - const response = await POST(buildRequest(KB_QUERY)) - - expect(response.status).toBe(403) - }) - - it('rejects unsupported file types before minting any URL', async () => { - mockValidateFileType.mockReturnValue({ - code: 'UNSUPPORTED_FILE_TYPE', - message: 'Unsupported file type: html.', - supportedTypes: ['pdf'], - }) - - const response = await POST( - buildRequest(KB_QUERY, [{ fileName: 'poc.html', contentType: 'text/html', fileSize: 41 }]) - ) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.code).toBe('UNSUPPORTED_FILE_TYPE') - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).not.toHaveBeenCalled() - }) - - it('mints knowledge-base URLs and records workspace ownership for a permitted caller', async () => { - const response = await POST(buildRequest(KB_QUERY)) - const data = await response.json() - - expect(response.status).toBe(200) - expect(mockGetUserEntityPermissions).toHaveBeenCalledWith('user-1', 'workspace', 'ws-1') - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).toHaveBeenCalledWith( - [{ fileName: 'doc.pdf', contentType: 'application/pdf', fileSize: 1024 }], - 'knowledge-base', - 'user-1', - 3600 - ) - expect(data.files).toHaveLength(1) - expect(data.files[0].fileInfo.key).toBe('knowledge-base/doc.pdf') - expect(data.files[0].fileInfo.path).toContain('?context=knowledge-base') - expect(data.directUploadSupported).toBe(true) - expect(mockRecordKnowledgeBaseFileOwnershipMany).toHaveBeenCalledWith([ - { - key: 'knowledge-base/doc.pdf', - userId: 'user-1', - workspaceId: 'ws-1', - originalName: 'doc.pdf', - contentType: 'application/pdf', - size: 1024, - }, - ]) - }) - - it('returns the fallback response when cloud storage is not configured', async () => { - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(false) - - const response = await POST(buildRequest(KB_QUERY)) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.directUploadSupported).toBe(false) - expect(data.files[0].presignedUrl).toBe('') - expect(storageServiceMockFns.mockGenerateBatchPresignedUploadUrls).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/app/api/files/presigned/batch/route.ts b/apps/sim/app/api/files/presigned/batch/route.ts deleted file mode 100644 index 226fdc9ed87..00000000000 --- a/apps/sim/app/api/files/presigned/batch/route.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { createLogger } from '@sim/logger' -import { type NextRequest, NextResponse } from 'next/server' -import { - batchPresignedUploadBodyContract, - batchPresignedUploadTypeSchema, - batchPresignedUploadTypes, -} from '@/lib/api/contracts/storage-transfer' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getServeStoragePrefix } from '@/lib/uploads/config' -import { - generateBatchPresignedUploadUrls, - hasCloudStorage, -} from '@/lib/uploads/core/storage-service' -import { recordKnowledgeBaseFileOwnershipMany } from '@/lib/uploads/server/metadata' -import { validateFileType } from '@/lib/uploads/utils/validation' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { createErrorResponse } from '@/app/api/files/utils' - -const logger = createLogger('BatchPresignedUploadAPI') - -/** - * Mints presigned upload URLs for knowledge-base ingest, the only context this - * endpoint can authorize. Every request must name a workspace the caller has - * write access to; other storage contexts are rejected rather than presigned, - * because a presigned PUT is a write grant into a bucket served from a trusted - * origin. - */ -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - batchPresignedUploadBodyContract, - request, - {}, - { - validationErrorResponse: (error) => - NextResponse.json( - { error: getValidationErrorMessage(error, 'Invalid request data') }, - { status: 400 } - ), - invalidJsonResponse: () => - NextResponse.json({ error: 'Invalid JSON in request body' }, { status: 400 }), - } - ) - if (!parsed.success) return parsed.response - - const { files } = parsed.data.body - - const uploadTypeParam = request.nextUrl.searchParams.get('type') - if (!uploadTypeParam) { - return NextResponse.json({ error: 'type query parameter is required' }, { status: 400 }) - } - - const uploadTypeResult = batchPresignedUploadTypeSchema.safeParse(uploadTypeParam) - if (!uploadTypeResult.success) { - return NextResponse.json( - { - error: `Invalid type parameter. Must be one of: ${batchPresignedUploadTypes.join(', ')}`, - }, - { status: 400 } - ) - } - - const uploadType = uploadTypeResult.data - const sessionUserId = session.user.id - - for (const file of files) { - const fileValidationError = validateFileType(file.fileName, file.contentType) - if (fileValidationError) { - return NextResponse.json( - { - error: fileValidationError.message, - code: fileValidationError.code, - supportedTypes: fileValidationError.supportedTypes, - }, - { status: 400 } - ) - } - } - - const workspaceId = request.nextUrl.searchParams.get('workspaceId') - if (!workspaceId?.trim()) { - return NextResponse.json( - { error: 'workspaceId query parameter is required for knowledge-base uploads' }, - { status: 400 } - ) - } - - const permission = await getUserEntityPermissions(sessionUserId, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json( - { error: 'Write or Admin access required for knowledge-base uploads' }, - { status: 403 } - ) - } - - if (!hasCloudStorage()) { - logger.info( - `Local storage detected - batch presigned URLs not available, client will use API fallback` - ) - return NextResponse.json({ - files: files.map((file) => ({ - fileName: file.fileName, - presignedUrl: '', // Empty URL signals fallback to API upload - fileInfo: { - path: '', - key: '', - name: file.fileName, - size: file.fileSize, - type: file.contentType, - }, - directUploadSupported: false, - })), - directUploadSupported: false, - }) - } - - logger.info(`Generating batch ${uploadType} presigned URLs for ${files.length} files`) - - const startTime = Date.now() - - const presignedUrls = await generateBatchPresignedUploadUrls( - files.map((file) => ({ - fileName: file.fileName, - contentType: file.contentType, - fileSize: file.fileSize, - })), - uploadType, - sessionUserId, - 3600 // 1 hour - ) - - const duration = Date.now() - startTime - logger.info( - `Generated ${files.length} presigned URLs in ${duration}ms (avg ${Math.round(duration / files.length)}ms per file)` - ) - - await recordKnowledgeBaseFileOwnershipMany( - presignedUrls.map((urlResponse, index) => ({ - key: urlResponse.key, - userId: sessionUserId, - workspaceId, - originalName: files[index].fileName, - contentType: files[index].contentType, - size: files[index].fileSize, - })) - ) - - const storagePrefix = getServeStoragePrefix() - - return NextResponse.json({ - files: presignedUrls.map((urlResponse, index) => { - const finalPath = `/api/files/serve/${storagePrefix}/${encodeURIComponent(urlResponse.key)}?context=${uploadType}` - const file = files[index] - - return { - fileName: file.fileName, - presignedUrl: urlResponse.url, - fileInfo: { - path: finalPath, - key: urlResponse.key, - name: file.fileName, - size: file.fileSize, - type: file.contentType, - }, - uploadHeaders: urlResponse.uploadHeaders, - directUploadSupported: true, - } - }), - directUploadSupported: true, - }) - } catch (error) { - logger.error('Error generating batch presigned URLs:', error) - return createErrorResponse( - error instanceof Error ? error : new Error('Failed to generate batch presigned URLs') - ) - } -}) diff --git a/apps/sim/app/api/files/presigned/route.test.ts b/apps/sim/app/api/files/presigned/route.test.ts deleted file mode 100644 index 3674ac70b76..00000000000 --- a/apps/sim/app/api/files/presigned/route.test.ts +++ /dev/null @@ -1,937 +0,0 @@ -/** - * Tests for file presigned API route - * - * @vitest-environment node - */ - -import { authMockFns, storageServiceMock, storageServiceMockFns } from '@sim/testing' -import { NextRequest } from 'next/server' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockVerifyFileAccess, - mockVerifyWorkspaceFileAccess, - mockUseBlobStorage, - mockUseS3Storage, - mockGetStorageConfig, - mockIsUsingCloudStorage, - mockGetStorageProvider, - mockValidateFileType, - mockValidateAttachmentFileType, - mockGenerateCopilotUploadUrl, - mockIsImageFileType, - mockGetStorageProviderUploads, - mockIsUsingCloudStorageUploads, - mockGetUserEntityPermissions, - mockGenerateWorkspaceFileKey, - mockGenerateExecutionFileKey, - mockInsertFileMetadata, - mockCheckStorageQuotaForBillingContext, - mockDecrementStorageUsageForBillingContext, - mockIncrementStorageUsageForBillingContext, - mockResolveStorageBillingContext, -} = vi.hoisted(() => ({ - mockVerifyFileAccess: vi.fn().mockResolvedValue(true), - mockVerifyWorkspaceFileAccess: vi.fn().mockResolvedValue(true), - mockUseBlobStorage: { value: false }, - mockUseS3Storage: { value: true }, - mockGetStorageConfig: vi.fn(), - mockIsUsingCloudStorage: vi.fn(), - mockGetStorageProvider: vi.fn(), - mockValidateFileType: vi.fn().mockReturnValue(null), - mockValidateAttachmentFileType: vi.fn().mockReturnValue(null), - mockGenerateCopilotUploadUrl: vi.fn().mockResolvedValue({ - url: 'https://example.com/presigned-url', - key: 'copilot/test-key.txt', - }), - mockIsImageFileType: vi.fn().mockReturnValue(true), - mockGetStorageProviderUploads: vi.fn(), - mockIsUsingCloudStorageUploads: vi.fn(), - mockGetUserEntityPermissions: vi.fn().mockResolvedValue('admin'), - mockGenerateWorkspaceFileKey: vi.fn( - (workspaceId: string, fileName: string) => `workspace/${workspaceId}/${fileName}` - ), - mockGenerateExecutionFileKey: vi.fn( - (ctx: { workspaceId: string; workflowId: string; executionId: string }, fileName: string) => - `execution/${ctx.workspaceId}/${ctx.workflowId}/${ctx.executionId}/${fileName}` - ), - mockInsertFileMetadata: vi.fn().mockResolvedValue({ id: 'wf_test' }), - mockCheckStorageQuotaForBillingContext: vi.fn(), - mockDecrementStorageUsageForBillingContext: vi.fn(), - mockIncrementStorageUsageForBillingContext: vi.fn(), - mockResolveStorageBillingContext: vi.fn(), -})) - -vi.mock('@/app/api/files/authorization', () => ({ - verifyFileAccess: mockVerifyFileAccess, - verifyWorkspaceFileAccess: mockVerifyWorkspaceFileAccess, -})) - -vi.mock('@/lib/uploads/config', () => ({ - get USE_BLOB_STORAGE() { - return mockUseBlobStorage.value - }, - get USE_S3_STORAGE() { - return mockUseS3Storage.value - }, - UPLOAD_DIR: '/uploads', - getServeStoragePrefix: () => (mockUseBlobStorage.value ? 'blob' : 's3'), - getStorageConfig: mockGetStorageConfig, - isUsingCloudStorage: mockIsUsingCloudStorage, - getStorageProvider: mockGetStorageProvider, -})) - -vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) - -vi.mock('@/lib/billing/storage', () => ({ - checkStorageQuotaForBillingContext: mockCheckStorageQuotaForBillingContext, - decrementStorageUsageForBillingContext: mockDecrementStorageUsageForBillingContext, - incrementStorageUsageForBillingContext: mockIncrementStorageUsageForBillingContext, - resolveStorageBillingContext: mockResolveStorageBillingContext, -})) - -vi.mock('@/lib/uploads/utils/validation', () => ({ - validateFileType: mockValidateFileType, - validateAttachmentFileType: mockValidateAttachmentFileType, -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => ({ - getUserEntityPermissions: mockGetUserEntityPermissions, -})) - -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - generateWorkspaceFileKey: mockGenerateWorkspaceFileKey, -})) - -vi.mock('@/lib/uploads/contexts/execution/utils', () => ({ - generateExecutionFileKey: mockGenerateExecutionFileKey, -})) - -vi.mock('@/lib/uploads/server/metadata', () => ({ - insertFileMetadata: mockInsertFileMetadata, - recordKnowledgeBaseFileOwnership: (ownership: Record) => - mockInsertFileMetadata({ ...ownership, context: 'knowledge-base' }), -})) - -vi.mock('@/lib/uploads/utils/file-utils', () => ({ - isImageFileType: mockIsImageFileType, -})) - -vi.mock('@/lib/uploads', () => ({ - CopilotFiles: { - generateCopilotUploadUrl: mockGenerateCopilotUploadUrl, - }, - getStorageProvider: mockGetStorageProviderUploads, - isUsingCloudStorage: mockIsUsingCloudStorageUploads, -})) - -import { POST } from '@/app/api/files/presigned/route' - -const defaultMockUser = { - id: 'test-user-id', - name: 'Test User', - email: 'test@example.com', -} - -function setupFileApiMocks( - options: { - authenticated?: boolean - storageProvider?: 's3' | 'blob' | 'local' - cloudEnabled?: boolean - } = {} -) { - const { authenticated = true, storageProvider = 's3', cloudEnabled = true } = options - - if (authenticated) { - authMockFns.mockGetSession.mockResolvedValue({ user: defaultMockUser }) - } else { - authMockFns.mockGetSession.mockResolvedValue(null) - } - - const useBlobStorage = storageProvider === 'blob' && cloudEnabled - const useS3Storage = storageProvider === 's3' && cloudEnabled - - mockUseBlobStorage.value = useBlobStorage - mockUseS3Storage.value = useS3Storage - - mockGetStorageConfig.mockReturnValue( - useBlobStorage - ? { - accountName: 'testaccount', - accountKey: 'testkey', - connectionString: 'testconnection', - containerName: 'testcontainer', - } - : { - bucket: 'test-bucket', - region: 'us-east-1', - } - ) - mockIsUsingCloudStorage.mockReturnValue(cloudEnabled) - mockGetStorageProvider.mockReturnValue( - storageProvider === 'blob' ? 'Azure Blob' : storageProvider === 's3' ? 'S3' : 'Local' - ) - - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(cloudEnabled) - storageServiceMockFns.mockGeneratePresignedUploadUrl.mockImplementation( - async (opts: { fileName: string; context: string; customKey?: string }) => { - const timestamp = Date.now() - const safeFileName = opts.fileName.replace(/[^a-zA-Z0-9.-]/g, '_') - const key = opts.customKey ?? `${opts.context}/${timestamp}-ik3a6w4-${safeFileName}` - return { - url: 'https://example.com/presigned-url', - key, - } - } - ) - storageServiceMockFns.mockGeneratePresignedDownloadUrl.mockResolvedValue( - 'https://example.com/presigned-url' - ) - - mockValidateFileType.mockReturnValue(null) - mockValidateAttachmentFileType.mockReturnValue(null) - mockGetUserEntityPermissions.mockResolvedValue('admin') - - mockGetStorageProviderUploads.mockReturnValue( - storageProvider === 'blob' ? 'Azure Blob' : storageProvider === 's3' ? 'S3' : 'Local' - ) - mockIsUsingCloudStorageUploads.mockReturnValue(cloudEnabled) -} - -describe('/api/files/presigned', () => { - beforeEach(() => { - vi.clearAllMocks() - vi.useFakeTimers() - vi.setSystemTime(new Date('2024-01-01T00:00:00Z')) - - vi.stubGlobal('crypto', { - randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'), - }) - }) - - afterEach(() => { - vi.useRealTimers() - }) - - describe('POST', () => { - it('should return graceful fallback response when cloud storage is not enabled', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 's3', - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'avatar.png', - contentType: 'image/png', - fileSize: 1024, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.directUploadSupported).toBe(false) - expect(data.presignedUrl).toBe('') - expect(data.fileName).toBe('avatar.png') - expect(data.fileInfo).toBeDefined() - expect(data.fileInfo.name).toBe('avatar.png') - expect(data.fileInfo.size).toBe(1024) - expect(data.fileInfo.type).toBe('image/png') - }) - - it('should return error when fileName is missing', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const request = new NextRequest('http://localhost:3000/api/files/presigned', { - method: 'POST', - body: JSON.stringify({ - contentType: 'text/plain', - fileSize: 1024, - }), - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('fileName is required and cannot be empty') - expect(data.code).toBe('VALIDATION_ERROR') - }) - - it('should return error when contentType is missing', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const request = new NextRequest('http://localhost:3000/api/files/presigned', { - method: 'POST', - body: JSON.stringify({ - fileName: 'test.txt', - fileSize: 1024, - }), - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('contentType is required and cannot be empty') - expect(data.code).toBe('VALIDATION_ERROR') - }) - - it('should return error when fileSize is invalid', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const request = new NextRequest('http://localhost:3000/api/files/presigned', { - method: 'POST', - body: JSON.stringify({ - fileName: 'test.txt', - contentType: 'text/plain', - fileSize: 0, - }), - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toBe('fileSize must be a positive number') - expect(data.code).toBe('VALIDATION_ERROR') - }) - - it('should return error when file size exceeds limit', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const largeFileSize = 150 * 1024 * 1024 // 150MB (exceeds 100MB limit) - const request = new NextRequest('http://localhost:3000/api/files/presigned', { - method: 'POST', - body: JSON.stringify({ - fileName: 'large-file.txt', - contentType: 'text/plain', - fileSize: largeFileSize, - }), - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('exceeds maximum allowed size') - expect(data.code).toBe('VALIDATION_ERROR') - }) - - it('should generate S3 presigned URL successfully', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'test avatar.png', - contentType: 'image/png', - fileSize: 1024, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.presignedUrl).toBe('https://example.com/presigned-url') - expect(data.fileInfo).toMatchObject({ - path: expect.stringMatching(/\/api\/files\/serve\/s3\/.+\?context=profile-pictures$/), - key: expect.stringMatching(/.*test.avatar\.png$/), - name: 'test avatar.png', - size: 1024, - type: 'image/png', - }) - expect(data.directUploadSupported).toBe(true) - }) - - it('should generate knowledge-base S3 presigned URL with kb prefix', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=knowledge-base&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'knowledge-doc.pdf', - contentType: 'application/pdf', - fileSize: 2048, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.fileInfo.key).toMatch(/^kb\/.*knowledge-doc\.pdf$/) - expect(data.directUploadSupported).toBe(true) - }) - - it('should generate profile-pictures S3 presigned URL with its prefix and direct path', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'avatar.png', - contentType: 'image/png', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.fileInfo.key).toMatch(/^profile-pictures\/.*avatar\.png$/) - expect(data.fileInfo.path).toMatch(/\/api\/files\/serve\/s3\/.+\?context=profile-pictures$/) - expect(data.presignedUrl).toBeTruthy() - expect(data.directUploadSupported).toBe(true) - }) - - it('should generate Azure Blob presigned URL successfully', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 'blob', - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'test avatar.png', - contentType: 'image/png', - fileSize: 1024, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.presignedUrl).toBeTruthy() - expect(typeof data.presignedUrl).toBe('string') - expect(data.fileInfo).toMatchObject({ - key: expect.stringMatching(/.*test.avatar\.png$/), - name: 'test avatar.png', - size: 1024, - type: 'image/png', - }) - expect(data.directUploadSupported).toBe(true) - }) - - it('should generate profile-pictures Azure Blob presigned URL with its prefix and direct path', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 'blob', - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'avatar.png', - contentType: 'image/png', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data.fileInfo.key).toMatch(/^profile-pictures\/.*avatar\.png$/) - expect(data.fileInfo.path).toMatch(/\/api\/files\/serve\/blob\/.+\?context=profile-pictures$/) - expect(data.presignedUrl).toBeTruthy() - expect(data.directUploadSupported).toBe(true) - }) - - it('should return error for unknown storage provider', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - storageServiceMockFns.mockGeneratePresignedUploadUrl.mockRejectedValue( - new Error('Unknown storage provider: unknown') - ) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'avatar.png', - contentType: 'image/png', - fileSize: 1024, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBeTruthy() - expect(typeof data.error).toBe('string') - }) - - it('should handle S3 errors gracefully', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - storageServiceMockFns.mockGeneratePresignedUploadUrl.mockRejectedValue( - new Error('S3 service unavailable') - ) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'avatar.png', - contentType: 'image/png', - fileSize: 1024, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBeTruthy() - expect(typeof data.error).toBe('string') - }) - - it('should handle Azure Blob errors gracefully', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 'blob', - }) - - storageServiceMockFns.mockGeneratePresignedUploadUrl.mockRejectedValue( - new Error('Azure service unavailable') - ) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=profile-pictures', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'avatar.png', - contentType: 'image/png', - fileSize: 1024, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(500) - expect(data.error).toBeTruthy() - expect(typeof data.error).toBe('string') - }) - - it('should handle malformed JSON gracefully', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const request = new NextRequest('http://localhost:3000/api/files/presigned', { - method: 'POST', - body: 'invalid json', - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(400) // Changed from 500 to 400 (ValidationError) - expect(data.error).toBe('Invalid JSON in request body') // Updated error message - expect(data.code).toBe('VALIDATION_ERROR') - }) - - it('rejects the unauthorizable chat context without minting a URL', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest('http://localhost:3000/api/files/presigned?type=chat', { - method: 'POST', - body: JSON.stringify({ - fileName: 'poc.html', - contentType: 'text/html', - fileSize: 41, - }), - }) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data.error).toContain('Invalid type parameter') - expect(storageServiceMockFns.mockGeneratePresignedUploadUrl).not.toHaveBeenCalled() - }) - }) - - describe('mothership uploads', () => { - it('uses validateAttachmentFileType (not validateFileType) — accepts images', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=mothership&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'screenshot.png', - contentType: 'image/png', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(200) - expect(mockValidateAttachmentFileType).toHaveBeenCalledWith('screenshot.png', { - allowArchives: true, - }) - expect(mockValidateFileType).not.toHaveBeenCalled() - }) - - it('rejects unsupported types when validator returns an error', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - mockValidateAttachmentFileType.mockReturnValue({ - code: 'UNSUPPORTED_FILE_TYPE', - message: 'Unsupported file type: exe.', - supportedTypes: [], - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=mothership&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'virus.exe', - contentType: 'application/octet-stream', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - expect(response.status).toBe(400) - expect(data.code).toBe('VALIDATION_ERROR') - expect(data.error).toContain('exe') - }) - - it('returns 403 when user lacks workspace write permission', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - mockGetUserEntityPermissions.mockResolvedValue('read') - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=mothership&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'doc.pdf', - contentType: 'application/pdf', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(403) - }) - - it('issues an unbilled pending mothership upload binding', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=mothership&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'screenshot.png', - contentType: 'image/png', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(mockInsertFileMetadata).toHaveBeenCalledTimes(1) - expect(mockInsertFileMetadata).toHaveBeenCalledWith({ - key: data.fileInfo.key, - userId: 'test-user-id', - workspaceId: 'ws-1', - context: 'mothership', - originalName: 'screenshot.png', - contentType: 'image/png', - size: 4096, - }) - expect(mockCheckStorageQuotaForBillingContext).not.toHaveBeenCalled() - expect(mockResolveStorageBillingContext).not.toHaveBeenCalled() - expect(mockIncrementStorageUsageForBillingContext).not.toHaveBeenCalled() - expect(mockDecrementStorageUsageForBillingContext).not.toHaveBeenCalled() - }) - - it('returns 500 when insertFileMetadata fails so callers do not get an unauthorizable URL', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - mockInsertFileMetadata.mockRejectedValueOnce(new Error('DB connection lost')) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=mothership&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'screenshot.png', - contentType: 'image/png', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(500) - }) - }) - - describe('execution uploads', () => { - it('uses validateAttachmentFileType — accepts video', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=execution&workspaceId=ws-1&workflowId=wf-1&executionId=exec-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'output.mp4', - contentType: 'video/mp4', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(200) - expect(mockValidateAttachmentFileType).toHaveBeenCalledWith('output.mp4') - expect(mockValidateFileType).not.toHaveBeenCalled() - }) - - it('rejects when validator returns an error', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - mockValidateAttachmentFileType.mockReturnValue({ - code: 'UNSUPPORTED_FILE_TYPE', - message: 'Unsupported file type: bin.', - supportedTypes: [], - }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=execution&workspaceId=ws-1&workflowId=wf-1&executionId=exec-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'blob.bin', - contentType: 'application/octet-stream', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - expect(response.status).toBe(400) - expect(data.code).toBe('VALIDATION_ERROR') - }) - - it('returns 400 when missing workflowId/executionId', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=execution&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'output.mp4', - contentType: 'video/mp4', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(400) - }) - - it('inserts a workspaceFiles row with context=execution so previews authorize', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=execution&workspaceId=ws-1&workflowId=wf-1&executionId=exec-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'output.mp4', - contentType: 'video/mp4', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(mockInsertFileMetadata).toHaveBeenCalledTimes(1) - expect(mockInsertFileMetadata).toHaveBeenCalledWith({ - key: data.fileInfo.key, - userId: 'test-user-id', - workspaceId: 'ws-1', - context: 'execution', - originalName: 'output.mp4', - contentType: 'video/mp4', - size: 4096, - }) - }) - }) - - describe('workspace-logos uploads', () => { - it('inserts a workspaceFiles row with context=workspace-logos so logos authorize', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=workspace-logos&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'logo.png', - contentType: 'image/png', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - const data = await response.json() - - expect(response.status).toBe(200) - expect(mockInsertFileMetadata).toHaveBeenCalledTimes(1) - expect(mockInsertFileMetadata).toHaveBeenCalledWith({ - key: data.fileInfo.key, - userId: 'test-user-id', - workspaceId: 'ws-1', - context: 'workspace-logos', - originalName: 'logo.png', - contentType: 'image/png', - size: 4096, - }) - }) - }) - - describe('knowledge-base uploads', () => { - it('uses validateFileType (docs-only), not validateAttachmentFileType', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=knowledge-base&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'doc.pdf', - contentType: 'application/pdf', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(200) - expect(mockValidateFileType).toHaveBeenCalledWith('doc.pdf', 'application/pdf') - expect(mockValidateAttachmentFileType).not.toHaveBeenCalled() - }) - - it('requires workspaceId for knowledge-base uploads', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=knowledge-base', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'doc.pdf', - contentType: 'application/pdf', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(400) - }) - - it('returns 403 when the user lacks write access to the workspace', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - mockGetUserEntityPermissions.mockResolvedValue('read') - - const request = new NextRequest( - 'http://localhost:3000/api/files/presigned?type=knowledge-base&workspaceId=ws-1', - { - method: 'POST', - body: JSON.stringify({ - fileName: 'doc.pdf', - contentType: 'application/pdf', - fileSize: 4096, - }), - } - ) - - const response = await POST(request) - expect(response.status).toBe(403) - }) - }) -}) diff --git a/apps/sim/app/api/files/presigned/route.ts b/apps/sim/app/api/files/presigned/route.ts deleted file mode 100644 index 49bec3aab16..00000000000 --- a/apps/sim/app/api/files/presigned/route.ts +++ /dev/null @@ -1,335 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { - presignedUploadBodyContract, - presignedUploadTypeSchema, - presignedUploadTypes, -} from '@/lib/api/contracts/storage-transfer' -import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { CopilotFiles } from '@/lib/uploads' -import { getServeStoragePrefix } from '@/lib/uploads/config' -import { generateExecutionFileKey } from '@/lib/uploads/contexts/execution/utils' -import { generateKnowledgeBaseFileKey } from '@/lib/uploads/contexts/knowledge-base/knowledge-base-file-manager' -import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { generatePresignedUploadUrl, hasCloudStorage } from '@/lib/uploads/core/storage-service' -import { insertFileMetadata, recordKnowledgeBaseFileOwnership } from '@/lib/uploads/server/metadata' -import { isImageFileType } from '@/lib/uploads/utils/file-utils' -import { validateAttachmentFileType, validateFileType } from '@/lib/uploads/utils/validation' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { createErrorResponse } from '@/app/api/files/utils' - -const logger = createLogger('PresignedUploadAPI') - -class PresignedUrlError extends Error { - constructor( - message: string, - public code: string, - public statusCode = 400 - ) { - super(message) - this.name = 'PresignedUrlError' - } -} - -class ValidationError extends PresignedUrlError { - constructor(message: string) { - super(message, 'VALIDATION_ERROR', 400) - } -} - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const parsed = await parseRequest( - presignedUploadBodyContract, - request, - {}, - { - validationErrorResponse: (error) => { - throw new ValidationError(getValidationErrorMessage(error, 'Invalid request data')) - }, - invalidJsonResponse: () => { - throw new ValidationError('Invalid JSON in request body') - }, - } - ) - if (!parsed.success) return parsed.response - - const { fileName, contentType, fileSize } = parsed.data.body - - const uploadTypeParam = request.nextUrl.searchParams.get('type') - if (!uploadTypeParam) { - throw new ValidationError('type query parameter is required') - } - - const uploadTypeResult = presignedUploadTypeSchema.safeParse(uploadTypeParam) - if (!uploadTypeResult.success) { - throw new ValidationError( - `Invalid type parameter. Must be one of: ${presignedUploadTypes.join(', ')}` - ) - } - - const uploadType = uploadTypeResult.data - - if (uploadType === 'knowledge-base') { - const fileValidationError = validateFileType(fileName, contentType) - if (fileValidationError) { - throw new ValidationError(`${fileValidationError.message}`) - } - } - - const sessionUserId = session.user.id - - if (!hasCloudStorage()) { - logger.info( - `Local storage detected - presigned URL not available for ${fileName}, client will use API fallback` - ) - return NextResponse.json({ - fileName, - presignedUrl: '', // Empty URL signals fallback to API upload - fileInfo: { - path: '', - key: '', - name: fileName, - size: fileSize, - type: contentType, - }, - directUploadSupported: false, - }) - } - - logger.info(`Generating ${uploadType} presigned URL for ${fileName}`) - - let presignedUrlResponse - - if (uploadType === 'copilot') { - try { - presignedUrlResponse = await CopilotFiles.generateCopilotUploadUrl({ - fileName, - contentType, - fileSize, - userId: sessionUserId, - expirationSeconds: 3600, - }) - } catch (error) { - throw new ValidationError(getErrorMessage(error, 'Chat validation failed')) - } - } else if (uploadType === 'mothership') { - const workspaceId = request.nextUrl.searchParams.get('workspaceId') - if (!workspaceId?.trim()) { - throw new ValidationError('workspaceId query parameter is required for chat uploads') - } - - const permission = await getUserEntityPermissions(sessionUserId, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json( - { error: 'Write or Admin access required for chat uploads' }, - { status: 403 } - ) - } - - const fileValidationError = validateAttachmentFileType(fileName, { allowArchives: true }) - if (fileValidationError) { - throw new ValidationError(fileValidationError.message) - } - - const customKey = generateWorkspaceFileKey(workspaceId, fileName) - presignedUrlResponse = await generatePresignedUploadUrl({ - fileName, - contentType, - fileSize, - context: 'mothership', - userId: sessionUserId, - customKey, - expirationSeconds: 3600, - metadata: { workspaceId }, - }) - - await insertFileMetadata({ - key: presignedUrlResponse.key, - userId: sessionUserId, - workspaceId, - context: 'mothership', - originalName: fileName, - contentType, - size: fileSize, - }) - } else if (uploadType === 'execution') { - const workflowId = request.nextUrl.searchParams.get('workflowId') - const executionId = request.nextUrl.searchParams.get('executionId') - const workspaceId = request.nextUrl.searchParams.get('workspaceId') - if (!workflowId?.trim() || !executionId?.trim() || !workspaceId?.trim()) { - throw new ValidationError( - 'workflowId, executionId, and workspaceId query parameters are required for execution uploads' - ) - } - - const permission = await getUserEntityPermissions(sessionUserId, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json( - { error: 'Write or Admin access required for execution uploads' }, - { status: 403 } - ) - } - - const fileValidationError = validateAttachmentFileType(fileName) - if (fileValidationError) { - throw new ValidationError(fileValidationError.message) - } - - const customKey = generateExecutionFileKey({ workspaceId, workflowId, executionId }, fileName) - presignedUrlResponse = await generatePresignedUploadUrl({ - fileName, - contentType, - fileSize, - context: 'execution', - userId: sessionUserId, - customKey, - expirationSeconds: 3600, - metadata: { workspaceId, workflowId, executionId }, - }) - - await insertFileMetadata({ - key: presignedUrlResponse.key, - userId: sessionUserId, - workspaceId, - context: 'execution', - originalName: fileName, - contentType, - size: fileSize, - }) - } else if (uploadType === 'workspace-logos') { - const workspaceId = request.nextUrl.searchParams.get('workspaceId') - if (!workspaceId?.trim()) { - throw new ValidationError( - 'workspaceId query parameter is required for workspace-logos uploads' - ) - } - - const permission = await getUserEntityPermissions(sessionUserId, 'workspace', workspaceId) - if (permission !== 'admin') { - return NextResponse.json( - { error: 'Admin access required for workspace logo uploads' }, - { status: 403 } - ) - } - - if (!isImageFileType(contentType)) { - throw new ValidationError( - 'Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for workspace logo uploads' - ) - } - - presignedUrlResponse = await generatePresignedUploadUrl({ - fileName, - contentType, - fileSize, - context: 'workspace-logos', - userId: sessionUserId, - expirationSeconds: 3600, - metadata: { workspaceId }, - }) - - await insertFileMetadata({ - key: presignedUrlResponse.key, - userId: sessionUserId, - workspaceId, - context: 'workspace-logos', - originalName: fileName, - contentType, - size: fileSize, - }) - } else if (uploadType === 'knowledge-base') { - const workspaceId = request.nextUrl.searchParams.get('workspaceId') - if (!workspaceId?.trim()) { - throw new ValidationError( - 'workspaceId query parameter is required for knowledge-base uploads' - ) - } - - const permission = await getUserEntityPermissions(sessionUserId, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json( - { error: 'Write or Admin access required for knowledge-base uploads' }, - { status: 403 } - ) - } - - const customKey = generateKnowledgeBaseFileKey(fileName) - presignedUrlResponse = await generatePresignedUploadUrl({ - fileName, - contentType, - fileSize, - context: 'knowledge-base', - userId: sessionUserId, - customKey, - expirationSeconds: 3600, - metadata: { workspaceId }, - }) - - await recordKnowledgeBaseFileOwnership({ - key: presignedUrlResponse.key, - userId: sessionUserId, - workspaceId, - originalName: fileName, - contentType, - size: fileSize, - }) - } else { - if (!isImageFileType(contentType)) { - throw new ValidationError( - 'Only image files (JPEG, PNG, GIF, WebP, SVG) are allowed for profile picture uploads' - ) - } - - presignedUrlResponse = await generatePresignedUploadUrl({ - fileName, - contentType, - fileSize, - context: uploadType, - userId: sessionUserId, - expirationSeconds: 3600, // 1 hour - }) - } - - const finalPath = `/api/files/serve/${getServeStoragePrefix()}/${encodeURIComponent(presignedUrlResponse.key)}?context=${uploadType}` - - return NextResponse.json({ - fileName, - presignedUrl: presignedUrlResponse.url, - fileInfo: { - path: finalPath, - key: presignedUrlResponse.key, - name: fileName, - size: fileSize, - type: contentType, - }, - uploadHeaders: presignedUrlResponse.uploadHeaders, - directUploadSupported: true, - }) - } catch (error) { - logger.error('Error generating presigned URL:', error) - - if (error instanceof PresignedUrlError) { - return NextResponse.json( - { - error: error.message, - code: error.code, - directUploadSupported: false, - }, - { status: error.statusCode } - ) - } - - return createErrorResponse( - error instanceof Error ? error : new Error('Failed to generate presigned URL') - ) - } -}) diff --git a/apps/sim/app/api/files/upload/route.test.ts b/apps/sim/app/api/files/upload/route.test.ts deleted file mode 100644 index 034efb5eb70..00000000000 --- a/apps/sim/app/api/files/upload/route.test.ts +++ /dev/null @@ -1,814 +0,0 @@ -/** - * Tests for file upload API route - * - * @vitest-environment node - */ -import { - authMockFns, - hybridAuthMockFns, - permissionsMock, - permissionsMockFns, - storageServiceMock, - storageServiceMockFns, -} from '@sim/testing' -import { NextRequest } from 'next/server' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => { - const mockVerifyFileAccess = vi.fn() - const mockVerifyWorkspaceFileAccess = vi.fn() - const mockVerifyKBFileAccess = vi.fn() - const mockVerifyCopilotFileAccess = vi.fn() - const mockUploadWorkspaceFile = vi.fn() - const mockGetStorageProvider = vi.fn() - const mockIsUsingCloudStorage = vi.fn() - const mockUploadFile = vi.fn() - const mockUploadExecutionFile = vi.fn() - const mockCheckStorageQuota = vi.fn() - const mockCheckStorageQuotaForBillingContext = vi.fn() - const mockDecrementStorageUsageForBillingContext = vi.fn() - const mockIncrementStorageUsageForBillingContext = vi.fn() - const mockResolveStorageBillingContext = vi.fn() - - return { - mockVerifyFileAccess, - mockVerifyWorkspaceFileAccess, - mockVerifyKBFileAccess, - mockVerifyCopilotFileAccess, - mockUploadWorkspaceFile, - mockGetStorageProvider, - mockIsUsingCloudStorage, - mockUploadFile, - mockUploadExecutionFile, - mockCheckStorageQuota, - mockCheckStorageQuotaForBillingContext, - mockDecrementStorageUsageForBillingContext, - mockIncrementStorageUsageForBillingContext, - mockResolveStorageBillingContext, - } -}) - -vi.mock('@sim/utils/id', () => ({ - generateId: vi.fn(() => 'test-uuid'), - generateShortId: vi.fn(() => 'mock-short-id'), - isValidUuid: vi.fn((v: string) => - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(v) - ), -})) - -vi.mock('@/app/api/files/authorization', () => ({ - verifyFileAccess: mocks.mockVerifyFileAccess, - verifyWorkspaceFileAccess: mocks.mockVerifyWorkspaceFileAccess, - verifyKBFileAccess: mocks.mockVerifyKBFileAccess, - verifyCopilotFileAccess: mocks.mockVerifyCopilotFileAccess, -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) - -vi.mock('@/lib/uploads/contexts/workspace', () => ({ - uploadWorkspaceFile: mocks.mockUploadWorkspaceFile, -})) - -vi.mock('@/lib/uploads/contexts/execution', () => ({ - uploadExecutionFile: mocks.mockUploadExecutionFile, -})) - -vi.mock('@/lib/uploads', () => ({ - getStorageProvider: mocks.mockGetStorageProvider, - isUsingCloudStorage: mocks.mockIsUsingCloudStorage, - uploadFile: mocks.mockUploadFile, -})) - -vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) - -vi.mock('@/lib/billing/storage', () => ({ - checkStorageQuota: mocks.mockCheckStorageQuota, - checkStorageQuotaForBillingContext: mocks.mockCheckStorageQuotaForBillingContext, - decrementStorageUsageForBillingContext: mocks.mockDecrementStorageUsageForBillingContext, - incrementStorageUsageForBillingContext: mocks.mockIncrementStorageUsageForBillingContext, - resolveStorageBillingContext: mocks.mockResolveStorageBillingContext, -})) - -vi.mock('@/lib/uploads/shared/types', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - MAX_WORKSPACE_FORMDATA_FILE_SIZE: 1024, - } -}) - -vi.mock('@/lib/uploads/setup.server', () => ({ - UPLOAD_DIR_SERVER: '/tmp/test-uploads', -})) - -import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' -import { POST } from '@/app/api/files/upload/route' - -/** - * Configure mocks for authenticated file upload tests - */ -function setupFileApiMocks( - options: { - authenticated?: boolean - storageProvider?: 's3' | 'blob' | 'local' - cloudEnabled?: boolean - } = {} -) { - const { authenticated = true, storageProvider = 's3', cloudEnabled = true } = options - - vi.stubGlobal('crypto', { - randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'), - }) - - if (authenticated) { - authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'test-user-id' } }) - } else { - authMockFns.mockGetSession.mockResolvedValue(null) - } - - hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValue({ - success: authenticated, - userId: authenticated ? 'test-user-id' : undefined, - error: authenticated ? undefined : 'Unauthorized', - }) - - mocks.mockVerifyFileAccess.mockResolvedValue(true) - mocks.mockVerifyWorkspaceFileAccess.mockResolvedValue(true) - mocks.mockVerifyKBFileAccess.mockResolvedValue(true) - mocks.mockVerifyCopilotFileAccess.mockResolvedValue(true) - - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('admin') - - mocks.mockUploadWorkspaceFile.mockResolvedValue({ - id: 'test-file-id', - name: 'test.txt', - url: '/api/files/serve/workspace/test-workspace-id/test-file.txt', - size: 100, - type: 'text/plain', - key: 'workspace/test-workspace-id/1234567890-test.txt', - uploadedAt: new Date().toISOString(), - expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(), - }) - - mocks.mockUploadExecutionFile.mockResolvedValue({ - id: 'test-execution-file-id', - name: 'test.txt', - url: '/api/files/serve/execution/test-workspace-id/test-file.txt', - size: 100, - type: 'text/plain', - key: 'execution/test-workspace-id/1234567890-test.txt', - uploadedAt: new Date().toISOString(), - expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(), - }) - - mocks.mockGetStorageProvider.mockReturnValue(storageProvider) - mocks.mockIsUsingCloudStorage.mockReturnValue(cloudEnabled) - mocks.mockUploadFile.mockResolvedValue({ - path: '/api/files/serve/test-key.txt', - key: 'test-key.txt', - name: 'test.txt', - size: 100, - type: 'text/plain', - }) - - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(cloudEnabled) - storageServiceMockFns.mockUploadFile.mockResolvedValue({ - key: 'test-key', - path: '/test/path', - }) - - mocks.mockCheckStorageQuota.mockResolvedValue({ - allowed: true, - currentUsage: 0, - limit: Number.MAX_SAFE_INTEGER, - }) -} - -describe('File Upload API Route', () => { - const createMockFormData = (files: File[], context = 'workspace'): FormData => { - const formData = new FormData() - formData.append('context', context) - formData.append('workspaceId', 'test-workspace-id') - files.forEach((file) => { - formData.append('file', file) - }) - return formData - } - - const createMockFile = ( - name = 'test.txt', - type = 'text/plain', - content = 'test content' - ): File => { - return new File([content], name, { type }) - } - - const createUploadRequest = (formData: FormData): NextRequest => - new NextRequest('http://localhost:3000/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - beforeEach(() => { - vi.clearAllMocks() - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - it('should upload a file to local storage', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - }) - - const mockFile = createMockFile() - const formData = createMockFormData([mockFile]) - - const req = createUploadRequest(formData) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data).toHaveProperty('url') - expect(data.url).toMatch(/\/api\/files\/serve\/.*\.txt$/) - expect(data).toHaveProperty('name', 'test.txt') - expect(data).toHaveProperty('size') - expect(data).toHaveProperty('type', 'text/plain') - expect(data).toHaveProperty('key') - - expect(uploadWorkspaceFile).toHaveBeenCalled() - }) - - it('should accept chunked multipart uploads without a content-length header', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - }) - - const formData = createMockFormData([createMockFile()]) - const req = new NextRequest('http://localhost:3000/api/files/upload', { - method: 'POST', - body: formData, - }) - - expect(req.headers.get('content-length')).toBeNull() - - const response = await POST(req) - - expect(response.status).toBe(200) - expect(uploadWorkspaceFile).toHaveBeenCalled() - }) - - it('should upload a file to S3 when in S3 mode', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - const mockFile = createMockFile() - const formData = createMockFormData([mockFile]) - - const req = createUploadRequest(formData) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(200) - expect(data).toHaveProperty('url') - expect(data.url).toContain('/api/files/serve/') - expect(data).toHaveProperty('name', 'test.txt') - expect(data).toHaveProperty('size') - expect(data).toHaveProperty('type', 'text/plain') - expect(data).toHaveProperty('key') - - expect(uploadWorkspaceFile).toHaveBeenCalled() - }) - - it('uploads a direct mothership attachment without workspace storage accounting', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - - const response = await POST( - createUploadRequest(createMockFormData([createMockFile('attachment.txt')], 'mothership')) - ) - - expect(response.status).toBe(200) - expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalledWith( - expect.objectContaining({ context: 'mothership' }) - ) - expect(mocks.mockCheckStorageQuotaForBillingContext).not.toHaveBeenCalled() - expect(mocks.mockResolveStorageBillingContext).not.toHaveBeenCalled() - expect(mocks.mockIncrementStorageUsageForBillingContext).not.toHaveBeenCalled() - expect(mocks.mockDecrementStorageUsageForBillingContext).not.toHaveBeenCalled() - }) - - it('does not mutate storage counters when a direct mothership upload fails', async () => { - setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' }) - storageServiceMockFns.mockUploadFile.mockRejectedValueOnce(new Error('storage unavailable')) - - const response = await POST( - createUploadRequest(createMockFormData([createMockFile('attachment.txt')], 'mothership')) - ) - - expect(response.status).toBe(500) - expect(mocks.mockCheckStorageQuotaForBillingContext).not.toHaveBeenCalled() - expect(mocks.mockResolveStorageBillingContext).not.toHaveBeenCalled() - expect(mocks.mockIncrementStorageUsageForBillingContext).not.toHaveBeenCalled() - expect(mocks.mockDecrementStorageUsageForBillingContext).not.toHaveBeenCalled() - }) - - it('should handle multiple file uploads', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - }) - - const mockFile1 = createMockFile('file1.txt', 'text/plain') - const mockFile2 = createMockFile('file2.txt', 'text/plain') - const formData = createMockFormData([mockFile1, mockFile2]) - - const req = createUploadRequest(formData) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBeGreaterThanOrEqual(200) - expect(response.status).toBeLessThan(600) - expect(data).toBeDefined() - }) - - it('rejects oversized workspace uploads before materializing file contents', async () => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - }) - - const mockFile = createMockFile('large.txt', 'text/plain', 'x'.repeat(1025)) - const arrayBufferSpy = vi.spyOn(mockFile, 'arrayBuffer') - const formData = { - getAll: (name: string) => (name === 'file' ? [mockFile] : []), - get: (name: string) => { - if (name === 'context') return 'workspace' - if (name === 'workspaceId') return 'test-workspace-id' - return null - }, - } as unknown as FormData - - const req = { - formData: async () => formData, - } as unknown as NextRequest - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(413) - expect(data.error).toBe('PayloadSizeLimitError') - expect(data.message).toContain('File exceeds the server upload limit') - expect(data.message).toContain('Use direct upload for larger workspace files') - expect(arrayBufferSpy).not.toHaveBeenCalled() - expect(uploadWorkspaceFile).not.toHaveBeenCalled() - }) - - it('should handle missing files', async () => { - setupFileApiMocks() - - const formData = new FormData() - - const req = createUploadRequest(formData) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(400) - expect(data).toHaveProperty('error', 'InvalidRequestError') - expect(data).toHaveProperty('message', 'No files provided') - }) - - it('should handle S3 upload errors', async () => { - setupFileApiMocks({ - cloudEnabled: true, - storageProvider: 's3', - }) - - mocks.mockUploadWorkspaceFile.mockRejectedValue(new Error('Storage limit exceeded')) - - const mockFile = createMockFile() - const formData = createMockFormData([mockFile]) - - const req = createUploadRequest(formData) - - const response = await POST(req) - const data = await response.json() - - expect(response.status).toBe(413) - expect(data).toHaveProperty('error') - expect(typeof data.error).toBe('string') - }) -}) - -describe('File Upload Security Tests', () => { - beforeEach(() => { - vi.clearAllMocks() - - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'test-user-id' }, - }) - - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(false) - storageServiceMockFns.mockUploadFile.mockResolvedValue({ - key: 'test-key', - path: '/test/path', - }) - mocks.mockIsUsingCloudStorage.mockReturnValue(false) - }) - - afterEach(() => { - vi.clearAllMocks() - }) - - describe('File Extension Validation', () => { - beforeEach(() => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - }) - }) - - it('should accept allowed file types', async () => { - const allowedTypes = [ - 'pdf', - 'doc', - 'docx', - 'txt', - 'md', - 'png', - 'jpg', - 'jpeg', - 'gif', - 'csv', - 'xlsx', - 'xls', - ] - - for (const ext of allowedTypes) { - const formData = new FormData() - const file = new File(['test content'], `test.${ext}`, { type: 'application/octet-stream' }) - formData.append('file', file) - formData.append('context', 'workspace') - formData.append('workspaceId', 'test-workspace-id') - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(200) - } - }) - - it('should accept HTML files (supported document type)', async () => { - const formData = new FormData() - const htmlContent = '

Hello World

' - const file = new File([htmlContent], 'document.html', { type: 'text/html' }) - formData.append('file', file) - formData.append('context', 'workspace') - formData.append('workspaceId', 'test-workspace-id') - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(200) - }) - - it('should accept SVG files (supported image type)', async () => { - const formData = new FormData() - const svgContent = - '' - const file = new File([svgContent], 'image.svg', { type: 'image/svg+xml' }) - formData.append('file', file) - formData.append('context', 'workspace') - formData.append('workspaceId', 'test-workspace-id') - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(200) - }) - - it('should reject unsupported file types', async () => { - const formData = new FormData() - const content = 'binary data' - const file = new File([content], 'archive.exe', { type: 'application/octet-stream' }) - formData.append('file', file) - formData.append('context', 'workspace') - formData.append('workspaceId', 'test-workspace-id') - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(400) - const data = await response.json() - expect(data.message).toContain("File type 'exe' is not allowed") - }) - - it('should reject files without extensions', async () => { - const formData = new FormData() - const file = new File(['test content'], 'noextension', { type: 'application/octet-stream' }) - formData.append('file', file) - formData.append('context', 'workspace') - formData.append('workspaceId', 'test-workspace-id') - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(400) - const data = await response.json() - expect(data.message).toContain("File type 'noextension' is not allowed") - }) - - it('should handle multiple files with mixed valid/invalid types', async () => { - const formData = new FormData() - - const validFile = new File(['valid content'], 'valid.pdf', { type: 'application/pdf' }) - formData.append('file', validFile) - - const invalidFile = new File(['binary content'], 'malicious.exe', { - type: 'application/x-msdownload', - }) - formData.append('file', invalidFile) - formData.append('context', 'workspace') - formData.append('workspaceId', 'test-workspace-id') - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(400) - const data = await response.json() - expect(data.message).toContain("File type 'exe' is not allowed") - }) - }) - - describe('Execution Context Permission Gate', () => { - const createExecutionFormData = ( - file: File, - workspaceId: string | null = 'test-workspace-id' - ) => { - const formData = new FormData() - formData.append('file', file) - formData.append('context', 'execution') - formData.append('workflowId', 'test-workflow-id') - formData.append('executionId', 'test-execution-id') - if (workspaceId !== null) formData.append('workspaceId', workspaceId) - return formData - } - - const postExecutionUpload = async (workspaceId: string | null = 'test-workspace-id') => { - const file = new File(['test content'], 'test.pdf', { type: 'application/pdf' }) - const formData = createExecutionFormData(file, workspaceId) - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - return POST(req as unknown as NextRequest) - } - - beforeEach(() => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - }) - }) - - it('rejects execution uploads without workspaceId', async () => { - const response = await postExecutionUpload(null) - - expect(response.status).toBe(400) - const data = await response.json() - expect(data.message).toContain('workflowId, executionId, and workspaceId') - expect(mocks.mockUploadExecutionFile).not.toHaveBeenCalled() - }) - - it('rejects execution uploads for a read-only workspace member', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('read') - - const response = await postExecutionUpload() - - expect(response.status).toBe(403) - const data = await response.json() - expect(data.error).toBe('Write or Admin access required for execution uploads') - expect(mocks.mockUploadExecutionFile).not.toHaveBeenCalled() - }) - - it('rejects execution uploads for a member with no workspace permission', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue(null) - - const response = await postExecutionUpload() - - expect(response.status).toBe(403) - expect(mocks.mockUploadExecutionFile).not.toHaveBeenCalled() - }) - - it('allows execution uploads for a write-permission workspace member', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - - const response = await postExecutionUpload() - - expect(response.status).toBe(200) - expect(mocks.mockUploadExecutionFile).toHaveBeenCalledWith( - { - workspaceId: 'test-workspace-id', - workflowId: 'test-workflow-id', - executionId: 'test-execution-id', - }, - expect.anything(), - 'test.pdf', - 'application/pdf', - 'test-user-id' - ) - }) - - it('allows execution uploads for an admin-permission workspace member', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('admin') - - const response = await postExecutionUpload() - - expect(response.status).toBe(200) - expect(mocks.mockUploadExecutionFile).toHaveBeenCalled() - }) - }) - - describe('Mothership Context Permission Gate', () => { - const postMothershipUpload = async (workspaceId: string | null = 'test-workspace-id') => { - const formData = new FormData() - const file = new File(['test content'], 'test.pdf', { type: 'application/pdf' }) - formData.append('file', file) - formData.append('context', 'mothership') - if (workspaceId !== null) formData.append('workspaceId', workspaceId) - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - return POST(req as unknown as NextRequest) - } - - beforeEach(() => { - setupFileApiMocks({ - cloudEnabled: false, - storageProvider: 'local', - }) - }) - - it('rejects mothership uploads without workspaceId', async () => { - const response = await postMothershipUpload(null) - - expect(response.status).toBe(400) - const data = await response.json() - expect(data.message).toContain('workspaceId') - expect(storageServiceMockFns.mockUploadFile).not.toHaveBeenCalled() - }) - - it('rejects mothership uploads for a workspace the caller does not belong to', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue(null) - - const response = await postMothershipUpload() - - expect(response.status).toBe(403) - const data = await response.json() - expect(data.error).toBe('Write or Admin access required for mothership uploads') - expect(storageServiceMockFns.mockUploadFile).not.toHaveBeenCalled() - }) - - it('rejects mothership uploads for a read-only workspace member', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('read') - - const response = await postMothershipUpload() - - expect(response.status).toBe(403) - expect(storageServiceMockFns.mockUploadFile).not.toHaveBeenCalled() - }) - - it('rejects mothership uploads over the caller storage quota', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - mocks.mockCheckStorageQuota.mockResolvedValue({ - allowed: false, - currentUsage: 100, - limit: 100, - error: 'Storage limit exceeded. Used: 0.00GB, Limit: 0GB', - }) - - const response = await postMothershipUpload() - - expect(response.status).toBe(413) - const data = await response.json() - expect(data.error).toContain('Storage limit exceeded') - expect(storageServiceMockFns.mockUploadFile).not.toHaveBeenCalled() - }) - - it('allows mothership uploads for a write-permission workspace member', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - - const response = await postMothershipUpload() - - expect(response.status).toBe(200) - expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledWith( - 'test-user-id', - 'workspace', - 'test-workspace-id' - ) - expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalled() - }) - - it('allows mothership uploads for an admin-permission workspace member', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('admin') - - const response = await postMothershipUpload() - - expect(response.status).toBe(200) - expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalled() - }) - - it('checks quota once against the combined size of a multi-file batch', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - - const formData = new FormData() - const fileA = new File(['a'.repeat(10)], 'a.pdf', { type: 'application/pdf' }) - const fileB = new File(['b'.repeat(20)], 'b.pdf', { type: 'application/pdf' }) - formData.append('file', fileA) - formData.append('file', fileB) - formData.append('context', 'mothership') - formData.append('workspaceId', 'test-workspace-id') - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(200) - expect(mocks.mockCheckStorageQuota).toHaveBeenCalledTimes(1) - expect(mocks.mockCheckStorageQuota).toHaveBeenCalledWith('test-user-id', 30) - expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledTimes(1) - }) - }) - - describe('Authentication Requirements', () => { - it('should reject uploads without authentication', async () => { - authMockFns.mockGetSession.mockResolvedValue(null) - - const formData = new FormData() - const file = new File(['test content'], 'test.pdf', { type: 'application/pdf' }) - formData.append('file', file) - - const req = new Request('http://localhost/api/files/upload', { - method: 'POST', - headers: { 'content-length': '1024' }, - body: formData, - }) - - const response = await POST(req as unknown as NextRequest) - - expect(response.status).toBe(401) - const data = await response.json() - expect(data.error).toBe('Unauthorized') - }) - }) -}) diff --git a/apps/sim/app/api/files/upload/route.ts b/apps/sim/app/api/files/upload/route.ts deleted file mode 100644 index 1c013d19c98..00000000000 --- a/apps/sim/app/api/files/upload/route.ts +++ /dev/null @@ -1,488 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { sanitizeFileName } from '@/executor/constants' -import '@/lib/uploads/core/setup.server' -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { - uploadFilesFormFieldsSchema, - uploadFilesFormFilesSchema, -} from '@/lib/api/contracts/storage-transfer' -import { getValidationErrorMessage } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { - assertKnownSizeWithinLimit, - isPayloadSizeLimitError, - MAX_MULTIPART_OVERHEAD_BYTES, - readFileToBufferWithLimit, - readFormDataWithLimit, -} from '@/lib/core/utils/stream-limits' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { captureServerEvent } from '@/lib/posthog/server' -import type { StorageContext } from '@/lib/uploads/config' -import { generateKnowledgeBaseFileKey } from '@/lib/uploads/contexts/knowledge-base/knowledge-base-file-manager' -import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { MAX_WORKSPACE_FORMDATA_FILE_SIZE } from '@/lib/uploads/shared/types' -import { isArchiveFileName, isImageFileType, resolveFileType } from '@/lib/uploads/utils/file-utils' -import { - SUPPORTED_ATTACHMENT_EXTENSIONS, - SUPPORTED_IMAGE_EXTENSIONS, - validateFileType, -} from '@/lib/uploads/utils/validation' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' -import { createErrorResponse, InvalidRequestError } from '@/app/api/files/utils' - -const ALLOWED_EXTENSIONS = new Set(SUPPORTED_ATTACHMENT_EXTENSIONS) - -function validateFileExtension(filename: string, context: StorageContext): boolean { - const extension = filename.split('.').pop()?.toLowerCase() - if (!extension) return false - // Archives are only extractable in the mothership copilot flow; every other - // context keeps rejecting them up front instead of failing downstream. - if (context === 'mothership' && isArchiveFileName(filename)) return true - return ALLOWED_EXTENSIONS.has(extension) -} - -export const dynamic = 'force-dynamic' - -const logger = createLogger('FilesUploadAPI') - -export const POST = withRouteHandler(async (request: NextRequest) => { - try { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const formData = await readFormDataWithLimit(request, { - maxBytes: MAX_WORKSPACE_FORMDATA_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES, - label: 'multipart upload body', - }) - - const rawFiles = formData.getAll('file') - const filesResult = uploadFilesFormFilesSchema.safeParse(rawFiles) - if (!filesResult.success) { - throw new InvalidRequestError('No files provided') - } - const files = filesResult.data - const totalFileSize = files.reduce((total, file) => total + file.size, 0) - assertKnownSizeWithinLimit(totalFileSize, MAX_WORKSPACE_FORMDATA_FILE_SIZE, 'uploaded files') - - const formFieldsResult = uploadFilesFormFieldsSchema.safeParse({ - workflowId: formData.get('workflowId'), - executionId: formData.get('executionId'), - workspaceId: formData.get('workspaceId'), - context: formData.get('context'), - }) - if (!formFieldsResult.success) { - throw new InvalidRequestError( - getValidationErrorMessage(formFieldsResult.error, 'Invalid upload form data') - ) - } - const formFields = formFieldsResult.data - const { workflowId, executionId, workspaceId, context: contextParam } = formFields - - // Context must be explicitly provided - if (!contextParam) { - throw new InvalidRequestError( - 'Upload requires explicit context parameter (knowledge-base, workspace, execution, copilot, chat, profile-pictures, or workspace-logos)' - ) - } - - const context = contextParam as StorageContext - - const storageService = await import('@/lib/uploads/core/storage-service') - const usingCloudStorage = storageService.hasCloudStorage() - logger.info(`Using storage mode: ${usingCloudStorage ? 'Cloud' : 'Local'} for file upload`) - - // Execution context requires a workspace write/admin permission check. Resolve it once per - // request (not per file) since workspaceId is invariant across all files in the upload. - let executionUploadContext: - | { workspaceId: string; workflowId: string; executionId: string } - | undefined - if (context === 'execution') { - if (!workflowId || !executionId || !workspaceId) { - throw new InvalidRequestError( - 'Execution context requires workflowId, executionId, and workspaceId parameters' - ) - } - - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json( - { error: 'Write or Admin access required for execution uploads' }, - { status: 403 } - ) - } - - executionUploadContext = { workspaceId, workflowId, executionId } - } - - // Mothership context requires the same workspace write/admin permission check, plus a - // storage quota check. Resolve both once per request (not per file) since workspaceId is - // invariant across all files in the upload and quota must account for the full batch size, - // not just one file. - let mothershipWorkspaceId: string | undefined - if (context === 'mothership') { - if (!workspaceId) { - throw new InvalidRequestError('Mothership context requires workspaceId parameter') - } - - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json( - { error: 'Write or Admin access required for mothership uploads' }, - { status: 403 } - ) - } - - const { checkStorageQuota } = await import('@/lib/billing/storage') - const quotaCheck = await checkStorageQuota(session.user.id, totalFileSize) - if (!quotaCheck.allowed) { - return NextResponse.json( - { error: quotaCheck.error || 'Storage limit exceeded' }, - { status: 413 } - ) - } - - mothershipWorkspaceId = workspaceId - } - - const uploadResults = [] - - for (const file of files) { - const originalName = file.name || 'untitled.md' - - if (!validateFileExtension(originalName, context)) { - const extension = originalName.split('.').pop()?.toLowerCase() || 'unknown' - throw new InvalidRequestError( - `File type '${extension}' is not allowed. Allowed types: ${Array.from(ALLOWED_EXTENSIONS).join(', ')}` - ) - } - - const buffer = await readFileToBufferWithLimit(file, { - maxBytes: MAX_WORKSPACE_FORMDATA_FILE_SIZE, - label: 'uploaded file', - }) - - // Handle execution context - if (context === 'execution' && executionUploadContext) { - const { uploadExecutionFile } = await import('@/lib/uploads/contexts/execution') - const userFile = await uploadExecutionFile( - executionUploadContext, - buffer, - originalName, - file.type, - session.user.id - ) - - uploadResults.push(userFile) - continue - } - - // Handle knowledge-base context - if (context === 'knowledge-base') { - // Validate file type for knowledge base - const validationError = validateFileType(originalName, file.type) - if (validationError) { - throw new InvalidRequestError(validationError.message) - } - - if (!workspaceId) { - throw new InvalidRequestError('workspaceId is required for knowledge-base uploads') - } - - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission !== 'write' && permission !== 'admin') { - return NextResponse.json( - { error: 'Write or Admin access required for knowledge-base uploads' }, - { status: 403 } - ) - } - - logger.info(`Uploading knowledge-base file: ${originalName}`) - - const storageKey = generateKnowledgeBaseFileKey(originalName) - - const metadata: Record = { - originalName: originalName, - uploadedAt: new Date().toISOString(), - purpose: 'knowledge-base', - userId: session.user.id, - workspaceId, - } - - const fileInfo = await storageService.uploadFile({ - file: buffer, - fileName: storageKey, - contentType: file.type, - context: 'knowledge-base', - preserveKey: true, - customKey: storageKey, - metadata, - }) - - const finalPath = usingCloudStorage - ? `${fileInfo.path}?context=knowledge-base` - : fileInfo.path - - const uploadResult = { - fileName: originalName, - presignedUrl: '', // Not used for server-side uploads - fileInfo: { - path: finalPath, - key: fileInfo.key, - name: originalName, - size: buffer.length, - type: file.type, - }, - directUploadSupported: false, - } - - logger.info(`Successfully uploaded knowledge-base file: ${fileInfo.key}`) - uploadResults.push(uploadResult) - continue - } - - // Handle workspace context - if (context === 'workspace') { - if (!workspaceId) { - throw new InvalidRequestError('Workspace context requires workspaceId parameter') - } - const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) - if (permission !== 'admin' && permission !== 'write') { - return NextResponse.json( - { error: 'Write or Admin access required for workspace uploads' }, - { status: 403 } - ) - } - - try { - const { uploadWorkspaceFile } = await import('@/lib/uploads/contexts/workspace') - const userFile = await uploadWorkspaceFile( - workspaceId, - session.user.id, - buffer, - originalName, - file.type || 'application/octet-stream' - ) - - uploadResults.push(userFile) - continue - } catch (workspaceError) { - const errorMessage = getErrorMessage(workspaceError, 'Upload failed') - const isDuplicate = errorMessage.includes('already exists') - const isStorageLimitError = - errorMessage.includes('Storage limit exceeded') || - errorMessage.includes('storage limit') - - logger.warn(`Workspace file upload failed: ${errorMessage}`) - - let statusCode = 500 - if (isDuplicate) statusCode = 409 - else if (isStorageLimitError) statusCode = 413 - - return NextResponse.json( - { - success: false, - error: errorMessage, - isDuplicate, - }, - { status: statusCode } - ) - } - } - - // Handle mothership context (chat-scoped uploads to workspace S3) - if (context === 'mothership' && mothershipWorkspaceId) { - logger.info(`Uploading mothership file: ${originalName}`) - - const storageKey = generateWorkspaceFileKey(mothershipWorkspaceId, originalName) - - const metadata: Record = { - originalName: originalName, - uploadedAt: new Date().toISOString(), - purpose: 'mothership', - userId: session.user.id, - workspaceId: mothershipWorkspaceId, - } - - const fileInfo = await storageService.uploadFile({ - file: buffer, - fileName: storageKey, - contentType: file.type || 'application/octet-stream', - context: 'mothership', - preserveKey: true, - customKey: storageKey, - metadata, - }) - - const finalPath = usingCloudStorage ? `${fileInfo.path}?context=mothership` : fileInfo.path - - uploadResults.push({ - fileName: originalName, - presignedUrl: '', - fileInfo: { - path: finalPath, - key: fileInfo.key, - name: originalName, - size: buffer.length, - type: file.type || 'application/octet-stream', - }, - directUploadSupported: false, - }) - - logger.info(`Successfully uploaded mothership file: ${fileInfo.key}`) - continue - } - - if ( - context === 'copilot' || - context === 'chat' || - context === 'profile-pictures' || - context === 'workspace-logos' - ) { - if (context !== 'copilot') { - const mimeType = file.type - const isGenericMime = !mimeType || mimeType === 'application/octet-stream' - const extension = originalName.split('.').pop()?.toLowerCase() ?? '' - const extensionIsImage = (SUPPORTED_IMAGE_EXTENSIONS as readonly string[]).includes( - extension - ) - const isImage = isGenericMime ? extensionIsImage : isImageFileType(mimeType) - if (!isImage) { - throw new InvalidRequestError(`Only image files are allowed for ${context} uploads`) - } - } - - if (context === 'workspace-logos') { - if (!workspaceId) { - throw new InvalidRequestError('workspace-logos context requires workspaceId parameter') - } - const permission = await getUserEntityPermissions( - session.user.id, - 'workspace', - workspaceId - ) - if (permission !== 'admin') { - return NextResponse.json( - { error: 'Admin access required for workspace logo uploads' }, - { status: 403 } - ) - } - } - - if (context === 'chat' && workspaceId) { - const permission = await getUserEntityPermissions( - session.user.id, - 'workspace', - workspaceId - ) - if (permission === null) { - return NextResponse.json( - { error: 'Insufficient permissions for workspace' }, - { status: 403 } - ) - } - } - - logger.info(`Uploading ${context} file: ${originalName}`) - - const resolvedContentType = resolveFileType({ type: file.type, name: originalName }) - - const timestamp = Date.now() - const safeFileName = sanitizeFileName(originalName) - const storageKey = `${context}/${timestamp}-${safeFileName}` - - const metadata: Record = { - originalName: originalName, - uploadedAt: new Date().toISOString(), - purpose: context, - userId: session.user.id, - } - - if (workspaceId && context === 'chat') { - metadata.workspaceId = workspaceId - } - - const fileInfo = await storageService.uploadFile({ - file: buffer, - fileName: storageKey, - contentType: resolvedContentType, - context, - preserveKey: true, - customKey: storageKey, - metadata, - }) - - const finalPath = usingCloudStorage ? `${fileInfo.path}?context=${context}` : fileInfo.path - - const uploadResult = { - fileName: originalName, - presignedUrl: '', // Not used for server-side uploads - fileInfo: { - path: finalPath, - key: fileInfo.key, - name: originalName, - size: buffer.length, - type: resolvedContentType, - }, - directUploadSupported: false, - } - - logger.info(`Successfully uploaded ${context} file: ${fileInfo.key}`) - - if (context === 'workspace-logos' && workspaceId) { - recordAudit({ - workspaceId, - actorId: session.user.id, - actorName: session.user.name, - actorEmail: session.user.email, - action: AuditAction.FILE_UPLOADED, - resourceType: AuditResourceType.WORKSPACE, - resourceId: workspaceId, - description: `Uploaded workspace logo "${originalName}"`, - metadata: { - fileName: originalName, - fileKey: fileInfo.key, - fileSize: buffer.length, - fileType: resolvedContentType, - }, - request, - }) - - captureServerEvent(session.user.id, 'workspace_logo_uploaded', { - workspace_id: workspaceId, - file_name: originalName, - file_size: buffer.length, - }) - } - - uploadResults.push(uploadResult) - continue - } - - // Unknown context - throw new InvalidRequestError( - `Unsupported context: ${context}. Use knowledge-base, workspace, execution, copilot, chat, profile-pictures, or workspace-logos` - ) - } - - if (uploadResults.length === 1) { - return NextResponse.json(uploadResults[0]) - } - return NextResponse.json({ files: uploadResults }) - } catch (error) { - logger.error('Error in file upload:', error) - if (isPayloadSizeLimitError(error)) { - return NextResponse.json( - { - error: 'PayloadSizeLimitError', - message: `File exceeds the server upload limit of ${Math.round(error.maxBytes / (1024 * 1024))}MB. Use direct upload for larger workspace files.`, - }, - { status: 413 } - ) - } - return createErrorResponse(error instanceof Error ? error : new Error('File upload failed')) - } -}) diff --git a/apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts b/apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts index ce66561f854..8f8e75574f5 100644 --- a/apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts +++ b/apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts @@ -1,62 +1,41 @@ import { type NextRequest, NextResponse } from 'next/server' -import { completeWorkspaceFileUploadContract } from '@/lib/api/contracts/upload-sessions' +import { completeInternalFileUploadContract } from '@/lib/api/contracts/upload-sessions' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' -import { getWorkspaceFile, registerUploadedWorkspaceFile } from '@/lib/uploads/contexts/workspace' -import { - completeUploadSession, - getOwnedUploadSession, -} from '@/lib/uploads/multipart-session/service' +import { completeUploadSession, getOwnedUploadSession } from '@/lib/uploads/upload-session/service' +import { finalizeUploadPurpose } from '@/app/api/files/uploads/finalizers' +import { reauthorizeUploadPurpose } from '@/app/api/files/uploads/purposes' import { requireUploadUser, - requireWorkspaceWrite, + toInternalUploadSession, uploadSessionErrorResponse, } from '@/app/api/files/uploads/utils' -import { toV2FileUpload } from '@/app/api/v2/files/uploads/utils' interface UploadRouteParams { params: Promise<{ uploadId: string }> } export const POST = withRouteHandler(async (request: NextRequest, context: UploadRouteParams) => { - const user = await requireUploadUser() - if (user instanceof NextResponse) return user - const parsed = await parseRequest(completeWorkspaceFileUploadContract, request, context) + const actor = await requireUploadUser() + if (actor instanceof NextResponse) return actor + const parsed = await parseRequest(completeInternalFileUploadContract, request, context) if (!parsed.success) return parsed.response - const { workspaceId } = parsed.data.query - const access = await requireWorkspaceWrite(user, workspaceId) - if (access) return access + try { - const upload = getOwnedUploadSession({ + const session = getOwnedUploadSession({ uploadId: parsed.data.params.uploadId, - workspaceId, - userId: user, - purpose: 'workspace_file', uploadToken: parsed.data.headers['upload-token'], + userId: actor.id, }) - const metadata = upload.metadata as { folderId?: string | null } + await reauthorizeUploadPurpose(actor.id, session) const completed = await completeUploadSession({ - session: upload, - parts: parsed.data.body.parts, - finalize: async (claimed) => { - const registered = await registerUploadedWorkspaceFile({ - workspaceId, - userId: user, - key: claimed.storageKey, - originalName: claimed.fileName, - contentType: claimed.contentType, - folderId: metadata.folderId, - }) - return { value: registered.file.id, completedFileId: registered.file.id } - }, + session, + completion: parsed.data.body, + finalize: (claimed) => finalizeUploadPurpose({ session: claimed, actor, request }), + }) + return NextResponse.json({ + data: toInternalUploadSession(completed.session, completed.value), }) - const fileId = completed.value - if (!fileId) throw new Error('Completed upload is missing its workspace file id') - const file = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) - if (!file) throw new Error(`Completed workspace file ${fileId} not found`) - if (!completed.alreadyCompleted) await notifyWorkspaceFilesChanged(workspaceId) - return NextResponse.json({ data: toV2FileUpload(completed.session, file) }) } catch (error) { const classified = uploadSessionErrorResponse(error) if (classified) return classified diff --git a/apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts b/apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts index 80e0edd9e12..5c79744cf75 100644 --- a/apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts +++ b/apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts @@ -1,39 +1,30 @@ import { type NextRequest, NextResponse } from 'next/server' -import { createWorkspaceFileUploadPartUrlsContract } from '@/lib/api/contracts/upload-sessions' +import { createInternalFileUploadPartUrlsContract } from '@/lib/api/contracts/upload-sessions' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createUploadPartUrls, - getOwnedUploadSession, -} from '@/lib/uploads/multipart-session/service' -import { - requireUploadUser, - requireWorkspaceWrite, - uploadSessionErrorResponse, -} from '@/app/api/files/uploads/utils' +import { createUploadPartUrls, getOwnedUploadSession } from '@/lib/uploads/upload-session/service' +import { reauthorizeUploadPurpose } from '@/app/api/files/uploads/purposes' +import { requireUploadUser, uploadSessionErrorResponse } from '@/app/api/files/uploads/utils' interface UploadRouteParams { params: Promise<{ uploadId: string }> } export const POST = withRouteHandler(async (request: NextRequest, context: UploadRouteParams) => { - const user = await requireUploadUser() - if (user instanceof NextResponse) return user - const parsed = await parseRequest(createWorkspaceFileUploadPartUrlsContract, request, context) + const actor = await requireUploadUser() + if (actor instanceof NextResponse) return actor + const parsed = await parseRequest(createInternalFileUploadPartUrlsContract, request, context) if (!parsed.success) return parsed.response - const { workspaceId } = parsed.data.query - const access = await requireWorkspaceWrite(user, workspaceId) - if (access) return access + try { - const upload = getOwnedUploadSession({ + const session = getOwnedUploadSession({ uploadId: parsed.data.params.uploadId, - workspaceId, - userId: user, - purpose: 'workspace_file', uploadToken: parsed.data.headers['upload-token'], + userId: actor.id, }) + await reauthorizeUploadPurpose(actor.id, session) const parts = await createUploadPartUrls({ - session: upload, + session, partNumbers: parsed.data.body.partNumbers, localOrigin: request.nextUrl.origin, }) diff --git a/apps/sim/app/api/files/uploads/[uploadId]/route.ts b/apps/sim/app/api/files/uploads/[uploadId]/route.ts index f33f3bb3004..e1cef9cf378 100644 --- a/apps/sim/app/api/files/uploads/[uploadId]/route.ts +++ b/apps/sim/app/api/files/uploads/[uploadId]/route.ts @@ -1,36 +1,34 @@ import { type NextRequest, NextResponse } from 'next/server' -import { abortWorkspaceFileUploadContract } from '@/lib/api/contracts/upload-sessions' +import { abortInternalFileUploadContract } from '@/lib/api/contracts/upload-sessions' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { abortUploadSession, getOwnedUploadSession } from '@/lib/uploads/multipart-session/service' +import { abortUploadSession, getOwnedUploadSession } from '@/lib/uploads/upload-session/service' +import { reauthorizeUploadPurpose } from '@/app/api/files/uploads/purposes' import { requireUploadUser, - requireWorkspaceWrite, + toInternalUploadSession, uploadSessionErrorResponse, } from '@/app/api/files/uploads/utils' -import { toV2FileUpload } from '@/app/api/v2/files/uploads/utils' interface UploadRouteParams { params: Promise<{ uploadId: string }> } export const DELETE = withRouteHandler(async (request: NextRequest, context: UploadRouteParams) => { - const user = await requireUploadUser() - if (user instanceof NextResponse) return user - const parsed = await parseRequest(abortWorkspaceFileUploadContract, request, context) + const actor = await requireUploadUser() + if (actor instanceof NextResponse) return actor + const parsed = await parseRequest(abortInternalFileUploadContract, request, context) if (!parsed.success) return parsed.response - const { workspaceId } = parsed.data.query - const access = await requireWorkspaceWrite(user, workspaceId) - if (access) return access + try { - const upload = getOwnedUploadSession({ + const session = getOwnedUploadSession({ uploadId: parsed.data.params.uploadId, - workspaceId, - userId: user, - purpose: 'workspace_file', uploadToken: parsed.data.headers['upload-token'], + userId: actor.id, }) - return NextResponse.json({ data: toV2FileUpload(await abortUploadSession(upload), null) }) + await reauthorizeUploadPurpose(actor.id, session) + const aborted = await abortUploadSession(session) + return NextResponse.json({ data: toInternalUploadSession(aborted, null) }) } catch (error) { const classified = uploadSessionErrorResponse(error) if (classified) return classified diff --git a/apps/sim/app/api/files/uploads/finalizers.test.ts b/apps/sim/app/api/files/uploads/finalizers.test.ts new file mode 100644 index 00000000000..a257244e0ef --- /dev/null +++ b/apps/sim/app/api/files/uploads/finalizers.test.ts @@ -0,0 +1,238 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockInsertReturning, + mockSelectLimit, + mockRecordAudit, + mockCaptureServerEvent, + mockGetWorkspaceFile, + mockRegisterUploadedWorkspaceFile, + mockNotifyWorkspaceFilesChanged, +} = vi.hoisted(() => ({ + mockInsertReturning: vi.fn(), + mockSelectLimit: vi.fn(), + mockRecordAudit: vi.fn(), + mockCaptureServerEvent: vi.fn(), + mockGetWorkspaceFile: vi.fn(), + mockRegisterUploadedWorkspaceFile: vi.fn(), + mockNotifyWorkspaceFilesChanged: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { + insert: vi.fn(() => ({ + values: vi.fn(() => ({ + onConflictDoNothing: vi.fn(() => ({ returning: mockInsertReturning })), + })), + })), + select: vi.fn(() => ({ + from: vi.fn(() => ({ + where: vi.fn(() => ({ + orderBy: vi.fn(() => ({ limit: mockSelectLimit })), + })), + })), + })), + }, +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { FILE_UPLOADED: 'file.uploaded' }, + AuditResourceType: { WORKSPACE: 'workspace' }, + recordAudit: mockRecordAudit, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) +vi.mock('@/lib/uploads/config', () => ({ getServeStoragePrefix: () => 's3' })) +vi.mock('@/lib/uploads/upload-session/service', () => ({ + UploadSessionError: class UploadSessionError extends Error { + constructor( + readonly code: string, + message: string + ) { + super(message) + } + }, +})) +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + getWorkspaceFile: mockGetWorkspaceFile, + registerUploadedWorkspaceFile: mockRegisterUploadedWorkspaceFile, +})) +vi.mock('@/lib/realtime/notify', () => ({ + notifyWorkspaceFilesChanged: mockNotifyWorkspaceFilesChanged, +})) + +import { finalizeUploadPurpose } from '@/app/api/files/uploads/finalizers' + +const now = new Date('2026-08-04T12:00:00.000Z') +const actor = { id: 'user-1', name: 'Ada', email: 'ada@example.com' } +const metadataRow = { + id: 'file-1', + key: 'workspace-logos/upload-1-logo.png', + userId: actor.id, + workspaceId: 'workspace-1', + folderId: null, + context: 'workspace-logos', + chatId: null, + messageId: null, + originalName: 'logo.png', + displayName: 'logo.png', + contentType: 'image/png', + size: 128, + sizeBytes: 128, + deletedAt: null, + uploadedAt: now, + updatedAt: now, + contentUpdatedAt: now, +} +const uploadSession = { + id: 'upload-1', + workspaceId: 'workspace-1', + userId: actor.id, + knowledgeBaseId: null, + workflowId: null, + executionId: null, + purpose: 'workspace_logo' as const, + method: 'put' as const, + storageContext: 'workspace-logos' as const, + storageKey: metadataRow.key, + storageProvider: 's3' as const, + providerUploadId: null, + fileName: 'logo.png', + contentType: 'image/png', + fileSize: 128, + status: 'uploading' as const, + metadata: {}, + uploadToken: 'signed-token', + createdAt: now, + expiresAt: new Date('2026-08-05T12:00:00.000Z'), + completedFileId: null, + error: null, + completedAt: null, + updatedAt: now, +} +const workspaceFile = { + id: 'wf-1', + workspaceId: 'workspace-1', + name: 'report.csv', + key: 'workspace/workspace-1/upload-1-report.csv', + path: '/api/files/serve/s3/workspace%2Fworkspace-1%2Fupload-1-report.csv?context=workspace', + size: 128, + type: 'text/csv', + uploadedBy: actor.id, + folderId: null, + deletedAt: null, + uploadedAt: now, + updatedAt: now, +} + +describe('upload purpose finalizers', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('emits workspace-logo side effects only for the metadata insert winner', async () => { + mockSelectLimit.mockResolvedValueOnce([]).mockResolvedValueOnce([metadataRow]) + mockInsertReturning.mockResolvedValueOnce([metadataRow]) + const request = new NextRequest('http://localhost/api/files/uploads/upload-1/complete') + + const first = await finalizeUploadPurpose({ session: uploadSession, actor, request }) + const retry = await finalizeUploadPurpose({ session: uploadSession, actor, request }) + + expect(first.value).toEqual({ + path: `/api/files/serve/s3/${encodeURIComponent(metadataRow.key)}?context=workspace-logos`, + key: metadataRow.key, + name: 'logo.png', + size: 128, + type: 'image/png', + }) + expect(retry.value).toEqual(first.value) + expect(mockRecordAudit).toHaveBeenCalledTimes(1) + expect(mockCaptureServerEvent).toHaveBeenCalledTimes(1) + }) + + it('rejects a storage key already bound to a different owner', async () => { + mockSelectLimit.mockResolvedValueOnce([{ ...metadataRow, userId: 'other-user' }]) + + await expect( + finalizeUploadPurpose({ + session: uploadSession, + actor, + request: new NextRequest('http://localhost/api/files/uploads/upload-1/complete'), + }) + ).rejects.toMatchObject({ code: 'conflict' }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) + + it('rejects a replay after its metadata was archived', async () => { + mockSelectLimit.mockResolvedValueOnce([ + { ...metadataRow, deletedAt: new Date('2026-08-04T13:00:00.000Z') }, + ]) + + await expect( + finalizeUploadPurpose({ + session: uploadSession, + actor, + request: new NextRequest('http://localhost/api/files/uploads/upload-1/complete'), + }) + ).rejects.toMatchObject({ code: 'conflict' }) + expect(mockInsertReturning).not.toHaveBeenCalled() + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) + + it('emits workspace-file side effects only for the metadata insert winner', async () => { + const workspaceSession = { + ...uploadSession, + purpose: 'workspace_file' as const, + storageContext: 'workspace' as const, + storageKey: workspaceFile.key, + fileName: workspaceFile.name, + contentType: workspaceFile.type, + } + mockRegisterUploadedWorkspaceFile + .mockResolvedValueOnce({ file: { id: workspaceFile.id }, created: true }) + .mockResolvedValueOnce({ file: { id: workspaceFile.id }, created: false }) + mockGetWorkspaceFile.mockResolvedValue(workspaceFile) + const request = new NextRequest('http://localhost/api/files/uploads/upload-1/complete') + + const first = await finalizeUploadPurpose({ session: workspaceSession, actor, request }) + const retry = await finalizeUploadPurpose({ session: workspaceSession, actor, request }) + + expect(retry.value).toEqual(first.value) + expect(mockNotifyWorkspaceFilesChanged).toHaveBeenCalledTimes(1) + expect(mockRecordAudit).toHaveBeenCalledTimes(1) + expect(mockCaptureServerEvent).toHaveBeenCalledTimes(1) + }) + + it('rejects a workspace-file replay after its metadata was archived', async () => { + const workspaceSession = { + ...uploadSession, + purpose: 'workspace_file' as const, + storageContext: 'workspace' as const, + storageKey: workspaceFile.key, + fileName: workspaceFile.name, + contentType: workspaceFile.type, + } + mockRegisterUploadedWorkspaceFile.mockResolvedValueOnce({ + file: { id: workspaceFile.id }, + created: false, + }) + mockGetWorkspaceFile.mockResolvedValueOnce({ ...workspaceFile, deletedAt: now }) + + await expect( + finalizeUploadPurpose({ + session: workspaceSession, + actor, + request: new NextRequest('http://localhost/api/files/uploads/upload-1/complete'), + }) + ).rejects.toMatchObject({ code: 'conflict' }) + expect(mockNotifyWorkspaceFilesChanged).not.toHaveBeenCalled() + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/files/uploads/finalizers.ts b/apps/sim/app/api/files/uploads/finalizers.ts new file mode 100644 index 00000000000..6673b9a60d4 --- /dev/null +++ b/apps/sim/app/api/files/uploads/finalizers.ts @@ -0,0 +1,362 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { db } from '@sim/db' +import { workspaceFiles } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { eq, sql } from 'drizzle-orm' +import type { NextRequest } from 'next/server' +import type { V2File } from '@/lib/api/contracts/v2/files' +import { captureServerEvent } from '@/lib/posthog/server' +import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' +import { getServeStoragePrefix } from '@/lib/uploads/config' +import { + getWorkspaceFile, + registerUploadedWorkspaceFile, + type WorkspaceFileRecord, +} from '@/lib/uploads/contexts/workspace' +import { type StorageContext, toLegacyWorkspaceFileSize } from '@/lib/uploads/shared/types' +import { UploadSessionError, type UploadSessionRecord } from '@/lib/uploads/upload-session/service' +import { toV2File } from '@/app/api/v2/files/utils' + +export interface UploadActor { + id: string + name?: string | null + email?: string | null +} + +export interface StoredUploadResult { + path: string + key: string + name: string + size: number + type: string +} + +export interface ExecutionUploadResult { + id: string + name: string + url: string + size: number + type: string + key: string + context: 'execution' +} + +export type UploadPurposeResult = V2File | StoredUploadResult | ExecutionUploadResult + +interface FinalizedWorkspaceFile { + file: WorkspaceFileRecord + created: boolean +} + +interface FinalizeUploadPurposeParams { + session: UploadSessionRecord + actor: UploadActor + request: NextRequest +} + +interface FinalizedUploadPurpose { + value: UploadPurposeResult + completedFileId?: string +} + +interface FinalizedMetadataInput { + key: string + userId: string + workspaceId: string + context: StorageContext + originalName: string + contentType: string + size: number +} + +type FileMetadataRecord = typeof workspaceFiles.$inferSelect + +/** + * Finalizes the domain resource represented by a verified upload object. + * Metadata-backed purposes use the storage key as their idempotency identity. + */ +export async function finalizeUploadPurpose({ + session, + actor, + request, +}: FinalizeUploadPurposeParams): Promise { + switch (session.purpose) { + case 'workspace_file': + return finalizeInternalWorkspaceFile(session, actor, request) + case 'profile_picture': + return { value: storedAssetResult(session, 'profile-pictures') } + case 'workspace_logo': + return finalizeWorkspaceLogo(session, actor, request) + case 'mothership_attachment': + return finalizeMothershipAttachment(session) + case 'execution_attachment': + return finalizeExecutionAttachment(session) + case 'table_import': + case 'knowledge_document': + throw new UploadSessionError( + 'validation', + `Purpose ${session.purpose} is not finalized by the internal files route` + ) + } +} + +async function finalizeInternalWorkspaceFile( + session: UploadSessionRecord, + actor: UploadActor, + request: NextRequest +): Promise { + const finalized = await finalizeWorkspaceFileUpload({ session, actor, request, source: 'ui' }) + return { + value: toV2File(finalized.file), + completedFileId: finalized.file.id, + } +} + +/** + * Registers a verified workspace object and emits its one-time domain side effects. + * The metadata insert winner is the only caller that notifies, audits, or records analytics. + */ +export async function finalizeWorkspaceFileUpload(params: { + session: UploadSessionRecord + actor: UploadActor + request: NextRequest + source: 'api' | 'ui' +}): Promise { + const { session, actor, request, source } = params + const workspaceId = requireWorkspaceId(session) + const metadata = session.metadata as { folderId?: string | null } + const registered = await registerUploadedWorkspaceFile({ + workspaceId, + userId: session.userId, + key: session.storageKey, + originalName: session.fileName, + contentType: session.contentType, + folderId: metadata.folderId, + }) + const file = await getWorkspaceFile(workspaceId, registered.file.id, { + includeDeleted: true, + throwOnError: true, + }) + if (!file) { + throw new Error(`Completed workspace file ${registered.file.id} not found`) + } + if (file.deletedAt) { + throw new UploadSessionError('conflict', 'Upload result was deleted') + } + if (registered.created) { + await notifyWorkspaceFilesChanged(workspaceId) + captureServerEvent( + actor.id, + 'file_uploaded', + { workspace_id: workspaceId, file_type: session.contentType }, + { groups: { workspace: workspaceId } } + ) + recordAudit({ + workspaceId, + actorId: actor.id, + actorName: actor.name, + actorEmail: actor.email, + action: AuditAction.FILE_UPLOADED, + resourceType: AuditResourceType.FILE, + resourceId: file.id, + resourceName: file.name, + description: `Uploaded file "${file.name}"${source === 'api' ? ' via API' : ''}`, + metadata: { fileSize: file.size, fileType: file.type }, + request, + }) + } + return { file, created: registered.created } +} + +async function finalizeWorkspaceLogo( + session: UploadSessionRecord, + actor: UploadActor, + request: NextRequest +): Promise { + const workspaceId = requireWorkspaceId(session) + const finalized = await insertOrLoadFileMetadata({ + key: session.storageKey, + userId: session.userId, + workspaceId, + context: 'workspace-logos', + originalName: session.fileName, + contentType: session.contentType, + size: session.fileSize, + }) + + if (finalized.created) { + recordAudit({ + workspaceId, + actorId: actor.id, + actorName: actor.name, + actorEmail: actor.email, + action: AuditAction.FILE_UPLOADED, + resourceType: AuditResourceType.WORKSPACE, + resourceId: workspaceId, + description: `Uploaded workspace logo "${session.fileName}"`, + metadata: { + fileName: session.fileName, + fileKey: session.storageKey, + fileSize: session.fileSize, + fileType: session.contentType, + }, + request, + }) + captureServerEvent(actor.id, 'workspace_logo_uploaded', { + workspace_id: workspaceId, + file_name: session.fileName, + file_size: session.fileSize, + }) + } + + return { value: storedAssetResult(session, 'workspace-logos') } +} + +async function finalizeMothershipAttachment( + session: UploadSessionRecord +): Promise { + const workspaceId = requireWorkspaceId(session) + await insertOrLoadFileMetadata({ + key: session.storageKey, + userId: session.userId, + workspaceId, + context: 'mothership', + originalName: session.fileName, + contentType: session.contentType, + size: session.fileSize, + }) + return { + value: storedAssetResult(session, 'mothership'), + } +} + +async function finalizeExecutionAttachment( + session: UploadSessionRecord +): Promise { + const workspaceId = requireWorkspaceId(session) + const finalized = await insertOrLoadFileMetadata({ + key: session.storageKey, + userId: session.userId, + workspaceId, + context: 'execution', + originalName: session.fileName, + contentType: session.contentType, + size: session.fileSize, + }) + return { + value: { + id: finalized.file.id, + name: session.fileName, + url: servePath(session.storageKey, 'execution'), + size: session.fileSize, + type: session.contentType, + key: session.storageKey, + context: 'execution', + }, + completedFileId: finalized.file.id, + } +} + +async function insertOrLoadFileMetadata( + input: FinalizedMetadataInput +): Promise<{ file: FileMetadataRecord; created: boolean }> { + const existing = await findFileMetadataByKey(input.key) + if (existing) { + assertMatchingMetadata(existing, input) + assertActiveFileMetadata(existing) + return { file: existing, created: false } + } + + const now = new Date() + const [inserted] = await db + .insert(workspaceFiles) + .values({ + id: generateId(), + key: input.key, + userId: input.userId, + workspaceId: input.workspaceId, + context: input.context, + originalName: input.originalName, + displayName: input.originalName, + contentType: input.contentType, + size: toLegacyWorkspaceFileSize(input.size), + sizeBytes: input.size, + deletedAt: null, + uploadedAt: now, + updatedAt: now, + contentUpdatedAt: now, + }) + .onConflictDoNothing() + .returning() + + if (inserted) return { file: inserted, created: true } + + const raceWinner = await findFileMetadataByKey(input.key) + if (!raceWinner) { + throw new UploadSessionError('conflict', `Storage key ${input.key} could not be registered`) + } + assertMatchingMetadata(raceWinner, input) + assertActiveFileMetadata(raceWinner) + return { file: raceWinner, created: false } +} + +async function findFileMetadataByKey(key: string): Promise { + const [file] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.key, key)) + .orderBy(sql`${workspaceFiles.deletedAt} IS NULL DESC`) + .limit(1) + return file +} + +function assertMatchingMetadata(existing: FileMetadataRecord, input: FinalizedMetadataInput): void { + const existingSize = existing.sizeBytes ?? existing.size + if ( + existing.key !== input.key || + existing.userId !== input.userId || + existing.workspaceId !== input.workspaceId || + existing.context !== input.context || + existing.originalName !== input.originalName || + existing.contentType !== input.contentType || + existingSize !== input.size + ) { + throw new UploadSessionError( + 'conflict', + `Storage key ${input.key} belongs to a different upload` + ) + } +} + +function assertActiveFileMetadata(file: FileMetadataRecord): void { + if (file.deletedAt) { + throw new UploadSessionError('conflict', 'Upload result was deleted') + } +} + +function servePath(key: string, context: StorageContext): string { + return `/api/files/serve/${getServeStoragePrefix()}/${encodeURIComponent(key)}?context=${context}` +} + +function storedAssetResult( + session: UploadSessionRecord, + context: 'profile-pictures' | 'workspace-logos' | 'mothership' +): StoredUploadResult { + return { + path: servePath(session.storageKey, context), + key: session.storageKey, + name: session.fileName, + size: session.fileSize, + type: session.contentType, + } +} + +function requireWorkspaceId(session: UploadSessionRecord): string { + if (!session.workspaceId) { + throw new UploadSessionError( + 'forbidden', + `Upload session ${session.id} is missing its workspace scope` + ) + } + return session.workspaceId +} diff --git a/apps/sim/app/api/files/uploads/purposes.ts b/apps/sim/app/api/files/uploads/purposes.ts new file mode 100644 index 00000000000..a67d2e3298c --- /dev/null +++ b/apps/sim/app/api/files/uploads/purposes.ts @@ -0,0 +1,188 @@ +import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow' +import type { CreateInternalFileUploadBody } from '@/lib/api/contracts/upload-sessions' +import { assertWorkspaceFileFolderTarget } from '@/lib/uploads/contexts/workspace' +import { + createUploadSession, + UploadSessionError, + type UploadSessionRecord, +} from '@/lib/uploads/upload-session/service' +import { isImageFileType } from '@/lib/uploads/utils/file-utils' +import { validateAttachmentFileType } from '@/lib/uploads/utils/validation' +import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' + +export type InternalUploadPurpose = CreateInternalFileUploadBody['purpose'] + +const INTERNAL_UPLOAD_PURPOSES = new Set([ + 'workspace_file', + 'profile_picture', + 'workspace_logo', + 'mothership_attachment', + 'execution_attachment', +]) + +export async function createPurposeUploadSession( + userId: string, + body: CreateInternalFileUploadBody, + localOrigin: string +) { + validatePurposeFile(body) + + switch (body.purpose) { + case 'workspace_file': { + await requireWorkspacePermission(userId, body.workspaceId, 'write') + const folderId = await assertWorkspaceFileFolderTarget(body.workspaceId, body.folderId) + return createUploadSession({ + purpose: body.purpose, + workspaceId: body.workspaceId, + userId, + fileName: body.name, + contentType: body.contentType, + fileSize: body.size, + metadata: { folderId }, + localOrigin, + }) + } + case 'profile_picture': + return createUploadSession({ + purpose: body.purpose, + userId, + fileName: body.name, + contentType: body.contentType, + fileSize: body.size, + localOrigin, + }) + case 'workspace_logo': + await requireWorkspacePermission(userId, body.workspaceId, 'admin') + return createUploadSession({ + purpose: body.purpose, + workspaceId: body.workspaceId, + userId, + fileName: body.name, + contentType: body.contentType, + fileSize: body.size, + localOrigin, + }) + case 'mothership_attachment': + await requireWorkspacePermission(userId, body.workspaceId, 'write') + return createUploadSession({ + purpose: body.purpose, + workspaceId: body.workspaceId, + userId, + fileName: body.name, + contentType: body.contentType, + fileSize: body.size, + localOrigin, + }) + case 'execution_attachment': + await requireExecutionPermission(userId, body.workflowId, body.workspaceId) + return createUploadSession({ + purpose: body.purpose, + workspaceId: body.workspaceId, + workflowId: body.workflowId, + executionId: body.executionId, + userId, + fileName: body.name, + contentType: body.contentType, + fileSize: body.size, + localOrigin, + }) + } +} + +/** + * Rechecks current domain authorization for every control-plane session request. + */ +export async function reauthorizeUploadPurpose( + userId: string, + session: UploadSessionRecord +): Promise { + if (session.userId !== userId || !isInternalUploadPurpose(session.purpose)) { + throw new UploadSessionError('not_found', 'Upload session not found') + } + + switch (session.purpose) { + case 'workspace_file': + case 'mothership_attachment': + await requireWorkspacePermission(userId, requireSessionScope(session.workspaceId), 'write') + return + case 'profile_picture': + return + case 'workspace_logo': + await requireWorkspacePermission(userId, requireSessionScope(session.workspaceId), 'admin') + return + case 'execution_attachment': + await requireExecutionPermission( + userId, + requireSessionScope(session.workflowId), + requireSessionScope(session.workspaceId) + ) + return + } +} + +export function isInternalUploadPurpose(purpose: string): purpose is InternalUploadPurpose { + return INTERNAL_UPLOAD_PURPOSES.has(purpose as InternalUploadPurpose) +} + +function validatePurposeFile(body: CreateInternalFileUploadBody): void { + if (body.purpose === 'profile_picture' || body.purpose === 'workspace_logo') { + if (!isImageFileType(body.contentType)) { + throw new UploadSessionError( + 'validation', + `Only image files are allowed for ${body.purpose.replace('_', ' ')} uploads` + ) + } + return + } + + if (body.purpose === 'mothership_attachment' || body.purpose === 'execution_attachment') { + const validation = validateAttachmentFileType(body.name, { + allowArchives: body.purpose === 'mothership_attachment', + }) + if (validation) throw new UploadSessionError('validation', validation.message) + } +} + +async function requireWorkspacePermission( + userId: string, + workspaceId: string, + action: 'write' | 'admin' +): Promise { + const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) + const allowed = + action === 'admin' ? permission === 'admin' : permission === 'write' || permission === 'admin' + if (!allowed) { + throw new UploadSessionError( + 'forbidden', + action === 'admin' ? 'Admin access required' : 'Write or Admin access required' + ) + } +} + +async function requireExecutionPermission( + userId: string, + workflowId: string, + signedWorkspaceId: string +): Promise { + const authorization = await authorizeWorkflowByWorkspacePermission({ + workflowId, + userId, + action: 'write', + }) + if (!authorization.workflow) { + throw new UploadSessionError('not_found', 'Workflow not found') + } + if (!authorization.allowed) { + throw new UploadSessionError('forbidden', authorization.message ?? 'Workflow access denied') + } + if (authorization.workflow.workspaceId !== signedWorkspaceId) { + throw new UploadSessionError('forbidden', 'Workflow does not belong to the upload workspace') + } +} + +function requireSessionScope(value: string | null, label = 'scope'): string { + if (!value) { + throw new UploadSessionError('forbidden', `Upload session is missing its ${label}`) + } + return value +} diff --git a/apps/sim/app/api/files/uploads/route.test.ts b/apps/sim/app/api/files/uploads/route.test.ts new file mode 100644 index 00000000000..1d6d3e0576f --- /dev/null +++ b/apps/sim/app/api/files/uploads/route.test.ts @@ -0,0 +1,307 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockGetSession, + mockCreateUploadSession, + mockGetOwnedUploadSession, + mockCompleteUploadSession, + mockGetUserEntityPermissions, + mockAuthorizeWorkflow, +} = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockCreateUploadSession: vi.fn(), + mockGetOwnedUploadSession: vi.fn(), + mockCompleteUploadSession: vi.fn(), + mockGetUserEntityPermissions: vi.fn(), + mockAuthorizeWorkflow: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ getSession: mockGetSession })) + +vi.mock('@/lib/uploads/upload-session/service', () => ({ + UploadSessionError: class UploadSessionError extends Error { + constructor( + readonly code: string, + message: string + ) { + super(message) + } + }, + createUploadSession: mockCreateUploadSession, + getOwnedUploadSession: mockGetOwnedUploadSession, + completeUploadSession: mockCompleteUploadSession, + createUploadPartUrls: vi.fn(), + abortUploadSession: vi.fn(), +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + getUserEntityPermissions: mockGetUserEntityPermissions, +})) + +vi.mock('@sim/platform-authz/workflow', () => ({ + authorizeWorkflowByWorkspacePermission: mockAuthorizeWorkflow, +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + assertWorkspaceFileFolderTarget: vi.fn(), + getWorkspaceFile: vi.fn(), + registerUploadedWorkspaceFile: vi.fn(), +})) + +vi.mock('@/lib/realtime/notify', () => ({ notifyWorkspaceFilesChanged: vi.fn() })) + +import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' +import { POST as completeUpload } from '@/app/api/files/uploads/[uploadId]/complete/route' +import { POST as createUpload } from '@/app/api/files/uploads/route' + +const actor = { id: 'user-1', name: 'Ada', email: 'ada@example.com' } +const now = new Date('2026-08-04T12:00:00.000Z') + +function session(overrides: Record = {}) { + return { + id: 'upload-1', + workspaceId: null, + userId: actor.id, + knowledgeBaseId: null, + workflowId: null, + executionId: null, + purpose: 'profile_picture', + method: 'put', + storageContext: 'profile-pictures', + storageKey: 'profile-pictures/upload-1-avatar.png', + storageProvider: 's3', + providerUploadId: null, + fileName: 'avatar.png', + contentType: 'image/png', + fileSize: 128, + status: 'uploading', + metadata: {}, + uploadToken: 'signed-token', + createdAt: now, + expiresAt: new Date('2026-08-05T12:00:00.000Z'), + completedFileId: null, + error: null, + completedAt: null, + updatedAt: now, + ...overrides, + } +} + +describe('/api/files/uploads', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: actor }) + mockGetUserEntityPermissions.mockResolvedValue('admin') + }) + + it('creates a purpose-scoped PUT session without exposing write capability in the session', async () => { + mockCreateUploadSession.mockResolvedValue({ + ...session(), + transfer: { + method: 'put', + url: 'https://storage.example.com/upload', + headers: { 'Content-Type': 'image/png' }, + }, + }) + const request = new NextRequest('http://localhost/api/files/uploads', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + purpose: 'profile_picture', + name: 'avatar.png', + contentType: 'image/png', + size: 128, + }), + }) + + const response = await createUpload(request) + const body = await response.json() + + expect(response.status).toBe(201) + expect(mockCreateUploadSession).toHaveBeenCalledWith( + expect.objectContaining({ + purpose: 'profile_picture', + userId: actor.id, + localOrigin: 'http://localhost', + }) + ) + expect(body.data).toMatchObject({ + session: { + id: 'upload-1', + purpose: 'profile_picture', + status: 'uploading', + result: null, + }, + uploadToken: 'signed-token', + transfer: { method: 'put' }, + }) + expect(body.data.session).not.toHaveProperty('uploadToken') + expect(body.data.session).not.toHaveProperty('transfer') + }) + + it('creates a PUT session for an empty workspace file', async () => { + mockCreateUploadSession.mockResolvedValue({ + ...session({ + workspaceId: 'workspace-1', + purpose: 'workspace_file', + storageContext: 'workspace', + storageKey: 'workspace/workspace-1/empty.md', + fileName: 'empty.md', + contentType: 'text/markdown', + fileSize: 0, + }), + transfer: { + method: 'put', + url: 'https://storage.example.com/upload', + headers: { 'Content-Type': 'text/markdown' }, + }, + }) + const request = new NextRequest('http://localhost/api/files/uploads', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + purpose: 'workspace_file', + workspaceId: 'workspace-1', + name: 'empty.md', + contentType: 'text/markdown', + size: 0, + }), + }) + + const response = await createUpload(request) + + expect(response.status).toBe(201) + expect(mockCreateUploadSession).toHaveBeenCalledWith( + expect.objectContaining({ purpose: 'workspace_file', fileSize: 0 }) + ) + await expect(response.json()).resolves.toMatchObject({ + data: { session: { purpose: 'workspace_file', size: 0 } }, + }) + }) + + it('preserves the 5 GiB direct-to-storage limit for mothership attachments', async () => { + mockCreateUploadSession.mockResolvedValue({ + ...session({ + workspaceId: 'workspace-1', + purpose: 'mothership_attachment', + method: 'multipart', + storageContext: 'mothership', + storageKey: 'mothership/workspace-1/archive.zip', + fileName: 'archive.zip', + contentType: 'application/zip', + fileSize: MAX_WORKSPACE_FILE_SIZE, + }), + transfer: { method: 'multipart', partSize: 8 * 1024 * 1024, partCount: 640 }, + }) + const request = new NextRequest('http://localhost/api/files/uploads', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + purpose: 'mothership_attachment', + workspaceId: 'workspace-1', + name: 'archive.zip', + contentType: 'application/zip', + size: MAX_WORKSPACE_FILE_SIZE, + }), + }) + + const response = await createUpload(request) + + expect(response.status).toBe(201) + expect(mockCreateUploadSession).toHaveBeenCalledWith( + expect.objectContaining({ + purpose: 'mothership_attachment', + fileSize: MAX_WORKSPACE_FILE_SIZE, + }) + ) + }) + + it('rejects mothership attachments above the 5 GiB direct-to-storage limit', async () => { + const request = new NextRequest('http://localhost/api/files/uploads', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + purpose: 'mothership_attachment', + workspaceId: 'workspace-1', + name: 'archive.zip', + contentType: 'application/zip', + size: MAX_WORKSPACE_FILE_SIZE + 1, + }), + }) + + const response = await createUpload(request) + + expect(response.status).toBe(400) + expect(mockCreateUploadSession).not.toHaveBeenCalled() + }) + + it('reauthorizes a terminal request and returns only the terminal-safe session', async () => { + const logoSession = session({ + workspaceId: 'workspace-1', + purpose: 'workspace_logo', + storageContext: 'workspace-logos', + storageKey: 'workspace-logos/upload-1-logo.png', + fileName: 'logo.png', + }) + const result = { + path: '/api/files/serve/s3/workspace-logos%2Fupload-1-logo.png?context=workspace-logos', + key: 'workspace-logos/upload-1-logo.png', + name: 'logo.png', + size: 128, + type: 'image/png', + } + mockGetOwnedUploadSession.mockReturnValue(logoSession) + mockCompleteUploadSession.mockResolvedValue({ + session: { ...logoSession, status: 'completed', completedAt: now }, + value: result, + alreadyCompleted: false, + }) + const request = new NextRequest('http://localhost/api/files/uploads/upload-1/complete', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'upload-token': 'signed-token', + }, + body: '{}', + }) + + const response = await completeUpload(request, { + params: Promise.resolve({ uploadId: 'upload-1' }), + }) + const body = await response.json() + + expect(response.status).toBe(200) + expect(mockGetUserEntityPermissions).toHaveBeenCalledWith(actor.id, 'workspace', 'workspace-1') + expect(mockCompleteUploadSession).toHaveBeenCalledWith( + expect.objectContaining({ session: logoSession, completion: {} }) + ) + expect(body).toEqual({ + data: expect.objectContaining({ + id: 'upload-1', + purpose: 'workspace_logo', + status: 'completed', + result, + }), + }) + expect(body.data).not.toHaveProperty('uploadToken') + expect(body.data).not.toHaveProperty('transfer') + }) + + it('authenticates before parsing the request body', async () => { + mockGetSession.mockResolvedValue(null) + const request = new NextRequest('http://localhost/api/files/uploads', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{not json', + }) + + const response = await createUpload(request) + + expect(response.status).toBe(401) + expect(mockCreateUploadSession).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/files/uploads/route.ts b/apps/sim/app/api/files/uploads/route.ts index 25385b370f9..85752ea6e82 100644 --- a/apps/sim/app/api/files/uploads/route.ts +++ b/apps/sim/app/api/files/uploads/route.ts @@ -1,36 +1,36 @@ import { type NextRequest, NextResponse } from 'next/server' -import { createWorkspaceFileUploadContract } from '@/lib/api/contracts/upload-sessions' +import { createInternalFileUploadContract } from '@/lib/api/contracts/upload-sessions' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { assertWorkspaceFileFolderTarget } from '@/lib/uploads/contexts/workspace' -import { createUploadSession } from '@/lib/uploads/multipart-session/service' +import { createPurposeUploadSession } from '@/app/api/files/uploads/purposes' import { requireUploadUser, - requireWorkspaceWrite, + toInternalUploadSession, uploadSessionErrorResponse, } from '@/app/api/files/uploads/utils' -import { toV2FileUpload } from '@/app/api/v2/files/uploads/utils' export const POST = withRouteHandler(async (request: NextRequest) => { - const user = await requireUploadUser() - if (user instanceof NextResponse) return user - const parsed = await parseRequest(createWorkspaceFileUploadContract, request, {}) + const actor = await requireUploadUser() + if (actor instanceof NextResponse) return actor + const parsed = await parseRequest(createInternalFileUploadContract, request, {}) if (!parsed.success) return parsed.response - const { workspaceId, name, contentType, size, folderId } = parsed.data.body - const access = await requireWorkspaceWrite(user, workspaceId) - if (access) return access + try { - const normalizedFolderId = await assertWorkspaceFileFolderTarget(workspaceId, folderId) - const upload = await createUploadSession({ - workspaceId, - userId: user, - purpose: 'workspace_file', - fileName: name, - contentType, - fileSize: size, - metadata: { folderId: normalizedFolderId }, - }) - return NextResponse.json({ data: toV2FileUpload(upload, null) }, { status: 201 }) + const created = await createPurposeUploadSession( + actor.id, + parsed.data.body, + request.nextUrl.origin + ) + return NextResponse.json( + { + data: { + session: toInternalUploadSession(created, null), + uploadToken: created.uploadToken, + transfer: created.transfer, + }, + }, + { status: 201 } + ) } catch (error) { const classified = uploadSessionErrorResponse(error) if (classified) return classified diff --git a/apps/sim/app/api/files/uploads/utils.ts b/apps/sim/app/api/files/uploads/utils.ts index b4530be82ca..ca8e9356cb3 100644 --- a/apps/sim/app/api/files/uploads/utils.ts +++ b/apps/sim/app/api/files/uploads/utils.ts @@ -1,21 +1,23 @@ import { NextResponse } from 'next/server' +import { + type InternalFileUploadSession, + internalFileUploadSessionSchema, +} from '@/lib/api/contracts/upload-sessions' import { getSession } from '@/lib/auth' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' +import type { UploadSessionRecord } from '@/lib/uploads/upload-session/service' +import type { UploadActor, UploadPurposeResult } from '@/app/api/files/uploads/finalizers' -export async function requireUploadUser(): Promise { +export async function requireUploadUser(): Promise { const session = await getSession() - return session?.user?.id ?? NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) -} - -export async function requireWorkspaceWrite( - userId: string, - workspaceId: string -): Promise { - const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) - return permission === 'write' || permission === 'admin' - ? null - : NextResponse.json({ error: 'Forbidden' }, { status: 403 }) + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + return { + id: session.user.id, + name: session.user.name, + email: session.user.email, + } } export function uploadSessionErrorResponse(error: unknown): NextResponse | null { @@ -27,3 +29,20 @@ export function uploadSessionErrorResponse(error: unknown): NextResponse | null ) : null } + +export function toInternalUploadSession( + session: UploadSessionRecord, + result: UploadPurposeResult | null +): InternalFileUploadSession { + return internalFileUploadSessionSchema.parse({ + id: session.id, + purpose: session.purpose, + status: session.status, + name: session.fileName, + contentType: session.contentType, + size: session.fileSize, + expiresAt: session.expiresAt.toISOString(), + error: session.error, + result, + }) +} diff --git a/apps/sim/app/api/help/route.ts b/apps/sim/app/api/help/route.ts index 3c10f68f4d4..88de0594622 100644 --- a/apps/sim/app/api/help/route.ts +++ b/apps/sim/app/api/help/route.ts @@ -20,7 +20,7 @@ const logger = createLogger('HelpAPI') /** * The form can carry several image attachments with no server-side count * cap, so this reuses the repo's largest existing per-request form-data - * bound (see files/upload route) rather than an arbitrary smaller limit + * multipart bound rather than an arbitrary smaller limit * that could reject a legitimate multi-image submission. */ const MAX_HELP_FORM_BYTES = MAX_WORKSPACE_FORMDATA_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts index 8d0cf3e8bf6..7426a000fbd 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts @@ -3,7 +3,7 @@ import { completeKnowledgeDocumentUploadContract } from '@/lib/api/contracts/kno import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { completeUploadSession } from '@/lib/uploads/multipart-session/service' +import { completeUploadSession } from '@/lib/uploads/upload-session/service' import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils' import { requireKnowledgeDocumentUploadAccess, @@ -45,7 +45,7 @@ export const POST = withRouteHandler( }) const completed = await completeUploadSession({ session: upload, - parts: parsed.data.body.parts, + completion: parsed.data.body, finalize: (claimed) => finalizeKnowledgeDocumentUpload({ claimed, diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts index 482705e5b62..38390d4c13e 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts @@ -2,7 +2,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { createKnowledgeDocumentUploadPartUrlsContract } from '@/lib/api/contracts/knowledge/upload-sessions' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createUploadPartUrls } from '@/lib/uploads/multipart-session/service' +import { createUploadPartUrls } from '@/lib/uploads/upload-session/service' import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils' import { requireKnowledgeDocumentUploadAccess, diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts index df28c35218f..ea79d0f4dc9 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts @@ -71,10 +71,13 @@ describe('POST /api/knowledge/[id]/documents/uploads', () => { fileName: 'guide.pdf', contentType: 'application/pdf', fileSize: 1024, - partSize: 8 * 1024 * 1024, - partCount: 1, uploadToken: 'token', error: null, + transfer: { + method: 'put', + url: 'https://storage.example/upload', + headers: { 'content-type': 'application/pdf' }, + }, }) }) @@ -98,6 +101,12 @@ describe('POST /api/knowledge/[id]/documents/uploads', () => { tag1: 'product', processingOptions: { recipe: 'default', lang: 'en' }, }, + localOrigin: 'http://localhost:3000', + }) + expect((await response.json()).data).toMatchObject({ + session: { id: 'upload-1', status: 'uploading', document: null }, + uploadToken: 'token', + transfer: { method: 'put', url: 'https://storage.example/upload' }, }) expect(mockRequireKnowledgeDocumentUploadBilling.mock.invocationCallOrder[0]).toBeLessThan( mockCreateKnowledgeDocumentUploadSession.mock.invocationCallOrder[0] diff --git a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts b/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts index 09b7aec9e59..58a1c69c253 100644 --- a/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts +++ b/apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts @@ -50,8 +50,18 @@ export const POST = withRouteHandler( contentType, fileSize: size, metadata, + localOrigin: request.nextUrl.origin, }) - return NextResponse.json({ data: toV2KnowledgeDocumentUpload(upload, null) }, { status: 201 }) + return NextResponse.json( + { + data: { + session: toV2KnowledgeDocumentUpload(upload, null), + uploadToken: upload.uploadToken, + transfer: upload.transfer, + }, + }, + { status: 201 } + ) } catch (error) { const classified = uploadSessionErrorResponse(error) if (classified) return classified diff --git a/apps/sim/app/api/table/imports/[importId]/complete/route.ts b/apps/sim/app/api/table/imports/[importId]/complete/route.ts index 4ca1b13974c..54b93e41b19 100644 --- a/apps/sim/app/api/table/imports/[importId]/complete/route.ts +++ b/apps/sim/app/api/table/imports/[importId]/complete/route.ts @@ -9,7 +9,10 @@ import { startUploadedTableImport, toV2TableImport, } from '@/lib/table/orchestration/import-resource' -import { completeUploadSession } from '@/lib/uploads/multipart-session/service' +import { + completeUploadSession, + validateUploadCompletion, +} from '@/lib/uploads/upload-session/service' import { orchestrationErrorResponse } from '@/app/api/table/utils' interface ImportRouteParams { @@ -30,15 +33,16 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Impor userId: auth.userId, uploadToken: parsed.data.headers['upload-token'], }) + validateUploadCompletion(upload, parsed.data.body) const existing = await findOwnedTableImport({ importId: upload.id, - workspaceId: upload.workspaceId, + workspaceId: parsed.data.query.workspaceId, userId: upload.userId, }) if (existing) return NextResponse.json({ data: toV2TableImport(existing) }) const completed = await completeUploadSession({ session: upload, - parts: parsed.data.body.parts, + completion: parsed.data.body, finalize: async () => ({ value: null }), }) return NextResponse.json({ diff --git a/apps/sim/app/api/table/imports/[importId]/parts/route.ts b/apps/sim/app/api/table/imports/[importId]/parts/route.ts index f3534b65958..130fe570a29 100644 --- a/apps/sim/app/api/table/imports/[importId]/parts/route.ts +++ b/apps/sim/app/api/table/imports/[importId]/parts/route.ts @@ -4,7 +4,7 @@ import { parseRequest } from '@/lib/api/server' import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getOwnedTableImportUpload } from '@/lib/table/orchestration/import-resource' -import { createUploadPartUrls } from '@/lib/uploads/multipart-session/service' +import { createUploadPartUrls } from '@/lib/uploads/upload-session/service' import { orchestrationErrorResponse } from '@/app/api/table/utils' interface ImportRouteParams { diff --git a/apps/sim/app/api/table/imports/route.ts b/apps/sim/app/api/table/imports/route.ts index 254420388fe..d7e42288f09 100644 --- a/apps/sim/app/api/table/imports/route.ts +++ b/apps/sim/app/api/table/imports/route.ts @@ -5,7 +5,7 @@ import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { createTableImportResource, - toV2TableImport, + toV2CreateTableImport, } from '@/lib/table/orchestration/import-resource' import { orchestrationErrorResponse } from '@/app/api/table/utils' @@ -17,8 +17,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const parsed = await parseRequest(createTableImportResourceContract, request, {}) if (!parsed.success) return parsed.response try { - const created = await createTableImportResource(parsed.data.body, auth.userId) - return NextResponse.json({ data: await toV2TableImport(created.record) }, { status: 201 }) + const created = await createTableImportResource( + parsed.data.body, + auth.userId, + request.nextUrl.origin + ) + return NextResponse.json({ data: toV2CreateTableImport(created) }, { status: 201 }) } catch (error) { const classified = orchestrationErrorResponse(error) if (classified) return classified diff --git a/apps/sim/app/api/v2/credentials/route.test.ts b/apps/sim/app/api/v2/credentials/route.test.ts index 2bfe1cbfad4..3833eb52c84 100644 --- a/apps/sim/app/api/v2/credentials/route.test.ts +++ b/apps/sim/app/api/v2/credentials/route.test.ts @@ -363,4 +363,22 @@ describe('POST /api/v2/credentials', () => { }) ) }) + + it('accepts and forwards an optional service-account data center', async () => { + const res = await callCreate({ + workspaceId: WORKSPACE_ID, + type: 'service_account', + providerId: 'zoho-desk-service-account', + clientId: 'zoho-client-id', + clientSecret: 'zoho-client-secret', + orgId: '600123456', + dataCenter: 'eu', + }) + + expect(res.status).toBe(201) + expect(mockPerformCreateCredential).toHaveBeenCalledWith( + expect.objectContaining({ dataCenter: 'eu' }) + ) + expect(JSON.stringify(await res.json())).not.toContain('dataCenter') + }) }) diff --git a/apps/sim/app/api/v2/files/[fileId]/content/route.test.ts b/apps/sim/app/api/v2/files/[fileId]/content/route.test.ts index b2fd918c924..860002a33e5 100644 --- a/apps/sim/app/api/v2/files/[fileId]/content/route.test.ts +++ b/apps/sim/app/api/v2/files/[fileId]/content/route.test.ts @@ -22,6 +22,7 @@ vi.mock('@/app/api/v2/lib/gate', () => ({ })) vi.mock('@/lib/workspace-files/orchestration', () => ({ + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES: 70 * 1024 * 1024, performUpdateWorkspaceFileContent: mockPerformUpdateContent, })) @@ -62,12 +63,15 @@ const RECORD = { updatedAt: new Date('2024-01-03T00:00:00Z'), } -const callPut = (body: unknown) => +const callPut = (body: unknown, contentLength?: number) => PUT( new NextRequest(`http://localhost:3000/api/v2/files/${FILE_ID}/content`, { method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), + headers: { + 'Content-Type': 'application/json', + ...(contentLength === undefined ? {} : { 'Content-Length': String(contentLength) }), + }, + body: typeof body === 'string' ? body : JSON.stringify(body), }), { params: Promise.resolve({ fileId: FILE_ID }) } ) @@ -104,6 +108,45 @@ describe('PUT /api/v2/files/[fileId]/content', () => { expect(mockPerformUpdateContent).not.toHaveBeenCalled() }) + it('400s malformed base64 in the v2 error envelope', async () => { + const res = await callPut({ workspaceId: WS, content: 'not-base64!', encoding: 'base64' }) + const body = await res.json() + + expect(res.status).toBe(400) + expect(body.error.code).toBe('BAD_REQUEST') + expect(body.error.message).toBe('content must be valid base64') + expect(mockPerformUpdateContent).not.toHaveBeenCalled() + }) + + it('accepts empty base64 as a zero-byte replacement', async () => { + const res = await callPut({ workspaceId: WS, content: '', encoding: 'base64' }) + + expect(res.status).toBe(200) + expect(mockPerformUpdateContent).toHaveBeenCalledWith( + expect.objectContaining({ content: '', encoding: 'base64' }) + ) + }) + + it('allows JSON bodies above the default 50 MiB cap for base64 expansion', async () => { + const res = await callPut( + { workspaceId: WS, content: 'TQ==', encoding: 'base64' }, + 60 * 1024 * 1024 + ) + + expect(res.status).toBe(200) + expect(mockPerformUpdateContent).toHaveBeenCalled() + }) + + it('returns an oversized JSON body in the canonical v2 413 envelope', async () => { + const res = await callPut({ workspaceId: WS, content: '' }, 70 * 1024 * 1024 + 1) + + expect(res.status).toBe(413) + await expect(res.json()).resolves.toEqual({ + error: { code: 'PAYLOAD_TOO_LARGE', message: 'Request body is too large' }, + }) + expect(mockPerformUpdateContent).not.toHaveBeenCalled() + }) + it('surfaces an access-denied failure in the v2 error envelope', async () => { mockResolveWorkspaceAccess.mockResolvedValue({ status: 403, diff --git a/apps/sim/app/api/v2/files/[fileId]/content/route.ts b/apps/sim/app/api/v2/files/[fileId]/content/route.ts index fb310ec2e94..bb03695aa5f 100644 --- a/apps/sim/app/api/v2/files/[fileId]/content/route.ts +++ b/apps/sim/app/api/v2/files/[fileId]/content/route.ts @@ -5,7 +5,10 @@ import { v2UpdateFileContentContract } from '@/lib/api/contracts/v2/files' import { parseRequest } from '@/lib/api/server' import { messageForOrchestrationError } from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performUpdateWorkspaceFileContent } from '@/lib/workspace-files/orchestration' +import { + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + performUpdateWorkspaceFileContent, +} from '@/lib/workspace-files/orchestration' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { toV2File } from '@/app/api/v2/files/utils' import { v2ApiGateError } from '@/app/api/v2/lib/gate' @@ -46,9 +49,15 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: FileRo if (gate) return gate const parsed = await parseRequest(v2UpdateFileContentContract, request, context, { + invalidJsonResponse: () => v2Error('BAD_REQUEST', 'Request body must be valid JSON'), + maxBodyBytes: MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, validationErrorResponse: v2ValidationError, }) - if (!parsed.success) return parsed.response + if (!parsed.success) { + return parsed.response.status === 413 + ? v2Error('PAYLOAD_TOO_LARGE', 'Request body is too large') + : parsed.response + } const { fileId } = parsed.data.params const { workspaceId, content, encoding } = parsed.data.body diff --git a/apps/sim/app/api/v2/files/route.test.ts b/apps/sim/app/api/v2/files/route.test.ts index ac6c3dca5bd..31d0eea1b59 100644 --- a/apps/sim/app/api/v2/files/route.test.ts +++ b/apps/sim/app/api/v2/files/route.test.ts @@ -6,13 +6,19 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCheckRateLimit, mockResolveWorkspaceAccess, mockQueryWorkspaceFiles } = vi.hoisted( - () => ({ - mockCheckRateLimit: vi.fn(), - mockResolveWorkspaceAccess: vi.fn(), - mockQueryWorkspaceFiles: vi.fn(), - }) -) +const { + mockCheckRateLimit, + mockPerformCreateWorkspaceFile, + mockQueryWorkspaceFiles, + mockResolveWorkspaceAccess, + mockV2ApiGateError, +} = vi.hoisted(() => ({ + mockCheckRateLimit: vi.fn(), + mockPerformCreateWorkspaceFile: vi.fn(), + mockResolveWorkspaceAccess: vi.fn(), + mockQueryWorkspaceFiles: vi.fn(), + mockV2ApiGateError: vi.fn().mockResolvedValue(null), +})) vi.mock('@/app/api/v1/middleware', () => ({ checkRateLimit: mockCheckRateLimit, @@ -20,15 +26,21 @@ vi.mock('@/app/api/v1/middleware', () => ({ })) vi.mock('@/app/api/v2/lib/gate', () => ({ - v2ApiGateError: vi.fn().mockResolvedValue(null), + v2ApiGateError: mockV2ApiGateError, })) vi.mock('@/lib/uploads/contexts/workspace', () => ({ queryWorkspaceFiles: mockQueryWorkspaceFiles, })) +vi.mock('@/lib/workspace-files/orchestration', () => ({ + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES: 70 * 1024 * 1024, + performCreateWorkspaceFile: mockPerformCreateWorkspaceFile, +})) + import { OrchestrationError } from '@/lib/core/orchestration/types' -import { GET } from '@/app/api/v2/files/route' +import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration' +import { GET, POST } from '@/app/api/v2/files/route' const WS = 'workspace-1' const FOLDER_ID = 'fold_1' @@ -82,6 +94,14 @@ const DEFAULT_LIST_ARGS = { const callList = (query: string) => GET(new NextRequest(`http://localhost:3000/api/v2/files?${query}`)) +function createRequest(body: Record) { + return new NextRequest('http://localhost:3000/api/v2/files', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }) +} + describe('GET /api/v2/files', () => { beforeEach(() => { vi.clearAllMocks() @@ -274,3 +294,212 @@ describe('GET /api/v2/files', () => { expect((await res.json()).nextCursor).toBeNull() }) }) + +describe('POST /api/v2/files', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue(RATE_LIMIT_OK) + mockV2ApiGateError.mockResolvedValue(null) + mockResolveWorkspaceAccess.mockResolvedValue(null) + mockPerformCreateWorkspaceFile.mockResolvedValue({ + success: true, + file: buildRecord({ name: 'untitled.md', size: 0, type: 'text/markdown' }), + }) + }) + + it('creates an empty exact-name file with an inferred MIME type', async () => { + const request = createRequest({ workspaceId: WS, name: 'untitled.md' }) + + const response = await POST(request) + + expect(response.status).toBe(201) + await expect(response.json()).resolves.toMatchObject({ + data: { id: 'wf_1', name: 'untitled.md', size: 0, type: 'text/markdown' }, + }) + expect(mockResolveWorkspaceAccess).toHaveBeenCalledWith(RATE_LIMIT_OK, 'user-1', WS, 'write') + expect(mockPerformCreateWorkspaceFile).toHaveBeenCalledWith({ + workspaceId: WS, + userId: 'user-1', + name: 'untitled.md', + contentType: 'text/markdown', + folderId: undefined, + content: Buffer.alloc(0), + exactName: true, + request, + }) + }) + + it('decodes initialized base64 content before orchestration', async () => { + mockPerformCreateWorkspaceFile.mockResolvedValue({ + success: true, + file: buildRecord({ + name: 'seed.bin', + size: 3, + type: 'application/octet-stream', + folderId: FOLDER_ID, + folderPath: 'Fixtures', + }), + }) + const request = createRequest({ + workspaceId: WS, + name: 'seed.bin', + contentType: 'application/octet-stream', + folderId: FOLDER_ID, + content: Buffer.from([1, 2, 3]).toString('base64'), + encoding: 'base64', + }) + + const response = await POST(request) + + expect(response.status).toBe(201) + expect(mockPerformCreateWorkspaceFile).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: WS, + name: 'seed.bin', + contentType: 'application/octet-stream', + folderId: FOLDER_ID, + content: Buffer.from([1, 2, 3]), + exactName: true, + }) + ) + }) + + it('rejects malformed base64 before workspace access or orchestration', async () => { + const response = await POST( + createRequest({ + workspaceId: WS, + name: 'seed.bin', + content: 'not-base64!', + encoding: 'base64', + }) + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: { code: 'BAD_REQUEST' }, + }) + expect(mockResolveWorkspaceAccess).not.toHaveBeenCalled() + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) + + it('accepts empty base64 as a zero-byte file', async () => { + const response = await POST( + createRequest({ workspaceId: WS, name: 'empty.bin', content: '', encoding: 'base64' }) + ) + + expect(response.status).toBe(201) + expect(mockPerformCreateWorkspaceFile).toHaveBeenCalledWith( + expect.objectContaining({ content: Buffer.alloc(0) }) + ) + }) + + it('returns the canonical v2 envelope when the JSON body exceeds the inline limit', async () => { + const request = new NextRequest('http://localhost:3000/api/v2/files', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': String(MAX_WORKSPACE_FILE_INLINE_BODY_BYTES + 1), + }, + body: '{}', + }) + + const response = await POST(request) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toMatchObject({ + error: { code: 'PAYLOAD_TOO_LARGE', message: 'Request body is too large' }, + }) + expect(mockResolveWorkspaceAccess).not.toHaveBeenCalled() + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) + + it('returns the canonical v2 envelope for malformed JSON', async () => { + const request = new NextRequest('http://localhost:3000/api/v2/files', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{not-json', + }) + + const response = await POST(request) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: { code: 'BAD_REQUEST', message: 'Request body must be valid JSON' }, + }) + expect(mockResolveWorkspaceAccess).not.toHaveBeenCalled() + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) + + it.each([ + { + label: 'name conflict', + result: { + success: false, + error: 'A file with this name already exists', + errorCode: 'conflict', + }, + status: 409, + code: 'CONFLICT', + message: 'A file with this name already exists', + }, + { + label: 'internal orchestration failure', + result: { success: false, error: 'database connection details', errorCode: 'internal' }, + status: 500, + code: 'INTERNAL_ERROR', + message: 'Internal server error', + }, + ])('maps a $label into the v2 error envelope', async ({ result, status, code, message }) => { + mockPerformCreateWorkspaceFile.mockResolvedValue(result) + + const response = await POST(createRequest({ workspaceId: WS, name: 'untitled.md' })) + + expect(response.status).toBe(status) + await expect(response.json()).resolves.toMatchObject({ error: { code, message } }) + }) + + it('returns the auth failure before gating, access checks, or orchestration', async () => { + mockCheckRateLimit.mockResolvedValue({ + allowed: false, + error: 'Invalid API key', + limit: 100, + remaining: 0, + resetAt: RATE_LIMIT_OK.resetAt, + }) + + const response = await POST(createRequest({ workspaceId: WS, name: 'untitled.md' })) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toMatchObject({ + error: { code: 'UNAUTHORIZED', message: 'Invalid API key' }, + }) + expect(mockV2ApiGateError).not.toHaveBeenCalled() + expect(mockResolveWorkspaceAccess).not.toHaveBeenCalled() + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) + + it('returns the v2 gate failure before access checks or orchestration', async () => { + const { v2Error } = await import('@/app/api/v2/lib/response') + mockV2ApiGateError.mockResolvedValueOnce(v2Error('NOT_FOUND', 'Not found')) + + const response = await POST(createRequest({ workspaceId: WS, name: 'untitled.md' })) + + expect(response.status).toBe(404) + expect(mockResolveWorkspaceAccess).not.toHaveBeenCalled() + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) + + it('requires workspace write access before orchestration', async () => { + mockResolveWorkspaceAccess.mockResolvedValue({ + status: 403, + code: 'FORBIDDEN', + message: 'Access denied', + }) + + const response = await POST(createRequest({ workspaceId: WS, name: 'untitled.md' })) + + expect(response.status).toBe(403) + expect(mockResolveWorkspaceAccess).toHaveBeenCalledWith(RATE_LIMIT_OK, 'user-1', WS, 'write') + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/v2/files/route.ts b/apps/sim/app/api/v2/files/route.ts index e6fd3e6698c..cc2e0177832 100644 --- a/apps/sim/app/api/v2/files/route.ts +++ b/apps/sim/app/api/v2/files/route.ts @@ -1,10 +1,20 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { NextRequest } from 'next/server' -import { type V2File, v2ListFilesContract } from '@/lib/api/contracts/v2/files' +import { + type V2File, + v2CreateFileContract, + v2ListFilesContract, +} from '@/lib/api/contracts/v2/files' import { parseRequest } from '@/lib/api/server' +import { messageForOrchestrationError } from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { queryWorkspaceFiles } from '@/lib/uploads/contexts/workspace' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' +import { + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + performCreateWorkspaceFile, +} from '@/lib/workspace-files/orchestration' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { toV2File } from '@/app/api/v2/files/utils' import { v2ApiGateError } from '@/app/api/v2/lib/gate' @@ -15,7 +25,9 @@ import { v2CaughtOrchestrationError, v2CursorList, v2CursorSortError, + v2Data, v2Error, + v2ErrorForOrchestration, v2RateLimitError, v2ValidationError, v2WorkspaceAccessError, @@ -90,3 +102,57 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return v2Error('INTERNAL_ERROR', 'Internal server error') } }) + +/** POST /api/v2/files — Create an authored workspace file, optionally with initial content. */ +export const POST = withRouteHandler(async (request: NextRequest) => { + try { + const rateLimit = await checkRateLimit(request, 'files') + if (!rateLimit.allowed) return v2RateLimitError(rateLimit) + + const userId = rateLimit.userId! + const gate = await v2ApiGateError(userId) + if (gate) return gate + + const parsed = await parseRequest( + v2CreateFileContract, + request, + {}, + { + invalidJsonResponse: () => v2Error('BAD_REQUEST', 'Request body must be valid JSON'), + maxBodyBytes: MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + validationErrorResponse: v2ValidationError, + } + ) + if (!parsed.success) { + return parsed.response.status === 413 + ? v2Error('PAYLOAD_TOO_LARGE', 'Request body is too large') + : parsed.response + } + + const { workspaceId, name, contentType, folderId, content, encoding } = parsed.data.body + const access = await resolveWorkspaceAccess(rateLimit, userId, workspaceId, 'write') + if (access) return v2WorkspaceAccessError(access) + + const result = await performCreateWorkspaceFile({ + workspaceId, + userId, + name, + contentType: contentType ?? getMimeTypeFromExtension(getFileExtension(name)), + folderId, + content: Buffer.from(content, encoding), + exactName: true, + request, + }) + if (!result.success || !result.file) { + return v2ErrorForOrchestration( + result.errorCode, + messageForOrchestrationError(result, 'Failed to create file') + ) + } + + return v2Data(toV2File(result.file), { rateLimit, status: 201 }) + } catch (error) { + logger.error('Error creating file', { error: getErrorMessage(error, 'Unknown error') }) + return v2Error('INTERNAL_ERROR', 'Internal server error') + } +}) diff --git a/apps/sim/app/api/v2/files/uploads/[uploadId]/complete/route.ts b/apps/sim/app/api/v2/files/uploads/[uploadId]/complete/route.ts index e4504fd952a..9739994d62d 100644 --- a/apps/sim/app/api/v2/files/uploads/[uploadId]/complete/route.ts +++ b/apps/sim/app/api/v2/files/uploads/[uploadId]/complete/route.ts @@ -1,15 +1,11 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { NextRequest } from 'next/server' import { v2CompleteFileUploadContract } from '@/lib/api/contracts/v2/files' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getWorkspaceFile, registerUploadedWorkspaceFile } from '@/lib/uploads/contexts/workspace' -import { - completeUploadSession, - getOwnedUploadSession, -} from '@/lib/uploads/multipart-session/service' +import { completeUploadSession, getOwnedUploadSession } from '@/lib/uploads/upload-session/service' +import { finalizeWorkspaceFileUpload } from '@/app/api/files/uploads/finalizers' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { toV2FileUpload } from '@/app/api/v2/files/uploads/utils' import { v2ApiGateError } from '@/app/api/v2/lib/gate' @@ -51,41 +47,20 @@ export const POST = withRouteHandler( purpose: 'workspace_file', uploadToken: parsed.data.headers['upload-token'], }) - const metadata = session.metadata as { folderId?: string | null } const result = await completeUploadSession({ session, - parts: parsed.data.body.parts, + completion: parsed.data.body, finalize: async (claimed) => { - const registered = await registerUploadedWorkspaceFile({ - workspaceId, - userId, - key: claimed.storageKey, - originalName: claimed.fileName, - contentType: claimed.contentType, - folderId: metadata.folderId, + const finalized = await finalizeWorkspaceFileUpload({ + session: claimed, + actor: { id: userId }, + request, + source: 'api', }) - return { value: registered.file.id, completedFileId: registered.file.id } + return { value: finalized.file, completedFileId: finalized.file.id } }, }) - const fileId = result.value - if (!fileId) throw new Error('Completed upload is missing its workspace file id') - const file = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) - if (!file) throw new Error(`Completed workspace file ${fileId} not found`) - - if (!result.alreadyCompleted) { - recordAudit({ - workspaceId, - actorId: userId, - action: AuditAction.FILE_UPLOADED, - resourceType: AuditResourceType.FILE, - resourceId: file.id, - resourceName: file.name, - description: `Uploaded file "${file.name}" via API`, - metadata: { fileSize: file.size, fileType: file.type }, - request, - }) - } - return v2Data(toV2FileUpload(result.session, file), { rateLimit }) + return v2Data(toV2FileUpload(result.session, result.value), { rateLimit }) } catch (error) { const classified = v2CaughtOrchestrationError(error) if (classified) return classified diff --git a/apps/sim/app/api/v2/files/uploads/[uploadId]/parts/route.ts b/apps/sim/app/api/v2/files/uploads/[uploadId]/parts/route.ts index c3148698c7b..f27f759faa7 100644 --- a/apps/sim/app/api/v2/files/uploads/[uploadId]/parts/route.ts +++ b/apps/sim/app/api/v2/files/uploads/[uploadId]/parts/route.ts @@ -4,10 +4,7 @@ import type { NextRequest } from 'next/server' import { v2CreateFileUploadPartUrlsContract } from '@/lib/api/contracts/v2/files' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { - createUploadPartUrls, - getOwnedUploadSession, -} from '@/lib/uploads/multipart-session/service' +import { createUploadPartUrls, getOwnedUploadSession } from '@/lib/uploads/upload-session/service' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' import { diff --git a/apps/sim/app/api/v2/files/uploads/[uploadId]/route.ts b/apps/sim/app/api/v2/files/uploads/[uploadId]/route.ts index dd07baa1d72..9561a1abcfa 100644 --- a/apps/sim/app/api/v2/files/uploads/[uploadId]/route.ts +++ b/apps/sim/app/api/v2/files/uploads/[uploadId]/route.ts @@ -4,7 +4,7 @@ import type { NextRequest } from 'next/server' import { v2AbortFileUploadContract } from '@/lib/api/contracts/v2/files' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { abortUploadSession, getOwnedUploadSession } from '@/lib/uploads/multipart-session/service' +import { abortUploadSession, getOwnedUploadSession } from '@/lib/uploads/upload-session/service' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { toV2FileUpload } from '@/app/api/v2/files/uploads/utils' import { v2ApiGateError } from '@/app/api/v2/lib/gate' diff --git a/apps/sim/app/api/v2/files/uploads/route.test.ts b/apps/sim/app/api/v2/files/uploads/route.test.ts index b70ac9aba3d..8065f99830b 100644 --- a/apps/sim/app/api/v2/files/uploads/route.test.ts +++ b/apps/sim/app/api/v2/files/uploads/route.test.ts @@ -29,7 +29,7 @@ vi.mock('@/lib/uploads/contexts/workspace', () => ({ assertWorkspaceFileFolderTarget: mockAssertFolder, })) -vi.mock('@/lib/uploads/multipart-session/service', () => ({ +vi.mock('@/lib/uploads/upload-session/service', () => ({ createUploadSession: mockCreateUploadSession, })) @@ -66,16 +66,21 @@ describe('POST /api/v2/files/uploads', () => { workspaceId: WORKSPACE_ID, userId: 'user-1', knowledgeBaseId: null, + workflowId: null, + executionId: null, purpose: 'workspace_file', + method: 'put', storageContext: 'workspace', storageKey: `${WORKSPACE_ID}/file.csv`, + finalKey: `${WORKSPACE_ID}/file.csv`, + stagingKey: 'upload-sessions/upload-1/file.csv', storageProvider: 's3', - providerUploadId: 'provider-1', + providerUploadId: null, fileName: 'file.csv', contentType: 'text/csv', fileSize: 10, - partSize: 8 * 1024 * 1024, - partCount: 1, + partSize: null, + partCount: null, status: 'uploading', uploadToken: 'signed-upload-token', metadata: {}, @@ -85,10 +90,15 @@ describe('POST /api/v2/files/uploads', () => { createdAt: new Date('2026-08-03T21:00:00.000Z'), updatedAt: new Date('2026-08-03T21:00:00.000Z'), completedAt: null, + transfer: { + method: 'put', + url: 'https://storage.example/upload', + headers: { 'content-type': 'text/csv' }, + }, }) }) - it('creates one signed multipart session for a small file', async () => { + it('creates one signed PUT session for a small file', async () => { const response = await request({ workspaceId: WORKSPACE_ID, name: 'file.csv', @@ -97,13 +107,16 @@ describe('POST /api/v2/files/uploads', () => { }) expect(response.status).toBe(201) - expect((await response.json()).data).toMatchObject({ - id: 'upload-1', - status: 'uploading', - partCount: 1, + const { data } = await response.json() + expect(data).toMatchObject({ + session: { id: 'upload-1', status: 'uploading', file: null }, uploadToken: 'signed-upload-token', - file: null, + transfer: { method: 'put', url: 'https://storage.example/upload' }, }) + expect(data.session).not.toHaveProperty('uploadToken') + expect(data.session).not.toHaveProperty('transfer') + expect(data.session).not.toHaveProperty('partSize') + expect(data.session).not.toHaveProperty('partCount') expect(mockCreateUploadSession).toHaveBeenCalledWith({ workspaceId: WORKSPACE_ID, userId: 'user-1', @@ -112,6 +125,7 @@ describe('POST /api/v2/files/uploads', () => { contentType: 'text/csv', fileSize: 10, metadata: { folderId: null }, + localOrigin: 'http://localhost:3000', }) }) @@ -134,7 +148,7 @@ describe('POST /api/v2/files/uploads', () => { expect(mockCreateUploadSession).not.toHaveBeenCalled() }) - it('rejects an empty file before creating provider state', async () => { + it('creates an upload session for an empty workspace file', async () => { const response = await request({ workspaceId: WORKSPACE_ID, name: 'file.csv', @@ -142,8 +156,9 @@ describe('POST /api/v2/files/uploads', () => { size: 0, }) - expect(response.status).toBe(400) - expect(mockResolveWorkspaceAccess).not.toHaveBeenCalled() - expect(mockCreateUploadSession).not.toHaveBeenCalled() + expect(response.status).toBe(201) + expect(mockCreateUploadSession).toHaveBeenCalledWith( + expect.objectContaining({ purpose: 'workspace_file', fileSize: 0 }) + ) }) }) diff --git a/apps/sim/app/api/v2/files/uploads/route.ts b/apps/sim/app/api/v2/files/uploads/route.ts index 8ee6777176d..10cc3089ef2 100644 --- a/apps/sim/app/api/v2/files/uploads/route.ts +++ b/apps/sim/app/api/v2/files/uploads/route.ts @@ -5,7 +5,7 @@ import { v2CreateFileUploadContract } from '@/lib/api/contracts/v2/files' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { assertWorkspaceFileFolderTarget } from '@/lib/uploads/contexts/workspace' -import { createUploadSession } from '@/lib/uploads/multipart-session/service' +import { createUploadSession } from '@/lib/uploads/upload-session/service' import { checkRateLimit, resolveWorkspaceAccess } from '@/app/api/v1/middleware' import { toV2FileUpload } from '@/app/api/v2/files/uploads/utils' import { v2ApiGateError } from '@/app/api/v2/lib/gate' @@ -50,8 +50,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => { contentType, fileSize: size, metadata: { folderId: normalizedFolderId }, + localOrigin: request.nextUrl.origin, }) - return v2Data(toV2FileUpload(session, null), { rateLimit, status: 201 }) + return v2Data( + { + session: toV2FileUpload(session, null), + uploadToken: session.uploadToken, + transfer: session.transfer, + }, + { rateLimit, status: 201 } + ) } catch (error) { const classified = v2CaughtOrchestrationError(error) if (classified) return classified diff --git a/apps/sim/app/api/v2/files/uploads/utils.ts b/apps/sim/app/api/v2/files/uploads/utils.ts index c79baf22bb1..45cebe61bab 100644 --- a/apps/sim/app/api/v2/files/uploads/utils.ts +++ b/apps/sim/app/api/v2/files/uploads/utils.ts @@ -1,7 +1,7 @@ import type { V2FileUpload } from '@/lib/api/contracts/v2/files' import type { V2UploadStatus } from '@/lib/api/contracts/v2/uploads' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' -import type { UploadSessionRecord } from '@/lib/uploads/multipart-session/service' +import type { UploadSessionRecord } from '@/lib/uploads/upload-session/service' import { toV2File } from '@/app/api/v2/files/utils' export function toV2FileUpload( @@ -14,9 +14,6 @@ export function toV2FileUpload( name: session.fileName, contentType: session.contentType, size: session.fileSize, - partSize: session.partSize, - partCount: session.partCount, - uploadToken: session.uploadToken, expiresAt: session.expiresAt.toISOString(), error: session.error, file: file ? toV2File(file) : null, diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.test.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.test.ts index b923f82a39c..91a07f307f5 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.test.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.test.ts @@ -22,7 +22,7 @@ vi.mock('@/app/api/v1/middleware', () => ({ checkRateLimit: mockCheckRateLimit } vi.mock('@/app/api/v2/lib/gate', () => ({ v2ApiGateError: vi.fn().mockResolvedValue(null), })) -vi.mock('@/lib/uploads/multipart-session/service', () => ({ +vi.mock('@/lib/uploads/upload-session/service', () => ({ completeUploadSession: mockCompleteUploadSession, })) vi.mock('@/app/api/v2/knowledge/[id]/documents/uploads/utils', () => ({ @@ -50,9 +50,14 @@ const SESSION = { workspaceId: WORKSPACE_ID, userId: 'user-1', knowledgeBaseId: 'kb-1', + workflowId: null, + executionId: null, purpose: 'knowledge_document', + method: 'multipart', storageContext: 'knowledge-base', storageKey: 'kb/guide.pdf', + finalKey: 'kb/guide.pdf', + stagingKey: 'upload-sessions/upload-1/guide.pdf', storageProvider: 's3', providerUploadId: 'provider-1', fileName: 'guide.pdf', @@ -146,6 +151,12 @@ describe('POST knowledge-document multipart completion', () => { source: 'api', }) ) + expect(mockCompleteUploadSession).toHaveBeenCalledWith( + expect.objectContaining({ + session: SESSION, + completion: { parts: [{ partNumber: 1, etag: 'etag-1' }] }, + }) + ) }) it('resolves the payer lazily, only when the finalizer asks for one', async () => { diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts index b4835d232ea..0da551ca60d 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts @@ -6,7 +6,7 @@ import { v2CompleteKnowledgeDocumentUploadContract } from '@/lib/api/contracts/v import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { completeUploadSession } from '@/lib/uploads/multipart-session/service' +import { completeUploadSession } from '@/lib/uploads/upload-session/service' import { checkRateLimit } from '@/app/api/v1/middleware' import { finalizeKnowledgeDocumentUpload, @@ -68,7 +68,7 @@ export const POST = withRouteHandler( }) const result = await completeUploadSession({ session, - parts: parsed.data.body.parts, + completion: parsed.data.body, finalize: (claimed) => finalizeKnowledgeDocumentUpload({ claimed, diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts index 80b079ad985..e69f1eece4b 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts @@ -5,7 +5,7 @@ import { NextResponse } from 'next/server' import { v2CreateKnowledgeDocumentUploadPartUrlsContract } from '@/lib/api/contracts/v2/knowledge' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { createUploadPartUrls } from '@/lib/uploads/multipart-session/service' +import { createUploadPartUrls } from '@/lib/uploads/upload-session/service' import { checkRateLimit } from '@/app/api/v1/middleware' import { getOwnedKnowledgeDocumentUpload, diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.test.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.test.ts index cf3ea905882..0d0b0462332 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.test.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.test.ts @@ -79,10 +79,13 @@ describe('POST /api/v2/knowledge/[id]/documents/uploads', () => { fileName: 'guide.pdf', contentType: 'application/pdf', fileSize: 1024, - partSize: 8 * 1024 * 1024, - partCount: 1, uploadToken: 'token', error: null, + transfer: { + method: 'put', + url: 'https://storage.example/upload', + headers: { 'content-type': 'application/pdf' }, + }, }) }) @@ -109,6 +112,12 @@ describe('POST /api/v2/knowledge/[id]/documents/uploads', () => { tag1: 'product', processingOptions: { recipe: 'default', lang: 'en' }, }, + localOrigin: 'http://localhost:3000', + }) + expect((await response.json()).data).toMatchObject({ + session: { id: 'upload-1', status: 'uploading', document: null }, + uploadToken: 'token', + transfer: { method: 'put', url: 'https://storage.example/upload' }, }) expect(mockResolveKnowledgeDocumentUploadBilling.mock.invocationCallOrder[0]).toBeLessThan( mockCreateKnowledgeDocumentUploadSession.mock.invocationCallOrder[0] diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.ts index fb15afddeea..e736548fd98 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/route.ts @@ -72,8 +72,16 @@ export const POST = withRouteHandler( contentType, fileSize: size, metadata, + localOrigin: request.nextUrl.origin, }) - return v2Data(toV2KnowledgeDocumentUpload(session, null), { rateLimit, status: 201 }) + return v2Data( + { + session: toV2KnowledgeDocumentUpload(session, null), + uploadToken: session.uploadToken, + transfer: session.transfer, + }, + { rateLimit, status: 201 } + ) } catch (error) { const classified = v2CaughtOrchestrationError(error) if (classified) return classified diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.test.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.test.ts index 708f666b2a9..564388df5d2 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.test.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.test.ts @@ -3,7 +3,7 @@ */ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { UploadSessionRecord } from '@/lib/uploads/multipart-session/service' +import type { UploadSessionRecord } from '@/lib/uploads/upload-session/service' const { mockAbortUploadSession, @@ -25,7 +25,7 @@ vi.mock('@/lib/knowledge/orchestration', () => ({ vi.mock('@/lib/knowledge/orchestration/documents', () => ({ findBoundKnowledgeDocument: mockFindBoundKnowledgeDocument, })) -vi.mock('@/lib/uploads/multipart-session/service', () => ({ +vi.mock('@/lib/uploads/upload-session/service', () => ({ abortUploadSession: mockAbortUploadSession, createUploadSession: mockCreateUploadSession, getOwnedUploadSession: vi.fn(), @@ -38,6 +38,7 @@ import { abortKnowledgeDocumentUpload, createKnowledgeDocumentUploadSession, finalizeKnowledgeDocumentUpload, + toV2KnowledgeDocumentUpload, } from '@/app/api/v2/knowledge/[id]/documents/uploads/utils' const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' @@ -46,9 +47,14 @@ const CLAIMED: UploadSessionRecord = { workspaceId: WORKSPACE_ID, userId: 'user-1', knowledgeBaseId: 'kb-1', + workflowId: null, + executionId: null, purpose: 'knowledge_document', + method: 'multipart', storageContext: 'knowledge-base', storageKey: 'kb/guide.pdf', + finalKey: 'kb/guide.pdf', + stagingKey: 'upload-sessions/upload-1/guide.pdf', storageProvider: 's3', providerUploadId: 'provider-1', fileName: 'guide.pdf', @@ -91,6 +97,7 @@ function createSession() { contentType: 'application/pdf', fileSize: 1024, metadata: { tag1: 'product' }, + localOrigin: 'http://localhost:3000', }) } @@ -114,6 +121,7 @@ describe('createKnowledgeDocumentUploadSession', () => { contentType: 'application/pdf', fileSize: 1024, metadata: { tag1: 'product' }, + localOrigin: 'http://localhost:3000', }) expect(mockRecordKnowledgeBaseFileOwnership).toHaveBeenCalledWith({ key: 'kb/guide.pdf', @@ -136,6 +144,17 @@ describe('createKnowledgeDocumentUploadSession', () => { }) }) +describe('toV2KnowledgeDocumentUpload', () => { + it('does not expose reusable upload capabilities after session creation', () => { + const serialized = toV2KnowledgeDocumentUpload(CLAIMED, null) + + expect(serialized).not.toHaveProperty('uploadToken') + expect(serialized).not.toHaveProperty('partSize') + expect(serialized).not.toHaveProperty('partCount') + expect(serialized).not.toHaveProperty('transfer') + }) +}) + describe('abortKnowledgeDocumentUpload', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.ts b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.ts index 2844b5245dc..4e4122221f0 100644 --- a/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.ts +++ b/apps/sim/app/api/v2/knowledge/[id]/documents/uploads/utils.ts @@ -16,13 +16,14 @@ import { performUploadKnowledgeDocument } from '@/lib/knowledge/orchestration' import type { CreatedKnowledgeDocument } from '@/lib/knowledge/orchestration/documents' import { findBoundKnowledgeDocument } from '@/lib/knowledge/orchestration/documents' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' +import { recordKnowledgeBaseFileOwnership } from '@/lib/uploads/server/metadata' import { abortUploadSession, + type CreatedUploadSession, createUploadSession, getOwnedUploadSession, type UploadSessionRecord, -} from '@/lib/uploads/multipart-session/service' -import { recordKnowledgeBaseFileOwnership } from '@/lib/uploads/server/metadata' +} from '@/lib/uploads/upload-session/service' import { resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' import type { RateLimitResult } from '@/app/api/v1/middleware' import { v2Error } from '@/app/api/v2/lib/response' @@ -110,7 +111,8 @@ export async function createKnowledgeDocumentUploadSession(params: { contentType: string fileSize: number metadata: Record -}): Promise { + localOrigin: string +}): Promise { const session = await createUploadSession({ ...params, purpose: 'knowledge_document', @@ -163,9 +165,6 @@ export function toV2KnowledgeDocumentUpload( name: session.fileName, contentType: session.contentType, size: session.fileSize, - partSize: session.partSize, - partCount: session.partCount, - uploadToken: session.uploadToken, expiresAt: session.expiresAt.toISOString(), error: session.error, document: document ? toV2KnowledgeDocumentSummary(document) : null, @@ -216,7 +215,7 @@ export async function abortKnowledgeDocumentUpload( } /** - * Binds a completed multipart session to its knowledge document. Shared by the public v2 + * Binds a completed upload session to its knowledge document. Shared by the public v2 * and session-authenticated routes so both get identical completion semantics. * * Ordering is load-bearing. A retry is answered from the already-bound document before any diff --git a/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.test.ts b/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.test.ts index 783a473fda5..637e56f8f99 100644 --- a/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.test.ts +++ b/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.test.ts @@ -3,6 +3,7 @@ */ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OrchestrationError } from '@/lib/core/orchestration/types' const { mockCheckRateLimit, @@ -12,6 +13,7 @@ const { mockStartUploadedTableImport, mockToV2TableImport, mockCompleteUploadSession, + mockValidateUploadCompletion, } = vi.hoisted(() => ({ mockCheckRateLimit: vi.fn(), mockResolveWorkspaceScope: vi.fn(), @@ -20,6 +22,7 @@ const { mockStartUploadedTableImport: vi.fn(), mockToV2TableImport: vi.fn(), mockCompleteUploadSession: vi.fn(), + mockValidateUploadCompletion: vi.fn(), })) vi.mock('@/app/api/v1/middleware', () => ({ @@ -42,8 +45,9 @@ vi.mock('@/lib/table/orchestration/import-resource', () => ({ toV2TableImport: mockToV2TableImport, })) -vi.mock('@/lib/uploads/multipart-session/service', () => ({ +vi.mock('@/lib/uploads/upload-session/service', () => ({ completeUploadSession: mockCompleteUploadSession, + validateUploadCompletion: mockValidateUploadCompletion, })) import { POST } from '@/app/api/v2/tables/imports/[importId]/complete/route' @@ -63,7 +67,7 @@ const UPLOAD = { userId: 'user-1', } -function request() { +function request(body: Record = { parts: [{ partNumber: 1, etag: 'etag-1' }] }) { return POST( new NextRequest( `http://localhost:3000/api/v2/tables/imports/import-1/complete?workspaceId=${WORKSPACE_ID}`, @@ -73,7 +77,7 @@ function request() { 'Content-Type': 'application/json', 'upload-token': 'signed-upload-token', }, - body: JSON.stringify({ parts: [{ partNumber: 1, etag: 'etag-1' }] }), + body: JSON.stringify(body), } ), { params: Promise.resolve({ importId: 'import-1' }) } @@ -109,7 +113,50 @@ describe('POST /api/v2/tables/imports/[importId]/complete', () => { workspaceId: WORKSPACE_ID, userId: 'user-1', }) + expect(mockValidateUploadCompletion).toHaveBeenCalledWith(UPLOAD, { + parts: [{ partNumber: 1, etag: 'etag-1' }], + }) expect(mockCompleteUploadSession).not.toHaveBeenCalled() expect(mockStartUploadedTableImport).not.toHaveBeenCalled() }) + + it('validates completion shape before returning an existing table job', async () => { + mockFindOwnedTableImport.mockResolvedValue({ id: 'import-1' }) + mockValidateUploadCompletion.mockImplementationOnce(() => { + throw new OrchestrationError('validation', 'Multipart completion requires parts') + }) + + const response = await request({}) + + expect(response.status).toBe(400) + expect(mockFindOwnedTableImport).not.toHaveBeenCalled() + expect(mockCompleteUploadSession).not.toHaveBeenCalled() + }) + + it.each([ + ['PUT', {}], + ['multipart', { parts: [{ partNumber: 1, etag: 'etag-1' }] }], + ])('forwards a %s completion body and starts the import job', async (_method, completion) => { + const started = { id: 'import-1', tableId: 'table-1', status: 'running' } + const responseBody = { id: 'import-1', tableId: 'table-1', status: 'processing' } + mockFindOwnedTableImport.mockResolvedValue(null) + mockCompleteUploadSession.mockResolvedValue({ + session: UPLOAD, + value: null, + alreadyCompleted: false, + }) + mockStartUploadedTableImport.mockResolvedValue(started) + mockToV2TableImport.mockReturnValue(responseBody) + + const response = await request(completion) + + expect(response.status).toBe(200) + expect(mockCompleteUploadSession).toHaveBeenCalledWith({ + session: UPLOAD, + completion, + finalize: expect.any(Function), + }) + expect(mockStartUploadedTableImport).toHaveBeenCalledWith(UPLOAD) + expect(await response.json()).toEqual({ data: responseBody }) + }) }) diff --git a/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.ts b/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.ts index 4e44d28dafe..17860487743 100644 --- a/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.ts +++ b/apps/sim/app/api/v2/tables/imports/[importId]/complete/route.ts @@ -10,7 +10,10 @@ import { startUploadedTableImport, toV2TableImport, } from '@/lib/table/orchestration/import-resource' -import { completeUploadSession } from '@/lib/uploads/multipart-session/service' +import { + completeUploadSession, + validateUploadCompletion, +} from '@/lib/uploads/upload-session/service' import { checkRateLimit, resolveWorkspaceScope } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' import { @@ -50,15 +53,16 @@ export const POST = withRouteHandler( userId, uploadToken: parsed.data.headers['upload-token'], }) + validateUploadCompletion(upload, parsed.data.body) const existing = await findOwnedTableImport({ importId: upload.id, - workspaceId: upload.workspaceId, + workspaceId, userId: upload.userId, }) if (existing) return v2Data(toV2TableImport(existing), { rateLimit }) const completed = await completeUploadSession({ session: upload, - parts: parsed.data.body.parts, + completion: parsed.data.body, finalize: async () => ({ value: null }), }) const started = await startUploadedTableImport(completed.session) diff --git a/apps/sim/app/api/v2/tables/imports/[importId]/parts/route.ts b/apps/sim/app/api/v2/tables/imports/[importId]/parts/route.ts index 74f3153a9be..1d419975ac5 100644 --- a/apps/sim/app/api/v2/tables/imports/[importId]/parts/route.ts +++ b/apps/sim/app/api/v2/tables/imports/[importId]/parts/route.ts @@ -5,7 +5,7 @@ import { v2CreateTableImportPartUrlsContract } from '@/lib/api/contracts/v2/tabl import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getOwnedTableImportUpload } from '@/lib/table/orchestration/import-resource' -import { createUploadPartUrls } from '@/lib/uploads/multipart-session/service' +import { createUploadPartUrls } from '@/lib/uploads/upload-session/service' import { checkRateLimit, resolveWorkspaceScope } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' import { diff --git a/apps/sim/app/api/v2/tables/imports/route.test.ts b/apps/sim/app/api/v2/tables/imports/route.test.ts new file mode 100644 index 00000000000..a45e7de4fce --- /dev/null +++ b/apps/sim/app/api/v2/tables/imports/route.test.ts @@ -0,0 +1,135 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCheckRateLimit, + mockResolveWorkspaceScope, + mockCreateTableImportResource, + mockToV2CreateTableImport, +} = vi.hoisted(() => ({ + mockCheckRateLimit: vi.fn(), + mockResolveWorkspaceScope: vi.fn(), + mockCreateTableImportResource: vi.fn(), + mockToV2CreateTableImport: vi.fn(), +})) + +vi.mock('@/app/api/v1/middleware', () => ({ + checkRateLimit: mockCheckRateLimit, + resolveWorkspaceScope: mockResolveWorkspaceScope, +})) + +vi.mock('@/app/api/v2/lib/gate', () => ({ + v2ApiGateError: vi.fn().mockResolvedValue(null), +})) + +vi.mock('@/app/api/v2/tables/utils', () => ({ + v2TableLockError: vi.fn().mockReturnValue(null), +})) + +vi.mock('@/lib/table/orchestration/import-resource', () => ({ + createTableImportResource: mockCreateTableImportResource, + toV2CreateTableImport: mockToV2CreateTableImport, +})) + +import { POST } from '@/app/api/v2/tables/imports/route' + +const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' +const RATE_LIMIT = { + allowed: true, + userId: 'user-1', + keyType: 'workspace', + limit: 100, + remaining: 99, + resetAt: new Date('2026-08-03T22:00:00.000Z'), +} + +describe('POST /api/v2/tables/imports', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCheckRateLimit.mockResolvedValue(RATE_LIMIT) + mockResolveWorkspaceScope.mockResolvedValue(null) + }) + + it.each([ + [ + 'upload', + { type: 'upload', name: 'data.csv', contentType: 'text/csv', size: 128 }, + { + session: { id: 'import-1', source: { type: 'upload' } }, + uploadToken: 'signed-token', + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + }, + ], + [ + 'workspace file', + { type: 'workspace_file', fileId: 'file-1' }, + { + session: { id: 'import-1', source: { type: 'workspace_file', fileId: 'file-1' } }, + uploadToken: null, + transfer: null, + }, + ], + ])('returns the create envelope for a %s source', async (_label, source, responseData) => { + const requestBody = { + workspaceId: WORKSPACE_ID, + source, + target: { type: 'new', name: 'imported_data' }, + } + const created = { record: { id: 'import-1' }, upload: null } + mockCreateTableImportResource.mockResolvedValue(created) + mockToV2CreateTableImport.mockReturnValue(responseData) + + const response = await POST( + new NextRequest('http://localhost:3000/api/v2/tables/imports', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(requestBody), + }) + ) + + expect(response.status).toBe(201) + expect(mockCreateTableImportResource).toHaveBeenCalledWith( + requestBody, + 'user-1', + 'http://localhost:3000' + ) + expect(mockToV2CreateTableImport).toHaveBeenCalledWith(created) + expect(await response.json()).toEqual({ data: responseData }) + }) + + it('accepts native JSON mapping and createColumns values', async () => { + const requestBody = { + workspaceId: WORKSPACE_ID, + source: { type: 'upload', name: 'data.csv', contentType: 'text/csv', size: 128 }, + target: { type: 'existing', tableId: 'table-1', mode: 'append' }, + mapping: { email: 'email_address', notes: null }, + createColumns: ['phone'], + } + const created = { record: { id: 'import-1' }, upload: null } + const responseData = { + session: { id: 'import-1', source: { type: 'upload' } }, + uploadToken: 'signed-token', + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + } + mockCreateTableImportResource.mockResolvedValue(created) + mockToV2CreateTableImport.mockReturnValue(responseData) + + const response = await POST( + new NextRequest('http://localhost:3000/api/v2/tables/imports', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(requestBody), + }) + ) + + expect(response.status).toBe(201) + expect(mockCreateTableImportResource).toHaveBeenCalledWith( + requestBody, + 'user-1', + 'http://localhost:3000' + ) + }) +}) diff --git a/apps/sim/app/api/v2/tables/imports/route.ts b/apps/sim/app/api/v2/tables/imports/route.ts index 2a0aeaf07a5..ad533a55e58 100644 --- a/apps/sim/app/api/v2/tables/imports/route.ts +++ b/apps/sim/app/api/v2/tables/imports/route.ts @@ -6,7 +6,7 @@ import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { createTableImportResource, - toV2TableImport, + toV2CreateTableImport, } from '@/lib/table/orchestration/import-resource' import { checkRateLimit, resolveWorkspaceScope } from '@/app/api/v1/middleware' import { v2ApiGateError } from '@/app/api/v2/lib/gate' @@ -40,8 +40,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => { if (!parsed.success) return parsed.response const scopeError = await resolveWorkspaceScope(rateLimit, parsed.data.body.workspaceId) if (scopeError) return v2WorkspaceAccessError(scopeError) - const created = await createTableImportResource(parsed.data.body, userId) - return v2Data(await toV2TableImport(created.record), { rateLimit, status: 201 }) + const created = await createTableImportResource( + parsed.data.body, + userId, + request.nextUrl.origin + ) + return v2Data(toV2CreateTableImport(created), { rateLimit, status: 201 }) } catch (error) { const lockError = v2TableLockError(error) if (lockError) return lockError diff --git a/apps/sim/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route.test.ts b/apps/sim/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route.test.ts new file mode 100644 index 00000000000..18fe9a6e23f --- /dev/null +++ b/apps/sim/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route.test.ts @@ -0,0 +1,100 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + MockLocalUploadBodyError, + mockExpectedUploadPartSize, + mockVerifyUploadSessionToken, + mockWriteLocalMultipartPart, +} = vi.hoisted(() => { + class MockLocalUploadBodyError extends Error {} + return { + MockLocalUploadBodyError, + mockExpectedUploadPartSize: vi.fn(), + mockVerifyUploadSessionToken: vi.fn(), + mockWriteLocalMultipartPart: vi.fn(), + } +}) + +vi.mock('@/lib/uploads/upload-session/provider', () => ({ + LocalUploadBodyError: MockLocalUploadBodyError, + writeLocalMultipartPart: mockWriteLocalMultipartPart, +})) + +vi.mock('@/lib/uploads/upload-session/service', () => ({ + expectedUploadPartSize: mockExpectedUploadPartSize, + verifyUploadSessionToken: mockVerifyUploadSessionToken, +})) + +import { PUT } from '@/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route' + +const SESSION = { + id: 'upload-1', + storageProvider: 'local', + method: 'multipart', + status: 'uploading', +} as const + +describe('PUT /api/v2/uploads/[uploadId]/parts/[partNumber]', () => { + beforeEach(() => { + vi.clearAllMocks() + mockVerifyUploadSessionToken.mockReturnValue(SESSION) + mockExpectedUploadPartSize.mockReturnValue(3) + mockWriteLocalMultipartPart.mockResolvedValue(undefined) + }) + + it('streams an exact-size local multipart part', async () => { + const response = await request() + + expect(response.status).toBe(204) + expect(mockVerifyUploadSessionToken).toHaveBeenCalledWith('signed-token') + expect(mockExpectedUploadPartSize).toHaveBeenCalledWith(SESSION, 1) + expect(mockWriteLocalMultipartPart).toHaveBeenCalledWith({ + uploadId: 'upload-1', + partNumber: 1, + body: expect.any(ReadableStream), + expectedSize: 3, + }) + }) + + it('maps a streamed-size failure to 400', async () => { + mockWriteLocalMultipartPart.mockRejectedValue( + new MockLocalUploadBodyError('Part 1 has 2 bytes; expected 3') + ) + + const response = await request({ contentLength: null }) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Part 1 has 2 bytes; expected 3', + }) + }) + + it('rejects PUT sessions before calculating a part size', async () => { + mockVerifyUploadSessionToken.mockReturnValue({ ...SESSION, method: 'put' }) + + const response = await request() + + expect(response.status).toBe(409) + expect(mockExpectedUploadPartSize).not.toHaveBeenCalled() + expect(mockWriteLocalMultipartPart).not.toHaveBeenCalled() + }) +}) + +function request(options?: { contentLength?: string | null }) { + const headers = new Headers({ 'Content-Type': 'application/octet-stream' }) + if (options?.contentLength !== null) { + headers.set('Content-Length', options?.contentLength ?? '3') + } + return PUT( + new NextRequest('http://localhost:3000/api/v2/uploads/upload-1/parts/1?token=signed-token', { + method: 'PUT', + headers, + body: new Uint8Array([1, 2, 3]), + }), + { params: Promise.resolve({ uploadId: 'upload-1', partNumber: '1' }) } + ) +} diff --git a/apps/sim/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route.ts b/apps/sim/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route.ts index 3baff93ec04..8e7c8cf4933 100644 --- a/apps/sim/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route.ts +++ b/apps/sim/app/api/v2/uploads/[uploadId]/parts/[partNumber]/route.ts @@ -2,12 +2,15 @@ import { type NextRequest, NextResponse } from 'next/server' import { localUploadPartContract } from '@/lib/api/contracts/upload-sessions' import { parseRequest } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { writeLocalMultipartPart } from '@/lib/uploads/multipart-session/provider' +import { + LocalUploadBodyError, + writeLocalMultipartPart, +} from '@/lib/uploads/upload-session/provider' import { expectedUploadPartSize, type UploadSessionRecord, verifyUploadSessionToken, -} from '@/lib/uploads/multipart-session/service' +} from '@/lib/uploads/upload-session/service' interface LocalPartRouteParams { params: Promise<{ uploadId: string; partNumber: string }> @@ -36,6 +39,9 @@ export const PUT = withRouteHandler( if (session.status !== 'uploading') { return NextResponse.json({ error: `Upload session is ${session.status}` }, { status: 409 }) } + if (session.method !== 'multipart') { + return NextResponse.json({ error: 'PUT upload sessions do not have parts' }, { status: 409 }) + } const { partNumber } = parsed.data.params const expectedSize = expectedUploadPartSize(session, partNumber) @@ -50,7 +56,14 @@ export const PUT = withRouteHandler( return NextResponse.json({ error: 'Upload part body is required' }, { status: 400 }) } - await writeLocalMultipartPart({ uploadId, partNumber, body: request.body, expectedSize }) + try { + await writeLocalMultipartPart({ uploadId, partNumber, body: request.body, expectedSize }) + } catch (error) { + if (error instanceof LocalUploadBodyError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + throw error + } return new NextResponse(null, { status: 204 }) } ) diff --git a/apps/sim/app/api/v2/uploads/[uploadId]/route.test.ts b/apps/sim/app/api/v2/uploads/[uploadId]/route.test.ts new file mode 100644 index 00000000000..22e18916d89 --- /dev/null +++ b/apps/sim/app/api/v2/uploads/[uploadId]/route.test.ts @@ -0,0 +1,142 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { MockLocalUploadBodyError, mockGetOwnedUploadSession, mockMetadata, mockWriteLocalPut } = + vi.hoisted(() => { + class MockLocalUploadBodyError extends Error {} + return { + MockLocalUploadBodyError, + mockGetOwnedUploadSession: vi.fn(), + mockMetadata: vi.fn(), + mockWriteLocalPut: vi.fn(), + } + }) + +vi.mock('@/lib/uploads/upload-session/provider', () => ({ + LocalUploadBodyError: MockLocalUploadBodyError, + writeLocalPutObject: mockWriteLocalPut, +})) + +vi.mock('@/lib/uploads/upload-session/service', () => ({ + getOwnedUploadSession: mockGetOwnedUploadSession, + uploadSessionObjectMetadata: mockMetadata, +})) + +import { PUT } from '@/app/api/v2/uploads/[uploadId]/route' + +const SESSION = { + id: 'upload-1', + workspaceId: 'workspace-1', + userId: 'user-1', + knowledgeBaseId: null, + workflowId: null, + executionId: null, + purpose: 'workspace_file', + method: 'put', + storageContext: 'workspace', + storageKey: 'workspace/workspace-1/file.bin', + finalKey: 'workspace/workspace-1/file.bin', + stagingKey: 'upload-sessions/upload-1/file.bin', + storageProvider: 'local', + providerUploadId: null, + fileName: 'file.bin', + contentType: 'application/octet-stream', + fileSize: 3, + partSize: null, + partCount: null, + status: 'uploading', + metadata: {}, + uploadToken: 'signed-token', + createdAt: new Date('2026-08-04T12:00:00.000Z'), + expiresAt: new Date('2099-08-05T12:00:00.000Z'), + completedFileId: null, + error: null, + completedAt: null, + updatedAt: new Date('2026-08-04T12:00:00.000Z'), +} as const + +describe('PUT /api/v2/uploads/[uploadId]', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetOwnedUploadSession.mockReturnValue(SESSION) + mockMetadata.mockReturnValue({ uploadId: 'upload-1', purpose: 'workspace_file' }) + mockWriteLocalPut.mockResolvedValue(undefined) + }) + + it('streams the local PUT with the signed session size and canonical metadata', async () => { + const response = await request() + + expect(response.status).toBe(204) + expect(mockGetOwnedUploadSession).toHaveBeenCalledWith({ + uploadId: 'upload-1', + uploadToken: 'signed-token', + }) + expect(mockWriteLocalPut).toHaveBeenCalledWith({ + uploadId: 'upload-1', + stagingKey: 'upload-sessions/upload-1/file.bin', + body: expect.any(ReadableStream), + expectedSize: 3, + contentType: 'application/octet-stream', + metadata: { uploadId: 'upload-1', purpose: 'workspace_file' }, + }) + }) + + it('streams an empty local PUT body for an empty workspace-file session', async () => { + mockGetOwnedUploadSession.mockReturnValue({ ...SESSION, fileSize: 0 }) + const response = await request({ contentLength: '0', body: new Uint8Array() }) + + expect(response.status).toBe(204) + expect(mockWriteLocalPut).toHaveBeenCalledWith( + expect.objectContaining({ expectedSize: 0, body: expect.any(ReadableStream) }) + ) + }) + + it('rejects a mismatched Content-Length before opening the local writer', async () => { + const response = await request({ contentLength: '2' }) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ + error: 'Upload must contain exactly 3 bytes', + }) + expect(mockWriteLocalPut).not.toHaveBeenCalled() + }) + + it('rejects a URL whose token names a non-local or multipart session', async () => { + mockGetOwnedUploadSession.mockReturnValue({ ...SESSION, method: 'multipart' }) + + const response = await request() + + expect(response.status).toBe(403) + expect(mockWriteLocalPut).not.toHaveBeenCalled() + }) + + it('maps exact-size streaming failures to a caller error', async () => { + mockWriteLocalPut.mockRejectedValue(new MockLocalUploadBodyError('Upload exceeds 3 bytes')) + + const response = await request({ contentLength: null }) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Upload exceeds 3 bytes' }) + }) +}) + +function request(options?: { contentLength?: string | null; body?: Uint8Array }) { + const headers = new Headers({ + 'Content-Type': 'application/octet-stream', + 'upload-token': 'signed-token', + }) + if (options?.contentLength !== null) { + headers.set('Content-Length', options?.contentLength ?? '3') + } + return PUT( + new NextRequest('http://localhost:3000/api/v2/uploads/upload-1', { + method: 'PUT', + headers, + body: options?.body ?? new Uint8Array([1, 2, 3]), + }), + { params: Promise.resolve({ uploadId: 'upload-1' }) } + ) +} diff --git a/apps/sim/app/api/v2/uploads/[uploadId]/route.ts b/apps/sim/app/api/v2/uploads/[uploadId]/route.ts new file mode 100644 index 00000000000..c44c33489a2 --- /dev/null +++ b/apps/sim/app/api/v2/uploads/[uploadId]/route.ts @@ -0,0 +1,73 @@ +import { type NextRequest, NextResponse } from 'next/server' +import { localPutUploadContract } from '@/lib/api/contracts/upload-sessions' +import { parseRequest } from '@/lib/api/server' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { LocalUploadBodyError, writeLocalPutObject } from '@/lib/uploads/upload-session/provider' +import { + getOwnedUploadSession, + uploadSessionObjectMetadata, +} from '@/lib/uploads/upload-session/service' + +interface LocalPutRouteParams { + params: Promise<{ uploadId: string }> +} + +/** Local-storage data plane for a signed whole-object PUT upload session. */ +export const PUT = withRouteHandler( + async (request: NextRequest, context: LocalPutRouteParams): Promise => { + const parsed = await parseRequest(localPutUploadContract, request, context) + if (!parsed.success) return parsed.response + + let session + try { + session = getOwnedUploadSession({ + uploadId: parsed.data.params.uploadId, + uploadToken: parsed.data.headers['upload-token'], + }) + } catch { + return NextResponse.json({ error: 'Invalid or expired upload token' }, { status: 403 }) + } + + if (session.storageProvider !== 'local' || session.method !== 'put') { + return NextResponse.json({ error: 'Upload URL does not match this session' }, { status: 403 }) + } + if (session.expiresAt.getTime() <= Date.now()) { + return NextResponse.json({ error: 'Upload session has expired' }, { status: 409 }) + } + + const contentType = request.headers.get('content-type') + if (contentType !== session.contentType) { + return NextResponse.json( + { error: `Content-Type must be ${session.contentType}` }, + { status: 400 } + ) + } + const contentLength = request.headers.get('content-length') + if (contentLength !== null && Number(contentLength) !== session.fileSize) { + return NextResponse.json( + { error: `Upload must contain exactly ${session.fileSize} bytes` }, + { status: 400 } + ) + } + if (!request.body) { + return NextResponse.json({ error: 'Upload body is required' }, { status: 400 }) + } + + try { + await writeLocalPutObject({ + uploadId: session.id, + stagingKey: session.stagingKey, + body: request.body, + expectedSize: session.fileSize, + contentType: session.contentType, + metadata: uploadSessionObjectMetadata(session), + }) + } catch (error) { + if (error instanceof LocalUploadBodyError) { + return NextResponse.json({ error: error.message }, { status: 400 }) + } + throw error + } + return new NextResponse(null, { status: 204 }) + } +) diff --git a/apps/sim/app/api/workspaces/[id]/files/[fileId]/content/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/[fileId]/content/route.test.ts new file mode 100644 index 00000000000..78f5cdb124f --- /dev/null +++ b/apps/sim/app/api/workspaces/[id]/files/[fileId]/content/route.test.ts @@ -0,0 +1,135 @@ +/** + * @vitest-environment node + */ +import { authMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetUserEntityPermissions, mockPerformUpdateContent } = vi.hoisted(() => ({ + mockGetUserEntityPermissions: vi.fn(), + mockPerformUpdateContent: vi.fn(), +})) + +vi.mock('@/lib/workspace-files/orchestration', () => ({ + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES: 70 * 1024 * 1024, + performUpdateWorkspaceFileContent: mockPerformUpdateContent, +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + getUserEntityPermissions: mockGetUserEntityPermissions, +})) + +import { PUT } from '@/app/api/workspaces/[id]/files/[fileId]/content/route' + +const WORKSPACE_ID = 'workspace-1' +const FILE_ID = 'wf_1' +const USER = { id: 'user-1', name: 'Test User', email: 'test@sim.ai' } +const RECORD = { + id: FILE_ID, + workspaceId: WORKSPACE_ID, + name: 'notes.md', + key: `workspace/${WORKSPACE_ID}/notes.md`, + path: '/api/files/serve/notes.md?context=workspace', + size: 5, + type: 'text/markdown', + uploadedBy: USER.id, + folderId: null, + folderPath: null, + uploadedAt: new Date('2026-08-04T00:00:00.000Z'), + updatedAt: new Date('2026-08-04T00:00:00.000Z'), +} + +const routeContext = { params: Promise.resolve({ id: WORKSPACE_ID, fileId: FILE_ID }) } + +function createRequest(body: unknown, contentLength?: number): NextRequest { + return new NextRequest( + `http://localhost:3000/api/workspaces/${WORKSPACE_ID}/files/${FILE_ID}/content`, + { + method: 'PUT', + headers: { + 'content-type': 'application/json', + ...(contentLength === undefined ? {} : { 'content-length': String(contentLength) }), + }, + body: typeof body === 'string' ? body : JSON.stringify(body), + } + ) +} + +describe('PUT /api/workspaces/[id]/files/[fileId]/content', () => { + beforeEach(() => { + vi.clearAllMocks() + authMockFns.mockGetSession.mockResolvedValue({ user: USER }) + mockGetUserEntityPermissions.mockResolvedValue('write') + mockPerformUpdateContent.mockResolvedValue({ success: true, file: RECORD }) + }) + + it('authenticates before parsing an invalid request body', async () => { + authMockFns.mockGetSession.mockResolvedValue(null) + + const response = await PUT(createRequest('{not-json'), routeContext) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(mockGetUserEntityPermissions).not.toHaveBeenCalled() + expect(mockPerformUpdateContent).not.toHaveBeenCalled() + }) + + it('authorizes the workspace before parsing the request body', async () => { + mockGetUserEntityPermissions.mockResolvedValue('read') + + const response = await PUT(createRequest('{not-json'), routeContext) + + expect(response.status).toBe(403) + await expect(response.json()).resolves.toEqual({ error: 'Insufficient permissions' }) + expect(mockGetUserEntityPermissions).toHaveBeenCalledWith(USER.id, 'workspace', WORKSPACE_ID) + expect(mockPerformUpdateContent).not.toHaveBeenCalled() + }) + + it('rejects malformed base64 after authorization', async () => { + const response = await PUT( + createRequest({ content: 'not-base64!', encoding: 'base64' }), + routeContext + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Validation error' }) + expect(mockPerformUpdateContent).not.toHaveBeenCalled() + }) + + it('accepts empty base64 as a zero-byte replacement', async () => { + const request = createRequest({ content: '', encoding: 'base64' }) + const response = await PUT(request, routeContext) + + expect(response.status).toBe(200) + expect(mockPerformUpdateContent).toHaveBeenCalledWith({ + workspaceId: WORKSPACE_ID, + fileId: FILE_ID, + userId: USER.id, + content: '', + encoding: 'base64', + actorName: USER.name, + actorEmail: USER.email, + request, + }) + }) + + it('allows JSON bodies above the default 50 MiB cap for base64 expansion', async () => { + const response = await PUT( + createRequest({ content: 'TQ==', encoding: 'base64' }, 60 * 1024 * 1024), + routeContext + ) + + expect(response.status).toBe(200) + expect(mockPerformUpdateContent).toHaveBeenCalled() + }) + + it('rejects a JSON body above the inline-content cap', async () => { + const response = await PUT(createRequest({ content: '' }, 70 * 1024 * 1024 + 1), routeContext) + + expect(response.status).toBe(413) + await expect(response.json()).resolves.toEqual({ + error: `Request body exceeds the maximum allowed size of ${70 * 1024 * 1024} bytes`, + }) + expect(mockPerformUpdateContent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/workspaces/[id]/files/[fileId]/content/route.ts b/apps/sim/app/api/workspaces/[id]/files/[fileId]/content/route.ts index e2963b00f94..a7d4934c783 100644 --- a/apps/sim/app/api/workspaces/[id]/files/[fileId]/content/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/[fileId]/content/route.ts @@ -1,14 +1,20 @@ import { createLogger } from '@sim/logger' import { type NextRequest, NextResponse } from 'next/server' -import { updateWorkspaceFileContentContract } from '@/lib/api/contracts/workspace-files' -import { parseRequest } from '@/lib/api/server' +import { + updateWorkspaceFileContentContract, + workspaceFileParamsSchema, +} from '@/lib/api/contracts/workspace-files' +import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { messageForOrchestrationError, statusForOrchestrationError, } from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performUpdateWorkspaceFileContent } from '@/lib/workspace-files/orchestration' +import { + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + performUpdateWorkspaceFileContent, +} from '@/lib/workspace-files/orchestration' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' export const dynamic = 'force-dynamic' @@ -26,10 +32,14 @@ export const PUT = withRouteHandler( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const parsed = await parseRequest(updateWorkspaceFileContentContract, request, context) - if (!parsed.success) return parsed.response - const { id: workspaceId, fileId } = parsed.data.params - const { content, encoding } = parsed.data.body + const paramsResult = workspaceFileParamsSchema.safeParse(await context.params) + if (!paramsResult.success) { + return NextResponse.json( + { error: getValidationErrorMessage(paramsResult.error, 'Invalid route parameters') }, + { status: 400 } + ) + } + const { id: workspaceId, fileId } = paramsResult.data const userPermission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId) if (userPermission !== 'admin' && userPermission !== 'write') { @@ -37,6 +47,12 @@ export const PUT = withRouteHandler( return NextResponse.json({ error: 'Insufficient permissions' }, { status: 403 }) } + const parsed = await parseRequest(updateWorkspaceFileContentContract, request, context, { + maxBodyBytes: MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + const { content, encoding } = parsed.data.body + const result = await performUpdateWorkspaceFileContent({ workspaceId, fileId, diff --git a/apps/sim/app/api/workspaces/[id]/files/presigned/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/presigned/route.test.ts deleted file mode 100644 index d69aa933389..00000000000 --- a/apps/sim/app/api/workspaces/[id]/files/presigned/route.test.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** - * @vitest-environment node - */ -import { - authMockFns, - permissionsMock, - permissionsMockFns, - storageServiceMock, - storageServiceMockFns, -} from '@sim/testing' -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockCheckStorageQuota, - mockGenerateWorkspaceFileKey, - mockResolveStorageBillingContext, - mockUseBlobStorage, -} = vi.hoisted(() => ({ - mockCheckStorageQuota: vi.fn(), - mockGenerateWorkspaceFileKey: vi.fn(), - mockResolveStorageBillingContext: vi.fn(), - mockUseBlobStorage: { value: false }, -})) - -vi.mock('@/lib/billing/storage', () => ({ - checkStorageQuotaForBillingContext: mockCheckStorageQuota, - resolveStorageBillingContext: mockResolveStorageBillingContext, -})) - -vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock) - -vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ - generateWorkspaceFileKey: mockGenerateWorkspaceFileKey, -})) - -vi.mock('@/lib/uploads/config', () => ({ - getServeStoragePrefix: () => (mockUseBlobStorage.value ? 'blob' : 's3'), -})) - -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) - -const WS = '7727ef3f-8cf6-4686-b063-2bb006a10785' -const STORAGE_CONTEXT = { - workspaceId: WS, - billedAccountUserId: 'workspace-owner', - billingEntity: { type: 'organization' as const, id: 'workspace-org' }, - plan: 'team_25000', - customStorageLimitGB: null, -} - -import { POST } from '@/app/api/workspaces/[id]/files/presigned/route' - -const params = (id = WS) => ({ params: Promise.resolve({ id }) }) - -const makeRequest = (body: unknown) => - new NextRequest(`http://localhost/api/workspaces/${WS}/files/presigned`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }) - -const validBody = { - fileName: 'video.mp4', - contentType: 'video/mp4', - fileSize: 10 * 1024 * 1024, -} - -describe('POST /api/workspaces/[id]/files/presigned', () => { - beforeEach(() => { - vi.clearAllMocks() - authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - mockCheckStorageQuota.mockResolvedValue({ allowed: true }) - mockResolveStorageBillingContext.mockResolvedValue(STORAGE_CONTEXT) - storageServiceMockFns.mockHasCloudStorage.mockReturnValue(true) - mockGenerateWorkspaceFileKey.mockReturnValue(`workspace/${WS}/123-abc-video.mp4`) - storageServiceMockFns.mockGeneratePresignedUploadUrl.mockResolvedValue({ - url: 'https://s3/presigned', - key: `workspace/${WS}/123-abc-video.mp4`, - uploadHeaders: { 'Content-Type': 'video/mp4' }, - }) - }) - - it('returns 401 when unauthenticated', async () => { - authMockFns.mockGetSession.mockResolvedValueOnce(null) - const res = await POST(makeRequest(validBody), params()) - expect(res.status).toBe(401) - }) - - it('returns 403 when user has read-only permission', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('read') - const res = await POST(makeRequest(validBody), params()) - expect(res.status).toBe(403) - }) - - it('returns 400 for missing fileName', async () => { - const res = await POST(makeRequest({ ...validBody, fileName: '' }), params()) - expect(res.status).toBe(400) - }) - - it('returns 400 for negative fileSize', async () => { - const res = await POST(makeRequest({ ...validBody, fileSize: -1 }), params()) - expect(res.status).toBe(400) - }) - - it('accepts fileSize === 0 (empty new files)', async () => { - const res = await POST(makeRequest({ ...validBody, fileSize: 0 }), params()) - expect(res.status).toBe(200) - }) - - it('returns 413 when fileSize exceeds 5 GiB ceiling', async () => { - const res = await POST( - makeRequest({ ...validBody, fileSize: 6 * 1024 * 1024 * 1024 }), - params() - ) - expect(res.status).toBe(413) - }) - - it('returns 413 when storage quota would be exceeded', async () => { - mockCheckStorageQuota.mockResolvedValueOnce({ allowed: false, error: 'Over quota' }) - const res = await POST(makeRequest(validBody), params()) - const body = await res.json() - expect(res.status).toBe(413) - expect(body.error).toBe('Over quota') - }) - - it('returns local fallback signal when cloud storage is not configured', async () => { - storageServiceMockFns.mockHasCloudStorage.mockReturnValueOnce(false) - const res = await POST(makeRequest(validBody), params()) - const body = await res.json() - expect(res.status).toBe(200) - expect(body.directUploadSupported).toBe(false) - expect(body.presignedUrl).toBe('') - expect(body.fileInfo.name).toBe('video.mp4') - expect(storageServiceMockFns.mockGeneratePresignedUploadUrl).not.toHaveBeenCalled() - }) - - it('issues a presigned URL bound to the workspace', async () => { - const res = await POST(makeRequest(validBody), params()) - const body = await res.json() - - expect(res.status).toBe(200) - expect(body.directUploadSupported).toBe(true) - expect(body.presignedUrl).toBe('https://s3/presigned') - expect(body.fileInfo.key).toBe(`workspace/${WS}/123-abc-video.mp4`) - expect(body.fileInfo.path).toContain('?context=workspace') - expect(body.fileInfo.path).toContain('s3') - expect(body.uploadHeaders).toEqual({ 'Content-Type': 'video/mp4' }) - - expect(mockGenerateWorkspaceFileKey).toHaveBeenCalledWith(WS, 'video.mp4') - expect(mockResolveStorageBillingContext).toHaveBeenCalledWith(WS) - expect(mockCheckStorageQuota).toHaveBeenCalledWith(STORAGE_CONTEXT, validBody.fileSize) - expect(storageServiceMockFns.mockGeneratePresignedUploadUrl).toHaveBeenCalledWith( - expect.objectContaining({ - context: 'workspace', - userId: 'user-1', - customKey: `workspace/${WS}/123-abc-video.mp4`, - metadata: { workspaceId: WS }, - }) - ) - }) - - it('serves blob path when blob storage is configured', async () => { - mockUseBlobStorage.value = true - try { - const res = await POST(makeRequest(validBody), params()) - const body = await res.json() - expect(body.fileInfo.path).toContain('/blob/') - } finally { - mockUseBlobStorage.value = false - } - }) -}) diff --git a/apps/sim/app/api/workspaces/[id]/files/presigned/route.ts b/apps/sim/app/api/workspaces/[id]/files/presigned/route.ts deleted file mode 100644 index 905ad938d98..00000000000 --- a/apps/sim/app/api/workspaces/[id]/files/presigned/route.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { workspacePresignedUploadContract } from '@/lib/api/contracts/workspace-files' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { - checkStorageQuotaForBillingContext, - resolveStorageBillingContext, -} from '@/lib/billing/storage' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { getServeStoragePrefix } from '@/lib/uploads/config' -import { assertWorkspaceFileFolderTarget } from '@/lib/uploads/contexts/workspace' -import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager' -import { generatePresignedUploadUrl, hasCloudStorage } from '@/lib/uploads/core/storage-service' -import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' - -const logger = createLogger('WorkspacePresignedAPI') - -/** - * POST /api/workspaces/[id]/files/presigned - * Returns a presigned PUT URL for a workspace-scoped object key. The client - * uploads the bytes directly to S3/Blob, then calls /files/register to - * insert metadata. - */ -export const POST = withRouteHandler( - async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = session.user.id - - const parsed = await parseRequest(workspacePresignedUploadContract, request, context) - if (!parsed.success) return parsed.response - const { params, body } = parsed.data - const workspaceId = params.id - const { fileName, contentType, fileSize, folderId } = body - - const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) - if (permission !== 'admin' && permission !== 'write') { - logger.warn(`User ${userId} lacks write permission for ${workspaceId}`) - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } - - if (fileSize > MAX_WORKSPACE_FILE_SIZE) { - return NextResponse.json( - { error: `File size exceeds maximum of ${MAX_WORKSPACE_FILE_SIZE} bytes` }, - { status: 413 } - ) - } - - let targetFolderId: string | null - try { - targetFolderId = await assertWorkspaceFileFolderTarget(workspaceId, folderId) - } catch (error) { - return NextResponse.json( - { error: getErrorMessage(error, 'Invalid target folder') }, - { status: 400 } - ) - } - - if (!hasCloudStorage()) { - logger.info(`Local storage detected, signaling API fallback for ${fileName}`) - return NextResponse.json({ - fileName, - presignedUrl: '', - fileInfo: { path: '', key: '', name: fileName, size: fileSize, type: contentType }, - directUploadSupported: false, - }) - } - - const storageBillingContext = await resolveStorageBillingContext(workspaceId) - const quotaCheck = await checkStorageQuotaForBillingContext(storageBillingContext, fileSize) - if (!quotaCheck.allowed) { - return NextResponse.json( - { error: quotaCheck.error || 'Storage limit exceeded' }, - { status: 413 } - ) - } - - const key = generateWorkspaceFileKey(workspaceId, fileName) - const presigned = await generatePresignedUploadUrl({ - fileName, - contentType, - fileSize, - context: 'workspace', - userId, - customKey: key, - expirationSeconds: 3600, - metadata: { workspaceId, ...(targetFolderId ? { folderId: targetFolderId } : {}) }, - }) - - const finalPath = `/api/files/serve/${getServeStoragePrefix()}/${encodeURIComponent(key)}?context=workspace` - - logger.info(`Issued workspace presigned URL for ${fileName} -> ${key}`) - - return NextResponse.json({ - fileName, - presignedUrl: presigned.url, - fileInfo: { - path: finalPath, - key: presigned.key, - name: fileName, - size: fileSize, - type: contentType, - }, - uploadHeaders: presigned.uploadHeaders, - directUploadSupported: true, - }) - } -) diff --git a/apps/sim/app/api/workspaces/[id]/files/register/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/register/route.test.ts deleted file mode 100644 index cce56f7b8e8..00000000000 --- a/apps/sim/app/api/workspaces/[id]/files/register/route.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -/** - * @vitest-environment node - */ -import { - auditMock, - auditMockFns, - authMockFns, - permissionsMock, - permissionsMockFns, - posthogServerMock, - posthogServerMockFns, -} from '@sim/testing' -import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockRegisterUploadedWorkspaceFile, mockParseWorkspaceFileKey, FileConflictErrorImpl } = - vi.hoisted(() => { - class FileConflictErrorImpl extends Error { - constructor(message: string) { - super(message) - this.name = 'FileConflictError' - } - } - return { - mockRegisterUploadedWorkspaceFile: vi.fn(), - mockParseWorkspaceFileKey: vi.fn(), - FileConflictErrorImpl, - } - }) - -vi.mock('@/lib/uploads/contexts/workspace', () => ({ - registerUploadedWorkspaceFile: mockRegisterUploadedWorkspaceFile, - parseWorkspaceFileKey: mockParseWorkspaceFileKey, - FileConflictError: FileConflictErrorImpl, -})) - -vi.mock('@/lib/posthog/server', () => posthogServerMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) -vi.mock('@sim/audit', () => auditMock) - -const WS = '7727ef3f-8cf6-4686-b063-2bb006a10785' -const VALID_KEY = `workspace/${WS}/123-abc-video.mp4` - -import { POST } from '@/app/api/workspaces/[id]/files/register/route' - -const params = (id = WS) => ({ params: Promise.resolve({ id }) }) - -const makeRequest = (body: unknown) => - new NextRequest(`http://localhost/api/workspaces/${WS}/files/register`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }) - -const validBody = { - key: VALID_KEY, - name: 'video.mp4', - contentType: 'video/mp4', -} - -describe('POST /api/workspaces/[id]/files/register', () => { - beforeEach(() => { - vi.clearAllMocks() - authMockFns.mockGetSession.mockResolvedValue({ - user: { id: 'user-1', name: 'User One', email: 'u@example.com' }, - }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') - mockParseWorkspaceFileKey.mockImplementation((key: string) => { - const match = key.match(/^workspace\/([^/]+)\//) - return match ? match[1] : null - }) - mockRegisterUploadedWorkspaceFile.mockResolvedValue({ - file: { - id: 'wf_123', - name: 'video.mp4', - size: 10 * 1024 * 1024, - type: 'video/mp4', - url: '/api/files/serve/...', - key: VALID_KEY, - context: 'workspace', - }, - created: true, - }) - }) - - it('returns 401 when unauthenticated', async () => { - authMockFns.mockGetSession.mockResolvedValueOnce(null) - const res = await POST(makeRequest(validBody), params()) - expect(res.status).toBe(401) - }) - - it('returns 403 when user lacks write permission', async () => { - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('read') - const res = await POST(makeRequest(validBody), params()) - expect(res.status).toBe(403) - }) - - it('rejects keys belonging to a different workspace', async () => { - const otherWsKey = `workspace/00000000-0000-0000-0000-000000000000/123-abc-video.mp4` - const res = await POST(makeRequest({ ...validBody, key: otherWsKey }), params()) - const body = await res.json() - expect(res.status).toBe(400) - expect(body.error).toContain('does not belong') - expect(mockRegisterUploadedWorkspaceFile).not.toHaveBeenCalled() - }) - - it('returns 400 for empty key/name', async () => { - const res = await POST(makeRequest({ ...validBody, key: '' }), params()) - expect(res.status).toBe(400) - }) - - it('returns 404 when storage object is missing', async () => { - mockRegisterUploadedWorkspaceFile.mockRejectedValueOnce( - new Error('Uploaded object not found in storage') - ) - const res = await POST(makeRequest(validBody), params()) - expect(res.status).toBe(404) - }) - - it('returns 409 on duplicate file conflict', async () => { - mockRegisterUploadedWorkspaceFile.mockRejectedValueOnce(new FileConflictErrorImpl('video.mp4')) - const res = await POST(makeRequest(validBody), params()) - const body = await res.json() - expect(res.status).toBe(409) - expect(body.isDuplicate).toBe(true) - }) - - it('skips audit + analytics on idempotent re-register (created=false)', async () => { - mockRegisterUploadedWorkspaceFile.mockResolvedValueOnce({ - file: { - id: 'wf_123', - name: 'video.mp4', - size: 10 * 1024 * 1024, - type: 'video/mp4', - url: '/api/files/serve/...', - key: VALID_KEY, - context: 'workspace', - }, - created: false, - }) - - const res = await POST(makeRequest(validBody), params()) - expect(res.status).toBe(200) - expect(posthogServerMockFns.mockCaptureServerEvent).not.toHaveBeenCalled() - expect(auditMockFns.mockRecordAudit).not.toHaveBeenCalled() - }) - - it('finalizes upload, records audit and analytics', async () => { - const res = await POST(makeRequest(validBody), params()) - const body = await res.json() - - expect(res.status).toBe(200) - expect(body.success).toBe(true) - expect(body.file).toMatchObject({ id: 'wf_123', key: VALID_KEY }) - - expect(mockRegisterUploadedWorkspaceFile).toHaveBeenCalledWith({ - workspaceId: WS, - userId: 'user-1', - key: VALID_KEY, - originalName: 'video.mp4', - contentType: 'video/mp4', - }) - - expect(posthogServerMockFns.mockCaptureServerEvent).toHaveBeenCalledWith( - 'user-1', - 'file_uploaded', - expect.objectContaining({ workspace_id: WS, file_type: 'video/mp4' }), - expect.any(Object) - ) - expect(auditMockFns.mockRecordAudit).toHaveBeenCalledWith( - expect.objectContaining({ - actorId: 'user-1', - workspaceId: WS, - }) - ) - }) -}) diff --git a/apps/sim/app/api/workspaces/[id]/files/register/route.ts b/apps/sim/app/api/workspaces/[id]/files/register/route.ts deleted file mode 100644 index 4ed0b90c285..00000000000 --- a/apps/sim/app/api/workspaces/[id]/files/register/route.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { type NextRequest, NextResponse } from 'next/server' -import { registerWorkspaceFileContract } from '@/lib/api/contracts/workspace-files' -import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' -import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { captureServerEvent } from '@/lib/posthog/server' -import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify' -import { - FileConflictError, - parseWorkspaceFileKey, - registerUploadedWorkspaceFile, -} from '@/lib/uploads/contexts/workspace' -import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' - -const logger = createLogger('WorkspaceRegisterAPI') - -/** - * POST /api/workspaces/[id]/files/register - * Finalize a direct-to-storage upload by inserting metadata, updating quota, - * and recording an audit log. Validates the storage key belongs to the - * caller's workspace to prevent cross-tenant key smuggling. - */ -export const POST = withRouteHandler( - async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - const userId = session.user.id - - const parsed = await parseRequest(registerWorkspaceFileContract, request, context) - if (!parsed.success) return parsed.response - const { params, body } = parsed.data - const workspaceId = params.id - const { key, name, contentType, folderId } = body - - const permission = await getUserEntityPermissions(userId, 'workspace', workspaceId) - if (permission !== 'admin' && permission !== 'write') { - logger.warn(`User ${userId} lacks write permission for ${workspaceId}`) - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } - - if (parseWorkspaceFileKey(key) !== workspaceId) { - logger.warn(`Key ${key} does not belong to workspace ${workspaceId}`) - return NextResponse.json( - { error: 'Storage key does not belong to this workspace' }, - { status: 400 } - ) - } - - try { - const { file: userFile, created } = await registerUploadedWorkspaceFile({ - workspaceId, - userId, - key, - originalName: name, - contentType, - folderId, - }) - - if (created) { - logger.info(`Registered direct upload ${name} -> ${key}`) - - await notifyWorkspaceFilesChanged(workspaceId) - - captureServerEvent( - userId, - 'file_uploaded', - { workspace_id: workspaceId, file_type: contentType }, - { groups: { workspace: workspaceId } } - ) - - recordAudit({ - workspaceId, - actorId: userId, - actorName: session.user.name, - actorEmail: session.user.email, - action: AuditAction.FILE_UPLOADED, - resourceType: AuditResourceType.FILE, - resourceId: userFile.id, - resourceName: name, - description: `Uploaded file "${name}"`, - metadata: { fileSize: userFile.size, fileType: contentType }, - request, - }) - } else { - logger.info(`Idempotent re-register for existing upload ${name} -> ${key}`) - } - - return NextResponse.json({ success: true, file: userFile }) - } catch (error) { - logger.error('Failed to register workspace file:', error) - - const errorMessage = getErrorMessage(error, 'Failed to register file') - const isDuplicate = - error instanceof FileConflictError || errorMessage.includes('already exists') - const isMissing = errorMessage.includes('not found in storage') - - const status = isDuplicate ? 409 : isMissing ? 404 : 500 - return NextResponse.json({ success: false, error: errorMessage, isDuplicate }, { status }) - } - } -) diff --git a/apps/sim/app/api/workspaces/[id]/files/route.test.ts b/apps/sim/app/api/workspaces/[id]/files/route.test.ts index 9d71894ecfa..7ce4f9f1a4c 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.test.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.test.ts @@ -1,143 +1,253 @@ /** - * Tests for the workspace files upload route's bounded multipart read. - * * @vitest-environment node */ -import { authMockFns, permissionsMock, permissionsMockFns, posthogServerMock } from '@sim/testing' +import { authMockFns } from '@sim/testing' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockUploadWorkspaceFile, mockGetWorkspaceShares, mockRecordAudit } = vi.hoisted(() => ({ - mockUploadWorkspaceFile: vi.fn(), +const { + mockGetUserEntityPermissions, + mockGetWorkspaceShares, + mockListWorkspaceFiles, + mockPerformCreateWorkspaceFile, +} = vi.hoisted(() => ({ + mockGetUserEntityPermissions: vi.fn(), mockGetWorkspaceShares: vi.fn(), - mockRecordAudit: vi.fn(), + mockListWorkspaceFiles: vi.fn(), + mockPerformCreateWorkspaceFile: vi.fn(), })) -vi.mock('@/lib/uploads/contexts/workspace', () => ({ - uploadWorkspaceFile: mockUploadWorkspaceFile, - FileConflictError: class FileConflictError extends Error {}, +vi.mock('@/lib/public-shares/share-manager', () => ({ + getWorkspaceShares: mockGetWorkspaceShares, })) -vi.mock('@/lib/uploads/shared/types', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - MAX_WORKSPACE_FORMDATA_FILE_SIZE: 1024, - } -}) +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + listWorkspaceFiles: mockListWorkspaceFiles, +})) -vi.mock('@/lib/public-shares/share-manager', () => ({ - getWorkspaceShares: mockGetWorkspaceShares, +vi.mock('@/lib/workspace-files/orchestration', () => ({ + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES: 70 * 1024 * 1024, + performCreateWorkspaceFile: mockPerformCreateWorkspaceFile, })) -vi.mock('@/lib/posthog/server', () => posthogServerMock) -vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + getUserEntityPermissions: mockGetUserEntityPermissions, +})) vi.mock('@/app/api/workflows/utils', () => ({ verifyWorkspaceMembership: vi.fn().mockResolvedValue('write'), })) -vi.mock('@sim/audit', () => ({ - recordAudit: mockRecordAudit, - AuditAction: { FILE_UPLOADED: 'file_uploaded' }, - AuditResourceType: { FILE: 'file' }, -})) - -const WS = '7727ef3f-8cf6-4686-b063-2bb006a10785' import { POST } from '@/app/api/workspaces/[id]/files/route' -const routeContext = { params: Promise.resolve({ id: WS }) } - -function buildFormData(file: File): FormData { - const formData = new FormData() - formData.append('file', file) - return formData +const WORKSPACE_ID = '7727ef3f-8cf6-4686-b063-2bb006a10785' +const USER = { id: 'user-1', name: 'Test User', email: 'test@sim.ai' } +const CREATED_FILE = { + id: 'wf_created', + workspaceId: WORKSPACE_ID, + name: 'untitled.md', + key: `workspace/${WORKSPACE_ID}/untitled.md`, + path: '/api/files/serve/untitled.md?context=workspace', + size: 0, + type: 'text/markdown', + uploadedBy: USER.id, + folderId: null, + folderPath: null, + deletedAt: null, + uploadedAt: new Date('2026-08-04T00:00:00.000Z'), + updatedAt: new Date('2026-08-04T00:00:00.000Z'), } -/** - * Builds a pull-based stream that emits fixed-size chunks on demand, so the - * size-capped reader's `reader.cancel()` simply stops future `pull` calls - * instead of racing an external (e.g. undici FormData) chunk producer. - */ -function makeChunkedOverLimitBody( - chunkBytes: number, - chunkCount: number -): ReadableStream { - let emitted = 0 - return new ReadableStream({ - pull(controller) { - if (emitted >= chunkCount) { - controller.close() - return - } - emitted++ - controller.enqueue(new Uint8Array(chunkBytes)) - }, +const routeContext = { params: Promise.resolve({ id: WORKSPACE_ID }) } + +function createRequest(body: unknown): NextRequest { + return new NextRequest(`http://localhost:3000/api/workspaces/${WORKSPACE_ID}/files`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: typeof body === 'string' ? body : JSON.stringify(body), }) } -describe('workspace files upload route', () => { +describe('POST /api/workspaces/[id]/files', () => { beforeEach(() => { vi.clearAllMocks() - authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) - permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') + authMockFns.mockGetSession.mockResolvedValue({ user: USER }) + mockGetUserEntityPermissions.mockResolvedValue('write') mockGetWorkspaceShares.mockResolvedValue(new Map()) - mockUploadWorkspaceFile.mockResolvedValue({ - id: 'file-1', - name: 'file.txt', - url: 'https://example.com/file.txt', - size: 11, - type: 'text/plain', - }) + mockListWorkspaceFiles.mockResolvedValue([]) + mockPerformCreateWorkspaceFile.mockResolvedValue({ success: true, file: CREATED_FILE }) }) - it('rejects a declared content-length above the limit before reading the body', async () => { - const formData = buildFormData(new File(['x'.repeat(10)], 'file.txt', { type: 'text/plain' })) - const req = new NextRequest(`http://localhost:3000/api/workspaces/${WS}/files`, { - method: 'POST', - headers: { 'content-length': String(10 * 1024 * 1024) }, - body: formData, - }) + it('authenticates before parsing an invalid request body', async () => { + authMockFns.mockGetSession.mockResolvedValue(null) + + const response = await POST(createRequest('{not-json'), routeContext) + + expect(response.status).toBe(401) + await expect(response.json()).resolves.toEqual({ error: 'Unauthorized' }) + expect(mockGetUserEntityPermissions).not.toHaveBeenCalled() + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) - const response = await POST(req, routeContext) - const data = await response.json() + it('authorizes the workspace before parsing the request body', async () => { + mockGetUserEntityPermissions.mockResolvedValue('read') - expect(response.status).toBe(413) - expect(data.error).toContain('exceeds maximum size') - expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + const response = await POST(createRequest({ content: 'missing a name' }), routeContext) + + expect(response.status).toBe(403) + await expect(response.json()).resolves.toEqual({ error: 'Insufficient permissions' }) + expect(mockGetUserEntityPermissions).toHaveBeenCalledWith(USER.id, 'workspace', WORKSPACE_ID) + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() }) - it('rejects a chunked body without content-length once the streamed size trips the cap', async () => { - const body = makeChunkedOverLimitBody(64 * 1024, 32) - const req = new NextRequest(`http://localhost:3000/api/workspaces/${WS}/files`, { - method: 'POST', - body, - // @ts-expect-error - duplex is required by undici for streamed bodies but missing from NextRequestInit types - duplex: 'half', + it('rejects an invalid body after workspace authorization', async () => { + const response = await POST(createRequest({ content: 'missing a name' }), routeContext) + const body = await response.json() + + expect(response.status).toBe(400) + expect(body.error).toBe('Validation error') + expect(mockGetUserEntityPermissions).toHaveBeenCalledWith(USER.id, 'workspace', WORKSPACE_ID) + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) + + it.each(['read', null])( + 'requires write or admin permission (%s is rejected)', + async (permission) => { + mockGetUserEntityPermissions.mockResolvedValue(permission) + + const response = await POST(createRequest({ name: 'untitled.md' }), routeContext) + + expect(response.status).toBe(403) + await expect(response.json()).resolves.toEqual({ error: 'Insufficient permissions' }) + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + } + ) + + it.each(['write', 'admin'])( + 'creates an empty file with defaults for %s users', + async (permission) => { + mockGetUserEntityPermissions.mockResolvedValue(permission) + const request = createRequest({ name: 'untitled.md' }) + + const response = await POST(request, routeContext) + const body = await response.json() + + expect(response.status).toBe(201) + expect(body).toMatchObject({ success: true, file: { id: CREATED_FILE.id } }) + expect(mockPerformCreateWorkspaceFile).toHaveBeenCalledTimes(1) + const params = mockPerformCreateWorkspaceFile.mock.calls[0][0] + expect(params).toMatchObject({ + workspaceId: WORKSPACE_ID, + userId: USER.id, + actorName: USER.name, + actorEmail: USER.email, + name: 'untitled.md', + contentType: 'text/markdown', + exactName: false, + }) + expect(params.folderId).toBeUndefined() + expect(params.content).toEqual(Buffer.alloc(0)) + expect(params.request).toBe(request) + } + ) + + it('decodes initialized base64 content and preserves folder and content type', async () => { + const content = Buffer.from([0, 1, 2, 255]) + const request = createRequest({ + name: 'data.bin', + contentType: 'application/octet-stream', + folderId: 'folder-1', + content: content.toString('base64'), + encoding: 'base64', + }) + mockPerformCreateWorkspaceFile.mockResolvedValue({ + success: true, + file: { + ...CREATED_FILE, + name: 'data.bin', + type: 'application/octet-stream', + size: content.length, + folderId: 'folder-1', + }, }) - expect(req.headers.get('content-length')).toBeNull() - const response = await POST(req, routeContext) - const data = await response.json() + const response = await POST(request, routeContext) + + expect(response.status).toBe(201) + expect(mockPerformCreateWorkspaceFile).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: WORKSPACE_ID, + name: 'data.bin', + contentType: 'application/octet-stream', + folderId: 'folder-1', + content, + exactName: false, + }) + ) + }) + + it('rejects malformed base64 after authorization and before orchestration', async () => { + const response = await POST( + createRequest({ name: 'data.bin', content: 'not-base64!', encoding: 'base64' }), + routeContext + ) + + expect(response.status).toBe(400) + await expect(response.json()).resolves.toMatchObject({ error: 'Validation error' }) + expect(mockGetUserEntityPermissions).toHaveBeenCalledWith(USER.id, 'workspace', WORKSPACE_ID) + expect(mockPerformCreateWorkspaceFile).not.toHaveBeenCalled() + }) + + it('accepts empty base64 as a zero-byte file', async () => { + const response = await POST( + createRequest({ name: 'empty.bin', content: '', encoding: 'base64' }), + routeContext + ) - expect(response.status).toBe(413) - expect(data.error).toContain('exceeds maximum size') - expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + expect(response.status).toBe(201) + expect(mockPerformCreateWorkspaceFile).toHaveBeenCalledWith( + expect.objectContaining({ content: Buffer.alloc(0) }) + ) }) - it('uploads a normal, well-under-limit file successfully', async () => { - const file = new File(['hello world'], 'file.txt', { type: 'text/plain' }) - const formData = buildFormData(file) - const req = new NextRequest(`http://localhost:3000/api/workspaces/${WS}/files`, { - method: 'POST', - headers: { 'content-length': '512' }, - body: formData, + it.each([ + ['validation', 400, 'Invalid file name'], + ['not_found', 404, 'Target folder not found'], + ['conflict', 409, 'A file with this name already exists'], + ['payload_too_large', 413, 'File size exceeds 50MB limit'], + ] as const)('maps a %s orchestration failure to %i', async (errorCode, expectedStatus, error) => { + mockPerformCreateWorkspaceFile.mockResolvedValue({ success: false, error, errorCode }) + + const response = await POST(createRequest({ name: 'untitled.md' }), routeContext) + + expect(response.status).toBe(expectedStatus) + await expect(response.json()).resolves.toEqual({ success: false, error }) + }) + + it('does not expose an internal orchestration error', async () => { + mockPerformCreateWorkspaceFile.mockResolvedValue({ + success: false, + error: 'update workspace_files set ... failed', + errorCode: 'internal', + }) + + const response = await POST(createRequest({ name: 'untitled.md' }), routeContext) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Failed to create file', }) + }) - const response = await POST(req, routeContext) - const data = await response.json() + it('maps an unexpected throw to a 500 response', async () => { + mockPerformCreateWorkspaceFile.mockRejectedValue(new Error('storage unavailable')) - expect(response.status).toBe(200) - expect(data.success).toBe(true) - expect(mockUploadWorkspaceFile).toHaveBeenCalledTimes(1) + const response = await POST(createRequest({ name: 'untitled.md' }), routeContext) + + expect(response.status).toBe(500) + await expect(response.json()).resolves.toEqual({ + success: false, + error: 'Failed to create file', + }) }) }) diff --git a/apps/sim/app/api/workspaces/[id]/files/route.ts b/apps/sim/app/api/workspaces/[id]/files/route.ts index b5d1d4f1fdc..9a370fb4f94 100644 --- a/apps/sim/app/api/workspaces/[id]/files/route.ts +++ b/apps/sim/app/api/workspaces/[id]/files/route.ts @@ -1,28 +1,26 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { + createWorkspaceFileContract, listWorkspaceFilesQuerySchema, workspaceFilesParamsSchema, } from '@/lib/api/contracts/workspace-files' -import { getValidationErrorMessage } from '@/lib/api/server' +import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { generateRequestId } from '@/lib/core/utils/request' import { - isPayloadSizeLimitError, - MAX_MULTIPART_OVERHEAD_BYTES, - readFormDataWithLimit, -} from '@/lib/core/utils/stream-limits' + messageForOrchestrationError, + statusForOrchestrationError, +} from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { captureServerEvent } from '@/lib/posthog/server' import { getWorkspaceShares } from '@/lib/public-shares/share-manager' +import { listWorkspaceFiles } from '@/lib/uploads/contexts/workspace' +import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils' import { - FileConflictError, - listWorkspaceFiles, - uploadWorkspaceFile, -} from '@/lib/uploads/contexts/workspace' -import { MAX_WORKSPACE_FORMDATA_FILE_SIZE } from '@/lib/uploads/shared/types' + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + performCreateWorkspaceFile, +} from '@/lib/workspace-files/orchestration' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' import { verifyWorkspaceMembership } from '@/app/api/workflows/utils' @@ -101,19 +99,11 @@ export const GET = withRouteHandler( /** * POST /api/workspaces/[id]/files - * Upload a new file to workspace storage (requires write permission) + * Create an authored workspace file (requires write permission) */ export const POST = withRouteHandler( - async (request: NextRequest, { params }: { params: Promise<{ id: string }> }) => { + async (request: NextRequest, context: { params: Promise<{ id: string }> }) => { const requestId = generateRequestId() - const paramsResult = workspaceFilesParamsSchema.safeParse(await params) - if (!paramsResult.success) { - return NextResponse.json( - { error: getValidationErrorMessage(paramsResult.error, 'Invalid route parameters') }, - { status: 400 } - ) - } - const { id: workspaceId } = paramsResult.data try { const session = await getSession() @@ -121,7 +111,15 @@ export const POST = withRouteHandler( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - // Check workspace permissions (requires write) + const paramsResult = workspaceFilesParamsSchema.safeParse(await context.params) + if (!paramsResult.success) { + return NextResponse.json( + { error: getValidationErrorMessage(paramsResult.error, 'Invalid route parameters') }, + { status: 400 } + ) + } + const { id: workspaceId } = paramsResult.data + const userPermission = await getUserEntityPermissions( session.user.id, 'workspace', @@ -134,93 +132,45 @@ export const POST = withRouteHandler( return NextResponse.json({ error: 'Insufficient permissions' }, { status: 403 }) } - let formData: FormData - try { - formData = await readFormDataWithLimit(request, { - maxBytes: MAX_WORKSPACE_FORMDATA_FILE_SIZE + MAX_MULTIPART_OVERHEAD_BYTES, - label: 'workspace file upload body', - }) - } catch (error) { - if (isPayloadSizeLimitError(error)) { - return NextResponse.json({ error: error.message }, { status: 413 }) - } - return NextResponse.json( - { error: 'Request body must be valid multipart form data' }, - { status: 400 } - ) - } - const rawFile = formData.get('file') - const rawFolderId = formData.get('folderId') - const folderId = - typeof rawFolderId === 'string' && rawFolderId.length > 0 ? rawFolderId : null - - if (!rawFile || !(rawFile instanceof File)) { - return NextResponse.json({ error: 'No file provided' }, { status: 400 }) - } - - const fileName = rawFile.name || 'untitled.md' - - if (rawFile.size > MAX_WORKSPACE_FORMDATA_FILE_SIZE) { - return NextResponse.json( - { - error: `File size exceeds maximum of ${MAX_WORKSPACE_FORMDATA_FILE_SIZE} bytes (${(rawFile.size / (1024 * 1024)).toFixed(2)}MB)`, - }, - { status: 413 } - ) - } - - const buffer = Buffer.from(await rawFile.arrayBuffer()) - - const userFile = await uploadWorkspaceFile( - workspaceId, - session.user.id, - buffer, - fileName, - rawFile.type || 'application/octet-stream', - { folderId } - ) - - logger.info(`[${requestId}] Uploaded workspace file: ${fileName}`) - - captureServerEvent( - session.user.id, - 'file_uploaded', - { workspace_id: workspaceId, file_type: rawFile.type || 'application/octet-stream' }, - { groups: { workspace: workspaceId } } - ) + const parsed = await parseRequest(createWorkspaceFileContract, request, context, { + maxBodyBytes: MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, + }) + if (!parsed.success) return parsed.response + const { name, contentType, folderId, content, encoding } = parsed.data.body - recordAudit({ + const result = await performCreateWorkspaceFile({ workspaceId, - actorId: session.user.id, + userId: session.user.id, + name, + contentType: contentType ?? getMimeTypeFromExtension(getFileExtension(name)), + folderId, + content: Buffer.from(content, encoding), + exactName: false, actorName: session.user.name, actorEmail: session.user.email, - action: AuditAction.FILE_UPLOADED, - resourceType: AuditResourceType.FILE, - resourceId: userFile.id, - resourceName: fileName, - description: `Uploaded file "${fileName}"`, - metadata: { fileSize: rawFile.size, fileType: rawFile.type || 'application/octet-stream' }, request, }) + if (!result.success || !result.file) { + return NextResponse.json( + { + success: false, + error: messageForOrchestrationError(result, 'Failed to create file'), + }, + { status: statusForOrchestrationError(result.errorCode) } + ) + } - return NextResponse.json({ - success: true, - file: userFile, - }) + logger.info(`[${requestId}] Created workspace file: ${result.file.name}`) + return NextResponse.json({ success: true, file: result.file }, { status: 201 }) } catch (error) { - logger.error(`[${requestId}] Error uploading workspace file:`, error) - - const errorMessage = getErrorMessage(error, 'Failed to upload file') - const isDuplicate = - error instanceof FileConflictError || errorMessage.includes('already exists') + logger.error(`[${requestId}] Error creating workspace file:`, error) return NextResponse.json( { success: false, - error: errorMessage, - isDuplicate, + error: 'Failed to create file', }, - { status: isDuplicate ? 409 : 500 } + { status: 500 } ) } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index 736ec48089e..d772c13c800 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -112,6 +112,7 @@ import { type WorkspaceFileFolderApi, } from '@/hooks/queries/workspace-file-folders' import { + useCreateWorkspaceFile, useDeleteWorkspaceFile, useRenameWorkspaceFile, useUploadWorkspaceFile, @@ -256,6 +257,7 @@ export function Files() { return map }, [members]) const uploadFile = useUploadWorkspaceFile() + const createWorkspaceFile = useCreateWorkspaceFile() const notifyLimit = useLimitUpgradeToast() const deleteFile = useDeleteWorkspaceFile() const renameFile = useRenameWorkspaceFile() @@ -1313,15 +1315,13 @@ export function Files() { const name = uniqueMarkdownName(DEFAULT_UNTITLED_NAME, existingNames) const mimeType = getMimeTypeFromExtension('md') - const blob = new Blob([''], { type: mimeType }) - const file = new File([blob], name, { type: mimeType }) - const result = await uploadFile.mutateAsync({ + const result = await createWorkspaceFile.mutateAsync({ workspaceId, - file, - folderId: currentFolderId, - skipToast: true, + name, + contentType: mimeType, + folderId: currentFolderId ?? undefined, }) - const fileId = result.file?.id + const fileId = result.file.id if (fileId) { justCreatedFileIdRef.current = fileId const params = new URLSearchParams({ new: '1' }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx index cfdc62d32af..ba29e66597d 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-documents-modal/add-documents-modal.tsx @@ -15,6 +15,10 @@ import { import { createLogger } from '@sim/logger' import { RotateCcw, X } from 'lucide-react' import { useParams } from 'next/navigation' +import { + assertMultiFileUploadAdmission, + MultiFileUploadAdmissionError, +} from '@/lib/uploads/client/admission' import { formatFileSize, validateKnowledgeBaseFile } from '@/lib/uploads/utils/file-utils' import { ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { useKnowledgeUpload } from '@/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload' @@ -77,11 +81,11 @@ export function AddDocumentsModal({ } const processFiles = (selectedFiles: File[]) => { - setFileError(null) - if (!selectedFiles || selectedFiles.length === 0) return try { + assertMultiFileUploadAdmission(selectedFiles, { existingFiles: files }) + setFileError(null) const newFiles: File[] = [] let hasError = false @@ -100,6 +104,10 @@ export function AddDocumentsModal({ setFiles((prev) => [...prev, ...newFiles]) } } catch (error) { + if (error instanceof MultiFileUploadAdmissionError) { + setFileError(error.message) + return + } logger.error('Error processing files:', error) setFileError('An error occurred while processing files. Please try again.') } @@ -156,7 +164,7 @@ export function AddDocumentsModal({ accept={ACCEPT_ATTRIBUTE} multiple onChange={processFiles} - description='PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, HTML, JSONL (max 100MB each)' + description='PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, HTML, JSONL (max 20 files, 100MB each, 500MB total)' error={fileError} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx index 5364d754044..88c4b5c8603 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx @@ -27,6 +27,10 @@ import { type FieldErrors, useForm } from 'react-hook-form' import { z } from 'zod' import type { StrategyOptions } from '@/lib/chunkers/types' import { KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH } from '@/lib/knowledge/constants' +import { + assertMultiFileUploadAdmission, + MultiFileUploadAdmissionError, +} from '@/lib/uploads/client/admission' import { formatFileSize, validateKnowledgeBaseFile } from '@/lib/uploads/utils/file-utils' import { ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { useKnowledgeUpload } from '@/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload' @@ -202,11 +206,11 @@ export const CreateBaseModal = memo(function CreateBaseModal({ }, [open, reset]) const processFiles = (selectedFiles: File[]) => { - setFileError(null) - if (!selectedFiles || selectedFiles.length === 0) return try { + assertMultiFileUploadAdmission(selectedFiles, { existingFiles: files }) + setFileError(null) const newFiles: File[] = [] let hasError = false @@ -225,6 +229,10 @@ export const CreateBaseModal = memo(function CreateBaseModal({ setFiles((prev) => [...prev, ...newFiles]) } } catch (error) { + if (error instanceof MultiFileUploadAdmissionError) { + setFileError(error.message) + return + } logger.error('Error processing files:', error) setFileError('An error occurred while processing files. Please try again.') } @@ -474,7 +482,7 @@ export const CreateBaseModal = memo(function CreateBaseModal({ accept={ACCEPT_ATTRIBUTE} multiple onChange={processFiles} - description='PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, HTML, JSONL (max 100MB each)' + description='PDF, DOC, DOCX, TXT, CSV, XLS, XLSX, MD, PPT, PPTX, HTML, JSONL (max 20 files, 100MB each, 500MB total)' error={fileError} /> diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx new file mode 100644 index 00000000000..c14cc716e7c --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.test.tsx @@ -0,0 +1,88 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const { mockInvalidateQueries, mockUploadKnowledgeDocumentSession } = vi.hoisted(() => ({ + mockInvalidateQueries: vi.fn(), + mockUploadKnowledgeDocumentSession: vi.fn(), +})) + +vi.mock('@tanstack/react-query', () => ({ + useQueryClient: () => ({ invalidateQueries: mockInvalidateQueries }), +})) + +vi.mock('@/lib/uploads/client/session-upload', () => ({ + uploadKnowledgeDocumentSession: mockUploadKnowledgeDocumentSession, +})) + +import { MULTI_FILE_UPLOAD_MAX_FILE_BYTES } from '@/lib/uploads/client/admission' +import { useKnowledgeUpload } from '@/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload' + +interface HookHarness { + result: () => ReturnType + unmount: () => void +} + +function renderKnowledgeUploadHook(onError: ReturnType): HookHarness { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + const root: Root = createRoot(document.createElement('div')) + let latest: ReturnType + + function Probe() { + latest = useKnowledgeUpload({ workspaceId: 'workspace-1', onError }) + return null + } + + act(() => root.render()) + return { + result: () => latest, + unmount: () => act(() => root.unmount()), + } +} + +function sizedFile(name: string, size: number): File { + const file = new File([], name, { type: 'application/octet-stream' }) + Object.defineProperty(file, 'size', { value: size }) + return file +} + +describe('useKnowledgeUpload admission', () => { + afterEach(() => { + vi.clearAllMocks() + }) + + it('rejects aggregate bytes before allocating upload progress or sessions', async () => { + const onError = vi.fn() + const { result, unmount } = renderKnowledgeUploadHook(onError) + const files = Array.from({ length: 6 }, (_, index) => + sizedFile(`file-${index}.bin`, MULTI_FILE_UPLOAD_MAX_FILE_BYTES) + ) + + await act(async () => { + await expect(result().uploadFiles(files, 'kb-1')).rejects.toMatchObject({ + code: 'UPLOAD_TOTAL_SIZE_EXCEEDED', + }) + }) + + expect(mockUploadKnowledgeDocumentSession).not.toHaveBeenCalled() + expect(mockInvalidateQueries).not.toHaveBeenCalled() + expect(result().isUploading).toBe(false) + expect(result().uploadProgress).toEqual({ + stage: 'idle', + filesCompleted: 0, + totalFiles: 0, + }) + expect(result().uploadError).toMatchObject({ + code: 'UPLOAD_TOTAL_SIZE_EXCEEDED', + message: 'Select files totaling 500 MiB or less.', + }) + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ code: 'UPLOAD_TOTAL_SIZE_EXCEEDED' }) + ) + + unmount() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts index d4270b5ae34..c9cd91416ef 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts @@ -4,11 +4,12 @@ import { getErrorMessage } from '@sim/utils/errors' import { useQueryClient } from '@tanstack/react-query' import type { V2KnowledgeDocumentSummary } from '@/lib/api/contracts/v2/knowledge' import { - runWithConcurrency, - type UploadProgressEvent, - WHOLE_FILE_PARALLEL_UPLOADS, -} from '@/lib/uploads/client/direct-upload' + assertMultiFileUploadAdmission, + MultiFileUploadAdmissionError, +} from '@/lib/uploads/client/admission' +import { runWithConcurrency, WHOLE_FILE_PARALLEL_UPLOADS } from '@/lib/uploads/client/concurrency' import { uploadKnowledgeDocumentSession } from '@/lib/uploads/client/session-upload' +import type { UploadProgressEvent } from '@/lib/uploads/client/types' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' const logger = createLogger('KnowledgeUpload') @@ -69,13 +70,13 @@ class KnowledgeUploadError extends Error { export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { const queryClient = useQueryClient() - const [isUploading, setIsUploading] = useState(false) const [uploadProgress, setUploadProgress] = useState({ stage: 'idle', filesCompleted: 0, totalFiles: 0, }) const [uploadError, setUploadError] = useState(null) + const isUploading = uploadProgress.stage !== 'idle' const updateFileStatus = (fileIndex: number, patch: Partial) => { setUploadProgress((prev) => ({ @@ -195,7 +196,7 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { } try { - setIsUploading(true) + assertMultiFileUploadAdmission(files) setUploadError(null) setUploadProgress({ stage: 'uploading', filesCompleted: 0, totalFiles: files.length }) @@ -217,15 +218,16 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) { const error: UploadError = err instanceof KnowledgeUploadError ? { message: err.message, code: err.code, details: err.details, timestamp: Date.now() } - : err instanceof Error - ? { message: err.message, timestamp: Date.now() } - : { message: 'Unknown error occurred during upload', timestamp: Date.now() } + : err instanceof MultiFileUploadAdmissionError + ? { message: err.message, code: err.code, timestamp: Date.now() } + : err instanceof Error + ? { message: err.message, timestamp: Date.now() } + : { message: 'Unknown error occurred during upload', timestamp: Date.now() } setUploadError(error) options.onError?.(error) throw err } finally { - setIsUploading(false) setUploadProgress({ stage: 'idle', filesCompleted: 0, totalFiles: 0 }) } } diff --git a/apps/sim/app/workspace/[workspaceId]/settings/hooks/use-profile-picture-upload.ts b/apps/sim/app/workspace/[workspaceId]/settings/hooks/use-profile-picture-upload.ts index 09fdbd75eff..879b875d637 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/hooks/use-profile-picture-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/settings/hooks/use-profile-picture-upload.ts @@ -1,8 +1,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { uploadViaApiFallback } from '@/lib/uploads/client/api-fallback' -import { DirectUploadError, runUploadStrategy } from '@/lib/uploads/client/direct-upload' +import { uploadInternalFileSession } from '@/lib/uploads/client/session-upload' const logger = createLogger('ProfilePictureUpload') const MAX_FILE_SIZE = 5 * 1024 * 1024 // 5MB @@ -66,28 +65,25 @@ export function useProfilePictureUpload({ const uploadFileToServer = useCallback( async (file: File): Promise => { - const presignedEndpoint = - context === 'workspace-logos' && workspaceId - ? `/api/files/presigned?type=workspace-logos&workspaceId=${encodeURIComponent(workspaceId)}` - : `/api/files/presigned?type=${context}` - - try { - const result = await runUploadStrategy({ + if (context === 'workspace-logos') { + if (!workspaceId) { + throw new Error('workspaceId is required for workspace logo upload') + } + const result = await uploadInternalFileSession({ + purpose: 'workspace_logo', + workspaceId, file, - workspaceId: workspaceId ?? '', - context, - presignedEndpoint, }) logger.info(`${context} uploaded successfully: ${result.path}`) return result.path - } catch (error) { - if (error instanceof DirectUploadError && error.code === 'FALLBACK_REQUIRED') { - const { path } = await uploadViaApiFallback(file, context, workspaceId) - logger.info(`${context} uploaded successfully via API fallback: ${path}`) - return path - } - throw error } + + const result = await uploadInternalFileSession({ + purpose: 'profile_picture', + file, + }) + logger.info(`${context} uploaded successfully: ${result.path}`) + return result.path }, [context, workspaceId] ) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments.test.tsx new file mode 100644 index 00000000000..9d1a3d26d63 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments.test.tsx @@ -0,0 +1,118 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockToastError, mockUploadInternalFileSession } = vi.hoisted(() => ({ + mockToastError: vi.fn(), + mockUploadInternalFileSession: vi.fn(), +})) + +vi.mock('@sim/emcn', () => ({ toast: { error: mockToastError } })) + +vi.mock('@/lib/uploads/client/session-upload', () => ({ + uploadInternalFileSession: mockUploadInternalFileSession, +})) + +import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' +import { useFileAttachments } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments' + +interface HookHarness { + result: () => ReturnType + unmount: () => void +} + +function renderFileAttachmentsHook(): HookHarness { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + const root: Root = createRoot(document.createElement('div')) + let latest: ReturnType + + function Probe() { + latest = useFileAttachments({ userId: 'user-1', workspaceId: 'workspace-1' }) + return null + } + + act(() => root.render()) + return { + result: () => latest, + unmount: () => act(() => root.unmount()), + } +} + +function sizedFile(name: string, size: number): File { + const file = new File([], name, { type: 'image/png' }) + Object.defineProperty(file, 'size', { value: size }) + return file +} + +function asFileList(files: File[]): FileList { + return Object.assign(files, { item: (index: number) => files[index] ?? null }) +} + +describe('useFileAttachments admission', () => { + const originalCreateObjectUrl = Object.getOwnPropertyDescriptor(URL, 'createObjectURL') + const createObjectUrl = vi.fn() + + beforeEach(() => { + Object.defineProperty(URL, 'createObjectURL', { + configurable: true, + value: createObjectUrl, + }) + }) + + afterEach(() => { + vi.clearAllMocks() + if (originalCreateObjectUrl) { + Object.defineProperty(URL, 'createObjectURL', originalCreateObjectUrl) + } else { + Reflect.deleteProperty(URL, 'createObjectURL') + } + }) + + it('rejects aggregate bytes before previews, placeholders, or sessions are allocated', async () => { + const { result, unmount } = renderFileAttachmentsHook() + const files = asFileList([ + ...Array.from({ length: 5 }, (_, index) => + sizedFile(`large-image-${index}.png`, MAX_WORKSPACE_FILE_SIZE) + ), + sizedFile('extra-image.png', 1), + ]) + + await act(async () => { + await result().processFiles(files) + }) + + expect(mockToastError).toHaveBeenCalledWith("Couldn't add files", { + description: 'Select files totaling 25 GiB or less.', + }) + expect(createObjectUrl).not.toHaveBeenCalled() + expect(mockUploadInternalFileSession).not.toHaveBeenCalled() + expect(result().attachedFiles).toEqual([]) + + unmount() + }) + + it('starts a mothership session for a file above the old FormData limit', async () => { + mockUploadInternalFileSession.mockResolvedValue({ + path: '/api/files/serve/s3/mothership%2Flarge-image.png?context=mothership', + key: 'mothership/large-image.png', + }) + const { result, unmount } = renderFileAttachmentsHook() + const file = sizedFile('large-image.png', 101 * 1024 * 1024) + + await act(async () => { + await result().processFiles(asFileList([file])) + }) + + expect(mockUploadInternalFileSession).toHaveBeenCalledWith( + expect.objectContaining({ purpose: 'mothership_attachment', file }) + ) + expect(result().attachedFiles).toEqual([ + expect.objectContaining({ name: file.name, uploading: false }), + ]) + + unmount() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments.ts index 4c40839d27b..ef68d2bb113 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments.ts @@ -5,8 +5,10 @@ import { toast } from '@sim/emcn' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { uploadViaApiFallback } from '@/lib/uploads/client/api-fallback' -import { DirectUploadError, runUploadStrategy } from '@/lib/uploads/client/direct-upload' +import { assertMultiFileUploadAdmission } from '@/lib/uploads/client/admission' +import { runWithConcurrency, WHOLE_FILE_PARALLEL_UPLOADS } from '@/lib/uploads/client/concurrency' +import { uploadInternalFileSession } from '@/lib/uploads/client/session-upload' +import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' import { resolveFileType } from '@/lib/uploads/utils/file-utils' const logger = createLogger('useFileAttachments') @@ -64,16 +66,26 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { const { userId, workspaceId, disabled, isLoading } = props const [attachedFiles, setAttachedFiles] = useState([]) - const [isDragging, setIsDragging] = useState(false) const [dragCounter, setDragCounter] = useState(0) + const isDragging = dragCounter > 0 const fileInputRef = useRef(null) + const attachedFilesRef = useRef([]) + const uploadControllersRef = useRef(new Map()) + + const updateAttachedFiles = useCallback((update: (files: AttachedFile[]) => AttachedFile[]) => { + const next = update(attachedFilesRef.current) + attachedFilesRef.current = next + setAttachedFiles(next) + }, []) /** * Cleanup preview URLs on unmount */ useEffect(() => { return () => { - attachedFiles.forEach((f) => { + for (const controller of uploadControllersRef.current.values()) controller.abort() + uploadControllersRef.current.clear() + attachedFilesRef.current.forEach((f) => { if (f.previewUrl) { URL.revokeObjectURL(f.previewUrl) } @@ -122,8 +134,18 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { return } + if (fileList.length === 0) return + try { + assertMultiFileUploadAdmission(fileList, { + existingFiles: attachedFilesRef.current, + maxFileBytes: MAX_WORKSPACE_FILE_SIZE, + }) + } catch (error) { + toast.error("Couldn't add files", { description: toError(error).message }) + return + } + const files = Array.from(fileList) - if (files.length === 0) return const placeholders: AttachedFile[] = files.map((file) => ({ id: generateId(), @@ -137,56 +159,48 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { ? URL.createObjectURL(file) : undefined, })) + const controllers = placeholders.map(() => new AbortController()) + placeholders.forEach((placeholder, index) => { + uploadControllersRef.current.set(placeholder.id, controllers[index]) + }) - setAttachedFiles((prev) => [...prev, ...placeholders]) - - const presignedEndpoint = `/api/files/presigned?type=mothership&workspaceId=${encodeURIComponent(workspaceId)}` - - await Promise.all( - files.map(async (file, i) => { - const placeholder = placeholders[i] - try { - let result: { path: string; key: string } - try { - result = await runUploadStrategy({ - file, - workspaceId, - context: 'mothership', - presignedEndpoint, - }) - } catch (error) { - if (error instanceof DirectUploadError && error.code === 'FALLBACK_REQUIRED') { - const fallback = await uploadViaApiFallback(file, 'mothership', workspaceId) - if (!fallback.key) { - throw new Error('Invalid upload response: missing key') - } - result = { path: fallback.path, key: fallback.key } - } else { - throw error - } - } - - logger.info(`File uploaded successfully: ${result.path}`) - - setAttachedFiles((prev) => - prev.map((f) => - f.id === placeholder.id - ? { ...f, path: result.path, key: result.key, uploading: false } - : f - ) + updateAttachedFiles((current) => [...current, ...placeholders]) + + await runWithConcurrency(files, WHOLE_FILE_PARALLEL_UPLOADS, async (file, i) => { + const placeholder = placeholders[i] + const controller = controllers[i] + try { + const result = await uploadInternalFileSession({ + purpose: 'mothership_attachment', + file, + workspaceId, + signal: controller.signal, + }) + + logger.info(`File uploaded successfully: ${result.path}`) + + updateAttachedFiles((current) => + current.map((f) => + f.id === placeholder.id + ? { ...f, path: result.path, key: result.key, uploading: false } + : f ) - } catch (error) { + ) + } catch (error) { + if (!controller.signal.aborted) { logger.error(`File upload failed: ${error}`) toast.error(`Couldn't upload "${file.name}"`, { description: toError(error).message, }) - if (placeholder.previewUrl) URL.revokeObjectURL(placeholder.previewUrl) - setAttachedFiles((prev) => prev.filter((f) => f.id !== placeholder.id)) } - }) - ) + if (placeholder.previewUrl) URL.revokeObjectURL(placeholder.previewUrl) + updateAttachedFiles((current) => current.filter((file) => file.id !== placeholder.id)) + } finally { + uploadControllersRef.current.delete(placeholder.id) + } + }) }, - [userId, workspaceId] + [userId, workspaceId, updateAttachedFiles] ) /** @@ -222,13 +236,15 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { */ const removeFile = useCallback( (fileId: string) => { - const file = attachedFiles.find((f) => f.id === fileId) + uploadControllersRef.current.get(fileId)?.abort() + uploadControllersRef.current.delete(fileId) + const file = attachedFilesRef.current.find((f) => f.id === fileId) if (file?.previewUrl) { URL.revokeObjectURL(file.previewUrl) } - setAttachedFiles((prev) => prev.filter((f) => f.id !== fileId)) + updateAttachedFiles((current) => current.filter((file) => file.id !== fileId)) }, - [attachedFiles] + [updateAttachedFiles] ) /** @@ -249,13 +265,7 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { const handleDragEnter = useCallback((e: React.DragEvent) => { e.preventDefault() e.stopPropagation() - setDragCounter((prev) => { - const newCount = prev + 1 - if (newCount === 1) { - setIsDragging(true) - } - return newCount - }) + setDragCounter((prev) => prev + 1) }, []) /** @@ -264,13 +274,7 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { const handleDragLeave = useCallback((e: React.DragEvent) => { e.preventDefault() e.stopPropagation() - setDragCounter((prev) => { - const newCount = prev - 1 - if (newCount === 0) { - setIsDragging(false) - } - return newCount - }) + setDragCounter((prev) => Math.max(0, prev - 1)) }, []) /** @@ -289,7 +293,6 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { async (e: React.DragEvent) => { e.preventDefault() e.stopPropagation() - setIsDragging(false) setDragCounter(0) if (e.dataTransfer.files && e.dataTransfer.files.length > 0) { @@ -303,26 +306,33 @@ export function useFileAttachments(props: UseFileAttachmentsProps) { * Clears all attached files and cleanup preview URLs */ const clearAttachedFiles = useCallback(() => { - attachedFiles.forEach((f) => { + for (const controller of uploadControllersRef.current.values()) controller.abort() + uploadControllersRef.current.clear() + attachedFilesRef.current.forEach((f) => { if (f.previewUrl) { URL.revokeObjectURL(f.previewUrl) } }) - setAttachedFiles([]) - }, [attachedFiles]) + updateAttachedFiles(() => []) + }, [updateAttachedFiles]) /** * Replaces the current attached files with a given set. * Cleans up preview URLs from the prior set before replacing. */ - const restoreAttachedFiles = useCallback((files: AttachedFile[]) => { - setAttachedFiles((prev) => { - prev.forEach((f) => { - if (f.previewUrl) URL.revokeObjectURL(f.previewUrl) + const restoreAttachedFiles = useCallback( + (files: AttachedFile[]) => { + for (const controller of uploadControllersRef.current.values()) controller.abort() + uploadControllersRef.current.clear() + updateAttachedFiles((current) => { + current.forEach((f) => { + if (f.previewUrl) URL.revokeObjectURL(f.previewUrl) + }) + return files }) - return files - }) - }, []) + }, + [updateAttachedFiles] + ) return { // State diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx index 3ed0c144334..3ba4638306d 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.test.tsx @@ -7,28 +7,17 @@ import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { - DirectUploadErrorMock, executionStoreState, mockExecute, mockExecuteFromBlock, mockFetch, mockResolveStartCandidates, - mockRunUploadStrategy, mockSelectBestTrigger, + mockUploadInternalFileSession, terminalStoreState, workflowBlocks, workflowStoreState, } = vi.hoisted(() => { - class DirectUploadErrorMock extends Error { - constructor( - message: string, - public code: string - ) { - super(message) - this.name = 'DirectUploadError' - } - } - const workflowBlocks = { start: { id: 'start', @@ -88,14 +77,13 @@ const { } return { - DirectUploadErrorMock, executionStoreState, mockExecute: vi.fn(), mockExecuteFromBlock: vi.fn(), mockFetch: vi.fn(), mockResolveStartCandidates: vi.fn(), - mockRunUploadStrategy: vi.fn(), mockSelectBestTrigger: vi.fn(), + mockUploadInternalFileSession: vi.fn(), terminalStoreState, workflowBlocks, workflowStoreState, @@ -127,9 +115,8 @@ vi.mock('@/lib/tokenization', () => ({ processStreamingBlockLogs: () => 0, })) -vi.mock('@/lib/uploads/client/direct-upload', () => ({ - DirectUploadError: DirectUploadErrorMock, - runUploadStrategy: mockRunUploadStrategy, +vi.mock('@/lib/uploads/client/session-upload', () => ({ + uploadInternalFileSession: mockUploadInternalFileSession, })) vi.mock('@/lib/workflows/input-format', () => ({ @@ -354,8 +341,8 @@ describe('useWorkflowExecution attachment uploads', () => { mockResolveStartCandidates.mockReturnValue([]) mockSelectBestTrigger.mockReturnValue([]) vi.stubGlobal('fetch', mockFetch) - mockRunUploadStrategy.mockRejectedValue( - new DirectUploadErrorMock('Server signaled fallback to API upload', 'FALLBACK_REQUIRED') + mockUploadInternalFileSession.mockRejectedValue( + new Error('Workspace file storage limit exceeded') ) mockFetch.mockResolvedValue( new Response(JSON.stringify({ error: 'Workspace file storage limit exceeded' }), { @@ -378,12 +365,14 @@ describe('useWorkflowExecution attachment uploads', () => { const file = new File(['report'], 'report.pdf', { type: 'application/pdf' }) let uploadError: unknown - mockRunUploadStrategy.mockResolvedValueOnce({ + mockUploadInternalFileSession.mockResolvedValueOnce({ + id: 'attachment-context', key: 'executions/context.txt', - path: '/uploads/context.txt', + url: '/uploads/context.txt', name: contextFile.name, size: contextFile.size, - contentType: contextFile.type, + type: contextFile.type, + context: 'execution', }) await act(async () => { @@ -437,12 +426,14 @@ describe('useWorkflowExecution attachment uploads', () => { } let runResult: unknown - mockRunUploadStrategy.mockResolvedValueOnce({ + mockUploadInternalFileSession.mockResolvedValueOnce({ + id: 'attachment-diagram', key: 'execution/diagram.png', - path: '/api/files/serve/execution%2Fdiagram.png', + url: '/api/files/serve/execution%2Fdiagram.png', name: file.name, size: file.size, - contentType: file.type, + type: file.type, + context: 'execution', }) await act(async () => { diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-attachment-upload.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-attachment-upload.ts index 938366c71bb..48a77406ed5 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-attachment-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-attachment-upload.ts @@ -1,11 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' -import { generateShortId } from '@sim/utils/id' -import { isRecordLike } from '@sim/utils/object' -import { - type ApiFallbackUploadMetadata, - uploadViaApiFallbackWithMetadata, -} from '@/lib/uploads/client/api-fallback' -import { DirectUploadError, runUploadStrategy } from '@/lib/uploads/client/direct-upload' +import { uploadInternalFileSession } from '@/lib/uploads/client/session-upload' export interface WorkflowAttachmentInput { name: string @@ -33,39 +27,6 @@ interface UploadWorkflowAttachmentsParams { executionId: string } -function getOptionalString(value: unknown): string | undefined { - if (typeof value !== 'string') return undefined - const trimmed = value.trim() - return trimmed || undefined -} - -function getDirectUploadFailureReason(error: unknown): string { - if (error instanceof DirectUploadError && isRecordLike(error.details)) { - const message = - getOptionalString(error.details.message) ?? getOptionalString(error.details.error) - if (message) return message - } - - return getErrorMessage(error, 'Unknown upload error') -} - -function normalizeFallbackUpload( - value: ApiFallbackUploadMetadata, - fallbackFile: WorkflowAttachmentInput -): UploadedWorkflowAttachment { - return { - id: value.id ?? `file_${Date.now()}_${generateShortId(7)}`, - name: value.name ?? fallbackFile.name, - url: value.path, - size: typeof value.size === 'number' ? value.size : fallbackFile.size, - type: value.type ?? fallbackFile.type, - key: value.key, - context: 'execution', - uploadedAt: value.uploadedAt, - expiresAt: value.expiresAt, - } -} - /** * Uploads every explicit workflow attachment before execution may begin. * @@ -78,46 +39,21 @@ export async function uploadWorkflowAttachments({ executionId, }: UploadWorkflowAttachmentsParams): Promise { const uploadedFiles: UploadedWorkflowAttachment[] = [] - const presignedEndpoint = `/api/files/presigned?type=execution&workflowId=${encodeURIComponent(workflowId)}&executionId=${encodeURIComponent(executionId)}&workspaceId=${encodeURIComponent(workspaceId)}` for (const fileData of files) { try { - const result = await runUploadStrategy({ + const result = await uploadInternalFileSession({ + purpose: 'execution_attachment', file: fileData.file, workspaceId, - context: 'execution', workflowId, executionId, - presignedEndpoint, - }) - uploadedFiles.push({ - id: `file_${Date.now()}_${generateShortId(7)}`, - name: fileData.file.name, - url: result.path, - size: fileData.file.size, - type: fileData.file.type, - key: result.key, - context: 'execution', }) + uploadedFiles.push(result) } catch (uploadError) { - if (!(uploadError instanceof DirectUploadError) || uploadError.code !== 'FALLBACK_REQUIRED') { - throw new Error( - `Failed to upload ${fileData.name}: ${getDirectUploadFailureReason(uploadError)}` - ) - } - - try { - const fallbackResult = await uploadViaApiFallbackWithMetadata(fileData.file, 'execution', { - workflowId, - executionId, - workspaceId, - }) - uploadedFiles.push(normalizeFallbackUpload(fallbackResult, fileData)) - } catch (error) { - throw new Error( - `Failed to upload ${fileData.name}: ${getErrorMessage(error, 'Network error')}` - ) - } + throw new Error( + `Failed to upload ${fileData.name}: ${getErrorMessage(uploadError, 'Network error')}` + ) } } diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/hooks/use-workspace-logo-upload.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/hooks/use-workspace-logo-upload.ts index 0d589d24996..a902e2f18bb 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/hooks/use-workspace-logo-upload.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/hooks/use-workspace-logo-upload.ts @@ -1,8 +1,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { uploadViaApiFallback } from '@/lib/uploads/client/api-fallback' -import { DirectUploadError, runUploadStrategy } from '@/lib/uploads/client/direct-upload' +import { uploadInternalFileSession } from '@/lib/uploads/client/session-upload' const logger = createLogger('WorkspaceLogoUpload') const MAX_FILE_SIZE = 5 * 1024 * 1024 // 5MB @@ -68,25 +67,13 @@ export function useWorkspaceLogoUpload({ throw new Error('workspaceId is required for workspace logo upload') } - const presignedEndpoint = `/api/files/presigned?type=workspace-logos&workspaceId=${encodeURIComponent(targetWorkspaceId)}` - - try { - const result = await runUploadStrategy({ - file, - workspaceId: targetWorkspaceId, - context: 'workspace-logos', - presignedEndpoint, - }) - logger.info(`Workspace logo uploaded successfully: ${result.path}`) - return result.path - } catch (error) { - if (error instanceof DirectUploadError && error.code === 'FALLBACK_REQUIRED') { - const { path } = await uploadViaApiFallback(file, 'workspace-logos', targetWorkspaceId) - logger.info(`Workspace logo uploaded via API fallback: ${path}`) - return path - } - throw error - } + const result = await uploadInternalFileSession({ + purpose: 'workspace_logo', + file, + workspaceId: targetWorkspaceId, + }) + logger.info(`Workspace logo uploaded successfully: ${result.path}`) + return result.path }, []) const processFile = useCallback( diff --git a/apps/sim/background/cleanup-soft-deletes.ts b/apps/sim/background/cleanup-soft-deletes.ts index 3227d82e92c..0dc4b025fea 100644 --- a/apps/sim/background/cleanup-soft-deletes.ts +++ b/apps/sim/background/cleanup-soft-deletes.ts @@ -702,9 +702,9 @@ const CLEANUP_TARGETS = [ ] as const /** - * Sweep abandoned knowledge-base ownership bindings. Presigned and multipart upload flows - * write a `workspace_files` binding before the object is stored and before any document is - * created. If the upload is never completed, that binding is orphaned — no + * Sweep abandoned knowledge-base ownership bindings. Knowledge upload sessions write a + * `workspace_files` binding before the object is stored and before any document is created. + * If the upload is never completed, that binding is orphaned — no * `document.storageKey` ever references its key. Such bindings are inert (read access requires * a live document, and the move re-point only follows referenced keys), but they accumulate, * so we drop the best-effort object and soft-delete the binding once they are older than the diff --git a/apps/sim/hooks/queries/tables.ts b/apps/sim/hooks/queries/tables.ts index f519258ad33..471e1620a93 100644 --- a/apps/sim/hooks/queries/tables.ts +++ b/apps/sim/hooks/queries/tables.ts @@ -87,6 +87,7 @@ import { updateWorkflowGroupContract, } from '@/lib/api/contracts/tables' import type { V2TableImportSource, V2TableImportTarget } from '@/lib/api/contracts/v2/tables' +import type { V2CompleteUploadBody } from '@/lib/api/contracts/v2/uploads' import { buildUpgradeHref } from '@/lib/billing/upgrade-reasons' import type { CsvHeaderMapping, @@ -111,7 +112,8 @@ import { optimisticallyScheduleNewlyEligibleGroups, } from '@/lib/table/deps' import { sanitizeName } from '@/lib/table/import' -import { uploadMultipartSession } from '@/lib/uploads/client/multipart-session' +import type { UploadProgressEvent } from '@/lib/uploads/client/types' +import { uploadFileSession } from '@/lib/uploads/client/upload-session' import { useTimezone } from '@/hooks/queries/general-settings' import { TABLE_LIST_STALE_TIME, @@ -1756,46 +1758,49 @@ async function createAndUploadTableImport(params: { timezone: params.timezone, }, }) - params.onCreated?.(created.data.id) - if (params.source.type === 'workspace_file') return created.data - if (!params.file || !created.data.upload) { + const { session, uploadToken, transfer } = created.data + params.onCreated?.(session.id) + if (params.source.type === 'workspace_file') return session + if (!params.file || !uploadToken || !transfer) { throw new Error('Upload-backed table import returned no upload session') } - const upload = created.data.upload - return uploadMultipartSession({ + const getPartUrls = async (partNumbers: number[]) => { + const response = await requestJson(createTableImportPartUrlsContract, { + params: { importId: session.id }, + query: { workspaceId: params.workspaceId }, + headers: { 'upload-token': uploadToken }, + body: { partNumbers }, + }) + return response.data.parts + } + const common = { file: params.file, - partSize: upload.partSize, - partCount: upload.partCount, - onProgress: params.onProgress ? (event) => params.onProgress?.(event.percent) : undefined, - getPartUrls: async (partNumbers) => { - const response = await requestJson(createTableImportPartUrlsContract, { - params: { importId: created.data.id }, - query: { workspaceId: params.workspaceId }, - headers: { 'upload-token': upload.uploadToken }, - body: { partNumbers }, - }) - return response.data.parts - }, - complete: async (parts) => { + onProgress: params.onProgress + ? (event: UploadProgressEvent) => params.onProgress?.(event.percent) + : undefined, + complete: async (body: V2CompleteUploadBody) => { const response = await requestJson(completeTableImportResourceContract, { - params: { importId: created.data.id }, + params: { importId: session.id }, query: { workspaceId: params.workspaceId }, - headers: { 'upload-token': upload.uploadToken }, - body: { parts }, + headers: { 'upload-token': uploadToken }, + body, }) return response.data }, abort: async () => { await requestJson(cancelTableImportResourceContract, { - params: { importId: created.data.id }, + params: { importId: session.id }, query: { workspaceId: params.workspaceId }, - headers: { 'upload-token': upload.uploadToken }, + headers: { 'upload-token': uploadToken }, }) }, - }) + } + return transfer.method === 'put' + ? uploadFileSession({ ...common, transfer }) + : uploadFileSession({ ...common, transfer, getPartUrls }) } -/** Uploads a CSV/TSV through a signed multipart session and creates a table from it. */ +/** Uploads a CSV/TSV through a signed upload session and creates a table from it. */ export function useImportCsv() { const queryClient = useQueryClient() const timezone = useTimezone() diff --git a/apps/sim/hooks/queries/workspace-files.test.tsx b/apps/sim/hooks/queries/workspace-files.test.tsx index db51e9fc452..64afac2dd03 100644 --- a/apps/sim/hooks/queries/workspace-files.test.tsx +++ b/apps/sim/hooks/queries/workspace-files.test.tsx @@ -12,11 +12,21 @@ import { sleep } from '@sim/utils/helpers' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { useWorkspaceFileContent } from '@/hooks/queries/workspace-files' +import { createWorkspaceFileContract } from '@/lib/api/contracts/workspace-files' +import { + useCreateWorkspaceFile, + useWorkspaceFileContent, + workspaceFilesKeys, +} from '@/hooks/queries/workspace-files' + +const { mockRequestJson } = vi.hoisted(() => ({ mockRequestJson: vi.fn() })) + +vi.mock('@/lib/api/client/request', () => ({ requestJson: mockRequestJson })) let fetchCount = 0 beforeEach(() => { + vi.clearAllMocks() ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true fetchCount = 0 vi.stubGlobal( @@ -63,6 +73,42 @@ function renderContentHook(options?: { } } +function renderCreateHook(): { + getMutation: () => ReturnType + queryClient: QueryClient + unmount: () => void +} { + const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } }) + const container = document.createElement('div') + const root: Root = createRoot(container) + let mutation: ReturnType | undefined + + function Probe() { + mutation = useCreateWorkspaceFile() + return null + } + + act(() => { + root.render( + + + + ) + }) + + return { + getMutation: () => { + if (!mutation) throw new Error('Create mutation did not render') + return mutation + }, + queryClient, + unmount: () => { + act(() => root.unmount()) + queryClient.clear() + }, + } +} + describe('useWorkspaceFileContent refetchInterval passthrough', () => { it('fetches once and does not poll by default', async () => { const { unmount } = renderContentHook() @@ -102,3 +148,34 @@ describe('useWorkspaceFileContent refetchInterval passthrough', () => { unmount() }) }) + +describe('useCreateWorkspaceFile', () => { + it('uses the create contract and reconciles workspace file caches', async () => { + const response = { success: true, file: { id: 'wf-created' } } + mockRequestJson.mockResolvedValue(response) + const { getMutation, queryClient, unmount } = renderCreateHook() + const invalidateQueries = vi.spyOn(queryClient, 'invalidateQueries') + let result: unknown + + await act(async () => { + result = await getMutation().mutateAsync({ + workspaceId: 'ws-1', + name: 'notes.md', + contentType: 'text/markdown', + }) + }) + + expect(result).toBe(response) + expect(mockRequestJson).toHaveBeenCalledWith(createWorkspaceFileContract, { + params: { id: 'ws-1' }, + body: { name: 'notes.md', contentType: 'text/markdown' }, + }) + expect(invalidateQueries).toHaveBeenCalledWith({ + queryKey: workspaceFilesKeys.workspaceLists('ws-1'), + }) + expect(invalidateQueries).toHaveBeenCalledWith({ + queryKey: workspaceFilesKeys.storageInfo(), + }) + unmount() + }) +}) diff --git a/apps/sim/hooks/queries/workspace-files.ts b/apps/sim/hooks/queries/workspace-files.ts index 922236c4918..0c5d0f09cbf 100644 --- a/apps/sim/hooks/queries/workspace-files.ts +++ b/apps/sim/hooks/queries/workspace-files.ts @@ -8,14 +8,16 @@ import { requestJson } from '@/lib/api/client/request' import { fileStorageStatusContract } from '@/lib/api/contracts/storage-transfer' import { getUsageLimitsContract } from '@/lib/api/contracts/usage-limits' import { + type CreateWorkspaceFileBody, + createWorkspaceFileContract, deleteWorkspaceFileContract, listWorkspaceFilesContract, renameWorkspaceFileContract, restoreWorkspaceFileContract, updateWorkspaceFileContentContract, } from '@/lib/api/contracts/workspace-files' -import type { UploadProgressEvent } from '@/lib/uploads/client/direct-upload' import { uploadWorkspaceFileSession } from '@/lib/uploads/client/session-upload' +import type { UploadProgressEvent } from '@/lib/uploads/client/types' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import type { UserFile } from '@/executor/types' import { useFileContentSource } from '@/hooks/use-file-content-source' @@ -387,6 +389,31 @@ export function useUploadWorkspaceFile() { }) } +type CreateWorkspaceFileParams = CreateWorkspaceFileBody & { + workspaceId: string +} + +export function useCreateWorkspaceFile() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async ({ workspaceId, ...body }: CreateWorkspaceFileParams) => + requestJson(createWorkspaceFileContract, { + params: { id: workspaceId }, + body, + }), + onSettled: (_data, _error, variables) => { + queryClient.invalidateQueries({ + queryKey: workspaceFilesKeys.workspaceLists(variables.workspaceId), + }) + queryClient.invalidateQueries({ queryKey: workspaceFilesKeys.storageInfo() }) + }, + onError: (error) => { + logger.error('Failed to create file:', error) + }, + }) +} + /** * Update workspace file content mutation */ diff --git a/apps/sim/lib/api/contracts/file-uploads.ts b/apps/sim/lib/api/contracts/file-uploads.ts deleted file mode 100644 index 10a1868ef6c..00000000000 --- a/apps/sim/lib/api/contracts/file-uploads.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts/types' - -export const fileUploadTypeSchema = z.enum([ - 'knowledge-base', - 'chat', - 'copilot', - 'profile-pictures', -]) - -export const fileUploadTypeQuerySchema = z.object({ - type: fileUploadTypeSchema, -}) - -export const presignedUploadBodySchema = z - .object({ - fileName: z.string().optional(), - contentType: z.string().optional(), - fileSize: z.number().optional(), - userId: z.string().optional(), - chatId: z.string().optional(), - }) - .passthrough() - -export const batchPresignedUploadBodySchema = z - .object({ - files: z - .array( - z - .object({ - fileName: z.string().optional(), - contentType: z.string().optional(), - fileSize: z.number().optional(), - }) - .passthrough() - ) - .optional(), - }) - .passthrough() - -export const presignedFileInfoSchema = z - .object({ - path: z.string(), - key: z.string(), - name: z.string(), - size: z.number(), - type: z.string(), - }) - .passthrough() - -export const presignedUploadResponseSchema = z - .object({ - fileName: z.string(), - presignedUrl: z.string(), - fileInfo: presignedFileInfoSchema, - uploadHeaders: z.record(z.string(), z.string()).optional(), - directUploadSupported: z.boolean(), - }) - .passthrough() - -export const batchPresignedUploadResponseSchema = z - .object({ - files: z.array(presignedUploadResponseSchema), - directUploadSupported: z.boolean(), - }) - .passthrough() - -export const createPresignedUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/presigned', - query: fileUploadTypeQuerySchema, - body: presignedUploadBodySchema, - response: { - mode: 'json', - schema: presignedUploadResponseSchema, - }, -}) - -export const createBatchPresignedUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/presigned/batch', - query: fileUploadTypeQuerySchema, - body: batchPresignedUploadBodySchema, - response: { - mode: 'json', - schema: batchPresignedUploadResponseSchema, - }, -}) diff --git a/apps/sim/lib/api/contracts/index.ts b/apps/sim/lib/api/contracts/index.ts index 10ad693347c..1b74fdb1567 100644 --- a/apps/sim/lib/api/contracts/index.ts +++ b/apps/sim/lib/api/contracts/index.ts @@ -12,7 +12,6 @@ export * from './desktop-auth' export * from './desktop-tool-authorization' export * from './environment' export * from './execution-payloads' -export * from './file-uploads' export * from './folders' export * from './hotspots' export * from './inbox' diff --git a/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts b/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts index 7a15defe3fd..805b40a6936 100644 --- a/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts +++ b/apps/sim/lib/api/contracts/knowledge/upload-sessions.ts @@ -1,6 +1,7 @@ import { defineRouteContract } from '@/lib/api/contracts/types' import { v2CreateKnowledgeDocumentUploadBodySchema, + v2CreateKnowledgeDocumentUploadDataSchema, v2KnowledgeDocumentUploadParamsSchema, v2KnowledgeDocumentUploadSchema, v2UploadKnowledgeDocumentQuerySchema, @@ -18,7 +19,10 @@ export const createKnowledgeDocumentUploadContract = defineRouteContract({ path: '/api/knowledge/[id]/documents/uploads', params: v2KnowledgeDocumentUploadParamsSchema.omit({ uploadId: true }), body: v2CreateKnowledgeDocumentUploadBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2KnowledgeDocumentUploadSchema) }, + response: { + mode: 'json', + schema: v2DataResponse(v2CreateKnowledgeDocumentUploadDataSchema), + }, }) export const abortKnowledgeDocumentUploadContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/primitives.test.ts b/apps/sim/lib/api/contracts/primitives.test.ts index 204e6e9d2fe..fe8ad146961 100644 --- a/apps/sim/lib/api/contracts/primitives.test.ts +++ b/apps/sim/lib/api/contracts/primitives.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from 'vitest' import { customPatternSchema, + isCanonicalBase64, organizationIdSchema, piiStagePolicySchema, piiStagesSchema, @@ -12,6 +13,19 @@ import { workspaceIdSchema, } from '@/lib/api/contracts/primitives' +describe('isCanonicalBase64', () => { + it.each(['', 'TQ==', 'TWE=', 'TWFu', 'AAEC/w=='])('accepts canonical base64 %j', (value) => { + expect(isCanonicalBase64(value)).toBe(true) + }) + + it.each(['TQ', 'TQ=', 'TQ===', 'T=Q=', 'TQ==\n', 'TR==', 'TWF='])( + 'rejects malformed or non-canonical base64 %j', + (value) => { + expect(isCanonicalBase64(value)).toBe(false) + } + ) +}) + describe('customPatternSchema', () => { it('accepts a well-formed pattern', () => { expect( diff --git a/apps/sim/lib/api/contracts/primitives.ts b/apps/sim/lib/api/contracts/primitives.ts index f899a27174f..142e0990a54 100644 --- a/apps/sim/lib/api/contracts/primitives.ts +++ b/apps/sim/lib/api/contracts/primitives.ts @@ -37,6 +37,50 @@ export function flattenFieldErrors( export const noInputSchema = z.object({}).strict() export type NoInput = z.output +/** + * Accepts canonical RFC 4648 base64, including the empty encoding used for a + * zero-byte file. Padding is required when the final quantum is incomplete, + * and non-zero unused pad bits are rejected. + */ +export function isCanonicalBase64(value: string): boolean { + if (value.length === 0) return true + if (value.length % 4 !== 0) return false + + let contentLength = value.length + while (contentLength > 0 && value.charCodeAt(contentLength - 1) === 61) { + contentLength -= 1 + } + + const paddingLength = value.length - contentLength + if (paddingLength > 2) return false + if (paddingLength === 1 && contentLength % 4 !== 3) return false + if (paddingLength === 2 && contentLength % 4 !== 2) return false + + let finalSextet = 0 + for (let index = 0; index < contentLength; index += 1) { + const code = value.charCodeAt(index) + const sextet = + code >= 65 && code <= 90 + ? code - 65 + : code >= 97 && code <= 122 + ? code - 71 + : code >= 48 && code <= 57 + ? code + 4 + : code === 43 + ? 62 + : code === 47 + ? 63 + : -1 + + if (sextet === -1) return false + finalSextet = sextet + } + + if (paddingLength === 1 && (finalSextet & 0b11) !== 0) return false + if (paddingLength === 2 && (finalSextet & 0b1111) !== 0) return false + return true +} + export const jobIdParamsSchema = z.object({ jobId: z.string().min(1), }) diff --git a/apps/sim/lib/api/contracts/storage-transfer.ts b/apps/sim/lib/api/contracts/storage-transfer.ts index b5ad82d362c..aaa0a54f5ba 100644 --- a/apps/sim/lib/api/contracts/storage-transfer.ts +++ b/apps/sim/lib/api/contracts/storage-transfer.ts @@ -1,8 +1,4 @@ import { z } from 'zod' -import { - batchPresignedUploadResponseSchema, - presignedUploadResponseSchema, -} from '@/lib/api/contracts/file-uploads' import { workspaceFileIdSchema } from '@/lib/api/contracts/primitives' import { type ContractBodyInput, @@ -19,56 +15,6 @@ import { const jsonResponseSchema = z.unknown() -function formatFileSize(bytes: number): string { - if (!Number.isFinite(bytes) || bytes <= 0) return '0 Bytes' - const k = 1024 - const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'] - const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1) - const value = bytes / k ** i - return `${value.toFixed(value >= 100 || i === 0 ? 0 : 1)} ${sizes[i]}` -} - -const multipartPartUrlSchema = z.object({ - partNumber: z.number(), - url: z.string(), - blockId: z.string().optional(), -}) - -const multipartCompletedUploadSchema = z.object({ - success: z.literal(true), - location: z.string(), - path: z.string(), - key: z.string(), -}) - -export const initiateMultipartResponseSchema = z.object({ - uploadId: z.string(), - key: z.string(), - uploadToken: z.string(), -}) - -export const getMultipartPartUrlsResponseSchema = z.object({ - presignedUrls: z.array(multipartPartUrlSchema), -}) - -export const completeMultipartResponseSchema = z.union([ - multipartCompletedUploadSchema, - z.object({ - results: z.array(multipartCompletedUploadSchema), - }), -]) - -export const abortMultipartResponseSchema = z.object({ - success: z.literal(true), -}) - -export const multipartUploadResponseSchema = z.union([ - initiateMultipartResponseSchema, - getMultipartPartUrlsResponseSchema, - completeMultipartResponseSchema, - abortMultipartResponseSchema, -]) - const connectionFields = { host: z.string().min(1, 'Host is required'), port: z.coerce.number().int().positive().default(22), @@ -325,168 +271,6 @@ export const fileDeleteBodySchema = z }) .passthrough() -const MAX_FILE_SIZE = 100 * 1024 * 1024 -export const validUploadTypes = [ - 'knowledge-base', - 'chat', - 'copilot', - 'profile-pictures', - 'mothership', - 'workspace-logos', - 'execution', -] as const - -export const uploadTypeSchema = z.enum(validUploadTypes) - -/** - * Storage contexts a client may mint a single presigned upload URL for. Each one - * has a per-context authorization predicate in `/api/files/presigned`; a context - * that cannot be authorized must not be listed here. `chat` is deliberately - * absent — it has no owning entity to authorize against and no client that mints - * one (chat assets go through the server-proxied `/api/files/upload`). - */ -export const presignedUploadTypes = [ - 'knowledge-base', - 'copilot', - 'profile-pictures', - 'mothership', - 'workspace-logos', - 'execution', -] as const - -export const presignedUploadTypeSchema = z.enum(presignedUploadTypes) - -/** - * Storage contexts `/api/files/presigned/batch` serves. Batching exists only for - * knowledge-base ingest; no other context has a batch client, and the batch - * endpoint carries no authorization predicate for one. - */ -export const batchPresignedUploadTypes = ['knowledge-base'] as const - -export const batchPresignedUploadTypeSchema = z.enum(batchPresignedUploadTypes) - -export const presignedUploadQuerySchema = z.object({ - type: presignedUploadTypeSchema, -}) - -export const presignedUrlBodySchema = z - .object({ - fileName: z - .string({ error: 'fileName is required and cannot be empty' }) - .refine((value) => value.trim().length > 0, { - message: 'fileName is required and cannot be empty', - }), - contentType: z - .string({ error: 'contentType is required and cannot be empty' }) - .refine((value) => value.trim().length > 0, { - message: 'contentType is required and cannot be empty', - }), - fileSize: z - .number({ error: 'fileSize must be a positive number' }) - .positive('fileSize must be a positive number') - .superRefine((val, ctx) => { - if (val > MAX_FILE_SIZE) { - ctx.addIssue({ - code: 'custom', - message: `File size ${formatFileSize(val)} exceeds maximum allowed size of ${formatFileSize(MAX_FILE_SIZE)}`, - }) - } - }), - userId: z.string().optional(), - chatId: z.string().optional(), - }) - .passthrough() - -export const batchPresignedUrlBodySchema = z - .object({ - files: z - .array( - z - .object({ - fileName: z.string().refine((value) => value.trim().length > 0, { - message: 'fileName is required for all files', - }), - contentType: z.string().refine((value) => value.trim().length > 0, { - message: 'contentType is required for all files', - }), - fileSize: z.number(), - }) - .passthrough() - .superRefine((file, ctx) => { - const name = typeof file.fileName === 'string' ? file.fileName : 'file' - if (!Number.isFinite(file.fileSize) || file.fileSize <= 0) { - ctx.addIssue({ - code: 'custom', - path: ['fileSize'], - message: `${name} is empty (fileSize must be greater than 0)`, - }) - } else if (file.fileSize > MAX_FILE_SIZE) { - ctx.addIssue({ - code: 'custom', - path: ['fileSize'], - message: `${name} (${formatFileSize(file.fileSize)}) exceeds maximum allowed size of ${formatFileSize(MAX_FILE_SIZE)}`, - }) - } - }) - ) - .min(1, 'files array is required and cannot be empty') - .max(100, 'Cannot process more than 100 files at once'), - }) - .passthrough() - -export const multipartActionSchema = z.enum(['initiate', 'get-part-urls', 'complete', 'abort']) - -export const initiateMultipartBodySchema = z - .object({ - fileName: z.string(), - contentType: z.string(), - fileSize: z.number(), - workspaceId: z.string({ error: 'workspaceId is required' }).min(1, 'workspaceId is required'), - context: z.string().optional(), - }) - .passthrough() - -export const tokenBoundMultipartBodySchema = z - .object({ - uploadToken: z.string().optional(), - }) - .passthrough() - -export const getMultipartPartUrlsBodySchema = tokenBoundMultipartBodySchema.extend({ - partNumbers: z.array(z.number()), -}) - -export const completeMultipartBodySchema = z - .object({ - uploadToken: z.string().optional(), - parts: z.unknown().optional(), - uploads: z - .array( - z - .object({ - uploadToken: z.string().optional(), - parts: z.unknown().optional(), - }) - .passthrough() - ) - .optional(), - }) - .passthrough() - -export type CompleteMultipartBody = z.output - -export const uploadFilesFormFilesSchema = z.preprocess( - (value) => (Array.isArray(value) ? value.filter((entry) => entry instanceof File) : value), - z.array(z.custom((value) => value instanceof File)).min(1, 'No files provided') -) - -export const uploadFilesFormFieldsSchema = z.object({ - workflowId: z.string().nullable(), - executionId: z.string().nullable(), - workspaceId: z.string().nullable(), - context: z.string().nullable(), -}) - export const fileServeParamsSchema = z.object({ path: z.array(z.string()).min(1), }) @@ -680,76 +464,6 @@ export const fileDeleteContract = defineRouteContract({ response: { mode: 'json', schema: jsonResponseSchema }, }) -export const fileUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/upload', - response: { mode: 'json', schema: jsonResponseSchema }, -}) - -export const presignedUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/presigned', - query: presignedUploadQuerySchema, - body: presignedUrlBodySchema, - response: { mode: 'json', schema: presignedUploadResponseSchema }, -}) - -export const presignedUploadBodyContract = defineRouteContract({ - method: 'POST', - path: '/api/files/presigned', - body: presignedUrlBodySchema, - response: { mode: 'json', schema: presignedUploadResponseSchema }, -}) - -export const batchPresignedUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/presigned/batch', - query: presignedUploadQuerySchema, - body: batchPresignedUrlBodySchema, - response: { mode: 'json', schema: batchPresignedUploadResponseSchema }, -}) - -export const batchPresignedUploadBodyContract = defineRouteContract({ - method: 'POST', - path: '/api/files/presigned/batch', - body: batchPresignedUrlBodySchema, - response: { mode: 'json', schema: batchPresignedUploadResponseSchema }, -}) - -export const multipartUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/multipart', - response: { mode: 'json', schema: multipartUploadResponseSchema }, -}) - -export const initiateMultipartUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/multipart', - body: initiateMultipartBodySchema, - response: { mode: 'json', schema: initiateMultipartResponseSchema }, -}) - -export const getMultipartPartUrlsContract = defineRouteContract({ - method: 'POST', - path: '/api/files/multipart', - body: getMultipartPartUrlsBodySchema, - response: { mode: 'json', schema: getMultipartPartUrlsResponseSchema }, -}) - -export const completeMultipartUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/multipart', - body: completeMultipartBodySchema, - response: { mode: 'json', schema: completeMultipartResponseSchema }, -}) - -export const abortMultipartUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/files/multipart', - body: tokenBoundMultipartBodySchema, - response: { mode: 'json', schema: abortMultipartResponseSchema }, -}) - export const fileServeContract = defineRouteContract({ method: 'GET', path: '/api/files/serve/[...path]', @@ -815,16 +529,6 @@ export type FileParseBody = ContractBodyInput export type FileParseResponse = ContractJsonResponse export type FileDeleteBody = ContractBodyInput export type FileDeleteResponse = ContractJsonResponse -export type PresignedUploadQuery = ContractQueryInput -export type PresignedUploadBody = ContractBodyInput -export type PresignedUploadResponse = ContractJsonResponse -export type BatchPresignedUploadQuery = ContractQueryInput -export type BatchPresignedUploadBody = ContractBodyInput -export type BatchPresignedUploadResponse = ContractJsonResponse -export type MultipartAction = z.output -export type InitiateMultipartBody = z.output -export type TokenBoundMultipartBody = z.output -export type GetMultipartPartUrlsBody = z.output export type FileServeParams = ContractParamsInput export type FileServeQuery = ContractQueryInput export type FileViewParams = ContractParamsInput diff --git a/apps/sim/lib/api/contracts/table-transfers.ts b/apps/sim/lib/api/contracts/table-transfers.ts index 69a5c4f4ec6..3b83e074603 100644 --- a/apps/sim/lib/api/contracts/table-transfers.ts +++ b/apps/sim/lib/api/contracts/table-transfers.ts @@ -3,6 +3,7 @@ import { defineRouteContract } from '@/lib/api/contracts/types' import { v2DataResponse } from '@/lib/api/contracts/v2/shared' import { v2CreateTableImportBodySchema, + v2CreateTableImportDataSchema, v2TableExportDownloadDataSchema, v2TableExportParamsSchema, v2TableExportSchema, @@ -22,7 +23,7 @@ export const createTableImportResourceContract = defineRouteContract({ method: 'POST', path: '/api/table/imports', body: v2CreateTableImportBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2TableImportSchema) }, + response: { mode: 'json', schema: v2DataResponse(v2CreateTableImportDataSchema) }, }) export const getTableImportResourceContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/tables.ts b/apps/sim/lib/api/contracts/tables.ts index 666b9d55552..d7abbcbe383 100644 --- a/apps/sim/lib/api/contracts/tables.ts +++ b/apps/sim/lib/api/contracts/tables.ts @@ -32,7 +32,7 @@ import { SORT_DIRECTIONS, TABLE_LIMITS, } from '@/lib/table/constants' -import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import' +import { CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE } from '@/lib/table/import' export const domainObjectSchema = () => z.custom(isRecordLike) @@ -1038,7 +1038,7 @@ export const csvFileSchema = z if (value.size > CSV_MAX_FILE_SIZE_BYTES) { ctx.addIssue({ code: 'custom', - message: `File exceeds maximum allowed size of ${CSV_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB`, + message: CSV_MAX_FILE_SIZE_MESSAGE, }) } }) diff --git a/apps/sim/lib/api/contracts/upload-sessions.ts b/apps/sim/lib/api/contracts/upload-sessions.ts index c56d8ca014b..4e3a5153f3f 100644 --- a/apps/sim/lib/api/contracts/upload-sessions.ts +++ b/apps/sim/lib/api/contracts/upload-sessions.ts @@ -1,53 +1,198 @@ import { z } from 'zod' +import { folderIdSchema, workflowIdSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' -import { - v2CreateFileUploadBodySchema, - v2FileUploadParamsSchema, - v2FileUploadSchema, - v2FileUploadWorkspaceQuerySchema, -} from '@/lib/api/contracts/v2/files' +import { v2FileSchema } from '@/lib/api/contracts/v2/files' import { v2DataResponse } from '@/lib/api/contracts/v2/shared' import { v2CompleteUploadBodySchema, v2PartUrlsBodySchema, v2PartUrlsDataSchema, + v2UploadStatusSchema, v2UploadTokenHeadersSchema, + v2UploadTransferSchema, } from '@/lib/api/contracts/v2/uploads' +import { executionIdSchema } from '@/lib/api/contracts/workflows' +import { + MAX_WORKSPACE_FILE_SIZE, + MAX_WORKSPACE_FORMDATA_FILE_SIZE, +} from '@/lib/uploads/shared/types' + +const MAX_ASSET_FILE_SIZE = 5 * 1024 * 1024 + +const internalFileUploadBaseShape = { + name: z.string().trim().min(1, 'name is required').max(255, 'name is too long'), + contentType: z + .string() + .trim() + .min(1, 'contentType is required') + .max(255, 'contentType is too long'), +} as const + +export const createInternalFileUploadBodySchema = z.discriminatedUnion('purpose', [ + z + .object({ + purpose: z.literal('workspace_file'), + ...internalFileUploadBaseShape, + size: z.number().int().nonnegative().max(MAX_WORKSPACE_FILE_SIZE), + workspaceId: workspaceIdSchema, + folderId: folderIdSchema.optional(), + }) + .strict(), + z + .object({ + purpose: z.literal('profile_picture'), + ...internalFileUploadBaseShape, + size: z.number().int().min(1).max(MAX_ASSET_FILE_SIZE), + }) + .strict(), + z + .object({ + purpose: z.literal('workspace_logo'), + ...internalFileUploadBaseShape, + size: z.number().int().min(1).max(MAX_ASSET_FILE_SIZE), + workspaceId: workspaceIdSchema, + }) + .strict(), + z + .object({ + purpose: z.literal('mothership_attachment'), + ...internalFileUploadBaseShape, + size: z.number().int().min(1).max(MAX_WORKSPACE_FILE_SIZE), + workspaceId: workspaceIdSchema, + }) + .strict(), + z + .object({ + purpose: z.literal('execution_attachment'), + ...internalFileUploadBaseShape, + size: z.number().int().min(1).max(MAX_WORKSPACE_FORMDATA_FILE_SIZE), + workspaceId: workspaceIdSchema, + workflowId: workflowIdSchema, + executionId: executionIdSchema, + }) + .strict(), +]) +export type CreateInternalFileUploadBody = z.input + +export const internalFileUploadParamsSchema = z.object({ + uploadId: z.string().min(1, 'uploadId is required'), +}) + +const internalFileUploadSessionBaseShape = { + id: z.string().min(1), + status: v2UploadStatusSchema, + name: z.string(), + contentType: z.string(), + expiresAt: z.string().datetime(), + error: z.string().nullable(), +} as const -export const createWorkspaceFileUploadContract = defineRouteContract({ +export const internalUploadedAssetSchema = z + .object({ + path: z.string().min(1), + key: z.string().min(1), + name: z.string().min(1), + size: z.number().int().positive(), + type: z.string().min(1), + }) + .strict() + +export const internalExecutionAttachmentSchema = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + size: z.number().int().positive(), + type: z.string().min(1), + url: z.string().min(1), + key: z.string().min(1), + context: z.literal('execution'), + }) + .strict() + +export const internalFileUploadSessionSchema = z.discriminatedUnion('purpose', [ + z + .object({ + ...internalFileUploadSessionBaseShape, + purpose: z.literal('workspace_file'), + size: z.number().int().nonnegative(), + result: v2FileSchema.nullable(), + }) + .strict(), + z + .object({ + ...internalFileUploadSessionBaseShape, + purpose: z.literal('profile_picture'), + size: z.number().int().positive(), + result: internalUploadedAssetSchema.nullable(), + }) + .strict(), + z + .object({ + ...internalFileUploadSessionBaseShape, + purpose: z.literal('workspace_logo'), + size: z.number().int().positive(), + result: internalUploadedAssetSchema.nullable(), + }) + .strict(), + z + .object({ + ...internalFileUploadSessionBaseShape, + purpose: z.literal('mothership_attachment'), + size: z.number().int().positive(), + result: internalUploadedAssetSchema.nullable(), + }) + .strict(), + z + .object({ + ...internalFileUploadSessionBaseShape, + purpose: z.literal('execution_attachment'), + size: z.number().int().positive(), + result: internalExecutionAttachmentSchema.nullable(), + }) + .strict(), +]) +export type InternalFileUploadSession = z.output + +export const createInternalFileUploadDataSchema = z + .object({ + session: internalFileUploadSessionSchema, + uploadToken: z.string().min(1), + transfer: v2UploadTransferSchema, + }) + .strict() +export type CreateInternalFileUploadData = z.output + +export const createInternalFileUploadContract = defineRouteContract({ method: 'POST', path: '/api/files/uploads', - body: v2CreateFileUploadBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2FileUploadSchema) }, + body: createInternalFileUploadBodySchema, + response: { mode: 'json', schema: v2DataResponse(createInternalFileUploadDataSchema) }, }) -export const abortWorkspaceFileUploadContract = defineRouteContract({ +export const abortInternalFileUploadContract = defineRouteContract({ method: 'DELETE', path: '/api/files/uploads/[uploadId]', - params: v2FileUploadParamsSchema, - query: v2FileUploadWorkspaceQuerySchema, + params: internalFileUploadParamsSchema, headers: v2UploadTokenHeadersSchema, - response: { mode: 'json', schema: v2DataResponse(v2FileUploadSchema) }, + response: { mode: 'json', schema: v2DataResponse(internalFileUploadSessionSchema) }, }) -export const createWorkspaceFileUploadPartUrlsContract = defineRouteContract({ +export const createInternalFileUploadPartUrlsContract = defineRouteContract({ method: 'POST', path: '/api/files/uploads/[uploadId]/parts', - params: v2FileUploadParamsSchema, - query: v2FileUploadWorkspaceQuerySchema, + params: internalFileUploadParamsSchema, headers: v2UploadTokenHeadersSchema, body: v2PartUrlsBodySchema, response: { mode: 'json', schema: v2DataResponse(v2PartUrlsDataSchema) }, }) -export const completeWorkspaceFileUploadContract = defineRouteContract({ +export const completeInternalFileUploadContract = defineRouteContract({ method: 'POST', path: '/api/files/uploads/[uploadId]/complete', - params: v2FileUploadParamsSchema, - query: v2FileUploadWorkspaceQuerySchema, + params: internalFileUploadParamsSchema, headers: v2UploadTokenHeadersSchema, body: v2CompleteUploadBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2FileUploadSchema) }, + response: { mode: 'json', schema: v2DataResponse(internalFileUploadSessionSchema) }, }) export const localUploadPartParamsSchema = z.object({ @@ -66,3 +211,11 @@ export const localUploadPartContract = defineRouteContract({ query: localUploadPartQuerySchema, response: { mode: 'empty', status: 204 }, }) + +export const localPutUploadContract = defineRouteContract({ + method: 'PUT', + path: '/api/v2/uploads/[uploadId]', + params: internalFileUploadParamsSchema, + headers: v2UploadTokenHeadersSchema, + response: { mode: 'empty', status: 204 }, +}) diff --git a/apps/sim/lib/api/contracts/v2/__tests__/tables.test.ts b/apps/sim/lib/api/contracts/v2/__tests__/tables.test.ts new file mode 100644 index 00000000000..9e9dca55d01 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/__tests__/tables.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from 'vitest' +import { + V2_TABLE_IMPORT_OPTIONS_MAX_BYTES, + v2CreateTableImportBodySchema, + v2TableUploadImportSourceSchema, +} from '@/lib/api/contracts/v2/tables' +import { TABLE_LIMITS } from '@/lib/table/constants' +import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import' + +const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' + +function uploadSource(size: number) { + return { + type: 'upload' as const, + name: 'data.csv', + contentType: 'text/csv', + size, + } +} + +function existingTableImport(overrides: Record = {}) { + return { + workspaceId: WORKSPACE_ID, + source: uploadSource(128), + target: { type: 'existing' as const, tableId: 'table-1', mode: 'append' as const }, + ...overrides, + } +} + +describe('v2 table import contracts', () => { + it('accepts the exact CSV byte limit and rejects one byte over it', () => { + expect( + v2TableUploadImportSourceSchema.safeParse(uploadSource(CSV_MAX_FILE_SIZE_BYTES)).success + ).toBe(true) + expect( + v2TableUploadImportSourceSchema.safeParse(uploadSource(CSV_MAX_FILE_SIZE_BYTES + 1)).success + ).toBe(false) + }) + + it('accepts native JSON mapping and createColumns values', () => { + const body = existingTableImport({ + mapping: { email: 'email_address', notes: null }, + createColumns: ['phone'], + }) + + expect(v2CreateTableImportBodySchema.parse(body)).toEqual(body) + }) + + it('rejects the legacy FormData JSON-string representation', () => { + expect( + v2CreateTableImportBodySchema.safeParse( + existingTableImport({ mapping: JSON.stringify({ email: 'email_address' }) }) + ).success + ).toBe(false) + expect( + v2CreateTableImportBodySchema.safeParse( + existingTableImport({ createColumns: JSON.stringify(['phone']) }) + ).success + ).toBe(false) + }) + + it('caps mapping entries and createColumns items at the table column limit', () => { + const mapping = Object.fromEntries( + Array.from({ length: TABLE_LIMITS.MAX_COLUMNS_PER_TABLE }, (_, index) => [ + `header_${index}`, + `column_${index}`, + ]) + ) + const createColumns = Array.from( + { length: TABLE_LIMITS.MAX_COLUMNS_PER_TABLE }, + (_, index) => `header_${index}` + ) + + expect(v2CreateTableImportBodySchema.safeParse(existingTableImport({ mapping })).success).toBe( + true + ) + expect( + v2CreateTableImportBodySchema.safeParse(existingTableImport({ createColumns })).success + ).toBe(true) + expect( + v2CreateTableImportBodySchema.safeParse( + existingTableImport({ mapping: { ...mapping, overflow: 'overflow' } }) + ).success + ).toBe(false) + expect( + v2CreateTableImportBodySchema.safeParse( + existingTableImport({ createColumns: [...createColumns, 'overflow'] }) + ).success + ).toBe(false) + }) + + it('bounds CSV header and mapped column names', () => { + const exact = 'x'.repeat(TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH) + const over = `${exact}x` + + expect( + v2CreateTableImportBodySchema.safeParse( + existingTableImport({ mapping: { [exact]: exact }, createColumns: [exact] }) + ).success + ).toBe(true) + expect( + v2CreateTableImportBodySchema.safeParse(existingTableImport({ mapping: { [over]: exact } })) + .success + ).toBe(false) + expect( + v2CreateTableImportBodySchema.safeParse(existingTableImport({ mapping: { header: over } })) + .success + ).toBe(false) + expect( + v2CreateTableImportBodySchema.safeParse(existingTableImport({ createColumns: [over] })) + .success + ).toBe(false) + }) + + it('caps aggregate mapping metadata before it is embedded in the signed upload token', () => { + const mapping = Object.fromEntries( + Array.from({ length: 30 }, (_, index) => [ + `header_${index}_${'h'.repeat(30)}`, + `column_${index}_${'c'.repeat(30)}`, + ]) + ) + const result = v2CreateTableImportBodySchema.safeParse(existingTableImport({ mapping })) + + expect(new TextEncoder().encode(JSON.stringify({ mapping })).byteLength).toBeGreaterThan( + V2_TABLE_IMPORT_OPTIONS_MAX_BYTES + ) + expect(result.success).toBe(false) + if (!result.success) { + expect(result.error.issues).toContainEqual( + expect.objectContaining({ + path: ['mapping'], + message: expect.stringMatching(/signed request token/), + }) + ) + } + }) +}) diff --git a/apps/sim/lib/api/contracts/v2/__tests__/uploads.test.ts b/apps/sim/lib/api/contracts/v2/__tests__/uploads.test.ts new file mode 100644 index 00000000000..b301ab84ea1 --- /dev/null +++ b/apps/sim/lib/api/contracts/v2/__tests__/uploads.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { v2CompleteUploadBodySchema, v2UploadTransferSchema } from '@/lib/api/contracts/v2/uploads' + +describe('v2 upload transfer contracts', () => { + it('accepts only an empty object for PUT completion', () => { + expect(v2CompleteUploadBodySchema.parse({})).toEqual({}) + expect(v2CompleteUploadBodySchema.safeParse({ method: 'put' }).success).toBe(false) + }) + + it('accepts a strict completed-parts body for multipart completion', () => { + expect( + v2CompleteUploadBodySchema.parse({ parts: [{ partNumber: 1, etag: 'etag-1' }] }) + ).toEqual({ parts: [{ partNumber: 1, etag: 'etag-1' }] }) + expect(v2CompleteUploadBodySchema.safeParse({ parts: [] }).success).toBe(false) + expect( + v2CompleteUploadBodySchema.safeParse({ + parts: [{ partNumber: 1 }], + ignored: true, + }).success + ).toBe(false) + }) + + it('discriminates a PUT transfer from multipart geometry', () => { + expect( + v2UploadTransferSchema.parse({ + method: 'put', + url: 'https://storage.example/upload', + headers: { 'Content-Type': 'application/octet-stream' }, + }) + ).toMatchObject({ method: 'put' }) + expect( + v2UploadTransferSchema.parse({ + method: 'multipart', + partSize: 8 * 1024 * 1024, + partCount: 7, + }) + ).toMatchObject({ method: 'multipart' }) + expect( + v2UploadTransferSchema.safeParse({ + method: 'put', + partSize: 8 * 1024 * 1024, + partCount: 1, + }).success + ).toBe(false) + }) +}) diff --git a/apps/sim/lib/api/contracts/v2/credentials.ts b/apps/sim/lib/api/contracts/v2/credentials.ts index 7c411ec532d..aea3f744be0 100644 --- a/apps/sim/lib/api/contracts/v2/credentials.ts +++ b/apps/sim/lib/api/contracts/v2/credentials.ts @@ -111,6 +111,7 @@ const credentialSecretFields = { clientId: z.string().trim().min(1, 'clientId cannot be empty').max(512).optional(), clientSecret: z.string().trim().min(1, 'clientSecret cannot be empty').max(1024).optional(), orgId: z.string().trim().min(1, 'orgId cannot be empty').max(255).optional(), + dataCenter: z.string().trim().min(1, 'dataCenter cannot be empty').max(32).optional(), } as const export const v2CreateCredentialBodySchema = z diff --git a/apps/sim/lib/api/contracts/v2/files.ts b/apps/sim/lib/api/contracts/v2/files.ts index 749d4a9bda7..e871a5438a3 100644 --- a/apps/sim/lib/api/contracts/v2/files.ts +++ b/apps/sim/lib/api/contracts/v2/files.ts @@ -1,5 +1,10 @@ import { z } from 'zod' -import { workspaceFileIdSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' +import { + folderIdSchema, + isCanonicalBase64, + workspaceFileIdSchema, + workspaceIdSchema, +} from '@/lib/api/contracts/primitives' import { shareAuthTypeSchema, shareRecordSchema } from '@/lib/api/contracts/public-shares' import { defineRouteContract } from '@/lib/api/contracts/types' import { @@ -14,15 +19,15 @@ import { v2PartUrlsDataSchema, v2UploadStatusSchema, v2UploadTokenHeadersSchema, + v2UploadTransferSchema, } from '@/lib/api/contracts/v2/uploads' import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' /** * v2 files contracts. v2 drops the v1 `{ success, data, limits }` envelope in * favor of the canonical v2 shapes (`{ data }` / `{ data, nextCursor }`) and - * adds cursor pagination to the list. The workspace is always carried as a query - * param — including on upload — so the route can authorize before reading the - * multipart body. + * adds cursor pagination to the list. List and item routes carry the workspace + * as a query parameter; upload-session creation carries it in the JSON body. * * Folders are referenced but not managed here. A file carries `folderId` / * `folderPath`, and `move` retargets it, but there are deliberately no @@ -65,8 +70,8 @@ export const v2CreateFileUploadBodySchema = z workspaceId: workspaceIdSchema, name: z.string().trim().min(1, 'name is required').max(255, 'name is too long'), contentType: z.string().trim().min(1, 'contentType is required').max(255), - size: z.number().int().min(1).max(MAX_WORKSPACE_FILE_SIZE), - folderId: z.string().min(1, 'folderId cannot be empty').optional(), + size: z.number().int().nonnegative().max(MAX_WORKSPACE_FILE_SIZE), + folderId: folderIdSchema.optional(), }) .strict() export type V2CreateFileUploadBody = z.input @@ -79,16 +84,22 @@ export const v2FileUploadSchema = z.object({ status: v2UploadStatusSchema, name: z.string(), contentType: z.string(), - size: z.number().int().positive(), - partSize: z.number().int().positive(), - partCount: z.number().int().positive(), - uploadToken: z.string().min(1), + size: z.number().int().nonnegative(), expiresAt: z.string().datetime(), error: z.string().nullable(), file: v2FileSchema.nullable(), }) export type V2FileUpload = z.output +export const v2CreateFileUploadDataSchema = z + .object({ + session: v2FileUploadSchema, + uploadToken: z.string().min(1), + transfer: v2UploadTransferSchema, + }) + .strict() +export type V2CreateFileUploadData = z.output + /** Acknowledgement returned by a successful archive (soft delete). */ export const v2DeleteFileResultSchema = z.object({ id: z.string(), @@ -131,6 +142,33 @@ const v2FileItemNameSchema = z 'name cannot contain path separators or dot segments' ) +export const v2CreateFileBodySchema = z + .object({ + workspaceId: workspaceIdSchema, + name: v2FileItemNameSchema, + contentType: z + .string() + .trim() + .min(1, 'contentType cannot be empty') + .max(255, 'contentType is too long') + .optional(), + folderId: folderIdSchema.optional(), + content: z.string().max(70_000_000, 'content is too large').default(''), + encoding: z.enum(['utf-8', 'base64']).default('utf-8'), + }) + .superRefine(({ content, encoding }, ctx) => { + if (encoding === 'base64' && !isCanonicalBase64(content)) { + ctx.addIssue({ + code: 'custom', + path: ['content'], + message: 'content must be valid base64', + }) + } + }) + .strict() + +export type V2CreateFileBody = z.input + /** Sortable file fields. `name` is the uploaded file name, not the storage key. */ export const v2FileSortFields = ['name', 'size', 'uploadedAt', 'updatedAt'] as const @@ -307,6 +345,15 @@ export const v2UpdateFileContentBodySchema = z content: z.string().max(70_000_000, 'content is too large'), encoding: z.enum(['utf-8', 'base64']).default('utf-8'), }) + .superRefine(({ content, encoding }, ctx) => { + if (encoding === 'base64' && !isCanonicalBase64(content)) { + ctx.addIssue({ + code: 'custom', + path: ['content'], + message: 'content must be valid base64', + }) + } + }) .strict() export type V2UpdateFileContentBody = z.input @@ -321,11 +368,21 @@ export const v2ListFilesContract = defineRouteContract({ }, }) +export const v2CreateFileContract = defineRouteContract({ + method: 'POST', + path: '/api/v2/files', + body: v2CreateFileBodySchema, + response: { + mode: 'json', + schema: v2DataResponse(v2FileSchema), + }, +}) + export const v2CreateFileUploadContract = defineRouteContract({ method: 'POST', path: '/api/v2/files/uploads', body: v2CreateFileUploadBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2FileUploadSchema) }, + response: { mode: 'json', schema: v2DataResponse(v2CreateFileUploadDataSchema) }, }) export const v2AbortFileUploadContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/v2/knowledge.ts b/apps/sim/lib/api/contracts/v2/knowledge.ts index bdca6fe4f01..0e7ee1b1967 100644 --- a/apps/sim/lib/api/contracts/v2/knowledge.ts +++ b/apps/sim/lib/api/contracts/v2/knowledge.ts @@ -28,6 +28,7 @@ import { v2PartUrlsDataSchema, v2UploadStatusSchema, v2UploadTokenHeadersSchema, + v2UploadTransferSchema, } from '@/lib/api/contracts/v2/uploads' import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' @@ -211,15 +212,23 @@ export const v2KnowledgeDocumentUploadSchema = z.object({ name: z.string(), contentType: z.string(), size: z.number().int().positive(), - partSize: z.number().int().positive(), - partCount: z.number().int().positive(), - uploadToken: z.string().min(1), expiresAt: z.string().datetime(), error: z.string().nullable(), document: v2KnowledgeDocumentSummarySchema.nullable(), }) export type V2KnowledgeDocumentUpload = z.output +export const v2CreateKnowledgeDocumentUploadDataSchema = z + .object({ + session: v2KnowledgeDocumentUploadSchema, + uploadToken: z.string().min(1), + transfer: v2UploadTransferSchema, + }) + .strict() +export type V2CreateKnowledgeDocumentUploadData = z.output< + typeof v2CreateKnowledgeDocumentUploadDataSchema +> + export const v2KnowledgeBaseSortFields = ['name', 'createdAt', 'updatedAt'] as const export type V2KnowledgeBaseSortBy = (typeof v2KnowledgeBaseSortFields)[number] @@ -345,7 +354,10 @@ export const v2CreateKnowledgeDocumentUploadContract = defineRouteContract({ path: '/api/v2/knowledge/[id]/documents/uploads', params: knowledgeBaseParamsSchema, body: v2CreateKnowledgeDocumentUploadBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2KnowledgeDocumentUploadSchema) }, + response: { + mode: 'json', + schema: v2DataResponse(v2CreateKnowledgeDocumentUploadDataSchema), + }, }) export const v2AbortKnowledgeDocumentUploadContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/v2/tables.ts b/apps/sim/lib/api/contracts/v2/tables.ts index 30b79fe9a37..3add52daf85 100644 --- a/apps/sim/lib/api/contracts/v2/tables.ts +++ b/apps/sim/lib/api/contracts/v2/tables.ts @@ -5,8 +5,6 @@ import { cancelTableRunsBodyBaseSchema, createTableColumnBodySchema, createTableViewBodySchema, - csvImportCreateColumnsSchema, - csvImportMappingSchema, deleteTableColumnBodySchema, deleteWorkflowGroupBodySchema, exportTableAsyncBodySchema, @@ -51,9 +49,10 @@ import { v2PartUrlsBodySchema, v2PartUrlsDataSchema, v2UploadTokenHeadersSchema, + v2UploadTransferSchema, } from '@/lib/api/contracts/v2/uploads' import { TABLE_LIMITS } from '@/lib/table/constants' -import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' +import { CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE } from '@/lib/table/import' /** * v2 tables contracts. @@ -81,6 +80,8 @@ import { MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types' export const V2_DEFAULT_ROW_LIMIT = 100 /** Hard cap on an explicit page `limit`. Larger pulls use `limit=0` (query) or an export resource. */ export const V2_MAX_ROW_LIMIT = 1000 +/** Keeps upload-token metadata comfortably below common 8 KiB request-header limits after signing. */ +export const V2_TABLE_IMPORT_OPTIONS_MAX_BYTES = 2 * 1024 /** * Public table shape emitted by `toApiTable` (timestamps ISO-serialized). @@ -928,16 +929,22 @@ export const v2TableImportParamsSchema = z.object({ importId: z.string().min(1) export const v2TableExportParamsSchema = z.object({ exportId: z.string().min(1) }) export const v2TableTransferWorkspaceQuerySchema = z.object({ workspaceId: workspaceIdSchema }) +export const v2TableUploadImportSourceSchema = z + .object({ + type: z.literal('upload'), + name: z.string().trim().min(1, 'name is required').max(255), + contentType: z.string().trim().min(1, 'contentType is required').max(255), + size: z.number().int().min(1).max(CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE), + }) + .strict() + +export const v2TableWorkspaceFileImportSourceSchema = z + .object({ type: z.literal('workspace_file'), fileId: z.string().min(1) }) + .strict() + export const v2TableImportSourceSchema = z.discriminatedUnion('type', [ - z - .object({ - type: z.literal('upload'), - name: z.string().trim().min(1, 'name is required').max(255), - contentType: z.string().trim().min(1, 'contentType is required').max(255), - size: z.number().int().min(1).max(MAX_WORKSPACE_FILE_SIZE), - }) - .strict(), - z.object({ type: z.literal('workspace_file'), fileId: z.string().min(1) }).strict(), + v2TableUploadImportSourceSchema, + v2TableWorkspaceFileImportSourceSchema, ]) export type V2TableImportSource = z.input @@ -959,13 +966,43 @@ export const v2TableImportTargetSchema = z.discriminatedUnion('type', [ ]) export type V2TableImportTarget = z.input +const v2CsvHeaderSchema = z + .string() + .min(1, 'CSV header must not be empty') + .max( + TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH, + `CSV header must be ${TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH} characters or less` + ) + +const v2CsvColumnNameSchema = z + .string() + .min(1, 'Column name must not be empty') + .max( + TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH, + `Column name must be ${TABLE_LIMITS.MAX_COLUMN_NAME_LENGTH} characters or less` + ) + +export const v2CsvImportMappingSchema = z + .record(v2CsvHeaderSchema, v2CsvColumnNameSchema.nullable()) + .refine( + (mapping) => Object.keys(mapping).length <= TABLE_LIMITS.MAX_COLUMNS_PER_TABLE, + `mapping cannot contain more than ${TABLE_LIMITS.MAX_COLUMNS_PER_TABLE} entries` + ) + +export const v2CsvImportCreateColumnsSchema = z + .array(v2CsvHeaderSchema) + .max( + TABLE_LIMITS.MAX_COLUMNS_PER_TABLE, + `createColumns cannot contain more than ${TABLE_LIMITS.MAX_COLUMNS_PER_TABLE} items` + ) + export const v2CreateTableImportBodySchema = z .object({ workspaceId: workspaceIdSchema, source: v2TableImportSourceSchema, target: v2TableImportTargetSchema, - mapping: csvImportMappingSchema.optional(), - createColumns: csvImportCreateColumnsSchema.optional(), + mapping: v2CsvImportMappingSchema.optional(), + createColumns: v2CsvImportCreateColumnsSchema.optional(), timezone: ianaTimezoneSchema.optional(), }) .strict() @@ -984,6 +1021,19 @@ export const v2CreateTableImportBodySchema = z message: 'createColumns is only supported for an existing table target', }) } + const serializedOptions = JSON.stringify({ + ...(body.mapping !== undefined ? { mapping: body.mapping } : {}), + ...(body.createColumns !== undefined ? { createColumns: body.createColumns } : {}), + }) + if ( + new TextEncoder().encode(serializedOptions).byteLength > V2_TABLE_IMPORT_OPTIONS_MAX_BYTES + ) { + ctx.addIssue({ + code: 'custom', + path: [body.mapping !== undefined ? 'mapping' : 'createColumns'], + message: `mapping and createColumns must serialize to at most ${V2_TABLE_IMPORT_OPTIONS_MAX_BYTES} bytes because upload metadata is carried in a signed request token`, + }) + } }) export type V2CreateTableImportBody = z.input @@ -998,13 +1048,6 @@ export const v2TableImportStatusSchema = z.enum([ ]) export type V2TableImportStatus = z.output -export const v2TableImportUploadSchema = z.object({ - uploadToken: z.string().min(1), - partSize: z.number().int().positive(), - partCount: z.number().int().positive(), - expiresAt: z.string().datetime(), -}) - export const v2TableImportSchema = z.object({ id: z.string(), workspaceId: z.string(), @@ -1014,18 +1057,43 @@ export const v2TableImportSchema = z.object({ tableId: z.string().nullable(), rowsProcessed: z.number().int().nonnegative(), error: z.string().nullable(), - upload: v2TableImportUploadSchema.nullable(), createdAt: z.string().datetime(), updatedAt: z.string().datetime(), completedAt: z.string().datetime().nullable(), }) export type V2TableImport = z.output +const v2UploadBackedTableImportSchema = v2TableImportSchema.extend({ + source: v2TableUploadImportSourceSchema, +}) + +const v2WorkspaceFileTableImportSchema = v2TableImportSchema.extend({ + source: v2TableWorkspaceFileImportSourceSchema, +}) + +export const v2CreateTableImportDataSchema = z.union([ + z + .object({ + session: v2UploadBackedTableImportSchema, + uploadToken: z.string().min(1), + transfer: v2UploadTransferSchema, + }) + .strict(), + z + .object({ + session: v2WorkspaceFileTableImportSchema, + uploadToken: z.null(), + transfer: z.null(), + }) + .strict(), +]) +export type V2CreateTableImportData = z.output + export const v2CreateTableImportContract = defineRouteContract({ method: 'POST', path: '/api/v2/tables/imports', body: v2CreateTableImportBodySchema, - response: { mode: 'json', schema: v2DataResponse(v2TableImportSchema) }, + response: { mode: 'json', schema: v2DataResponse(v2CreateTableImportDataSchema) }, }) export const v2GetTableImportContract = defineRouteContract({ diff --git a/apps/sim/lib/api/contracts/v2/uploads.ts b/apps/sim/lib/api/contracts/v2/uploads.ts index d18f9124692..0e91d768b83 100644 --- a/apps/sim/lib/api/contracts/v2/uploads.ts +++ b/apps/sim/lib/api/contracts/v2/uploads.ts @@ -27,13 +27,44 @@ export const v2CompletedPartSchema = z .strict() export type V2CompletedPart = z.input -export const v2CompleteUploadBodySchema = z +const v2CompleteMultipartUploadBodySchema = z .object({ parts: z.array(v2CompletedPartSchema).min(1).max(640), }) .strict() + +const v2CompletePutUploadBodySchema = z.object({}).strict() + +export const v2CompleteUploadBodySchema = z.union([ + v2CompleteMultipartUploadBodySchema, + v2CompletePutUploadBodySchema, +]) export type V2CompleteUploadBody = z.input +export const v2PutUploadTransferSchema = z + .object({ + method: z.literal('put'), + url: z.string().url(), + headers: z.record(z.string(), z.string()), + }) + .strict() +export type V2PutUploadTransfer = z.output + +export const v2MultipartUploadTransferSchema = z + .object({ + method: z.literal('multipart'), + partSize: z.number().int().positive(), + partCount: z.number().int().positive().max(640), + }) + .strict() +export type V2MultipartUploadTransfer = z.output + +export const v2UploadTransferSchema = z.discriminatedUnion('method', [ + v2PutUploadTransferSchema, + v2MultipartUploadTransferSchema, +]) +export type V2UploadTransfer = z.output + export const v2PartUrlsBodySchema = z .object({ partNumbers: z.array(z.number().int().min(1)).min(1).max(100), diff --git a/apps/sim/lib/api/contracts/workspace-files.ts b/apps/sim/lib/api/contracts/workspace-files.ts index 1351e5f9527..e917e7f5ffb 100644 --- a/apps/sim/lib/api/contracts/workspace-files.ts +++ b/apps/sim/lib/api/contracts/workspace-files.ts @@ -1,5 +1,10 @@ import { z } from 'zod' -import { inlineFileRefQuerySchema } from '@/lib/api/contracts/primitives' +import { + folderIdSchema, + inlineFileRefQuerySchema, + isCanonicalBase64, + workspaceIdSchema, +} from '@/lib/api/contracts/primitives' import { shareRecordSchema } from '@/lib/api/contracts/public-shares' import { defineRouteContract } from '@/lib/api/contracts/types' @@ -12,7 +17,7 @@ import { defineRouteContract } from '@/lib/api/contracts/types' export const workspaceFileScopeSchema = z.enum(['active', 'archived']) export const workspaceFilesParamsSchema = z.object({ - id: z.string({ error: 'Workspace ID is required' }).min(1, 'Workspace ID is required'), + id: workspaceIdSchema, }) export const workspaceFileParamsSchema = workspaceFilesParamsSchema.extend({ @@ -38,10 +43,11 @@ export const getInlineWorkspaceFileContract = defineRouteContract({ }, }) -const workspaceFileNameSchema = z +export const workspaceFileNameSchema = z .string({ error: 'Name is required' }) .trim() .min(1, 'Name is required') + .max(255, 'Name is too long') .refine( (name) => name !== '.' && name !== '..' && !name.includes('/') && !name.includes('\\'), 'Name cannot contain path separators or dot segments' @@ -51,10 +57,46 @@ export const renameWorkspaceFileBodySchema = z.object({ name: workspaceFileNameSchema, }) -export const updateWorkspaceFileContentBodySchema = z.object({ - content: z.string(), - encoding: z.enum(['base64', 'utf-8']).optional(), -}) +export const updateWorkspaceFileContentBodySchema = z + .object({ + content: z.string().max(70_000_000, 'Content is too large'), + encoding: z.enum(['base64', 'utf-8']).optional(), + }) + .superRefine(({ content, encoding }, ctx) => { + if (encoding === 'base64' && !isCanonicalBase64(content)) { + ctx.addIssue({ + code: 'custom', + path: ['content'], + message: 'Content must be valid base64', + }) + } + }) + +export const createWorkspaceFileBodySchema = z + .object({ + name: workspaceFileNameSchema, + contentType: z + .string() + .trim() + .min(1, 'Content type cannot be empty') + .max(255, 'Content type is too long') + .optional(), + folderId: folderIdSchema.optional(), + content: z.string().max(70_000_000, 'Content is too large').default(''), + encoding: z.enum(['utf-8', 'base64']).default('utf-8'), + }) + .superRefine(({ content, encoding }, ctx) => { + if (encoding === 'base64' && !isCanonicalBase64(content)) { + ctx.addIssue({ + code: 'custom', + path: ['content'], + message: 'Content must be valid base64', + }) + } + }) + .strict() + +export type CreateWorkspaceFileBody = z.input export const workspaceFileRecordSchema = z.object({ id: z.string(), @@ -96,6 +138,19 @@ export const listWorkspaceFilesContract = defineRouteContract({ }, }) +export const createWorkspaceFileContract = defineRouteContract({ + method: 'POST', + path: '/api/workspaces/[id]/files', + params: workspaceFilesParamsSchema, + body: createWorkspaceFileBodySchema, + response: { + mode: 'json', + schema: workspaceFileSuccessSchema.extend({ + file: workspaceFileRecordSchema, + }), + }, +}) + export const renameWorkspaceFileContract = defineRouteContract({ method: 'PATCH', path: '/api/workspaces/[id]/files/[fileId]', @@ -195,78 +250,3 @@ export const workspaceFileCompiledCheckContract = defineRouteContract({ schema: compiledCheckResponseSchema, }, }) - -export const workspacePresignedUploadBodySchema = z.object({ - fileName: workspaceFileNameSchema, - contentType: z.string().min(1, 'contentType is required'), - fileSize: z.number().nonnegative('fileSize must be a non-negative number'), - folderId: z.string().nullable().optional(), -}) - -export type WorkspacePresignedUploadBody = z.input - -const workspacePresignedFileInfoSchema = z.object({ - path: z.string(), - key: z.string(), - name: z.string(), - size: z.number(), - type: z.string(), -}) - -const workspacePresignedUploadResponseSchema = z.object({ - fileName: z.string(), - presignedUrl: z.string(), - fileInfo: workspacePresignedFileInfoSchema, - uploadHeaders: z.record(z.string(), z.string()).optional(), - directUploadSupported: z.boolean(), -}) - -export const workspacePresignedUploadContract = defineRouteContract({ - method: 'POST', - path: '/api/workspaces/[id]/files/presigned', - params: workspaceFilesParamsSchema, - body: workspacePresignedUploadBodySchema, - response: { - mode: 'json', - schema: workspacePresignedUploadResponseSchema, - }, -}) - -export const registerWorkspaceFileBodySchema = z.object({ - key: z.string().min(1, 'key is required'), - name: workspaceFileNameSchema, - contentType: z.string().min(1, 'contentType is required'), - folderId: z.string().nullable().optional(), -}) - -export type RegisterWorkspaceFileBody = z.input - -const registeredWorkspaceFileSchema = z.object({ - id: z.string(), - name: z.string(), - url: z.string(), - size: z.number(), - type: z.string(), - key: z.string(), - context: z.string().optional(), -}) - -const registerWorkspaceFileResponseSchema = z.object({ - success: z.boolean(), - file: registeredWorkspaceFileSchema.optional(), - error: z.string().optional(), - isDuplicate: z.boolean().optional(), -}) - -export type RegisterWorkspaceFileResponse = z.output - -export const registerWorkspaceFileContract = defineRouteContract({ - method: 'POST', - path: '/api/workspaces/[id]/files/register', - params: workspaceFilesParamsSchema, - body: registerWorkspaceFileBodySchema, - response: { - mode: 'json', - schema: registerWorkspaceFileResponseSchema, - }, -}) diff --git a/apps/sim/lib/table/import-runner.test.ts b/apps/sim/lib/table/import-runner.test.ts index df66a5e0984..b0fae115c83 100644 --- a/apps/sim/lib/table/import-runner.test.ts +++ b/apps/sim/lib/table/import-runner.test.ts @@ -59,6 +59,7 @@ vi.mock('@/app/api/table/utils', () => ({ normalizeColumn: (col: unknown) => col, })) +import { CSV_MAX_BATCH_SIZE_BYTES } from '@/lib/table/import' import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' const table = { @@ -136,4 +137,27 @@ describe('runTableImport source-file cleanup', () => { expect(mockMarkJobReady).toHaveBeenCalled() expect(mockDeleteFile).not.toHaveBeenCalled() }) + + it('flushes retained records before the serialized batch byte budget is exceeded', async () => { + const cell = 'x'.repeat(390 * 1024) + const csv = `name\n${Array.from({ length: 14 }, () => cell).join('\n')}\n` + mockHeadObject.mockResolvedValue({ size: Buffer.byteLength(csv) }) + mockDownloadFileStream.mockResolvedValue(Readable.from(csv)) + mockBulkInsertImportBatch.mockImplementation(async ({ rows }) => ({ + inserted: rows.length, + lastOrderKey: 'a1', + })) + + await runTableImport(buildPayload()) + + expect(mockBulkInsertImportBatch).toHaveBeenCalledTimes(2) + for (const [input] of mockBulkInsertImportBatch.mock.calls) { + const retainedBytes = input.rows.reduce( + (total: number, row: Record) => + total + Buffer.byteLength(JSON.stringify(row), 'utf8'), + 0 + ) + expect(retainedBytes).toBeLessThanOrEqual(CSV_MAX_BATCH_SIZE_BYTES) + } + }) }) diff --git a/apps/sim/lib/table/import-runner.ts b/apps/sim/lib/table/import-runner.ts index b0eba05c14d..669fe7169ee 100644 --- a/apps/sim/lib/table/import-runner.ts +++ b/apps/sim/lib/table/import-runner.ts @@ -7,6 +7,7 @@ import { captureServerEvent } from '@/lib/posthog/server' import { buildAutoMapping, CSV_MAX_BATCH_SIZE, + CSV_MAX_BATCH_SIZE_BYTES, CSV_SCHEMA_SAMPLE_SIZE, type CsvHeaderMapping, coerceRowsForTable, @@ -178,8 +179,10 @@ export async function runTableImport(payload: TableImportPayload): Promise let headerToColumn: Map | null = null let inserted = 0 let lastReported = 0 - const sample: Record[] = [] + let sample: Record[] = [] + let sampleBytes = 0 let batch: Record[] = [] + let batchBytes = 0 /** * Resolve the schema + header→column mapping from the buffered sample (runs once). @@ -311,19 +314,43 @@ export async function runTableImport(payload: TableImportPayload): Promise let ready = false for await (const record of parser as AsyncIterable>) { + const recordBytes = Buffer.byteLength(JSON.stringify(record), 'utf8') + if (recordBytes > CSV_MAX_BATCH_SIZE_BYTES) { + throw new Error(`CSV record exceeds ${CSV_MAX_BATCH_SIZE_BYTES} serialized bytes`) + } + if (!ready) { - sample.push(record) - if (sample.length >= CSV_SCHEMA_SAMPLE_SIZE) { + if (sample.length > 0 && sampleBytes + recordBytes > CSV_MAX_BATCH_SIZE_BYTES) { await resolveSetup() await flush(sample) + sample = [] + sampleBytes = 0 ready = true + } else { + sample.push(record) + sampleBytes += recordBytes + if (sample.length >= CSV_SCHEMA_SAMPLE_SIZE || sampleBytes >= CSV_MAX_BATCH_SIZE_BYTES) { + await resolveSetup() + await flush(sample) + sample = [] + sampleBytes = 0 + ready = true + } + continue } - continue + } + + if (batch.length > 0 && batchBytes + recordBytes > CSV_MAX_BATCH_SIZE_BYTES) { + await flush(batch) + batch = [] + batchBytes = 0 } batch.push(record) - if (batch.length >= CSV_MAX_BATCH_SIZE) { + batchBytes += recordBytes + if (batch.length >= CSV_MAX_BATCH_SIZE || batchBytes >= CSV_MAX_BATCH_SIZE_BYTES) { await flush(batch) batch = [] + batchBytes = 0 } } diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index 4ef9a6ef375..f58565ee118 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -8,6 +8,7 @@ import { sniffCsvDelimiterFromStream } from '@/lib/table/csv-delimiter-stream' import { buildAutoMapping, CSV_DELIMITER_SNIFF_BYTES, + CSV_MAX_RECORD_SIZE_BYTES, CsvImportValidationError, coerceRowsForTable, coerceValue, @@ -336,12 +337,24 @@ describe('import', () => { const rows = await parseViaStream('name,age\nAlice,30\n') expect(Object.keys(rows[0])).toEqual(['name', 'age']) }) + + it('rejects a record larger than the parser byte budget', async () => { + const oversizedValue = 'x'.repeat(CSV_MAX_RECORD_SIZE_BYTES * 2) + + await expect(parseViaStream(`value\n${oversizedValue}\n`)).rejects.toThrow( + new RegExp(`maximum number of tolerated bytes of ${CSV_MAX_RECORD_SIZE_BYTES}`) + ) + }) }) describe('csvParseOptions', () => { it('sets bom and the delimiter, with a header-capturing columns callback', () => { const options = csvParseOptions('\t') - expect(options).toMatchObject({ bom: true, delimiter: '\t' }) + expect(options).toMatchObject({ + bom: true, + delimiter: '\t', + max_record_size: CSV_MAX_RECORD_SIZE_BYTES, + }) expect(typeof options.columns).toBe('function') }) diff --git a/apps/sim/lib/table/import.ts b/apps/sim/lib/table/import.ts index 3759e4eefe0..e6f597e67b7 100644 --- a/apps/sim/lib/table/import.ts +++ b/apps/sim/lib/table/import.ts @@ -34,6 +34,9 @@ export type CsvDelimiter = (typeof CSV_DELIMITER_CANDIDATES)[number] */ export const CSV_DELIMITER_SNIFF_BYTES = 64 * 1024 +/** Maximum characters buffered for one CSV record before parsing fails. */ +export const CSV_MAX_RECORD_SIZE_BYTES = 1024 * 1024 + /** * Single source of truth for the `csv-parse` options used by both the buffered * sync parser and the streaming parser. @@ -62,9 +65,13 @@ export function csvParseOptions( relax_column_count: true, relax_quotes: true, skip_records_with_error: true, + on_skip(error) { + if (error?.code === 'CSV_MAX_RECORD_SIZE') throw error + }, cast: false, bom: true, delimiter, + max_record_size: CSV_MAX_RECORD_SIZE_BYTES, } } @@ -220,9 +227,14 @@ export const CSV_SCHEMA_SAMPLE_SIZE = 100 */ export const CSV_MAX_BATCH_SIZE = 5000 +/** Maximum serialized CSV row data retained before an import batch is flushed. */ +export const CSV_MAX_BATCH_SIZE_BYTES = 5 * 1024 * 1024 + /** Maximum CSV/TSV file size accepted by import routes (25 MB). */ export const CSV_MAX_FILE_SIZE_BYTES = 25 * 1024 * 1024 +export const CSV_MAX_FILE_SIZE_MESSAGE = `File exceeds maximum allowed size of ${CSV_MAX_FILE_SIZE_BYTES / (1024 * 1024)} MB` + /** * Error thrown when the user-supplied mapping or CSV does not line up with the * target table. Callers should translate this into a 400 response. diff --git a/apps/sim/lib/table/orchestration/import-resource.test.ts b/apps/sim/lib/table/orchestration/import-resource.test.ts new file mode 100644 index 00000000000..b281965477d --- /dev/null +++ b/apps/sim/lib/table/orchestration/import-resource.test.ts @@ -0,0 +1,191 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCreateTable, + mockCreateUploadSession, + mockDbLimit, + mockGetUserEntityPermissions, + mockGetUserSettings, + mockGetWorkspaceFile, + mockGetWorkspaceTableLimits, + mockRunDetached, +} = vi.hoisted(() => ({ + mockCreateTable: vi.fn(), + mockCreateUploadSession: vi.fn(), + mockDbLimit: vi.fn(), + mockGetUserEntityPermissions: vi.fn(), + mockGetUserSettings: vi.fn(), + mockGetWorkspaceFile: vi.fn(), + mockGetWorkspaceTableLimits: vi.fn(), + mockRunDetached: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { + select: () => ({ + from: () => ({ + where: () => ({ limit: mockDbLimit }), + }), + }), + }, +})) +vi.mock('@/lib/core/config/env-flags', () => ({ isTriggerDevEnabled: false })) +vi.mock('@/lib/core/utils/background', () => ({ runDetached: mockRunDetached })) +vi.mock('@/lib/table/billing', () => ({ getWorkspaceTableLimits: mockGetWorkspaceTableLimits })) +vi.mock('@/lib/table/import-runner', () => ({ runTableImport: vi.fn() })) +vi.mock('@/lib/table/service', () => ({ + createTable: mockCreateTable, + getTableById: vi.fn(), +})) +vi.mock('@/lib/uploads/contexts/workspace', () => ({ getWorkspaceFile: mockGetWorkspaceFile })) +vi.mock('@/lib/uploads/upload-session/service', () => ({ + abortUploadSession: vi.fn(), + createUploadSession: mockCreateUploadSession, + getOwnedUploadSession: vi.fn(), +})) +vi.mock('@/lib/users/queries', () => ({ getUserSettings: mockGetUserSettings })) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + getUserEntityPermissions: mockGetUserEntityPermissions, +})) + +import { CSV_MAX_FILE_SIZE_BYTES } from '@/lib/table/import' +import { createTableImportResource } from '@/lib/table/orchestration/import-resource' + +const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' +const SOURCE = { type: 'workspace_file' as const, fileId: 'file-1' } +const TARGET = { type: 'new' as const, name: 'imported_data' } + +function workspaceFile(size: number) { + return { + id: 'file-1', + workspaceId: WORKSPACE_ID, + name: 'data.csv', + key: 'workspace/data.csv', + path: '/api/files/serve/workspace/data.csv', + size, + type: 'text/csv', + uploadedBy: 'user-1', + uploadedAt: new Date('2026-08-04T12:00:00.000Z'), + updatedAt: new Date('2026-08-04T12:00:00.000Z'), + } +} + +describe('createTableImportResource workspace file size', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetUserEntityPermissions.mockResolvedValue('write') + mockGetWorkspaceTableLimits.mockResolvedValue({ maxTables: 100, maxRowsPerTable: 10_000 }) + mockCreateTable.mockResolvedValue({ id: 'table-1' }) + mockGetUserSettings.mockResolvedValue({ timezone: 'UTC' }) + mockDbLimit.mockResolvedValue([ + { + id: 'import-1', + workspaceId: WORKSPACE_ID, + tableId: 'table-1', + type: 'import', + status: 'running', + rowsProcessed: 0, + error: null, + payload: { + kind: 'table_import', + userId: 'user-1', + source: SOURCE, + target: TARGET, + options: {}, + }, + startedAt: new Date('2026-08-04T12:00:00.000Z'), + updatedAt: new Date('2026-08-04T12:00:00.000Z'), + completedAt: null, + }, + ]) + }) + + it('accepts a workspace CSV at the exact byte limit', async () => { + mockGetWorkspaceFile.mockResolvedValue(workspaceFile(CSV_MAX_FILE_SIZE_BYTES)) + + const result = await createTableImportResource( + { workspaceId: WORKSPACE_ID, source: SOURCE, target: TARGET }, + 'user-1', + 'http://localhost:3000' + ) + + expect(result.upload).toBeNull() + expect(mockCreateTable).toHaveBeenCalledOnce() + expect(mockRunDetached).toHaveBeenCalledOnce() + }) + + it('rejects a workspace CSV one byte over the limit before creating a table', async () => { + mockGetWorkspaceFile.mockResolvedValue(workspaceFile(CSV_MAX_FILE_SIZE_BYTES + 1)) + + await expect( + createTableImportResource( + { workspaceId: WORKSPACE_ID, source: SOURCE, target: TARGET }, + 'user-1', + 'http://localhost:3000' + ) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mockCreateTable).not.toHaveBeenCalled() + expect(mockRunDetached).not.toHaveBeenCalled() + }) +}) + +describe('createTableImportResource upload size', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetUserEntityPermissions.mockResolvedValue('write') + mockCreateUploadSession.mockResolvedValue({ + id: 'import-1', + userId: 'user-1', + status: 'uploading', + uploadToken: 'signed-token', + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + createdAt: new Date('2026-08-04T12:00:00.000Z'), + updatedAt: new Date('2026-08-04T12:00:00.000Z'), + completedAt: null, + }) + }) + + it('creates an upload session for a CSV at the exact byte limit', async () => { + await createTableImportResource( + { + workspaceId: WORKSPACE_ID, + source: { + type: 'upload', + name: 'data.csv', + contentType: 'text/csv', + size: CSV_MAX_FILE_SIZE_BYTES, + }, + target: TARGET, + }, + 'user-1', + 'http://localhost:3000' + ) + + expect(mockCreateUploadSession).toHaveBeenCalledWith( + expect.objectContaining({ fileSize: CSV_MAX_FILE_SIZE_BYTES, purpose: 'table_import' }) + ) + }) + + it('rejects an upload one byte over the limit before creating a session', async () => { + await expect( + createTableImportResource( + { + workspaceId: WORKSPACE_ID, + source: { + type: 'upload', + name: 'data.csv', + contentType: 'text/csv', + size: CSV_MAX_FILE_SIZE_BYTES + 1, + }, + target: TARGET, + }, + 'user-1', + 'http://localhost:3000' + ) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mockCreateUploadSession).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/table/orchestration/import-resource.ts b/apps/sim/lib/table/orchestration/import-resource.ts index 699ac1ead3f..2f1e88ef8d1 100644 --- a/apps/sim/lib/table/orchestration/import-resource.ts +++ b/apps/sim/lib/table/orchestration/import-resource.ts @@ -5,10 +5,12 @@ import { generateId } from '@sim/utils/id' import { and, eq } from 'drizzle-orm' import { type V2CreateTableImportBody, + type V2CreateTableImportData, type V2TableImport, type V2TableImportSource, type V2TableImportTarget, v2CreateTableImportBodySchema, + v2CreateTableImportDataSchema, v2TableImportSourceSchema, v2TableImportTargetSchema, } from '@/lib/api/contracts/v2/tables' @@ -18,6 +20,7 @@ import { runDetached } from '@/lib/core/utils/background' import { generateRequestId } from '@/lib/core/utils/request' import { findActiveFolder } from '@/lib/folders/queries' import { getWorkspaceTableLimits } from '@/lib/table/billing' +import { CSV_MAX_FILE_SIZE_BYTES, CSV_MAX_FILE_SIZE_MESSAGE } from '@/lib/table/import' import { runTableImport, type TableImportPayload } from '@/lib/table/import-runner' import { markJobCanceled, markJobFailed, markTableJobRunning } from '@/lib/table/jobs/service' import { assertRowDelete, assertRowInsert } from '@/lib/table/mutation-locks' @@ -26,10 +29,11 @@ import type { TableImportJobPayload } from '@/lib/table/types' import { getWorkspaceFile, type WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { abortUploadSession, + type CreatedUploadSession, createUploadSession, getOwnedUploadSession, type UploadSessionRecord, -} from '@/lib/uploads/multipart-session/service' +} from '@/lib/uploads/upload-session/service' import { getUserSettings } from '@/lib/users/queries' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' @@ -46,7 +50,6 @@ interface TableImportResource { status: TableImportStatus rowsProcessed: number error: string | null - upload: UploadSessionRecord | null createdAt: Date updatedAt: Date completedAt: Date | null @@ -54,12 +57,13 @@ interface TableImportResource { interface CreateTableImportResult { record: TableImportResource - upload: UploadSessionRecord | null + upload: CreatedUploadSession | null } export async function createTableImportResource( body: V2CreateTableImportBody, - userId: string + userId: string, + localOrigin: string ): Promise { await assertWorkspaceWrite(userId, body.workspaceId) await validateTarget(body.workspaceId, body.target) @@ -68,6 +72,9 @@ export async function createTableImportResource( if (body.source.type === 'upload') { assertCsvFileName(body.source.name) + if (body.source.size > CSV_MAX_FILE_SIZE_BYTES) { + throw new OrchestrationError('validation', CSV_MAX_FILE_SIZE_MESSAGE) + } const upload = await createUploadSession({ id: importId, workspaceId: body.workspaceId, @@ -77,6 +84,7 @@ export async function createTableImportResource( contentType: body.source.contentType, fileSize: body.source.size, metadata: { tableImport: body }, + localOrigin, }) return { record: resourceFromUpload(upload, body), upload } } @@ -104,15 +112,16 @@ export async function startUploadedTableImport( upload: UploadSessionRecord ): Promise { const body = tableImportBodyFromUpload(upload) + const workspaceId = body.workspaceId const existing = await findOwnedTableImport({ importId: upload.id, - workspaceId: upload.workspaceId, + workspaceId, userId: upload.userId, }) if (existing) return existing return startTableImport({ id: upload.id, - workspaceId: upload.workspaceId, + workspaceId, userId: upload.userId, source: body.source, target: body.target, @@ -192,7 +201,6 @@ export async function findOwnedTableImport(params: { status: tableImportStatus(job.status), rowsProcessed: job.rowsProcessed, error: job.error, - upload: null, createdAt: job.startedAt, updatedAt: job.updatedAt, completedAt: job.completedAt, @@ -224,20 +232,20 @@ export function toV2TableImport(record: TableImportResource): V2TableImport { tableId: record.tableId, rowsProcessed: record.rowsProcessed, error: record.error, - upload: record.upload - ? { - uploadToken: record.upload.uploadToken, - partSize: record.upload.partSize, - partCount: record.upload.partCount, - expiresAt: record.upload.expiresAt.toISOString(), - } - : null, createdAt: record.createdAt.toISOString(), updatedAt: record.updatedAt.toISOString(), completedAt: record.completedAt?.toISOString() ?? null, } } +export function toV2CreateTableImport(result: CreateTableImportResult): V2CreateTableImportData { + return v2CreateTableImportDataSchema.parse({ + session: toV2TableImport(result.record), + uploadToken: result.upload?.uploadToken ?? null, + transfer: result.upload?.transfer ?? null, + }) +} + interface StartTableImportParams { id: string workspaceId: string @@ -340,7 +348,7 @@ function resourceFromUpload( ): TableImportResource { return { id: upload.id, - workspaceId: upload.workspaceId, + workspaceId: body.workspaceId, userId: upload.userId, source: body.source, target: body.target, @@ -349,7 +357,6 @@ function resourceFromUpload( status: upload.status === 'aborted' ? 'canceled' : 'uploading', rowsProcessed: 0, error: null, - upload, createdAt: upload.createdAt, updatedAt: upload.updatedAt, completedAt: upload.completedAt, @@ -425,6 +432,9 @@ async function requireWorkspaceSource( ): Promise { const file = await getWorkspaceFile(workspaceId, fileId, { throwOnError: true }) if (!file) throw new OrchestrationError('not_found', 'Workspace file not found') + if (file.size > CSV_MAX_FILE_SIZE_BYTES) { + throw new OrchestrationError('validation', CSV_MAX_FILE_SIZE_MESSAGE) + } return file } diff --git a/apps/sim/lib/uploads/client/admission.test.ts b/apps/sim/lib/uploads/client/admission.test.ts new file mode 100644 index 00000000000..889678594e6 --- /dev/null +++ b/apps/sim/lib/uploads/client/admission.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest' +import { + assertMultiFileUploadAdmission, + MULTI_FILE_UPLOAD_MAX_FILE_BYTES, + MULTI_FILE_UPLOAD_MAX_FILES, + MULTI_FILE_UPLOAD_MAX_TOTAL_BYTES, + type MultiFileUploadAdmissionError, +} from '@/lib/uploads/client/admission' + +function files(count: number, size: number) { + return Array.from({ length: count }, (_, index) => ({ name: `file-${index}.bin`, size })) +} + +describe('multi-file upload admission', () => { + it('accepts the exact aggregate-byte boundary', () => { + expect(() => + assertMultiFileUploadAdmission(files(5, MULTI_FILE_UPLOAD_MAX_FILE_BYTES)) + ).not.toThrow() + expect(MULTI_FILE_UPLOAD_MAX_TOTAL_BYTES).toBe(5 * MULTI_FILE_UPLOAD_MAX_FILE_BYTES) + }) + + it('rejects a selection whose combined count exceeds the action cap', () => { + expect(() => + assertMultiFileUploadAdmission([{ name: 'new.bin', size: 1 }], { + existingFiles: files(MULTI_FILE_UPLOAD_MAX_FILES, 1), + }) + ).toThrow( + expect.objectContaining>({ + code: 'UPLOAD_FILE_COUNT_EXCEEDED', + }) + ) + }) + + it('rejects one file above the shared per-file ceiling', () => { + expect(() => + assertMultiFileUploadAdmission([ + { name: 'oversized.bin', size: MULTI_FILE_UPLOAD_MAX_FILE_BYTES + 1 }, + ]) + ).toThrow( + expect.objectContaining>({ + code: 'UPLOAD_FILE_SIZE_EXCEEDED', + }) + ) + }) + + it('rejects aggregate bytes across existing and newly selected files', () => { + expect(() => + assertMultiFileUploadAdmission(files(1, MULTI_FILE_UPLOAD_MAX_FILE_BYTES), { + existingFiles: files(5, MULTI_FILE_UPLOAD_MAX_FILE_BYTES), + }) + ).toThrow( + expect.objectContaining>({ + code: 'UPLOAD_TOTAL_SIZE_EXCEEDED', + }) + ) + }) + + it('supports a larger direct-to-storage limit without weakening aggregate admission', () => { + expect(() => + assertMultiFileUploadAdmission([{ name: 'archive.zip', size: 1024 }], { + maxFileBytes: 1024, + maxTotalBytes: 2048, + }) + ).not.toThrow() + + expect(() => + assertMultiFileUploadAdmission(files(3, 1024), { + maxFileBytes: 1024, + maxTotalBytes: 2048, + }) + ).toThrow( + expect.objectContaining>({ + code: 'UPLOAD_TOTAL_SIZE_EXCEEDED', + }) + ) + }) +}) diff --git a/apps/sim/lib/uploads/client/admission.ts b/apps/sim/lib/uploads/client/admission.ts new file mode 100644 index 00000000000..670a570a2fd --- /dev/null +++ b/apps/sim/lib/uploads/client/admission.ts @@ -0,0 +1,102 @@ +import { + MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE, + MAX_WORKSPACE_FORMDATA_FILE_SIZE, +} from '@/lib/uploads/shared/types' + +export const MULTI_FILE_UPLOAD_MAX_FILES = 20 +const MULTI_FILE_UPLOAD_MAX_TOTAL_FILE_EQUIVALENTS = 5 +export const MULTI_FILE_UPLOAD_MAX_FILE_BYTES = Math.min( + MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE, + MAX_WORKSPACE_FORMDATA_FILE_SIZE +) +export const MULTI_FILE_UPLOAD_MAX_TOTAL_BYTES = + MULTI_FILE_UPLOAD_MAX_TOTAL_FILE_EQUIVALENTS * MULTI_FILE_UPLOAD_MAX_FILE_BYTES + +export type MultiFileUploadAdmissionErrorCode = + | 'UPLOAD_FILE_COUNT_EXCEEDED' + | 'UPLOAD_FILE_SIZE_EXCEEDED' + | 'UPLOAD_TOTAL_SIZE_EXCEEDED' + +export class MultiFileUploadAdmissionError extends Error { + constructor( + message: string, + readonly code: MultiFileUploadAdmissionErrorCode + ) { + super(message) + this.name = 'MultiFileUploadAdmissionError' + } +} + +interface UploadAdmissionFile { + readonly name?: string + readonly size: number +} + +interface MultiFileUploadAdmissionOptions { + existingFiles?: ArrayLike + maxFileBytes?: number + maxTotalBytes?: number +} + +/** + * Bounds one user upload action before previews, UI rows, or upload sessions are allocated. + * Knowledge uploads use the shared 100 MiB / 500 MiB defaults. Direct-to-storage consumers may + * provide their larger server-side limit while retaining the same count and aggregate bounds. + */ +export function assertMultiFileUploadAdmission( + files: ArrayLike, + options: MultiFileUploadAdmissionOptions = {} +): void { + const existingFiles = options.existingFiles + const maxFileBytes = options.maxFileBytes ?? MULTI_FILE_UPLOAD_MAX_FILE_BYTES + const maxTotalBytes = + options.maxTotalBytes ?? MULTI_FILE_UPLOAD_MAX_TOTAL_FILE_EQUIVALENTS * maxFileBytes + if (!Number.isSafeInteger(maxFileBytes) || maxFileBytes < 1) { + throw new Error('Invalid per-file upload limit') + } + if (!Number.isSafeInteger(maxTotalBytes) || maxTotalBytes < maxFileBytes) { + throw new Error('Invalid aggregate upload limit') + } + const existingCount = existingFiles?.length ?? 0 + const totalCount = existingCount + files.length + if (totalCount > MULTI_FILE_UPLOAD_MAX_FILES) { + throw new MultiFileUploadAdmissionError( + `Select up to ${MULTI_FILE_UPLOAD_MAX_FILES} files at a time.`, + 'UPLOAD_FILE_COUNT_EXCEEDED' + ) + } + + let totalBytes = 0 + const groups = existingFiles ? [existingFiles, files] : [files] + for (const group of groups) { + for (let index = 0; index < group.length; index++) { + const file = group[index] + if (!file || !Number.isSafeInteger(file.size) || file.size < 0) { + throw new Error('Invalid file size in upload selection') + } + if (file.size > maxFileBytes) { + const label = file.name ? `"${file.name}"` : 'A selected file' + throw new MultiFileUploadAdmissionError( + `${label} is too large. Each file must be ${formatBinaryBytes(maxFileBytes)} or smaller.`, + 'UPLOAD_FILE_SIZE_EXCEEDED' + ) + } + totalBytes += file.size + } + } + + if (totalBytes > maxTotalBytes) { + throw new MultiFileUploadAdmissionError( + `Select files totaling ${formatBinaryBytes(maxTotalBytes)} or less.`, + 'UPLOAD_TOTAL_SIZE_EXCEEDED' + ) + } +} + +function formatBinaryBytes(bytes: number): string { + const gibibyte = 1024 ** 3 + if (bytes % gibibyte === 0) return `${bytes / gibibyte} GiB` + const mebibyte = 1024 ** 2 + if (bytes % mebibyte === 0) return `${bytes / mebibyte} MiB` + return `${bytes} bytes` +} diff --git a/apps/sim/lib/uploads/client/api-fallback.test.ts b/apps/sim/lib/uploads/client/api-fallback.test.ts deleted file mode 100644 index 0c0d4196144..00000000000 --- a/apps/sim/lib/uploads/client/api-fallback.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { uploadViaApiFallbackWithMetadata } from '@/lib/uploads/client/api-fallback' - -const mockFetch = vi.fn() - -describe('uploadViaApiFallbackWithMetadata', () => { - beforeEach(() => { - mockFetch.mockReset() - vi.stubGlobal('fetch', mockFetch) - }) - - afterEach(() => { - vi.unstubAllGlobals() - }) - - it('builds execution multipart fields and normalizes an array response', async () => { - mockFetch.mockResolvedValue( - new Response( - JSON.stringify({ - files: [ - { - id: 'file-1', - name: 'diagram.png', - url: '/api/files/serve/execution%2Fdiagram.png', - size: 7, - type: 'image/png', - key: 'execution/diagram.png', - }, - ], - }), - { status: 200, headers: { 'Content-Type': 'application/json' } } - ) - ) - const file = new File(['diagram'], 'diagram.png', { type: 'image/png' }) - - const result = await uploadViaApiFallbackWithMetadata(file, 'execution', { - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - }) - - expect(result).toEqual({ - id: 'file-1', - name: 'diagram.png', - path: '/api/files/serve/execution%2Fdiagram.png', - size: 7, - type: 'image/png', - key: 'execution/diagram.png', - }) - - const request = mockFetch.mock.calls[0]?.[1] as RequestInit - const formData = request.body as FormData - expect(formData.get('file')).toBe(file) - expect(formData.get('context')).toBe('execution') - expect(formData.get('workspaceId')).toBe('workspace-1') - expect(formData.get('workflowId')).toBe('workflow-1') - expect(formData.get('executionId')).toBe('execution-1') - }) - - it('throws the exact server upload error', async () => { - mockFetch.mockResolvedValue( - new Response(JSON.stringify({ error: 'Workspace file storage limit exceeded' }), { - status: 413, - headers: { 'Content-Type': 'application/json' }, - }) - ) - - await expect( - uploadViaApiFallbackWithMetadata( - new File(['report'], 'report.pdf', { type: 'application/pdf' }), - 'execution', - { - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - } - ) - ).rejects.toThrow('Workspace file storage limit exceeded') - }) -}) diff --git a/apps/sim/lib/uploads/client/api-fallback.ts b/apps/sim/lib/uploads/client/api-fallback.ts deleted file mode 100644 index 2281fd05e61..00000000000 --- a/apps/sim/lib/uploads/client/api-fallback.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { isRecordLike } from '@sim/utils/object' -import type { StorageContext } from '@/lib/uploads/shared/types' - -export interface ApiFallbackUploadOptions { - workspaceId?: string - workflowId?: string - executionId?: string -} - -export interface ApiFallbackUploadMetadata { - path: string - key?: string - id?: string - name?: string - size?: number - type?: string - uploadedAt?: string - expiresAt?: string -} - -function getOptionalString(value: unknown): string | undefined { - if (typeof value !== 'string') return undefined - const trimmed = value.trim() - return trimmed || undefined -} - -function getOptionalNumber(value: unknown): number | undefined { - return typeof value === 'number' && Number.isFinite(value) ? value : undefined -} - -function getUploadFailureReason(value: unknown, response: Response): string { - if (isRecordLike(value)) { - const message = getOptionalString(value.message) ?? getOptionalString(value.error) - if (message) return message - } - - return `Failed to upload file: ${response.status}` -} - -function normalizeUploadMetadata(value: unknown): ApiFallbackUploadMetadata { - if (!isRecordLike(value)) { - throw new Error('Invalid upload response: expected file metadata') - } - - const fileInfo = isRecordLike(value.fileInfo) ? value.fileInfo : undefined - const path = - getOptionalString(fileInfo?.path) ?? - getOptionalString(value.path) ?? - getOptionalString(value.url) - if (!path) { - throw new Error('Invalid upload response: missing path') - } - - return { - path, - key: getOptionalString(fileInfo?.key) ?? getOptionalString(value.key), - id: getOptionalString(fileInfo?.id) ?? getOptionalString(value.id), - name: - getOptionalString(fileInfo?.name) ?? - getOptionalString(value.name) ?? - getOptionalString(value.fileName), - size: getOptionalNumber(fileInfo?.size) ?? getOptionalNumber(value.size), - type: getOptionalString(fileInfo?.type) ?? getOptionalString(value.type), - uploadedAt: getOptionalString(fileInfo?.uploadedAt) ?? getOptionalString(value.uploadedAt), - expiresAt: getOptionalString(fileInfo?.expiresAt) ?? getOptionalString(value.expiresAt), - } -} - -async function parseUploadResponse(response: Response): Promise { - if (!response.ok) { - const errorData: unknown = await response.json().catch(() => null) - throw new Error(getUploadFailureReason(errorData, response)) - } - - let data: unknown - try { - data = await response.json() - } catch { - throw new Error('Invalid upload response: response was not JSON') - } - - const results = isRecordLike(data) && Array.isArray(data.files) ? data.files : [data] - if (results.length === 0) { - throw new Error('Invalid upload response: no files returned') - } - if (results.length > 1) { - throw new Error('Invalid upload response: multiple files returned for a single-file upload') - } - - return normalizeUploadMetadata(results[0]) -} - -/** - * Uploads one file through the server-proxied multipart fallback and returns - * normalized metadata for either the singular or `{ files: [...] }` response. - */ -export async function uploadViaApiFallbackWithMetadata( - file: File, - context: StorageContext, - options: ApiFallbackUploadOptions = {} -): Promise { - const formData = new FormData() - formData.append('file', file) - formData.append('context', context) - if (options.workspaceId) { - formData.append('workspaceId', options.workspaceId) - } - if (options.workflowId) { - formData.append('workflowId', options.workflowId) - } - if (options.executionId) { - formData.append('executionId', options.executionId) - } - - // boundary-raw-fetch: local-dev fallback when cloud storage is not configured; multipart upload incompatible with requestJson - const response = await fetch('/api/files/upload', { method: 'POST', body: formData }) - return parseUploadResponse(response) -} - -/** - * Server-proxied fallback used only when cloud storage isn't configured (local dev). - * Production always takes the presigned PUT path. - */ -export async function uploadViaApiFallback( - file: File, - context: StorageContext, - workspaceId?: string -): Promise<{ path: string; key?: string }> { - const { path, key } = await uploadViaApiFallbackWithMetadata(file, context, { workspaceId }) - return { path, key } -} diff --git a/apps/sim/lib/uploads/client/concurrency.test.ts b/apps/sim/lib/uploads/client/concurrency.test.ts new file mode 100644 index 00000000000..b0880ef091d --- /dev/null +++ b/apps/sim/lib/uploads/client/concurrency.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from 'vitest' +import { runWithConcurrency } from '@/lib/uploads/client/concurrency' + +describe('runWithConcurrency', () => { + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects invalid concurrency limit %s', + async (limit) => { + await expect(runWithConcurrency([1], limit, async (item) => item)).rejects.toThrow( + 'Concurrency limit must be a positive safe integer' + ) + } + ) + + it('preserves input order while bounding concurrency', async () => { + let active = 0 + let peak = 0 + const release: Array<() => void> = [] + const worker = vi.fn(async (item: number) => { + active += 1 + peak = Math.max(peak, active) + await new Promise((resolve) => release.push(resolve)) + active -= 1 + return item * 2 + }) + + const resultPromise = runWithConcurrency([1, 2, 3], 2, worker) + await Promise.resolve() + expect(worker).toHaveBeenCalledTimes(2) + + release.shift()?.() + release.shift()?.() + await Promise.resolve() + await Promise.resolve() + expect(worker).toHaveBeenCalledTimes(3) + release.shift()?.() + + await expect(resultPromise).resolves.toEqual([ + { status: 'fulfilled', value: 2 }, + { status: 'fulfilled', value: 4 }, + { status: 'fulfilled', value: 6 }, + ]) + expect(peak).toBe(2) + }) +}) diff --git a/apps/sim/lib/uploads/client/concurrency.ts b/apps/sim/lib/uploads/client/concurrency.ts new file mode 100644 index 00000000000..307e145776f --- /dev/null +++ b/apps/sim/lib/uploads/client/concurrency.ts @@ -0,0 +1,38 @@ +export const WHOLE_FILE_PARALLEL_UPLOADS = 3 + +/** + * Runs a worker with bounded concurrency and preserves input ordering in the settled results. + */ +export async function runWithConcurrency( + items: T[], + limit: number, + worker: (item: T, index: number) => Promise +): Promise>> { + if (!Number.isSafeInteger(limit) || limit <= 0) { + throw new Error('Concurrency limit must be a positive safe integer') + } + + const results: Array> = Array(items.length) + if (items.length === 0) return results + + const concurrency = Math.min(limit, items.length) + let nextIndex = 0 + + const runners = Array.from({ length: concurrency }, async () => { + while (true) { + const currentIndex = nextIndex++ + if (currentIndex >= items.length) return + try { + results[currentIndex] = { + status: 'fulfilled', + value: await worker(items[currentIndex], currentIndex), + } + } catch (error) { + results[currentIndex] = { status: 'rejected', reason: error } + } + } + }) + + await Promise.all(runners) + return results +} diff --git a/apps/sim/lib/uploads/client/direct-upload.test.ts b/apps/sim/lib/uploads/client/direct-upload.test.ts deleted file mode 100644 index da2b2aca65f..00000000000 --- a/apps/sim/lib/uploads/client/direct-upload.test.ts +++ /dev/null @@ -1,266 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { - DirectUploadError, - type PresignedUploadInfo, - runUploadStrategy, -} from '@/lib/uploads/client/direct-upload' - -const ONE_MB = 1024 * 1024 -const LARGE_THRESHOLD = 50 * ONE_MB - -const makeFile = (size: number, name = 'test.bin', type = 'application/octet-stream'): File => { - const file = new File([new Uint8Array(0)], name, { type }) - Object.defineProperty(file, 'size', { value: size }) - return file -} - -const presigned = (overrides?: Partial): PresignedUploadInfo => ({ - fileName: 'test.bin', - presignedUrl: 'https://s3/presigned', - fileInfo: { - path: '/api/files/serve/test', - key: 'workspace/ws-1/test.bin', - name: 'test.bin', - size: ONE_MB, - type: 'application/octet-stream', - }, - uploadHeaders: undefined, - directUploadSupported: true, - ...overrides, -}) - -class MockXHR { - static instances: MockXHR[] = [] - upload = { addEventListener: vi.fn() } - status = 200 - statusText = 'OK' - private listeners: Record void>> = {} - open = vi.fn() - setRequestHeader = vi.fn() - abort = vi.fn() - send = vi.fn(() => { - queueMicrotask(() => this.listeners.load?.forEach((cb) => cb())) - }) - addEventListener = (event: string, cb: () => void) => { - ;(this.listeners[event] ??= []).push(cb) - } - removeEventListener = vi.fn() - constructor() { - MockXHR.instances.push(this) - } -} - -describe('runUploadStrategy', () => { - let originalXHR: typeof XMLHttpRequest - - beforeEach(() => { - MockXHR.instances = [] - originalXHR = globalThis.XMLHttpRequest - globalThis.XMLHttpRequest = MockXHR as unknown as typeof XMLHttpRequest - }) - - afterEach(() => { - globalThis.XMLHttpRequest = originalXHR - vi.restoreAllMocks() - }) - - it('uses presigned PUT for files at or below the multipart threshold', async () => { - const file = makeFile(LARGE_THRESHOLD) - - const result = await runUploadStrategy({ - file, - workspaceId: 'ws-1', - context: 'workspace', - presignedOverride: presigned(), - }) - - expect(result.key).toBe('workspace/ws-1/test.bin') - expect(MockXHR.instances).toHaveLength(1) - expect(MockXHR.instances[0].open).toHaveBeenCalledWith('PUT', 'https://s3/presigned') - }) - - it('sets Content-Type exactly once when uploadHeaders already carry it (GCS signed uploads)', async () => { - const file = makeFile(1024) - - await runUploadStrategy({ - file, - workspaceId: 'ws-1', - context: 'workspace', - presignedOverride: presigned({ - uploadHeaders: { - 'Content-Type': 'application/octet-stream', - 'x-goog-meta-workspaceid': 'ws-1', - }, - }), - }) - - const calls = MockXHR.instances[0].setRequestHeader.mock.calls - const contentTypeCalls = calls.filter( - ([k]: [string, string]) => k.toLowerCase() === 'content-type' - ) - expect(contentTypeCalls).toHaveLength(1) - expect(contentTypeCalls[0][1]).toBe('application/octet-stream') - expect(calls.some(([k]: [string, string]) => k === 'x-goog-meta-workspaceid')).toBe(true) - }) - - it('falls back to the file content type when uploadHeaders omit Content-Type', async () => { - const file = makeFile(1024) - - await runUploadStrategy({ - file, - workspaceId: 'ws-1', - context: 'workspace', - presignedOverride: presigned({ uploadHeaders: { 'x-ms-blob-type': 'BlockBlob' } }), - }) - - const calls = MockXHR.instances[0].setRequestHeader.mock.calls - const contentTypeCalls = calls.filter( - ([k]: [string, string]) => k.toLowerCase() === 'content-type' - ) - expect(contentTypeCalls).toHaveLength(1) - }) - - it('throws FALLBACK_REQUIRED when server signals no cloud storage', async () => { - const file = makeFile(ONE_MB) - - await expect( - runUploadStrategy({ - file, - workspaceId: 'ws-1', - context: 'workspace', - presignedOverride: presigned({ presignedUrl: '', directUploadSupported: false }), - }) - ).rejects.toMatchObject({ - name: 'DirectUploadError', - code: 'FALLBACK_REQUIRED', - }) - }) - - it('takes the multipart path for files larger than the threshold and posts unified parts', async () => { - const file = makeFile(LARGE_THRESHOLD + ONE_MB) - const calls: Array<{ url: string; body: unknown }> = [] - - const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { - const url = typeof input === 'string' ? input : input.toString() - const rawBody = init?.body - const body = typeof rawBody === 'string' ? JSON.parse(rawBody) : undefined - calls.push({ url, body }) - - if (url.includes('action=initiate')) { - return new Response( - JSON.stringify({ uploadId: 'u1', key: 'workspace/ws-1/big.bin', uploadToken: 't' }), - { status: 200 } - ) - } - if (url.includes('action=get-part-urls')) { - return new Response( - JSON.stringify({ - presignedUrls: [ - { partNumber: 1, url: 'https://s3/part1' }, - { partNumber: 2, url: 'https://s3/part2' }, - ], - }), - { status: 200 } - ) - } - if (url.startsWith('https://s3/part')) { - return new Response(null, { status: 200, headers: { ETag: '"etag-x"' } }) - } - if (url.includes('action=complete')) { - return new Response(JSON.stringify({ path: '/api/files/serve/big' }), { status: 200 }) - } - throw new Error(`unexpected url ${url}`) - }) - - vi.stubGlobal('fetch', fetchMock) - - const result = await runUploadStrategy({ - file, - workspaceId: 'ws-1', - context: 'workspace', - }) - - expect(result.path).toBe('/api/files/serve/big') - - const completeCall = calls.find((c) => c.url.includes('action=complete'))! - expect(completeCall.body).toMatchObject({ - uploadToken: 't', - parts: [ - { partNumber: 1, etag: 'etag-x' }, - { partNumber: 2, etag: 'etag-x' }, - ], - }) - }) - - it('rejects with ABORTED when signal is already aborted before PUT begins', async () => { - const file = makeFile(ONE_MB) - const controller = new AbortController() - controller.abort() - - await expect( - runUploadStrategy({ - file, - workspaceId: 'ws-1', - context: 'workspace', - presignedOverride: presigned(), - signal: controller.signal, - }) - ).rejects.toMatchObject({ name: 'DirectUploadError', code: 'ABORTED' }) - }) - - it('fires action=abort when the multipart complete call fails', async () => { - const file = makeFile(LARGE_THRESHOLD + ONE_MB) - const calls: Array<{ url: string }> = [] - - const fetchMock = vi.fn(async (input: RequestInfo | URL) => { - const url = typeof input === 'string' ? input : input.toString() - calls.push({ url }) - - if (url.includes('action=initiate')) { - return new Response( - JSON.stringify({ uploadId: 'u1', key: 'workspace/ws-1/big.bin', uploadToken: 't' }), - { status: 200 } - ) - } - if (url.includes('action=get-part-urls')) { - return new Response( - JSON.stringify({ - presignedUrls: [ - { partNumber: 1, url: 'https://s3/part1' }, - { partNumber: 2, url: 'https://s3/part2' }, - ], - }), - { status: 200 } - ) - } - if (url.startsWith('https://s3/part')) { - return new Response(null, { status: 200, headers: { ETag: '"etag-x"' } }) - } - if (url.includes('action=complete')) { - return new Response(JSON.stringify({ error: 'kaboom' }), { status: 500 }) - } - if (url.includes('action=abort')) { - return new Response(null, { status: 200 }) - } - throw new Error(`unexpected url ${url}`) - }) - - vi.stubGlobal('fetch', fetchMock) - - await expect( - runUploadStrategy({ file, workspaceId: 'ws-1', context: 'workspace' }) - ).rejects.toBeInstanceOf(DirectUploadError) - - expect(calls.some((c) => c.url.includes('action=abort'))).toBe(true) - }) - - it('throws when neither presignedEndpoint nor presignedOverride is supplied', async () => { - const file = makeFile(ONE_MB) - await expect( - runUploadStrategy({ file, workspaceId: 'ws-1', context: 'workspace' }) - ).rejects.toBeInstanceOf(DirectUploadError) - }) -}) diff --git a/apps/sim/lib/uploads/client/direct-upload.ts b/apps/sim/lib/uploads/client/direct-upload.ts deleted file mode 100644 index 41e9911c26d..00000000000 --- a/apps/sim/lib/uploads/client/direct-upload.ts +++ /dev/null @@ -1,637 +0,0 @@ -import { createLogger } from '@sim/logger' -import { sleep } from '@sim/utils/helpers' -import { getFileContentType, isAbortError } from '@/lib/uploads/utils/file-utils' - -const logger = createLogger('DirectUpload') - -const CHUNK_SIZE = 8 * 1024 * 1024 -export const LARGE_FILE_THRESHOLD = 50 * 1024 * 1024 -const BASE_TIMEOUT_MS = 2 * 60 * 1000 -const TIMEOUT_PER_MB_MS = 1500 -const MAX_TIMEOUT_MS = 10 * 60 * 1000 -export const MULTIPART_PART_CONCURRENCY = 3 -export const MULTIPART_MAX_RETRIES = 3 -export const MULTIPART_RETRY_DELAY_MS = 2000 -export const MULTIPART_RETRY_BACKOFF = 2 -export const WHOLE_FILE_PARALLEL_UPLOADS = 3 - -interface PresignedFileInfo { - path: string - key: string - name: string - size: number - type: string -} - -export interface PresignedUploadInfo { - fileName: string - presignedUrl: string - fileInfo: PresignedFileInfo - uploadHeaders?: Record - directUploadSupported: boolean -} - -export interface UploadStrategyResult { - key: string - path: string - name: string - size: number - contentType: string -} - -export interface UploadProgressEvent { - loaded: number - total: number - percent: number -} - -export type DirectUploadErrorCode = - | 'PRESIGNED_URL_ERROR' - | 'DIRECT_UPLOAD_ERROR' - | 'MULTIPART_ERROR' - | 'ABORTED' - | 'FALLBACK_REQUIRED' - -export class DirectUploadError extends Error { - constructor( - message: string, - public code: DirectUploadErrorCode, - public details?: unknown, - public status?: number - ) { - super(message) - this.name = 'DirectUploadError' - } -} - -/** - * Transport-level upload errors worth retrying at the outer level: timeouts, - * network failures, and 5xx from the storage backend. Excludes deterministic - * client failures (4xx, `PRESIGNED_URL_ERROR`, `FALLBACK_REQUIRED`) and aborts. - */ -export const isTransientUploadError = (error: unknown): boolean => { - if (!(error instanceof DirectUploadError)) return false - if (error.code !== 'DIRECT_UPLOAD_ERROR' && error.code !== 'MULTIPART_ERROR') return false - if (error.status === undefined) return true - return error.status >= 500 && error.status < 600 -} - -export const calculateUploadTimeoutMs = (fileSize: number): number => { - const sizeInMb = fileSize / (1024 * 1024) - const dynamicBudget = BASE_TIMEOUT_MS + sizeInMb * TIMEOUT_PER_MB_MS - return Math.min(dynamicBudget, MAX_TIMEOUT_MS) -} - -/** - * Run `worker` over `items` with at most `limit` concurrent invocations. - * Returns a settled result per item (never rejects), so callers can handle - * partial failures explicitly. - */ -export const runWithConcurrency = async ( - items: T[], - limit: number, - worker: (item: T, index: number) => Promise -): Promise>> => { - const results: Array> = Array(items.length) - if (items.length === 0) return results - - const concurrency = Math.max(1, Math.min(limit, items.length)) - let nextIndex = 0 - - const runners = Array.from({ length: concurrency }, async () => { - while (true) { - const currentIndex = nextIndex++ - if (currentIndex >= items.length) break - try { - const value = await worker(items[currentIndex], currentIndex) - results[currentIndex] = { status: 'fulfilled', value } - } catch (error) { - results[currentIndex] = { status: 'rejected', reason: error } - } - } - }) - - await Promise.all(runners) - return results -} - -/** - * Normalize a presigned-upload server response into a {@link PresignedUploadInfo}. - * Accepts both single (`/api/files/presigned`) and batch entry shapes, tolerates - * `presignedUrl` vs `uploadUrl` aliases, and short-circuits when the server - * signals no cloud storage (`directUploadSupported: false`) so callers can fall - * back to a server-proxied upload path. - * - * @throws {@link DirectUploadError} with code `PRESIGNED_URL_ERROR` if the - * response is missing a presigned URL or `fileInfo.path`. - */ -export const normalizePresignedData = (data: unknown, context: string): PresignedUploadInfo => { - const d = (data ?? {}) as Record - const presignedUrl = (d.presignedUrl as string) || (d.uploadUrl as string) || '' - const fileInfo = d.fileInfo as Record | undefined - const directUploadSupported = d.directUploadSupported !== false - - if (!directUploadSupported) { - return { - fileName: (d.fileName as string) || context, - presignedUrl: '', - fileInfo: { path: '', key: '', name: context, size: 0, type: '' }, - directUploadSupported: false, - } - } - - if (!presignedUrl || !fileInfo?.path) { - throw new DirectUploadError( - `Invalid presigned response for ${context}`, - 'PRESIGNED_URL_ERROR', - data - ) - } - - return { - fileName: (d.fileName as string) || (fileInfo.name as string) || context, - presignedUrl, - fileInfo: { - path: fileInfo.path as string, - key: (fileInfo.key as string) || '', - name: (fileInfo.name as string) || context, - size: (fileInfo.size as number) || (d.fileSize as number) || 0, - type: (fileInfo.type as string) || (d.contentType as string) || '', - }, - uploadHeaders: (d.uploadHeaders as Record) || undefined, - directUploadSupported: true, - } -} - -interface GetPresignedOptions { - endpoint: string - file: File - body?: Record - signal?: AbortSignal -} - -/** - * Fetch a single presigned upload URL from a server endpoint that follows the - * `{ presignedUrl, fileInfo, uploadHeaders?, directUploadSupported }` contract. - */ -export const getPresignedUploadInfo = async ( - opts: GetPresignedOptions -): Promise => { - const { endpoint, file, body, signal } = opts - const response = await fetch(endpoint, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - fileName: file.name, - contentType: getFileContentType(file), - fileSize: file.size, - ...body, - }), - signal, - }) - - if (!response.ok) { - let errorDetails: unknown = null - try { - errorDetails = await response.json() - } catch {} - const serverMessage = - errorDetails != null && - typeof errorDetails === 'object' && - typeof (errorDetails as Record).error === 'string' - ? ((errorDetails as Record).error as string) - : null - throw new DirectUploadError( - serverMessage || - `Failed to get presigned URL for ${file.name}: ${response.status} ${response.statusText}`, - 'PRESIGNED_URL_ERROR', - errorDetails - ) - } - - return normalizePresignedData(await response.json(), file.name) -} - -interface UploadViaPutOptions { - file: File - presignedUrl: string - uploadHeaders?: Record - signal?: AbortSignal - onProgress?: (event: UploadProgressEvent) => void -} - -const uploadViaPresignedPut = (opts: UploadViaPutOptions): Promise => { - const { file, presignedUrl, uploadHeaders, signal, onProgress } = opts - - return new Promise((resolve, reject) => { - const xhr = new XMLHttpRequest() - let isCompleted = false - const timeoutMs = calculateUploadTimeoutMs(file.size) - - const timeoutId = setTimeout(() => { - if (isCompleted) return - isCompleted = true - signal?.removeEventListener('abort', abortHandler) - xhr.abort() - reject(new DirectUploadError(`Upload timeout for ${file.name}`, 'DIRECT_UPLOAD_ERROR')) - }, timeoutMs) - - const abortHandler = () => { - if (isCompleted) return - isCompleted = true - clearTimeout(timeoutId) - xhr.abort() - reject(new DirectUploadError(`Upload aborted for ${file.name}`, 'ABORTED')) - } - - if (signal) { - if (signal.aborted) { - abortHandler() - return - } - signal.addEventListener('abort', abortHandler) - } - - xhr.upload.addEventListener('progress', (event) => { - if (event.lengthComputable && !isCompleted) { - onProgress?.({ - loaded: event.loaded, - total: event.total, - percent: Math.round((event.loaded / event.total) * 100), - }) - } - }) - - xhr.addEventListener('load', () => { - if (isCompleted) return - isCompleted = true - clearTimeout(timeoutId) - signal?.removeEventListener('abort', abortHandler) - - if (xhr.status >= 200 && xhr.status < 300) { - onProgress?.({ loaded: file.size, total: file.size, percent: 100 }) - resolve() - } else { - reject( - new DirectUploadError( - `Direct upload failed for ${file.name}: ${xhr.status} ${xhr.statusText}`, - 'DIRECT_UPLOAD_ERROR', - undefined, - xhr.status - ) - ) - } - }) - - xhr.addEventListener('error', () => { - if (isCompleted) return - isCompleted = true - clearTimeout(timeoutId) - signal?.removeEventListener('abort', abortHandler) - reject(new DirectUploadError(`Network error uploading ${file.name}`, 'DIRECT_UPLOAD_ERROR')) - }) - - xhr.open('PUT', presignedUrl) - const providesContentType = - uploadHeaders && - Object.keys(uploadHeaders).some((key) => key.toLowerCase() === 'content-type') - if (!providesContentType) { - xhr.setRequestHeader('Content-Type', getFileContentType(file)) - } - if (uploadHeaders) { - for (const [key, value] of Object.entries(uploadHeaders)) { - xhr.setRequestHeader(key, value) - } - } - xhr.send(file) - }) -} - -interface MultipartUploadOptions { - file: File - workspaceId: string - context: - | 'workspace' - | 'knowledge-base' - | 'mothership' - | 'profile-pictures' - | 'workspace-logos' - | 'execution' - workflowId?: string - executionId?: string - signal?: AbortSignal - onProgress?: (event: UploadProgressEvent) => void -} - -interface CompletedPart { - partNumber: number - etag?: string -} - -interface PartUrl { - partNumber: number - url: string -} - -const uploadViaMultipart = async ( - opts: MultipartUploadOptions -): Promise<{ key: string; path: string }> => { - const { file, workspaceId, context, workflowId, executionId, signal, onProgress } = opts - - // boundary-raw-fetch: multipart upload control plane uses action query strings; client lifecycle (initiate/get-part-urls/complete/abort) is sequenced manually and not modeled by a single contract - const initiateResponse = await fetch('/api/files/multipart?action=initiate', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - fileName: file.name, - contentType: getFileContentType(file), - fileSize: file.size, - workspaceId, - context, - ...(workflowId ? { workflowId } : {}), - ...(executionId ? { executionId } : {}), - }), - signal, - }) - - if (!initiateResponse.ok) { - let errorBody: { error?: string } | null = null - try { - errorBody = (await initiateResponse.clone().json()) as { error?: string } - } catch {} - if ( - initiateResponse.status === 400 && - typeof errorBody?.error === 'string' && - errorBody.error.toLowerCase().includes('cloud storage') - ) { - throw new DirectUploadError( - 'Server signaled fallback to API upload', - 'FALLBACK_REQUIRED', - errorBody - ) - } - throw new DirectUploadError( - `Failed to initiate multipart upload: ${initiateResponse.statusText}`, - 'MULTIPART_ERROR', - undefined, - initiateResponse.status - ) - } - - const { key, uploadToken } = (await initiateResponse.json()) as { - uploadId: string - key: string - uploadToken: string - } - - const numParts = Math.ceil(file.size / CHUNK_SIZE) - const partNumbers = Array.from({ length: numParts }, (_, i) => i + 1) - - const abortMultipart = async () => { - try { - // boundary-raw-fetch: fire-and-forget abort during multipart cleanup; intentionally avoids contract response parsing so cleanup cannot mask the original error - await fetch('/api/files/multipart?action=abort', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ uploadToken }), - }) - } catch (err) { - logger.warn('Failed to abort multipart upload:', err) - } - } - - let presignedUrls: PartUrl[] - try { - // boundary-raw-fetch: multipart upload control plane uses action query strings; sequenced with initiate/complete/abort outside the contract layer - const partUrlsResponse = await fetch('/api/files/multipart?action=get-part-urls', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ uploadToken, partNumbers }), - signal, - }) - - if (!partUrlsResponse.ok) { - throw new DirectUploadError( - `Failed to get part URLs: ${partUrlsResponse.statusText}`, - 'MULTIPART_ERROR', - undefined, - partUrlsResponse.status - ) - } - - ;({ presignedUrls } = (await partUrlsResponse.json()) as { presignedUrls: PartUrl[] }) - } catch (err) { - await abortMultipart() - throw err - } - - const completedBytes = new Array(numParts).fill(0) - const reportProgress = () => { - const loaded = completedBytes.reduce((a, b) => a + b, 0) - onProgress?.({ - loaded, - total: file.size, - percent: Math.min(100, Math.round((loaded / file.size) * 100)), - }) - } - - const uploadedParts: CompletedPart[] = [] - - try { - const uploadPart = async ({ partNumber, url }: PartUrl): Promise => { - const start = (partNumber - 1) * CHUNK_SIZE - const end = Math.min(start + CHUNK_SIZE, file.size) - const chunk = file.slice(start, end) - - for (let attempt = 0; attempt <= MULTIPART_MAX_RETRIES; attempt++) { - try { - const partResponse = await fetch(url, { - method: 'PUT', - body: chunk, - signal, - headers: { 'Content-Type': getFileContentType(file) }, - }) - - if (!partResponse.ok) { - throw new DirectUploadError( - `Failed to upload part ${partNumber}: ${partResponse.statusText}`, - 'MULTIPART_ERROR', - undefined, - partResponse.status - ) - } - - const etag = partResponse.headers.get('ETag') || undefined - completedBytes[partNumber - 1] = end - start - reportProgress() - - return { partNumber, etag: etag?.replace(/"/g, '') } - } catch (partError) { - const isClientError = - partError instanceof DirectUploadError && - partError.status !== undefined && - partError.status >= 400 && - partError.status < 500 - if (isAbortError(partError) || isClientError || attempt >= MULTIPART_MAX_RETRIES) { - throw partError - } - const delay = MULTIPART_RETRY_DELAY_MS * MULTIPART_RETRY_BACKOFF ** attempt - logger.warn( - `Part ${partNumber} failed (attempt ${attempt + 1}), retrying in ${Math.round(delay / 1000)}s` - ) - await sleep(delay) - } - } - - throw new DirectUploadError(`Retries exhausted for part ${partNumber}`, 'MULTIPART_ERROR') - } - - const partResults = await runWithConcurrency( - presignedUrls, - MULTIPART_PART_CONCURRENCY, - uploadPart - ) - - for (const result of partResults) { - if (result?.status === 'fulfilled') { - uploadedParts.push(result.value) - } else if (result?.status === 'rejected') { - throw result.reason - } - } - } catch (error) { - await abortMultipart() - throw error - } - - let path: string - try { - // boundary-raw-fetch: multipart upload control plane uses action query strings; sequenced with initiate/get-part-urls/abort outside the contract layer - const completeResponse = await fetch('/api/files/multipart?action=complete', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ uploadToken, parts: uploadedParts }), - signal, - }) - - if (!completeResponse.ok) { - throw new DirectUploadError( - `Failed to complete multipart upload: ${completeResponse.statusText}`, - 'MULTIPART_ERROR', - undefined, - completeResponse.status - ) - } - - ;({ path } = (await completeResponse.json()) as { path: string }) - } catch (err) { - await abortMultipart() - throw err - } - return { key, path } -} - -export interface RunUploadStrategyOptions { - file: File - workspaceId: string - context: - | 'workspace' - | 'knowledge-base' - | 'mothership' - | 'profile-pictures' - | 'workspace-logos' - | 'execution' - /** Endpoint to mint a presigned PUT URL. Required unless `presignedOverride` is provided. */ - presignedEndpoint?: string - /** Pre-fetched presigned data (e.g. from a batch endpoint). Skips per-file fetch. */ - presignedOverride?: PresignedUploadInfo - /** Extra JSON body fields for the presigned endpoint. */ - presignedBody?: Record - /** Required when context is `execution`; forwarded to the multipart route to scope the storage key. */ - workflowId?: string - /** Required when context is `execution`; forwarded to the multipart route to scope the storage key. */ - executionId?: string - signal?: AbortSignal - onProgress?: (event: UploadProgressEvent) => void -} - -/** - * Strategy ladder for client-side uploads: - * - Files larger than {@link LARGE_FILE_THRESHOLD} use multipart S3/Blob with chunked PUTs. - * - Smaller files use a presigned PUT URL (fetched per-file, or supplied via - * `presignedOverride` for batched flows like KB). - * - If the server signals no cloud storage is configured, a {@link DirectUploadError} - * with code `FALLBACK_REQUIRED` is thrown so callers can fall back to a server-proxied path. - */ -export const runUploadStrategy = async ( - opts: RunUploadStrategyOptions -): Promise => { - const { - file, - presignedEndpoint, - presignedOverride, - presignedBody, - workspaceId, - context, - workflowId, - executionId, - signal, - onProgress, - } = opts - const contentType = getFileContentType(file) - - if (presignedOverride && !presignedOverride.directUploadSupported) { - throw new DirectUploadError('Server signaled fallback to API upload', 'FALLBACK_REQUIRED') - } - - if (file.size > LARGE_FILE_THRESHOLD) { - const { key, path } = await uploadViaMultipart({ - file, - workspaceId, - context, - workflowId, - executionId, - signal, - onProgress, - }) - return { key, path, name: file.name, size: file.size, contentType } - } - - let presigned: PresignedUploadInfo - if (presignedOverride) { - presigned = presignedOverride - } else { - if (!presignedEndpoint) { - throw new DirectUploadError( - 'runUploadStrategy requires either presignedEndpoint or presignedOverride', - 'PRESIGNED_URL_ERROR' - ) - } - presigned = await getPresignedUploadInfo({ - endpoint: presignedEndpoint, - file, - body: presignedBody, - signal, - }) - } - - if (!presigned.directUploadSupported) { - throw new DirectUploadError('Server signaled fallback to API upload', 'FALLBACK_REQUIRED') - } - - await uploadViaPresignedPut({ - file, - presignedUrl: presigned.presignedUrl, - uploadHeaders: presigned.uploadHeaders, - signal, - onProgress, - }) - - return { - key: presigned.fileInfo.key, - path: presigned.fileInfo.path, - name: file.name, - size: file.size, - contentType, - } -} diff --git a/apps/sim/lib/uploads/client/multipart-session.test.ts b/apps/sim/lib/uploads/client/multipart-session.test.ts deleted file mode 100644 index 0f8b495a2b1..00000000000 --- a/apps/sim/lib/uploads/client/multipart-session.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -/** - * @vitest-environment node - */ -import { afterEach, describe, expect, it, vi } from 'vitest' -import type { V2CompletedPart } from '@/lib/api/contracts/v2/uploads' -import { uploadMultipartSession } from '@/lib/uploads/client/multipart-session' - -describe('uploadMultipartSession', () => { - afterEach(() => { - vi.unstubAllGlobals() - }) - - it('requests fresh URL batches and completes with every uploaded part in order', async () => { - const file = new File(['abcdefghijklmnopqrstuvwxyz'], 'letters.txt') - const getPartUrls = vi.fn(async (partNumbers: number[]) => - partNumbers.map((partNumber) => ({ - partNumber, - url: `https://storage.example/part/${partNumber}`, - headers: { 'Content-Type': 'application/octet-stream' }, - expiresAt: '2026-08-03T22:00:00.000Z', - })) - ) - const complete = vi.fn(async (parts: V2CompletedPart[]) => parts) - const abort = vi.fn(async () => {}) - const onProgress = vi.fn() - vi.stubGlobal( - 'fetch', - vi.fn( - async (_url: string) => new Response(null, { status: 200, headers: { etag: '"etag"' } }) - ) - ) - - const result = await uploadMultipartSession({ - file, - partSize: 1, - partCount: 26, - getPartUrls, - complete, - abort, - onProgress, - }) - - expect(getPartUrls).toHaveBeenCalledTimes(2) - expect(getPartUrls.mock.calls[0][0]).toEqual( - Array.from({ length: 25 }, (_, index) => index + 1) - ) - expect(getPartUrls.mock.calls[1][0]).toEqual([26]) - expect(result).toHaveLength(26) - expect(result[0]).toEqual({ partNumber: 1, etag: 'etag' }) - expect(result[25]).toEqual({ partNumber: 26, etag: 'etag' }) - expect(onProgress).toHaveBeenLastCalledWith({ loaded: 26, total: 26, percent: 100 }) - expect(abort).not.toHaveBeenCalled() - }) - - it('aborts the signed session when a part upload is aborted', async () => { - const file = new File(['part'], 'part.txt') - const complete = vi.fn() - const abort = vi.fn(async () => {}) - vi.stubGlobal( - 'fetch', - vi.fn(async () => { - throw new DOMException('The operation was aborted', 'AbortError') - }) - ) - - await expect( - uploadMultipartSession({ - file, - partSize: 4, - partCount: 1, - getPartUrls: async () => [ - { - partNumber: 1, - url: 'https://storage.example/part/1', - headers: {}, - expiresAt: '2026-08-03T22:00:00.000Z', - }, - ], - complete, - abort, - }) - ).rejects.toMatchObject({ name: 'AbortError' }) - expect(abort).toHaveBeenCalledTimes(1) - expect(complete).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/uploads/client/multipart-session.ts b/apps/sim/lib/uploads/client/multipart-session.ts deleted file mode 100644 index 2c3b1da05fd..00000000000 --- a/apps/sim/lib/uploads/client/multipart-session.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { sleep } from '@sim/utils/helpers' -import type { V2CompletedPart, V2UploadPartUrl } from '@/lib/api/contracts/v2/uploads' -import { - MULTIPART_MAX_RETRIES, - MULTIPART_PART_CONCURRENCY, - MULTIPART_RETRY_BACKOFF, - MULTIPART_RETRY_DELAY_MS, - runWithConcurrency, - type UploadProgressEvent, -} from '@/lib/uploads/client/direct-upload' -import { isAbortError } from '@/lib/uploads/utils/file-utils' - -interface UploadMultipartSessionParams { - file: File - partSize: number - partCount: number - signal?: AbortSignal - onProgress?: (event: UploadProgressEvent) => void - getPartUrls: (partNumbers: number[]) => Promise - complete: (parts: V2CompletedPart[]) => Promise - abort: () => Promise -} - -export async function uploadMultipartSession( - params: UploadMultipartSessionParams -): Promise { - const { file, partSize, partCount, signal, onProgress } = params - const completedBytes = new Array(partCount).fill(0) - const completedParts: V2CompletedPart[] = [] - try { - for (let start = 1; start <= partCount; start += 25) { - const partNumbers = Array.from( - { length: Math.min(25, partCount - start + 1) }, - (_, index) => start + index - ) - const partUrls = await params.getPartUrls(partNumbers) - const results = await runWithConcurrency( - partUrls, - MULTIPART_PART_CONCURRENCY, - async (part): Promise => { - const partStart = (part.partNumber - 1) * partSize - const end = Math.min(partStart + partSize, file.size) - const chunk = file.slice(partStart, end) - for (let attempt = 0; attempt <= MULTIPART_MAX_RETRIES; attempt++) { - try { - // boundary-raw-fetch: signed multipart data-plane URL may target cloud storage or local Sim - const response = await fetch(part.url, { - method: 'PUT', - body: chunk, - headers: part.headers, - signal, - }) - if (!response.ok) { - throw new Error(`Part ${part.partNumber} failed (${response.status})`) - } - completedBytes[part.partNumber - 1] = end - partStart - const loaded = completedBytes.reduce((sum, bytes) => sum + bytes, 0) - onProgress?.({ - loaded, - total: file.size, - percent: Math.min(100, Math.round((loaded / file.size) * 100)), - }) - const etag = response.headers.get('etag') - return { - partNumber: part.partNumber, - ...(etag ? { etag: etag.replaceAll('"', '') } : {}), - } - } catch (error) { - if (isAbortError(error) || attempt >= MULTIPART_MAX_RETRIES) throw error - await sleep(MULTIPART_RETRY_DELAY_MS * MULTIPART_RETRY_BACKOFF ** attempt) - } - } - throw new Error(`Retries exhausted for part ${part.partNumber}`) - } - ) - completedParts.push( - ...results.map((result) => { - if (result.status === 'rejected') throw result.reason - return result.value - }) - ) - } - return await params.complete(completedParts) - } catch (error) { - await params.abort().catch(() => {}) - throw error - } -} diff --git a/apps/sim/lib/uploads/client/session-upload.test.ts b/apps/sim/lib/uploads/client/session-upload.test.ts index 790f15ae108..026c3f55a0e 100644 --- a/apps/sim/lib/uploads/client/session-upload.test.ts +++ b/apps/sim/lib/uploads/client/session-upload.test.ts @@ -2,24 +2,26 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' -import type { V2CompletedPart, V2UploadPartUrl } from '@/lib/api/contracts/v2/uploads' +import type { V2CompleteUploadBody } from '@/lib/api/contracts/v2/uploads' -interface MultipartMockParams { - getPartUrls: (partNumbers: number[]) => Promise - complete: (parts: V2CompletedPart[]) => Promise +interface UploadClientMockParams { + complete: (body: V2CompleteUploadBody) => Promise } -const { mockRequestJson, mockUploadMultipartSession } = vi.hoisted(() => ({ +const { mockRequestJson, mockUploadFileSession } = vi.hoisted(() => ({ mockRequestJson: vi.fn(), - mockUploadMultipartSession: vi.fn(), + mockUploadFileSession: vi.fn(), })) vi.mock('@/lib/api/client/request', () => ({ requestJson: mockRequestJson })) -vi.mock('@/lib/uploads/client/multipart-session', () => ({ - uploadMultipartSession: mockUploadMultipartSession, +vi.mock('@/lib/uploads/client/upload-session', () => ({ + uploadFileSession: mockUploadFileSession, })) -import { uploadKnowledgeDocumentSession } from '@/lib/uploads/client/session-upload' +import { + uploadInternalFileSession, + uploadKnowledgeDocumentSession, +} from '@/lib/uploads/client/session-upload' const DOCUMENT = { id: 'upload-1', @@ -35,60 +37,90 @@ const DOCUMENT = { createdAt: '2026-08-04T21:00:00.000Z', } as const -describe('uploadKnowledgeDocumentSession', () => { +describe('session upload domain clients', () => { beforeEach(() => { vi.clearAllMocks() + }) + + it('uses the PUT knowledge session without requesting part URLs', async () => { mockRequestJson .mockResolvedValueOnce({ data: { - id: 'upload-1', - partSize: 8 * 1024 * 1024, - partCount: 1, + session: { id: 'upload-1' }, uploadToken: 'token', + transfer: { + method: 'put', + url: 'https://storage.example/upload', + headers: { 'Content-Type': 'application/pdf' }, + }, }, }) - .mockResolvedValueOnce({ - data: { parts: [{ partNumber: 1, url: 'https://storage.example/part-1', headers: {} }] }, - }) .mockResolvedValueOnce({ data: { document: DOCUMENT } }) - mockUploadMultipartSession.mockImplementation( - async (params: MultipartMockParams) => { - await params.getPartUrls([1]) - return params.complete([{ partNumber: 1, etag: 'etag-1' }]) - } + mockUploadFileSession.mockImplementation( + async (params: UploadClientMockParams) => params.complete({}) ) - }) - - it('uses the first-party session routes and preserves signed processing metadata', async () => { - const file = { - name: 'guide.pdf', - type: 'application/pdf', - size: 1024, - } as File + const file = { name: 'guide.pdf', type: 'application/pdf', size: 1024 } as File await expect( uploadKnowledgeDocumentSession({ - workspaceId: '6fc7631d-88cd-46f8-9f0a-d4764daef7f8', + workspaceId: 'workspace-1', knowledgeBaseId: 'kb-1', file, tag1: 'product', - processingOptions: { recipe: 'default', lang: 'en' }, }) ).resolves.toEqual(DOCUMENT) expect(mockRequestJson.mock.calls[0][0].path).toBe('/api/knowledge/[id]/documents/uploads') - expect(mockRequestJson.mock.calls[0][1].body).toMatchObject({ - name: 'guide.pdf', - contentType: 'application/pdf', - size: 1024, - tag1: 'product', - processingOptions: { recipe: 'default', lang: 'en' }, - }) expect(mockRequestJson.mock.calls[1][0].path).toBe( - '/api/knowledge/[id]/documents/uploads/[uploadId]/parts' - ) - expect(mockRequestJson.mock.calls[2][0].path).toBe( '/api/knowledge/[id]/documents/uploads/[uploadId]/complete' ) + expect(mockRequestJson.mock.calls[1][1].body).toEqual({}) + expect(mockRequestJson.mock.calls.some(([contract]) => contract.path.endsWith('/parts'))).toBe( + false + ) + }) + + it('returns the purpose-specific result from the generic internal session', async () => { + const result = { + path: '/api/files/serve/logo.png', + key: 'workspace-logos/logo.png', + name: 'logo.png', + size: 100, + type: 'image/png', + } + mockRequestJson + .mockResolvedValueOnce({ + data: { + session: { id: 'upload-2', purpose: 'workspace_logo' }, + uploadToken: 'token', + transfer: { + method: 'put', + url: 'https://storage.example/upload', + headers: { 'Content-Type': 'image/png' }, + }, + }, + }) + .mockResolvedValueOnce({ + data: { id: 'upload-2', purpose: 'workspace_logo', result }, + }) + mockUploadFileSession.mockImplementation( + async (params: UploadClientMockParams) => params.complete({}) + ) + + await expect( + uploadInternalFileSession({ + purpose: 'workspace_logo', + workspaceId: 'workspace-1', + file: { name: 'logo.png', type: 'image/png', size: 100 } as File, + }) + ).resolves.toEqual(result) + + expect(mockRequestJson.mock.calls[0][1].body).toEqual({ + purpose: 'workspace_logo', + workspaceId: 'workspace-1', + name: 'logo.png', + contentType: 'image/png', + size: 100, + }) }) }) diff --git a/apps/sim/lib/uploads/client/session-upload.ts b/apps/sim/lib/uploads/client/session-upload.ts index e06d9c57860..dabb59b893e 100644 --- a/apps/sim/lib/uploads/client/session-upload.ts +++ b/apps/sim/lib/uploads/client/session-upload.ts @@ -6,17 +6,24 @@ import { createKnowledgeDocumentUploadPartUrlsContract, } from '@/lib/api/contracts/knowledge/upload-sessions' import { - abortWorkspaceFileUploadContract, - completeWorkspaceFileUploadContract, - createWorkspaceFileUploadContract, - createWorkspaceFileUploadPartUrlsContract, + abortInternalFileUploadContract, + type CreateInternalFileUploadBody, + completeInternalFileUploadContract, + createInternalFileUploadContract, + createInternalFileUploadPartUrlsContract, + type InternalFileUploadSession, } from '@/lib/api/contracts/upload-sessions' import type { V2KnowledgeDocumentSummary, V2KnowledgeDocumentUploadMetadata, } from '@/lib/api/contracts/v2/knowledge' -import type { UploadProgressEvent } from '@/lib/uploads/client/direct-upload' -import { uploadMultipartSession } from '@/lib/uploads/client/multipart-session' +import type { + V2CompleteUploadBody, + V2UploadPartUrl, + V2UploadTransfer, +} from '@/lib/api/contracts/v2/uploads' +import type { UploadProgressEvent } from '@/lib/uploads/client/types' +import { uploadFileSession } from '@/lib/uploads/client/upload-session' import { getFileContentType } from '@/lib/uploads/utils/file-utils' interface UploadWorkspaceFileSessionParams { @@ -27,6 +34,31 @@ interface UploadWorkspaceFileSessionParams { onProgress?: (event: UploadProgressEvent) => void } +interface InternalUploadCommonParams { + file: File + signal?: AbortSignal + onProgress?: (event: UploadProgressEvent) => void +} + +type InternalUploadContext = + | { purpose: 'workspace_file'; workspaceId: string; folderId?: string | null } + | { purpose: 'profile_picture' } + | { purpose: 'workspace_logo'; workspaceId: string } + | { purpose: 'mothership_attachment'; workspaceId: string } + | { + purpose: 'execution_attachment' + workspaceId: string + workflowId: string + executionId: string + } + +type InternalUploadPurpose = InternalUploadContext['purpose'] +type InternalUploadResult = NonNullable< + Extract['result'] +> + +export type UploadInternalFileSessionParams = InternalUploadCommonParams & InternalUploadContext + interface UploadKnowledgeDocumentSessionParams extends V2KnowledgeDocumentUploadMetadata { workspaceId: string knowledgeBaseId: string @@ -35,56 +67,124 @@ interface UploadKnowledgeDocumentSessionParams extends V2KnowledgeDocumentUpload onProgress?: (event: UploadProgressEvent) => void } +interface RunCreatedUploadParams { + file: File + transfer: V2UploadTransfer + signal?: AbortSignal + onProgress?: (event: UploadProgressEvent) => void + getPartUrls: (partNumbers: number[]) => Promise + complete: (body: V2CompleteUploadBody) => Promise + abort: () => Promise +} + +function runCreatedUpload(params: RunCreatedUploadParams): Promise { + const common = { + file: params.file, + signal: params.signal, + onProgress: params.onProgress, + complete: params.complete, + abort: params.abort, + } + return params.transfer.method === 'put' + ? uploadFileSession({ ...common, transfer: params.transfer }) + : uploadFileSession({ + ...common, + transfer: params.transfer, + getPartUrls: params.getPartUrls, + }) +} + export async function uploadWorkspaceFileSession(params: UploadWorkspaceFileSessionParams) { - const { workspaceId, folderId, file, signal, onProgress } = params - const created = await requestJson(createWorkspaceFileUploadContract, { - body: { - workspaceId, - name: file.name, - contentType: getFileContentType(file), - size: file.size, - ...(folderId ? { folderId } : {}), - }, + return uploadInternalFileSession({ + purpose: 'workspace_file', + ...params, + }) +} + +export async function uploadInternalFileSession( + params: InternalUploadCommonParams & Extract +): Promise> { + const { file, signal, onProgress } = params + const created = await requestJson(createInternalFileUploadContract, { + body: internalUploadBody(params), signal, }) - const upload = created.data - return uploadMultipartSession({ + const { session, uploadToken, transfer } = created.data + if (session.purpose !== params.purpose) { + throw new Error(`Expected ${params.purpose} upload session; received ${session.purpose}`) + } + return runCreatedUpload({ file, - partSize: upload.partSize, - partCount: upload.partCount, + transfer, signal, onProgress, getPartUrls: async (partNumbers) => { - const batch = await requestJson(createWorkspaceFileUploadPartUrlsContract, { - params: { uploadId: upload.id }, - query: { workspaceId }, - headers: { 'upload-token': upload.uploadToken }, + const batch = await requestJson(createInternalFileUploadPartUrlsContract, { + params: { uploadId: session.id }, + headers: { 'upload-token': uploadToken }, body: { partNumbers }, signal, }) return batch.data.parts }, - complete: async (parts) => { - const completed = await requestJson(completeWorkspaceFileUploadContract, { - params: { uploadId: upload.id }, - query: { workspaceId }, - headers: { 'upload-token': upload.uploadToken }, - body: { parts }, + complete: async (body) => { + const completed = await requestJson(completeInternalFileUploadContract, { + params: { uploadId: session.id }, + headers: { 'upload-token': uploadToken }, + body, signal, }) - if (!completed.data.file) throw new Error('Completed upload returned no workspace file') - return completed.data.file + if (completed.data.purpose !== params.purpose) { + throw new Error(`Expected ${params.purpose} completion; received ${completed.data.purpose}`) + } + if (!completed.data.result) { + throw new Error(`Completed ${params.purpose} upload returned no result`) + } + return completed.data.result as InternalUploadResult }, abort: async () => { - await requestJson(abortWorkspaceFileUploadContract, { - params: { uploadId: upload.id }, - query: { workspaceId }, - headers: { 'upload-token': upload.uploadToken }, + await requestJson(abortInternalFileUploadContract, { + params: { uploadId: session.id }, + headers: { 'upload-token': uploadToken }, }) }, }) } +function internalUploadBody(params: UploadInternalFileSessionParams): CreateInternalFileUploadBody { + const fileFields = { + name: params.file.name, + contentType: getFileContentType(params.file), + size: params.file.size, + } + switch (params.purpose) { + case 'workspace_file': + return { + purpose: params.purpose, + workspaceId: params.workspaceId, + ...fileFields, + ...(params.folderId ? { folderId: params.folderId } : {}), + } + case 'profile_picture': + return { purpose: params.purpose, ...fileFields } + case 'workspace_logo': + case 'mothership_attachment': + return { + purpose: params.purpose, + workspaceId: params.workspaceId, + ...fileFields, + } + case 'execution_attachment': + return { + purpose: params.purpose, + workspaceId: params.workspaceId, + workflowId: params.workflowId, + executionId: params.executionId, + ...fileFields, + } + } +} + export async function uploadKnowledgeDocumentSession( params: UploadKnowledgeDocumentSessionParams ): Promise { @@ -100,29 +200,28 @@ export async function uploadKnowledgeDocumentSession( }, signal, }) - const upload = created.data - return uploadMultipartSession({ + const { session, uploadToken, transfer } = created.data + return runCreatedUpload({ file, - partSize: upload.partSize, - partCount: upload.partCount, + transfer, signal, onProgress, getPartUrls: async (partNumbers) => { const batch = await requestJson(createKnowledgeDocumentUploadPartUrlsContract, { - params: { id: knowledgeBaseId, uploadId: upload.id }, + params: { id: knowledgeBaseId, uploadId: session.id }, query: { workspaceId }, - headers: { 'upload-token': upload.uploadToken }, + headers: { 'upload-token': uploadToken }, body: { partNumbers }, signal, }) return batch.data.parts }, - complete: async (parts) => { + complete: async (body) => { const completed = await requestJson(completeKnowledgeDocumentUploadContract, { - params: { id: knowledgeBaseId, uploadId: upload.id }, + params: { id: knowledgeBaseId, uploadId: session.id }, query: { workspaceId }, - headers: { 'upload-token': upload.uploadToken }, - body: { parts }, + headers: { 'upload-token': uploadToken }, + body, signal, }) if (!completed.data.document) { @@ -132,9 +231,9 @@ export async function uploadKnowledgeDocumentSession( }, abort: async () => { await requestJson(abortKnowledgeDocumentUploadContract, { - params: { id: knowledgeBaseId, uploadId: upload.id }, + params: { id: knowledgeBaseId, uploadId: session.id }, query: { workspaceId }, - headers: { 'upload-token': upload.uploadToken }, + headers: { 'upload-token': uploadToken }, }) }, }) diff --git a/apps/sim/lib/uploads/client/types.ts b/apps/sim/lib/uploads/client/types.ts new file mode 100644 index 00000000000..28d4c02ad98 --- /dev/null +++ b/apps/sim/lib/uploads/client/types.ts @@ -0,0 +1,5 @@ +export interface UploadProgressEvent { + loaded: number + total: number + percent: number +} diff --git a/apps/sim/lib/uploads/client/upload-session.test.ts b/apps/sim/lib/uploads/client/upload-session.test.ts new file mode 100644 index 00000000000..8df72f6b24f --- /dev/null +++ b/apps/sim/lib/uploads/client/upload-session.test.ts @@ -0,0 +1,325 @@ +/** + * @vitest-environment jsdom + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { V2CompleteUploadBody } from '@/lib/api/contracts/v2/uploads' +import { calculateUploadTimeoutMs, uploadFileSession } from '@/lib/uploads/client/upload-session' + +const MIB = 1024 * 1024 +const PUT_THRESHOLD = 50 * MIB + +function sizedFile(size: number): File { + const file = new File([], 'data.bin', { type: 'application/octet-stream' }) + Object.defineProperty(file, 'size', { value: size }) + return file +} + +class MockXhr extends EventTarget { + static instances: MockXhr[] = [] + static onSend: (xhr: MockXhr) => void = (xhr) => { + queueMicrotask(() => xhr.dispatchEvent(new Event('load'))) + } + + readonly upload = new EventTarget() + readonly requestHeaders = new Map() + status = 200 + statusText = 'OK' + timeout = 0 + responseHeaders = new Map() + open = vi.fn() + abort = vi.fn() + send = vi.fn(() => MockXhr.onSend(this)) + + constructor() { + super() + MockXhr.instances.push(this) + } + + setRequestHeader(name: string, value: string) { + this.requestHeaders.set(name, value) + } + + getResponseHeader(name: string) { + return this.responseHeaders.get(name) ?? null + } +} + +describe('uploadFileSession', () => { + const originalXhr = globalThis.XMLHttpRequest + + beforeEach(() => { + MockXhr.instances = [] + MockXhr.onSend = (xhr) => queueMicrotask(() => xhr.dispatchEvent(new Event('load'))) + globalThis.XMLHttpRequest = MockXhr as unknown as typeof XMLHttpRequest + }) + + afterEach(() => { + globalThis.XMLHttpRequest = originalXhr + vi.useRealTimers() + vi.unstubAllGlobals() + vi.restoreAllMocks() + }) + + it('uploads an exact-threshold file with PUT and completes with an empty body', async () => { + const file = sizedFile(PUT_THRESHOLD) + const complete = vi.fn(async (_body: V2CompleteUploadBody) => 'done') + const abort = vi.fn(async () => undefined) + const onProgress = vi.fn() + + await expect( + uploadFileSession({ + file, + transfer: { + method: 'put', + url: 'https://storage.example/upload', + headers: { 'Content-Type': 'application/octet-stream', 'x-upload': 'signed' }, + }, + complete, + abort, + onProgress, + }) + ).resolves.toBe('done') + + expect(MockXhr.instances).toHaveLength(1) + expect(MockXhr.instances[0].open).toHaveBeenCalledWith('PUT', 'https://storage.example/upload') + expect(MockXhr.instances[0].requestHeaders).toEqual( + new Map([ + ['Content-Type', 'application/octet-stream'], + ['x-upload', 'signed'], + ]) + ) + expect(MockXhr.instances[0].timeout).toBe(calculateUploadTimeoutMs(file.size)) + expect(complete).toHaveBeenCalledWith({}) + expect(onProgress).toHaveBeenLastCalledWith({ + loaded: PUT_THRESHOLD, + total: PUT_THRESHOLD, + percent: 100, + }) + expect(abort).not.toHaveBeenCalled() + }) + + it('uploads an empty file with PUT and reports finite completion progress', async () => { + const complete = vi.fn(async (_body: V2CompleteUploadBody) => 'done') + const onProgress = vi.fn() + + await expect( + uploadFileSession({ + file: sizedFile(0), + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + complete, + abort: vi.fn(async () => undefined), + onProgress, + }) + ).resolves.toBe('done') + + expect(complete).toHaveBeenCalledWith({}) + expect(onProgress).toHaveBeenLastCalledWith({ loaded: 0, total: 0, percent: 100 }) + }) + + it('uploads a file above the threshold through bounded multipart batches', async () => { + const file = sizedFile(PUT_THRESHOLD + 1) + const partSize = 8 * MIB + const partCount = Math.ceil(file.size / partSize) + const getPartUrls = vi.fn(async (partNumbers: number[]) => + [...partNumbers].reverse().map((partNumber) => ({ + partNumber, + url: `https://storage.example/parts/${partNumber}`, + headers: { 'Content-Type': 'application/octet-stream' }, + expiresAt: '2026-08-05T00:00:00.000Z', + })) + ) + const complete = vi.fn(async (_body: V2CompleteUploadBody) => 'done') + const abort = vi.fn(async () => undefined) + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(null, { status: 200, headers: { etag: '"part-etag"' } })) + ) + + await expect( + uploadFileSession({ + file, + transfer: { method: 'multipart', partSize, partCount }, + getPartUrls, + complete, + abort, + }) + ).resolves.toBe('done') + + expect(getPartUrls).toHaveBeenCalledWith(Array.from({ length: partCount }, (_, i) => i + 1)) + expect(complete).toHaveBeenCalledWith({ + parts: Array.from({ length: partCount }, (_, index) => ({ + partNumber: index + 1, + etag: 'part-etag', + })), + }) + expect(abort).not.toHaveBeenCalled() + }) + + it('does not retry a deterministic PUT 4xx', async () => { + MockXhr.onSend = (xhr) => { + xhr.status = 403 + xhr.statusText = 'Forbidden' + queueMicrotask(() => xhr.dispatchEvent(new Event('load'))) + } + const abort = vi.fn(async () => undefined) + + await expect( + uploadFileSession({ + file: sizedFile(1), + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + complete: vi.fn(), + abort, + }) + ).rejects.toMatchObject({ status: 403 }) + + expect(MockXhr.instances).toHaveLength(1) + expect(abort).toHaveBeenCalledTimes(1) + }) + + it('retains a completed PUT transfer when completion fails', async () => { + const abort = vi.fn(async () => undefined) + + await expect( + uploadFileSession({ + file: sizedFile(1), + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + complete: async () => { + throw new Error('finalizer unavailable') + }, + abort, + }) + ).rejects.toThrow('finalizer unavailable') + + expect(abort).not.toHaveBeenCalled() + }) + + it('retries transient PUT failures and keeps progress monotonic', async () => { + vi.useFakeTimers() + const onProgress = vi.fn() + MockXhr.onSend = (xhr) => { + const attempt = MockXhr.instances.length + if (attempt === 1) { + xhr.upload.dispatchEvent( + new ProgressEvent('progress', { lengthComputable: true, loaded: 8, total: 10 }) + ) + queueMicrotask(() => xhr.dispatchEvent(new Event('error'))) + return + } + xhr.upload.dispatchEvent( + new ProgressEvent('progress', { lengthComputable: true, loaded: 2, total: 10 }) + ) + queueMicrotask(() => xhr.dispatchEvent(new Event('load'))) + } + + const promise = uploadFileSession({ + file: sizedFile(10), + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + complete: async () => 'done', + abort: async () => undefined, + onProgress, + }) + await vi.runAllTimersAsync() + + await expect(promise).resolves.toBe('done') + expect(MockXhr.instances).toHaveLength(2) + expect(onProgress.mock.calls.map(([event]) => event.loaded)).toEqual([8, 8, 10]) + }) + + it('aborts XHR and the control session when the caller cancels', async () => { + const controller = new AbortController() + MockXhr.onSend = () => undefined + const abort = vi.fn(async () => undefined) + const promise = uploadFileSession({ + file: sizedFile(10), + transfer: { method: 'put', url: 'https://storage.example/upload', headers: {} }, + complete: vi.fn(), + abort, + signal: controller.signal, + }) + + controller.abort() + + await expect(promise).rejects.toMatchObject({ name: 'AbortError' }) + expect(MockXhr.instances[0].abort).toHaveBeenCalledTimes(1) + expect(abort).toHaveBeenCalledTimes(1) + }) + + it('fails before uploading when a multipart URL batch is incomplete', async () => { + const abort = vi.fn(async () => undefined) + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + + await expect( + uploadFileSession({ + file: sizedFile(2), + transfer: { method: 'multipart', partSize: 1, partCount: 2 }, + getPartUrls: async () => [ + { + partNumber: 1, + url: 'https://storage.example/parts/1', + headers: {}, + expiresAt: '2026-08-05T00:00:00.000Z', + }, + ], + complete: vi.fn(), + abort, + }) + ).rejects.toThrow('Expected 2 part URLs; received 1') + + expect(fetchMock).not.toHaveBeenCalled() + expect(abort).toHaveBeenCalledTimes(1) + }) + + it('does not retry a multipart 4xx response', async () => { + const fetchMock = vi.fn(async () => new Response(null, { status: 403 })) + vi.stubGlobal('fetch', fetchMock) + + await expect( + uploadFileSession({ + file: sizedFile(1), + transfer: { method: 'multipart', partSize: 1, partCount: 1 }, + getPartUrls: async () => [ + { + partNumber: 1, + url: 'https://storage.example/parts/1', + headers: {}, + expiresAt: '2026-08-05T00:00:00.000Z', + }, + ], + complete: vi.fn(), + abort: async () => undefined, + }) + ).rejects.toMatchObject({ status: 403 }) + + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('retains a completed multipart transfer when completion fails', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => new Response(null, { status: 200 })) + ) + const abort = vi.fn(async () => undefined) + + await expect( + uploadFileSession({ + file: sizedFile(1), + transfer: { method: 'multipart', partSize: 1, partCount: 1 }, + getPartUrls: async () => [ + { + partNumber: 1, + url: 'https://storage.example/parts/1', + headers: {}, + expiresAt: '2026-08-05T00:00:00.000Z', + }, + ], + complete: async () => { + throw new Error('finalizer unavailable') + }, + abort, + }) + ).rejects.toThrow('finalizer unavailable') + + expect(abort).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/uploads/client/upload-session.ts b/apps/sim/lib/uploads/client/upload-session.ts new file mode 100644 index 00000000000..7ad7695b3ee --- /dev/null +++ b/apps/sim/lib/uploads/client/upload-session.ts @@ -0,0 +1,374 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' +import type { + V2CompletedPart, + V2CompleteUploadBody, + V2MultipartUploadTransfer, + V2PutUploadTransfer, + V2UploadPartUrl, +} from '@/lib/api/contracts/v2/uploads' +import { runWithConcurrency } from '@/lib/uploads/client/concurrency' +import type { UploadProgressEvent } from '@/lib/uploads/client/types' +import { isAbortError } from '@/lib/uploads/utils/file-utils' + +const BASE_TIMEOUT_MS = 2 * 60 * 1000 +const TIMEOUT_PER_MB_MS = 1500 +const MAX_TIMEOUT_MS = 10 * 60 * 1000 +const PART_URL_BATCH_SIZE = 25 +const PART_UPLOAD_CONCURRENCY = 3 +const MAX_RETRIES = 3 +const RETRY_BASE_MS = 500 +const RETRY_MAX_MS = 8000 + +const logger = createLogger('UploadSessionClient') + +export function calculateUploadTimeoutMs(fileSize: number): number { + const sizeInMb = fileSize / (1024 * 1024) + return Math.min(BASE_TIMEOUT_MS + sizeInMb * TIMEOUT_PER_MB_MS, MAX_TIMEOUT_MS) +} + +export class UploadSessionTransportError extends Error { + constructor( + message: string, + readonly status?: number, + readonly retryAfterMs: number | null = null, + readonly transient = false + ) { + super(message) + this.name = 'UploadSessionTransportError' + } +} + +interface UploadFileSessionCommon { + file: File + signal?: AbortSignal + onProgress?: (event: UploadProgressEvent) => void + complete: (body: V2CompleteUploadBody) => Promise + abort: () => Promise +} + +interface UploadPutFileSession extends UploadFileSessionCommon { + transfer: V2PutUploadTransfer + getPartUrls?: never +} + +interface UploadMultipartFileSession extends UploadFileSessionCommon { + transfer: V2MultipartUploadTransfer + getPartUrls: (partNumbers: number[]) => Promise +} + +export type UploadFileSessionParams = UploadPutFileSession | UploadMultipartFileSession + +function isPutFileSession( + params: UploadFileSessionParams +): params is UploadPutFileSession { + return params.transfer.method === 'put' +} + +export async function uploadFileSession(params: UploadFileSessionParams): Promise { + let completion: V2CompleteUploadBody + try { + if (isPutFileSession(params)) { + await uploadPut(params) + completion = {} + } else { + completion = { parts: await uploadMultipart(params) } + } + } catch (error) { + await params.abort().catch((abortError) => { + logger.warn('Failed to abort upload session after an upload error', { + error: getErrorMessage(abortError), + }) + }) + throw error + } + + return params.complete(completion) +} + +async function uploadPut(params: UploadPutFileSession): Promise { + let reportedLoaded = 0 + const reportProgress = (loaded: number) => { + reportedLoaded = Math.max(reportedLoaded, loaded) + params.onProgress?.({ + loaded: reportedLoaded, + total: params.file.size, + percent: + params.file.size === 0 + ? 100 + : Math.min(100, Math.round((reportedLoaded / params.file.size) * 100)), + }) + } + + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + await uploadPutAttempt({ + file: params.file, + transfer: params.transfer, + signal: params.signal, + onProgress: reportProgress, + }) + reportProgress(params.file.size) + return + } catch (error) { + if (isAbortError(error) || !isRetryableUploadError(error) || attempt >= MAX_RETRIES) { + throw error + } + await waitForRetry(attempt + 1, error.retryAfterMs, params.file.name, params.signal) + } + } + throw new Error(`PUT upload retries exhausted for ${params.file.name}`) +} + +interface UploadPutAttemptParams { + file: File + transfer: V2PutUploadTransfer + signal?: AbortSignal + onProgress: (loaded: number) => void +} + +function uploadPutAttempt(params: UploadPutAttemptParams): Promise { + if (params.signal?.aborted) return Promise.reject(uploadAbortError(params.file.name)) + + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest() + let settled = false + + const cleanup = () => { + params.signal?.removeEventListener('abort', handleSignalAbort) + xhr.upload.removeEventListener('progress', handleProgress) + xhr.removeEventListener('load', handleLoad) + xhr.removeEventListener('error', handleError) + xhr.removeEventListener('timeout', handleTimeout) + } + + const resolveOnce = () => { + if (settled) return + settled = true + cleanup() + resolve() + } + + const rejectOnce = (error: Error) => { + if (settled) return + settled = true + cleanup() + reject(error) + } + + const handleSignalAbort = () => { + xhr.abort() + rejectOnce(uploadAbortError(params.file.name)) + } + + const handleProgress = (event: ProgressEvent) => { + if (event.lengthComputable && !settled) params.onProgress(event.loaded) + } + + const handleLoad = () => { + if (xhr.status >= 200 && xhr.status < 300) { + resolveOnce() + return + } + rejectOnce( + new UploadSessionTransportError( + `PUT upload failed for ${params.file.name}: ${xhr.status} ${xhr.statusText}`, + xhr.status, + parseRetryAfter(xhr.getResponseHeader('Retry-After'), RETRY_MAX_MS), + isRetryableStatus(xhr.status) + ) + ) + } + + const handleError = () => { + rejectOnce( + new UploadSessionTransportError( + `Network error uploading ${params.file.name}`, + undefined, + null, + true + ) + ) + } + + const handleTimeout = () => { + rejectOnce( + new UploadSessionTransportError( + `Upload timed out for ${params.file.name}`, + undefined, + null, + true + ) + ) + } + + xhr.open('PUT', params.transfer.url) + xhr.timeout = calculateUploadTimeoutMs(params.file.size) + for (const [key, value] of Object.entries(params.transfer.headers)) { + xhr.setRequestHeader(key, value) + } + xhr.upload.addEventListener('progress', handleProgress) + xhr.addEventListener('load', handleLoad) + xhr.addEventListener('error', handleError) + xhr.addEventListener('timeout', handleTimeout) + params.signal?.addEventListener('abort', handleSignalAbort, { once: true }) + xhr.send(params.file) + }) +} + +async function uploadMultipart( + params: UploadMultipartFileSession +): Promise { + const { file, transfer, signal, onProgress } = params + const expectedPartCount = Math.ceil(file.size / transfer.partSize) + if (expectedPartCount !== transfer.partCount) { + throw new Error( + `Upload session expected ${transfer.partCount} parts, but file requires ${expectedPartCount}` + ) + } + + const completedBytes = new Array(transfer.partCount).fill(0) + const completedParts: V2CompletedPart[] = [] + for (let start = 1; start <= transfer.partCount; start += PART_URL_BATCH_SIZE) { + const partNumbers = Array.from( + { length: Math.min(PART_URL_BATCH_SIZE, transfer.partCount - start + 1) }, + (_, index) => start + index + ) + const partUrls = validatePartUrlBatch(partNumbers, await params.getPartUrls(partNumbers)) + const results = await runWithConcurrency( + partUrls, + PART_UPLOAD_CONCURRENCY, + async (part): Promise => { + const partStart = (part.partNumber - 1) * transfer.partSize + const end = Math.min(partStart + transfer.partSize, file.size) + const chunk = file.slice(partStart, end) + const etag = await uploadMultipartPart({ part, chunk, fileName: file.name, signal }) + completedBytes[part.partNumber - 1] = end - partStart + const loaded = completedBytes.reduce((sum, bytes) => sum + bytes, 0) + onProgress?.({ + loaded, + total: file.size, + percent: Math.min(100, Math.round((loaded / file.size) * 100)), + }) + return { partNumber: part.partNumber, ...(etag ? { etag } : {}) } + } + ) + completedParts.push( + ...results.map((result) => { + if (result.status === 'rejected') throw result.reason + return result.value + }) + ) + } + return completedParts +} + +function validatePartUrlBatch(requested: number[], received: V2UploadPartUrl[]): V2UploadPartUrl[] { + if (received.length !== requested.length) { + throw new Error(`Expected ${requested.length} part URLs; received ${received.length}`) + } + const byPartNumber = new Map() + for (const part of received) { + if (byPartNumber.has(part.partNumber)) { + throw new Error(`Received duplicate URL for part ${part.partNumber}`) + } + byPartNumber.set(part.partNumber, part) + } + return requested.map((partNumber) => { + const part = byPartNumber.get(partNumber) + if (!part) throw new Error(`Missing upload URL for part ${partNumber}`) + return part + }) +} + +async function uploadMultipartPart(params: { + part: V2UploadPartUrl + chunk: Blob + fileName: string + signal?: AbortSignal +}): Promise { + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + // boundary-raw-fetch: signed multipart data-plane URL may target cloud storage or local Sim + const response = await fetch(params.part.url, { + method: 'PUT', + body: params.chunk, + headers: params.part.headers, + signal: params.signal, + }) + if (!response.ok) { + throw new UploadSessionTransportError( + `Part ${params.part.partNumber} failed (${response.status})`, + response.status, + parseRetryAfter(response.headers.get('Retry-After'), RETRY_MAX_MS), + isRetryableStatus(response.status) + ) + } + return response.headers.get('etag')?.replaceAll('"', '') + } catch (error) { + if (isAbortError(error)) throw error + const classified = + error instanceof TypeError + ? new UploadSessionTransportError( + `Network error uploading part ${params.part.partNumber}`, + undefined, + null, + true + ) + : error + if (!isRetryableUploadError(classified) || attempt >= MAX_RETRIES) { + throw classified + } + await waitForRetry(attempt + 1, classified.retryAfterMs, params.fileName, params.signal) + } + } + throw new Error(`Part ${params.part.partNumber} upload retries exhausted`) +} + +function isRetryableUploadError(error: unknown): error is UploadSessionTransportError { + return error instanceof UploadSessionTransportError && error.transient +} + +function isRetryableStatus(status: number): boolean { + return status === 408 || status === 429 || (status >= 500 && status < 600) +} + +async function waitForRetry( + attempt: number, + retryAfterMs: number | null, + fileName: string, + signal?: AbortSignal +): Promise { + if (signal?.aborted) throw uploadAbortError(fileName) + const delay = backoffWithJitter(attempt, retryAfterMs, { + baseMs: RETRY_BASE_MS, + maxMs: RETRY_MAX_MS, + }) + if (!signal) { + await sleep(delay) + return + } + await new Promise((resolve, reject) => { + const handleAbort = () => { + signal.removeEventListener('abort', handleAbort) + reject(uploadAbortError(fileName)) + } + signal.addEventListener('abort', handleAbort, { once: true }) + sleep(delay).then( + () => { + signal.removeEventListener('abort', handleAbort) + resolve() + }, + (error) => { + signal.removeEventListener('abort', handleAbort) + reject(error) + } + ) + }) +} + +function uploadAbortError(fileName: string): DOMException { + return new DOMException(`Upload aborted for ${fileName}`, 'AbortError') +} diff --git a/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts b/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts index 2e9b3fdf93d..f0406e73601 100644 --- a/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/copilot/copilot-file-manager.ts @@ -4,11 +4,8 @@ import { deleteFile, downloadFile, generatePresignedDownloadUrl, - generatePresignedUploadUrl, uploadFile, } from '@/lib/uploads/core/storage-service' -import type { PresignedUrlResponse } from '@/lib/uploads/shared/types' -import { isImageFileType } from '@/lib/uploads/utils/file-utils' const logger = createLogger('CopilotFileManager') @@ -49,14 +46,6 @@ interface CopilotFileAttachment { media_type: string } -export interface GenerateCopilotUploadUrlOptions { - fileName: string - contentType: string - fileSize: number - userId: string - expirationSeconds?: number -} - export interface CopilotStoredFile { id: string key: string @@ -68,48 +57,6 @@ export interface CopilotStoredFile { mimeType: string } -/** - * Generate a presigned URL for copilot file upload - * - * Images and document files are allowed for copilot uploads. - * Requires authenticated user session. - * - * @param options Upload URL generation options - * @returns Presigned URL response with upload URL and file key - * @throws Error if file type is unsupported or user is not authenticated - */ -export async function generateCopilotUploadUrl( - options: GenerateCopilotUploadUrlOptions -): Promise { - const { fileName, contentType, fileSize, userId, expirationSeconds = 3600 } = options - - if (!userId?.trim()) { - throw new Error('Authenticated user session is required for copilot uploads') - } - - if (!isSupportedFileType(contentType) && !isImageFileType(contentType)) { - throw new Error( - 'Unsupported file type. Allowed: images (JPEG, PNG, GIF, WebP), PDF, and text files (TXT, CSV, MD, HTML, JSON, XML).' - ) - } - - const presignedUrlResponse = await generatePresignedUploadUrl({ - fileName, - contentType, - fileSize, - context: 'copilot', - userId, - expirationSeconds, - }) - - logger.info(`Generated copilot upload URL for: ${fileName}`, { - key: presignedUrlResponse.key, - userId, - }) - - return presignedUrlResponse -} - export async function uploadCopilotFile(options: { buffer: Buffer fileName: string diff --git a/apps/sim/lib/uploads/contexts/copilot/index.ts b/apps/sim/lib/uploads/contexts/copilot/index.ts index d4b1c93a3e2..5fdfc829dc4 100644 --- a/apps/sim/lib/uploads/contexts/copilot/index.ts +++ b/apps/sim/lib/uploads/contexts/copilot/index.ts @@ -1,6 +1,5 @@ export type { CopilotStoredFile } from './copilot-file-manager' export { downloadCopilotFile, - generateCopilotUploadUrl, uploadCopilotFile, } from './copilot-file-manager' diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-folder-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-folder-manager.ts index ca2b80229cc..3c561c8bd1f 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-folder-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-folder-manager.ts @@ -338,10 +338,10 @@ export async function getWorkspaceFileFolder( return mapFolder(folder, paths) } -export async function assertWorkspaceFileFolderTarget( +export async function resolveWorkspaceFileFolderTarget( workspaceId: string, folderId?: string | null -): Promise { +): Promise { const normalized = normalizeParentId(folderId) if (!normalized) return null @@ -350,7 +350,15 @@ export async function assertWorkspaceFileFolderTarget( throw new OrchestrationError('not_found', 'Target folder not found') } - return normalized + return folder +} + +export async function assertWorkspaceFileFolderTarget( + workspaceId: string, + folderId?: string | null +): Promise { + const folder = await resolveWorkspaceFileFolderTarget(workspaceId, folderId) + return folder?.id ?? null } export async function createWorkspaceFileFolder(params: { diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts index 3a7afab9538..ebd22f89f83 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts @@ -43,6 +43,7 @@ import { getServePathPrefix } from '@/lib/uploads' import { deleteFile, downloadFile, + hasCloudStorage, headObject, uploadFile, } from '@/lib/uploads/core/storage-service' @@ -60,6 +61,7 @@ import { getWorkspaceFileFolderPath, listWorkspaceFileFolders, normalizeWorkspaceFileItemName, + resolveWorkspaceFileFolderTarget, } from './workspace-file-folder-manager' const logger = createLogger('WorkspaceFileStorage') @@ -106,6 +108,14 @@ export interface WorkspaceFileRecord { share?: ShareRecord | null } +export interface UploadedWorkspaceFileRecord extends WorkspaceFileRecord { + url: string + context: 'workspace' + folderId: string | null + folderPath: string | null + deletedAt: Date | null +} + interface ListWorkspaceFilesOptions { scope?: WorkspaceFileScope folders?: WorkspaceFileFolderRecord[] @@ -213,18 +223,19 @@ class WorkspaceFileRegistrationConflictError extends Error { } /** - * Reads one active metadata row by its unique storage key. + * Reads metadata by upload-operation key across its full lifecycle, preferring an active row. */ -async function findActiveWorkspaceFileByKey( +async function findWorkspaceFileByRegistrationKey( executor: DbOrTx, key: string ): Promise { - const [file] = await executor + const files = await executor .select() .from(workspaceFiles) - .where(and(eq(workspaceFiles.key, key), isNull(workspaceFiles.deletedAt))) + .where(eq(workspaceFiles.key, key)) + .orderBy(sql`${workspaceFiles.deletedAt} IS NULL DESC`) .limit(1) - return file + return files[0] } /** @@ -250,7 +261,7 @@ async function findWorkspaceFileForLifecycle( } /** - * Confirms that an active-key conflict belongs to the same direct-upload + * Confirms that a key belongs to the same upload-session * operation. The generated storage key is the operation identity; immutable * ownership and object attributes prevent unrelated callers from reusing it. */ @@ -260,7 +271,6 @@ function isSameWorkspaceFileRegistration( workspaceId: string userId: string key: string - folderId: string | null contentType: string size: number } @@ -269,11 +279,9 @@ function isSameWorkspaceFileRegistration( file.key === params.key && file.workspaceId === params.workspaceId && file.userId === params.userId && - file.folderId === params.folderId && file.context === 'workspace' && file.contentType === params.contentType && - workspaceFileSize(file) === params.size && - file.deletedAt === null + workspaceFileSize(file) === params.size ) } @@ -331,10 +339,12 @@ export async function uploadWorkspaceFile( fileName: string, contentType: string, options?: { folderId?: string | null; exactName?: boolean } -): Promise { +): Promise { logger.info(`Uploading workspace file: ${fileName} for workspace ${workspaceId}`) - const folderId = await assertWorkspaceFileFolderTarget(workspaceId, options?.folderId) + const folderTarget = await resolveWorkspaceFileFolderTarget(workspaceId, options?.folderId) + const folderId = folderTarget?.id ?? null + const folderPath = folderTarget?.path ?? null const normalizedFileName = normalizeWorkspaceFileItemName(fileName, 'File') const exactName = options?.exactName ?? false const storageBillingContext = await resolveStorageBillingContext(workspaceId) @@ -349,7 +359,7 @@ export async function uploadWorkspaceFile( throw new FileConflictError(uniqueName) } const storageKey = generateWorkspaceFileKey(workspaceId, uniqueName) - let fileId = `wf_${generateShortId()}` + const fileId = `wf_${generateShortId()}` try { logger.info(`Generated storage key: ${storageKey}`) @@ -376,9 +386,12 @@ export async function uploadWorkspaceFile( logger.info(`Upload returned key: ${uploadResult.key}`) - let updatedUsage: number | undefined + let finalized: { + inserted: typeof workspaceFiles.$inferSelect + updatedUsage: number | undefined + } try { - const finalized = await db.transaction(async (tx) => { + finalized = await db.transaction(async (tx) => { const inserted = await insertWorkspaceFileMetadataInTx(tx, { id: fileId, key: uploadResult.key, @@ -399,36 +412,22 @@ export async function uploadWorkspaceFile( ) return { inserted, updatedUsage: usage } }) - fileId = finalized.inserted.id - updatedUsage = finalized.updatedUsage } catch (finalizationError) { await cleanupWorkspaceStorageObject(uploadResult.key, 'metadata finalization failure') throw finalizationError } - void maybeNotifyStorageLimitForBillingContext(storageBillingContext, updatedUsage) + void maybeNotifyStorageLimitForBillingContext(storageBillingContext, finalized.updatedUsage) logger.info( `Successfully uploaded workspace file: ${uniqueName} with key: ${uploadResult.key}` ) - const pathPrefix = getServePathPrefix() - const serveUrl = `${pathPrefix}${encodeURIComponent(uploadResult.key)}?context=workspace` - - // Fan out the live-tree signal for the direct-upload paths (multipart - // fallback, copilot create, /api/files/upload, v1 files) — the presigned - // path already notifies from its register route. + // Fan out the live-tree signal for this server-buffered path. Upload-session + // finalization sends its own notification after registering metadata. await notifyWorkspaceFilesChanged(workspaceId) - return { - id: fileId, - name: uniqueName, - size: fileBuffer.length, - type: contentType, - url: serveUrl, - key: uploadResult.key, - context: 'workspace', - } + return mapUploadedWorkspaceFileRecord(finalized.inserted, workspaceId, folderPath) } catch (error) { lastError = error if (error instanceof FileConflictError) { @@ -469,8 +468,8 @@ export async function uploadWorkspaceFile( } /** - * Finalize a workspace file that was uploaded directly to cloud storage - * (presigned PUT or completed multipart). Verifies the object exists, + * Finalize a workspace file that was uploaded through a transfer session + * (signed PUT or completed multipart). Verifies the object exists, * checks quota, allocates a non-colliding display name, inserts metadata, * and increments storage usage. * @@ -503,7 +502,6 @@ export async function registerUploadedWorkspaceFile(params: { throw new Error('Uploaded object not found in storage') } const verifiedSize = head.size - const folderId = await assertWorkspaceFileFolderTarget(workspaceId, params.folderId) if (verifiedSize > MAX_WORKSPACE_FILE_SIZE) { await cleanupWorkspaceStorageObject(key, 'size-cap rejection') @@ -514,16 +512,16 @@ export async function registerUploadedWorkspaceFile(params: { workspaceId, userId, key, - folderId, contentType, size: verifiedSize, } - const existing = await findActiveWorkspaceFileByKey(db, key) + const existing = await findWorkspaceFileByRegistrationKey(db, key) if (existing) { if (!isSameWorkspaceFileRegistration(existing, registrationIdentity)) { throw new WorkspaceFileRegistrationConflictError(key) } - logger.info(`Using existing metadata record for direct upload: ${key}`) + assertActiveWorkspaceFileRegistration(existing) + logger.info(`Using existing metadata record for upload session: ${key}`) const pathPrefix = getServePathPrefix() return { file: { @@ -539,6 +537,8 @@ export async function registerUploadedWorkspaceFile(params: { } } + const folderId = await assertWorkspaceFileFolderTarget(workspaceId, params.folderId) + const storageBillingContext = await resolveStorageBillingContext(workspaceId) for (let attempt = 0; attempt < MAX_UPLOAD_UNIQUE_RETRIES; attempt++) { const fileId = `wf_${generateShortId()}` @@ -560,11 +560,12 @@ export async function registerUploadedWorkspaceFile(params: { size: verifiedSize, }) if (!inserted) { - const raceWinner = await findActiveWorkspaceFileByKey(tx, key) + const raceWinner = await findWorkspaceFileByRegistrationKey(tx, key) if (!raceWinner) return { kind: 'name-conflict' } as const if (!isSameWorkspaceFileRegistration(raceWinner, registrationIdentity)) { throw new WorkspaceFileRegistrationConflictError(key) } + assertActiveWorkspaceFileRegistration(raceWinner) return { kind: 'existing', file: raceWinner } as const } @@ -605,6 +606,12 @@ export async function registerUploadedWorkspaceFile(params: { throw new FileConflictError(normalizedOriginalName) } +function assertActiveWorkspaceFileRegistration(file: typeof workspaceFiles.$inferSelect): void { + if (file.deletedAt) { + throw new OrchestrationError('conflict', 'Upload result was deleted') + } +} + /** * Like `withCopySuffix` but with `n=1` meaning "no suffix" — used by retry loops where * the first attempt should try the original name (`image.png`, `image (2).png`, ...). @@ -867,6 +874,26 @@ function mapWorkspaceFileRecord( } } +function mapUploadedWorkspaceFileRecord( + file: typeof workspaceFiles.$inferSelect, + workspaceId: string, + folderPath: string | null +): UploadedWorkspaceFileRecord { + const record = mapWorkspaceFileRecord( + file, + workspaceId, + file.folderId && folderPath ? new Map([[file.folderId, folderPath]]) : new Map() + ) + return { + ...record, + url: record.path, + context: 'workspace', + folderId: record.folderId ?? null, + folderPath: record.folderPath ?? null, + deletedAt: record.deletedAt ?? null, + } +} + async function mapSingleWorkspaceFileRecord( file: typeof workspaceFiles.$inferSelect, workspaceId: string diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-query.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-query.test.ts index 21b5be978f6..f4c81ef8bc3 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-query.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-query.test.ts @@ -43,6 +43,7 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => getWorkspaceFileFolderPath: vi.fn(), listWorkspaceFileFolders: vi.fn(async () => []), normalizeWorkspaceFileItemName: vi.fn((name: string) => name), + resolveWorkspaceFileFolderTarget: vi.fn(async () => null), })) import { queryWorkspaceFiles } from '@/lib/uploads/contexts/workspace/workspace-file-manager' diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts index cf0b5cf7189..cb101d5455e 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-accounting.test.ts @@ -15,6 +15,7 @@ const { mockMergeEditIntoLiveFileDoc, mockNotifyWorkspaceFilesChanged, mockResolveStorageBillingContext, + mockResolveWorkspaceFileFolderTarget, mockUploadFile, } = vi.hoisted(() => ({ mockDecrementStorageUsageForBillingContextInTx: vi.fn(), @@ -27,6 +28,7 @@ const { mockMergeEditIntoLiveFileDoc: vi.fn(), mockNotifyWorkspaceFilesChanged: vi.fn(), mockResolveStorageBillingContext: vi.fn(), + mockResolveWorkspaceFileFolderTarget: vi.fn(), mockUploadFile: vi.fn(), })) @@ -62,6 +64,7 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => getWorkspaceFileFolderPath: vi.fn(), listWorkspaceFileFolders: vi.fn(async () => []), normalizeWorkspaceFileItemName: vi.fn((name: string) => name), + resolveWorkspaceFileFolderTarget: mockResolveWorkspaceFileFolderTarget, })) vi.mock('@/lib/workspaces/permissions/utils', () => ({ @@ -108,6 +111,7 @@ describe('workspace file metadata and storage accounting', () => { vi.clearAllMocks() resetDbChainMock() mockResolveStorageBillingContext.mockResolvedValue(STORAGE_CONTEXT) + mockResolveWorkspaceFileFolderTarget.mockResolvedValue(null) mockHasCloudStorage.mockReturnValue(false) mockHeadObject.mockResolvedValue({ size: FILE_ROW.size }) mockUploadFile.mockResolvedValue({ key: FILE_ROW.key }) @@ -120,6 +124,46 @@ describe('workspace file metadata and storage accounting', () => { mockNotifyWorkspaceFilesChanged.mockResolvedValue(undefined) }) + it('returns the canonical inserted record with the pre-resolved folder path', async () => { + const folderId = 'folder-1' + const folderPath = 'Docs/Notes' + const inserted = { ...FILE_ROW, folderId } + mockResolveWorkspaceFileFolderTarget.mockResolvedValueOnce({ id: folderId, path: folderPath }) + dbChainMockFns.returning.mockResolvedValueOnce([inserted]) + + const uploaded = await uploadWorkspaceFile( + FILE_ROW.workspaceId, + FILE_ROW.userId, + Buffer.from('hello'), + FILE_ROW.originalName, + FILE_ROW.contentType, + { folderId } + ) + + const serveUrl = `/api/files/serve/s3/${encodeURIComponent(FILE_ROW.key)}?context=workspace` + expect(uploaded).toEqual( + expect.objectContaining({ + id: FILE_ROW.id, + workspaceId: FILE_ROW.workspaceId, + name: FILE_ROW.originalName, + key: FILE_ROW.key, + path: serveUrl, + url: serveUrl, + size: FILE_ROW.size, + type: FILE_ROW.contentType, + uploadedBy: FILE_ROW.userId, + folderId, + folderPath, + deletedAt: null, + uploadedAt: FILE_ROW.uploadedAt, + updatedAt: FILE_ROW.updatedAt, + contentUpdatedAt: FILE_ROW.contentUpdatedAt, + context: 'workspace', + }) + ) + expect(mockResolveWorkspaceFileFolderTarget).toHaveBeenCalledOnce() + }) + it('cleans up a newly uploaded object when atomic metadata finalization rolls back', async () => { dbChainMockFns.returning.mockResolvedValueOnce([FILE_ROW]) mockIncrementStorageUsageForBillingContextInTx.mockRejectedValueOnce( @@ -196,7 +240,7 @@ describe('workspace file metadata and storage accounting', () => { expect(mockDeleteFile).not.toHaveBeenCalled() }) - it('does not delete a direct-upload object when atomic finalization rolls back', async () => { + it('does not delete an upload-session object when atomic finalization rolls back', async () => { mockHasCloudStorage.mockReturnValue(true) dbChainMockFns.limit.mockResolvedValueOnce([]) dbChainMockFns.returning.mockResolvedValueOnce([FILE_ROW]) @@ -220,6 +264,28 @@ describe('workspace file metadata and storage accounting', () => { ) }) + it('rejects archived upload metadata without charging storage again', async () => { + const archivedFile = { + ...FILE_ROW, + deletedAt: new Date('2026-07-02T00:00:00.000Z'), + } + mockHasCloudStorage.mockReturnValue(true) + dbChainMockFns.limit.mockResolvedValueOnce([archivedFile]) + + await expect( + registerUploadedWorkspaceFile({ + workspaceId: FILE_ROW.workspaceId, + userId: FILE_ROW.userId, + key: FILE_ROW.key, + originalName: FILE_ROW.originalName, + contentType: FILE_ROW.contentType, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + expect(dbChainMockFns.returning).not.toHaveBeenCalled() + expect(mockIncrementStorageUsageForBillingContextInTx).not.toHaveBeenCalled() + expect(mockMaybeNotifyStorageLimitForBillingContext).not.toHaveBeenCalled() + }) + it('archives metadata without changing stored-byte counters', async () => { dbChainMockFns.limit.mockResolvedValueOnce([FILE_ROW]) dbChainMockFns.returning.mockResolvedValueOnce([FILE_ROW]) diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-billing.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-billing.test.ts index eaf463774bc..4f3b0af4102 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-billing.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-storage-billing.test.ts @@ -8,11 +8,13 @@ const { mockIncrementStorageUsageForBillingContextInTx, mockMaybeNotifyStorageLimitForBillingContext, mockResolveStorageBillingContext, + mockResolveWorkspaceFileFolderTarget, mockUploadFile, } = vi.hoisted(() => ({ mockIncrementStorageUsageForBillingContextInTx: vi.fn(), mockMaybeNotifyStorageLimitForBillingContext: vi.fn(), mockResolveStorageBillingContext: vi.fn(), + mockResolveWorkspaceFileFolderTarget: vi.fn(), mockUploadFile: vi.fn(), })) @@ -43,6 +45,7 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-folder-manager', () => getWorkspaceFileFolderPath: vi.fn(), listWorkspaceFileFolders: vi.fn(async () => []), normalizeWorkspaceFileItemName: vi.fn((name: string) => name), + resolveWorkspaceFileFolderTarget: mockResolveWorkspaceFileFolderTarget, })) import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-file-manager' @@ -60,6 +63,7 @@ describe('workspace file storage attribution', () => { vi.clearAllMocks() resetDbChainMock() mockResolveStorageBillingContext.mockResolvedValue(STORAGE_CONTEXT) + mockResolveWorkspaceFileFolderTarget.mockResolvedValue(null) mockIncrementStorageUsageForBillingContextInTx.mockResolvedValue(5) mockMaybeNotifyStorageLimitForBillingContext.mockResolvedValue(undefined) mockUploadFile.mockResolvedValue({ diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts index 619116c72ae..70bbb925ceb 100644 --- a/apps/sim/lib/uploads/core/storage-service.ts +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -30,6 +30,9 @@ import { const logger = createLogger('StorageService') +/** Sidecar attached to local objects promoted through the upload-session transport. */ +export const LOCAL_UPLOAD_METADATA_SUFFIX = '.upload-metadata.json' + /** * Create a Blob config from StorageConfig * @throws Error if required properties are missing @@ -556,7 +559,7 @@ export async function deleteFile(options: DeleteFileOptions): Promise { } } - const { unlink } = await import('fs/promises') + const { rm, unlink } = await import('fs/promises') const { join } = await import('path') const { UPLOAD_DIR_SERVER } = await import('./setup.server') @@ -564,6 +567,7 @@ export async function deleteFile(options: DeleteFileOptions): Promise { const filePath = join(UPLOAD_DIR_SERVER, safeKey) await unlink(filePath) + await rm(`${filePath}${LOCAL_UPLOAD_METADATA_SUFFIX}`, { force: true }) } /** AWS SDK v3 silently caps HTTP connections at 50/endpoint — stay well under. */ @@ -846,36 +850,6 @@ async function generateBlobPresignedUrl( } } -/** - * Generate multiple presigned URLs at once (batch operation) - */ -export async function generateBatchPresignedUploadUrls( - files: Array<{ - fileName: string - contentType: string - fileSize: number - }>, - context: StorageContext, - userId?: string, - expirationSeconds?: number -): Promise { - const results: PresignedUrlResponse[] = [] - - for (const file of files) { - const result = await generatePresignedUploadUrl({ - fileName: file.fileName, - contentType: file.contentType, - fileSize: file.fileSize, - context, - userId, - expirationSeconds, - }) - results.push(result) - } - - return results -} - /** * Generate a presigned URL for downloading/accessing an existing file */ diff --git a/apps/sim/lib/uploads/core/upload-token.test.ts b/apps/sim/lib/uploads/core/upload-token.test.ts index 532985f5411..85beca8acbc 100644 --- a/apps/sim/lib/uploads/core/upload-token.test.ts +++ b/apps/sim/lib/uploads/core/upload-token.test.ts @@ -4,57 +4,121 @@ import { describe, expect, it } from 'vitest' import { signUploadToken, verifyUploadToken } from '@/lib/uploads/core/upload-token' +const TIMESTAMPS = { + createdAt: '2099-08-03T20:00:00.000Z', + expiresAt: '2099-08-04T20:00:00.000Z', +} as const + describe('upload token', () => { - it('round-trips stateless multipart session state', () => { - const token = signUploadToken({ + it('round-trips strict multipart session state', () => { + const payload = { uploadId: 'upload-1', - key: 'workspace-1/file.csv', - userId: 'user-1', + actorId: 'user-1', workspaceId: 'workspace-1', - context: 'workspace', + purpose: 'knowledge_document', + knowledgeBaseId: 'kb-1', + context: 'knowledge-base', + finalKey: 'kb/final-file.csv', + stagingKey: 'upload-sessions/upload-1/file.csv', + provider: 's3', + providerUploadId: 'provider-upload-1', + method: 'multipart', fileName: 'file.csv', contentType: 'text/csv', fileSize: 12, - purpose: 'workspace_file', - provider: 's3', - providerUploadId: 'provider-upload-1', partSize: 8, partCount: 2, - metadata: { folderId: 'folder-1' }, - createdAt: '2026-08-03T20:00:00.000Z', - expiresAt: '2026-08-04T20:00:00.000Z', - }) + metadata: { tag1: 'product' }, + ...TIMESTAMPS, + } as const + const token = signUploadToken(payload) - expect(verifyUploadToken(token)).toEqual({ - valid: true, - payload: { - uploadId: 'upload-1', - key: 'workspace-1/file.csv', - userId: 'user-1', - workspaceId: 'workspace-1', - context: 'workspace', - fileName: 'file.csv', - contentType: 'text/csv', - fileSize: 12, - purpose: 'workspace_file', - provider: 's3', - providerUploadId: 'provider-upload-1', - partSize: 8, - partCount: 2, - metadata: { folderId: 'folder-1' }, - createdAt: '2026-08-03T20:00:00.000Z', - expiresAt: '2026-08-04T20:00:00.000Z', - }, - }) + expect(verifyUploadToken(token)).toEqual({ valid: true, payload }) + }) + + it('round-trips a user-scoped PUT without a synthetic workspace', () => { + const payload = { + uploadId: 'upload-2', + actorId: 'user-1', + workspaceId: null, + purpose: 'profile_picture', + context: 'profile-pictures', + finalKey: 'profile-pictures/upload-2-avatar.png', + stagingKey: 'upload-sessions/upload-2/avatar.png', + provider: 'local', + providerUploadId: null, + method: 'put', + fileName: 'avatar.png', + contentType: 'image/png', + fileSize: 12, + metadata: {}, + ...TIMESTAMPS, + } as const + + expect(verifyUploadToken(signUploadToken(payload))).toEqual({ valid: true, payload }) + }) + + it('round-trips an empty workspace-file PUT', () => { + const payload = { + uploadId: 'upload-empty', + actorId: 'user-1', + workspaceId: 'workspace-1', + purpose: 'workspace_file', + context: 'workspace', + finalKey: 'workspace/workspace-1/empty.md', + stagingKey: 'upload-sessions/upload-empty/empty.md', + provider: 'local', + providerUploadId: null, + method: 'put', + fileName: 'empty.md', + contentType: 'text/markdown', + fileSize: 0, + metadata: {}, + ...TIMESTAMPS, + } as const + + expect(verifyUploadToken(signUploadToken(payload))).toEqual({ valid: true, payload }) + }) + + it('rejects an empty PUT for non-workspace-file purposes', () => { + expect(() => + signUploadToken({ + uploadId: 'upload-empty', + actorId: 'user-1', + workspaceId: null, + purpose: 'profile_picture', + context: 'profile-pictures', + finalKey: 'profile-pictures/empty.png', + stagingKey: 'upload-sessions/upload-empty/empty.png', + provider: 'local', + providerUploadId: null, + method: 'put', + fileName: 'empty.png', + contentType: 'image/png', + fileSize: 0, + metadata: {}, + ...TIMESTAMPS, + }) + ).toThrow('Upload token payload has invalid object state') }) it('rejects a modified token', () => { const token = signUploadToken({ uploadId: 'upload-1', - key: 'workspace-1/file.csv', - userId: 'user-1', + actorId: 'user-1', workspaceId: 'workspace-1', + purpose: 'workspace_file', context: 'workspace', + finalKey: 'workspace/workspace-1/final.csv', + stagingKey: 'upload-sessions/upload-1/file.csv', + provider: 'local', + providerUploadId: null, + method: 'put', + fileName: 'file.csv', + contentType: 'text/csv', + fileSize: 12, + metadata: {}, + ...TIMESTAMPS, }) const [payload, signature] = token.split('.') diff --git a/apps/sim/lib/uploads/core/upload-token.ts b/apps/sim/lib/uploads/core/upload-token.ts index da058708536..86fba13110e 100644 --- a/apps/sim/lib/uploads/core/upload-token.ts +++ b/apps/sim/lib/uploads/core/upload-token.ts @@ -1,44 +1,113 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Base64 } from '@sim/security/hmac' import { env } from '@/lib/core/config/env' -import type { StorageContext } from '@/lib/uploads/shared/types' -export interface UploadTokenPayload { +export type UploadSessionPurpose = + | 'workspace_file' + | 'table_import' + | 'knowledge_document' + | 'profile_picture' + | 'workspace_logo' + | 'mothership_attachment' + | 'execution_attachment' + +export type UploadStorageProvider = 's3' | 'blob' | 'gcs' | 'local' +export type UploadTransferMethod = 'put' | 'multipart' + +type UploadPurposeScope = + | { + purpose: 'workspace_file' + workspaceId: string + context: 'workspace' + } + | { + purpose: 'table_import' + workspaceId: string + context: 'table-import' + } + | { + purpose: 'knowledge_document' + workspaceId: string + context: 'knowledge-base' + knowledgeBaseId: string + } + | { + purpose: 'profile_picture' + workspaceId: null + context: 'profile-pictures' + } + | { + purpose: 'workspace_logo' + workspaceId: string + context: 'workspace-logos' + } + | { + purpose: 'mothership_attachment' + workspaceId: string + context: 'mothership' + } + | { + purpose: 'execution_attachment' + workspaceId: string + context: 'execution' + workflowId: string + executionId: string + } + +type UploadTransferState = + | { + method: 'put' + providerUploadId: null + } + | { + method: 'multipart' + providerUploadId: string | null + partSize: number + partCount: number + } + +interface UploadTokenBase { uploadId: string - key: string - userId: string - workspaceId: string - context: StorageContext - /** Knowledge base bound to a knowledge-document multipart session. */ - knowledgeBaseId?: string - /** Original file name, carried so the completion handler can record ownership metadata. */ - fileName?: string - /** File MIME type, carried for ownership metadata at completion. */ - contentType?: string - /** File size in bytes, carried for ownership metadata at completion. */ - fileSize?: number - /** Multipart-session purpose. Omitted by the legacy multipart endpoint. */ - purpose?: 'workspace_file' | 'table_import' | 'knowledge_document' - /** Storage provider that owns the multipart upload state. */ - provider?: 's3' | 'blob' | 'gcs' | 'local' - /** Provider-issued multipart upload id. Local and block-blob uploads do not need one. */ - providerUploadId?: string | null - /** Fixed byte size of every part except the final part. */ - partSize?: number - /** Exact number of parts the client must complete. */ - partCount?: number - /** Signed purpose-specific data needed during finalization. */ - metadata?: Record - /** ISO timestamps used to reconstruct the stateless session response. */ - createdAt?: string - expiresAt?: string + actorId: string + finalKey: string + stagingKey: string + provider: UploadStorageProvider + fileName: string + contentType: string + fileSize: number + metadata: Record + createdAt: string + expiresAt: string } -interface SignedPayload extends UploadTokenPayload { +export type UploadTokenPayload = UploadTokenBase & UploadPurposeScope & UploadTransferState + +type SignedPayload = UploadTokenPayload & { exp: number - v: 1 + v: 2 } +const BASE_KEYS = [ + 'uploadId', + 'actorId', + 'finalKey', + 'stagingKey', + 'provider', + 'providerUploadId', + 'method', + 'purpose', + 'workspaceId', + 'context', + 'fileName', + 'contentType', + 'fileSize', + 'metadata', + 'createdAt', + 'expiresAt', + 'exp', + 'v', +] as const + const toBase64Url = (input: string): string => Buffer.from(input, 'utf8').toString('base64url') const fromBase64Url = (input: string): string => Buffer.from(input, 'base64url').toString('utf8') @@ -46,17 +115,19 @@ const fromBase64Url = (input: string): string => Buffer.from(input, 'base64url') const sign = (payload: string): string => hmacSha256Base64(payload, env.INTERNAL_API_SECRET) /** - * Sign an upload session token binding every supplied field to its signature. - * Multipart sessions include the caller, workspace, storage context and key, - * purpose, provider state, file metadata, part geometry, and—for knowledge - * documents—the target knowledge base. Follow-up calls reconstruct their - * complete trusted session exclusively from this signed state. + * Signs the complete, immutable state of one upload session. + * + * Version 2 intentionally has no compatibility parser for legacy multipart tokens. A token must + * carry a purpose-specific scope, transfer method, staging and final keys, provider state, exact + * object identity, and one canonical expiry. */ -export function signUploadToken(payload: UploadTokenPayload, expiresInSeconds = 60 * 60): string { +export function signUploadToken(payload: UploadTokenPayload): string { + assertUploadTokenPayload(payload) + const expiresAt = new Date(payload.expiresAt) const signed: SignedPayload = { ...payload, - exp: Math.floor(Date.now() / 1000) + expiresInSeconds, - v: 1, + exp: Math.floor(expiresAt.getTime() / 1000), + v: 2, } const encoded = toBase64Url(JSON.stringify(signed)) return `${encoded}.${sign(encoded)}` @@ -67,74 +138,172 @@ export type UploadTokenVerification = | { valid: false } export function verifyUploadToken(token: string): UploadTokenVerification { - if (typeof token !== 'string') { - return { valid: false } - } + if (typeof token !== 'string') return { valid: false } const parts = token.split('.') if (parts.length !== 2) return { valid: false } const [encoded, signature] = parts - if (!encoded || !signature) return { valid: false } + if (!encoded || !signature || !safeCompare(signature, sign(encoded))) return { valid: false } - const expected = sign(encoded) - if (!safeCompare(signature, expected)) { + let parsed: unknown + try { + parsed = JSON.parse(fromBase64Url(encoded)) + } catch { return { valid: false } } - let parsed: SignedPayload + if (!isRecord(parsed) || parsed.v !== 2 || !isSafePositiveInteger(parsed.exp)) { + return { valid: false } + } + if (parsed.exp <= Math.floor(Date.now() / 1000)) return { valid: false } + try { - parsed = JSON.parse(fromBase64Url(encoded)) as SignedPayload + assertUploadTokenPayload(parsed) } catch { return { valid: false } } + if (Math.floor(new Date(parsed.expiresAt).getTime() / 1000) !== parsed.exp) { + return { valid: false } + } + + const { exp: _exp, v: _version, ...payload } = parsed + return { valid: true, payload } +} + +function assertUploadTokenPayload(value: unknown): asserts value is UploadTokenPayload { + if (!isRecord(value)) throw new Error('Upload token payload must be an object') + + const purposeKeys = + value.purpose === 'knowledge_document' + ? ['knowledgeBaseId'] + : value.purpose === 'execution_attachment' + ? ['workflowId', 'executionId'] + : [] + const methodKeys = value.method === 'multipart' ? ['partSize', 'partCount'] : [] + const allowedKeys = new Set([...BASE_KEYS, ...purposeKeys, ...methodKeys]) + if (Object.keys(value).some((key) => !allowedKeys.has(key))) { + throw new Error('Upload token payload contains unexpected state') + } + if ( - parsed.v !== 1 || - typeof parsed.exp !== 'number' || - parsed.exp < Math.floor(Date.now() / 1000) || - typeof parsed.uploadId !== 'string' || - typeof parsed.key !== 'string' || - typeof parsed.userId !== 'string' || - typeof parsed.workspaceId !== 'string' || - typeof parsed.context !== 'string' + !isNonEmptyString(value.uploadId) || + !isNonEmptyString(value.actorId) || + !isNonEmptyString(value.finalKey) || + !isNonEmptyString(value.stagingKey) || + value.finalKey === value.stagingKey || + !value.stagingKey.startsWith(`upload-sessions/${value.uploadId}/`) || + !isNonEmptyString(value.fileName) || + !isNonEmptyString(value.contentType) || + !isValidFileSize(value.purpose, value.fileSize) || + !isPlainRecord(value.metadata) ) { - return { valid: false } + throw new Error('Upload token payload has invalid object state') + } + + if ( + value.provider !== 's3' && + value.provider !== 'blob' && + value.provider !== 'gcs' && + value.provider !== 'local' + ) { + throw new Error('Upload token payload has an invalid provider') + } + + if (value.method === 'put') { + if (value.providerUploadId !== null || 'partSize' in value || 'partCount' in value) { + throw new Error('PUT upload token has multipart state') + } + } else if (value.method === 'multipart') { + if (!isSafePositiveInteger(value.partSize) || !isSafePositiveInteger(value.partCount)) { + throw new Error('Multipart upload token has invalid geometry') + } + if (value.provider === 'local') { + if (value.providerUploadId !== null) { + throw new Error('Local multipart upload token has a provider upload id') + } + } else if (!isNonEmptyString(value.providerUploadId)) { + throw new Error('Cloud multipart upload token is missing its provider upload id') + } + } else { + throw new Error('Upload token payload has an invalid transfer method') + } + + assertPurposeScope(value) + + if (!isNonEmptyString(value.createdAt) || !isNonEmptyString(value.expiresAt)) { + throw new Error('Upload token payload is missing timestamps') + } + const createdAt = new Date(value.createdAt).getTime() + const expiresAt = new Date(value.expiresAt).getTime() + if (!Number.isFinite(createdAt) || !Number.isFinite(expiresAt) || expiresAt <= createdAt) { + throw new Error('Upload token payload has invalid timestamps') } +} - return { - valid: true, - payload: { - uploadId: parsed.uploadId, - key: parsed.key, - userId: parsed.userId, - workspaceId: parsed.workspaceId, - context: parsed.context as StorageContext, - ...(typeof parsed.knowledgeBaseId === 'string' - ? { knowledgeBaseId: parsed.knowledgeBaseId } - : {}), - ...(typeof parsed.fileName === 'string' ? { fileName: parsed.fileName } : {}), - ...(typeof parsed.contentType === 'string' ? { contentType: parsed.contentType } : {}), - ...(typeof parsed.fileSize === 'number' ? { fileSize: parsed.fileSize } : {}), - ...(parsed.purpose === 'workspace_file' || - parsed.purpose === 'table_import' || - parsed.purpose === 'knowledge_document' - ? { purpose: parsed.purpose } - : {}), - ...(parsed.provider === 's3' || - parsed.provider === 'blob' || - parsed.provider === 'gcs' || - parsed.provider === 'local' - ? { provider: parsed.provider } - : {}), - ...(typeof parsed.providerUploadId === 'string' || parsed.providerUploadId === null - ? { providerUploadId: parsed.providerUploadId } - : {}), - ...(typeof parsed.partSize === 'number' ? { partSize: parsed.partSize } : {}), - ...(typeof parsed.partCount === 'number' ? { partCount: parsed.partCount } : {}), - ...(parsed.metadata && typeof parsed.metadata === 'object' && !Array.isArray(parsed.metadata) - ? { metadata: parsed.metadata } - : {}), - ...(typeof parsed.createdAt === 'string' ? { createdAt: parsed.createdAt } : {}), - ...(typeof parsed.expiresAt === 'string' ? { expiresAt: parsed.expiresAt } : {}), - }, +function assertPurposeScope(value: Record): void { + switch (value.purpose) { + case 'workspace_file': + assertWorkspacePurpose(value, 'workspace') + break + case 'table_import': + assertWorkspacePurpose(value, 'table-import') + break + case 'knowledge_document': + assertWorkspacePurpose(value, 'knowledge-base') + if (!isNonEmptyString(value.knowledgeBaseId)) { + throw new Error('Knowledge upload token is missing knowledgeBaseId') + } + break + case 'profile_picture': + if (value.workspaceId !== null || value.context !== 'profile-pictures') { + throw new Error('Profile-picture upload token has invalid scope') + } + break + case 'workspace_logo': + assertWorkspacePurpose(value, 'workspace-logos') + break + case 'mothership_attachment': + assertWorkspacePurpose(value, 'mothership') + break + case 'execution_attachment': + assertWorkspacePurpose(value, 'execution') + if (!isNonEmptyString(value.workflowId) || !isNonEmptyString(value.executionId)) { + throw new Error('Execution upload token is missing workflow scope') + } + break + default: + throw new Error('Upload token payload has an invalid purpose') } } + +function assertWorkspacePurpose(value: Record, context: string): void { + if (!isNonEmptyString(value.workspaceId) || value.context !== context) { + throw new Error('Upload token payload has invalid workspace scope') + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function isPlainRecord(value: unknown): value is Record { + if (!isRecord(value)) return false + const prototype = Object.getPrototypeOf(value) + return prototype === Object.prototype || prototype === null +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0 +} + +function isSafePositiveInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 +} + +function isValidFileSize(purpose: unknown, value: unknown): value is number { + return ( + typeof value === 'number' && + Number.isSafeInteger(value) && + (value > 0 || (purpose === 'workspace_file' && value === 0)) + ) +} diff --git a/apps/sim/lib/uploads/multipart-session/provider.ts b/apps/sim/lib/uploads/multipart-session/provider.ts deleted file mode 100644 index 64e258cc879..00000000000 --- a/apps/sim/lib/uploads/multipart-session/provider.ts +++ /dev/null @@ -1,319 +0,0 @@ -import { createReadStream, createWriteStream } from 'node:fs' -import { mkdir, rename, rm, stat } from 'node:fs/promises' -import { dirname, join } from 'node:path' -import { pipeline } from 'node:stream/promises' -import { getErrorMessage } from '@sim/utils/errors' -import { - getStorageConfig, - USE_BLOB_STORAGE, - USE_GCS_STORAGE, - USE_S3_STORAGE, -} from '@/lib/uploads/config' -import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' -import { - createBlobConfig, - createGcsConfig, - createS3Config, -} from '@/lib/uploads/core/storage-service' -import type { StorageContext } from '@/lib/uploads/shared/types' -import { sanitizeFileKey } from '@/lib/uploads/utils/file-utils' - -export type MultipartStorageProvider = 's3' | 'blob' | 'gcs' | 'local' - -export interface CompletedUploadPart { - partNumber: number - etag?: string -} - -export interface MultipartPartUrl { - partNumber: number - url: string - headers: Record - expiresAt: string -} - -export function multipartStorageProvider(): MultipartStorageProvider { - if (USE_BLOB_STORAGE) return 'blob' - if (USE_S3_STORAGE) return 's3' - if (USE_GCS_STORAGE) return 'gcs' - return 'local' -} - -export async function initiateMultipartProviderUpload(params: { - key: string - fileName: string - contentType: string - fileSize: number - context: StorageContext - localUploadId: string -}): Promise<{ provider: MultipartStorageProvider; providerUploadId: string | null }> { - const { key, fileName, contentType, fileSize, context, localUploadId } = params - const provider = multipartStorageProvider() - const config = getStorageConfig(context) - - if (provider === 's3') { - const { initiateS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') - const result = await initiateS3MultipartUpload({ - fileName, - contentType, - fileSize, - customConfig: createS3Config(config), - customKey: key, - purpose: context, - }) - return { provider, providerUploadId: result.uploadId } - } - if (provider === 'blob') { - const { initiateMultipartUpload } = await import('@/lib/uploads/providers/blob/client') - const result = await initiateMultipartUpload({ - fileName, - contentType, - fileSize, - customConfig: createBlobConfig(config), - customKey: key, - }) - return { provider, providerUploadId: result.uploadId } - } - if (provider === 'gcs') { - const { initiateGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') - const result = await initiateGcsMultipartUpload({ - fileName, - contentType, - fileSize, - customConfig: createGcsConfig(config), - customKey: key, - purpose: context, - }) - return { provider, providerUploadId: result.uploadId } - } - - await mkdir(localPartsDirectory(localUploadId), { recursive: true }) - return { provider, providerUploadId: null } -} - -export async function getMultipartProviderPartUrls(params: { - provider: MultipartStorageProvider - providerUploadId: string | null - key: string - context: StorageContext - partNumbers: number[] - localUrl: (partNumber: number) => string -}): Promise { - const expiresAt = new Date(Date.now() + 60 * 60 * 1000).toISOString() - const { provider, providerUploadId, key, context, partNumbers } = params - if (provider === 'local') { - return partNumbers.map((partNumber) => ({ - partNumber, - url: params.localUrl(partNumber), - headers: { 'Content-Type': 'application/octet-stream' }, - expiresAt, - })) - } - if (!providerUploadId) throw new Error(`Missing ${provider} multipart upload id`) - const config = getStorageConfig(context) - - if (provider === 's3') { - const { getS3MultipartPartUrls } = await import('@/lib/uploads/providers/s3/client') - const urls = await getS3MultipartPartUrls( - key, - providerUploadId, - partNumbers, - createS3Config(config) - ) - return urls.map(({ partNumber, url }) => ({ - partNumber, - url, - headers: { 'Content-Type': 'application/octet-stream' }, - expiresAt, - })) - } - if (provider === 'blob') { - const { getMultipartPartUrls } = await import('@/lib/uploads/providers/blob/client') - const urls = await getMultipartPartUrls(key, partNumbers, createBlobConfig(config)) - return urls.map(({ partNumber, url }) => ({ - partNumber, - url, - headers: { 'Content-Type': 'application/octet-stream' }, - expiresAt, - })) - } - const { getGcsMultipartPartUrls } = await import('@/lib/uploads/providers/gcs/client') - const urls = await getGcsMultipartPartUrls( - key, - providerUploadId, - partNumbers, - createGcsConfig(config) - ) - return urls.map(({ partNumber, url }) => ({ - partNumber, - url, - headers: { 'Content-Type': 'application/octet-stream' }, - expiresAt, - })) -} - -export async function completeMultipartProviderUpload(params: { - provider: MultipartStorageProvider - providerUploadId: string | null - uploadId: string - key: string - contentType: string - context: StorageContext - parts: CompletedUploadPart[] -}): Promise { - const { provider, providerUploadId, uploadId, key, contentType, context, parts } = params - if (provider === 'local') { - await assembleLocalParts(uploadId, key, parts) - return - } - if (!providerUploadId) throw new Error(`Missing ${provider} multipart upload id`) - const config = getStorageConfig(context) - if (provider === 's3') { - const { completeS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') - await completeS3MultipartUpload( - key, - providerUploadId, - parts.map((part) => ({ - PartNumber: part.partNumber, - ETag: requiredEtag(provider, part), - })), - createS3Config(config) - ) - return - } - if (provider === 'blob') { - const { completeMultipartUpload, deriveBlobBlockId } = await import( - '@/lib/uploads/providers/blob/client' - ) - await completeMultipartUpload( - key, - parts.map((part) => ({ - partNumber: part.partNumber, - blockId: deriveBlobBlockId(part.partNumber), - })), - createBlobConfig(config), - contentType - ) - return - } - const { completeGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') - await completeGcsMultipartUpload( - key, - providerUploadId, - parts.map((part) => ({ - PartNumber: part.partNumber, - ETag: requiredEtag(provider, part), - })), - createGcsConfig(config) - ) -} - -export async function abortMultipartProviderUpload(params: { - provider: MultipartStorageProvider - providerUploadId: string | null - uploadId: string - key: string - context: StorageContext -}): Promise { - const { provider, providerUploadId, uploadId, key, context } = params - if (provider === 'local') { - await rm(localPartsDirectory(uploadId), { recursive: true, force: true }) - const destination = join(UPLOAD_DIR_SERVER, sanitizeFileKey(key)) - await rm(`${destination}.uploading-${uploadId}`, { force: true }) - return - } - if (!providerUploadId) throw new Error(`Missing ${provider} multipart upload id`) - const config = getStorageConfig(context) - if (provider === 's3') { - const { abortS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') - await abortS3MultipartUpload(key, providerUploadId, createS3Config(config)) - return - } - if (provider === 'blob') { - const { abortMultipartUpload } = await import('@/lib/uploads/providers/blob/client') - await abortMultipartUpload(key, createBlobConfig(config)) - return - } - const { abortGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') - await abortGcsMultipartUpload(key, providerUploadId, createGcsConfig(config)) -} - -export async function writeLocalMultipartPart(params: { - uploadId: string - partNumber: number - body: ReadableStream - expectedSize: number -}): Promise { - const { Readable, Transform } = await import('node:stream') - const directory = localPartsDirectory(params.uploadId) - await mkdir(directory, { recursive: true }) - const destination = localPartPath(params.uploadId, params.partNumber) - let bytes = 0 - const counter = new Transform({ - transform(chunk: Buffer, _encoding, callback) { - bytes += chunk.length - if (bytes > params.expectedSize) { - callback(new Error(`Part ${params.partNumber} exceeds ${params.expectedSize} bytes`)) - return - } - callback(null, chunk) - }, - }) - try { - await pipeline( - Readable.fromWeb(params.body as Parameters[0]), - counter, - createWriteStream(destination, { flags: 'w' }) - ) - if (bytes !== params.expectedSize) { - throw new Error( - `Part ${params.partNumber} has ${bytes} bytes; expected ${params.expectedSize}` - ) - } - } catch (error) { - await rm(destination, { force: true }).catch(() => {}) - throw new Error(getErrorMessage(error, `Failed to store part ${params.partNumber}`), { - cause: error, - }) - } -} - -function localPartsDirectory(uploadId: string): string { - return join(UPLOAD_DIR_SERVER, '.multipart', uploadId) -} - -function localPartPath(uploadId: string, partNumber: number): string { - return join(localPartsDirectory(uploadId), `${partNumber}.part`) -} - -async function assembleLocalParts( - uploadId: string, - key: string, - parts: CompletedUploadPart[] -): Promise { - const safeKey = sanitizeFileKey(key) - const destination = join(UPLOAD_DIR_SERVER, safeKey) - const temporary = `${destination}.uploading-${uploadId}` - await mkdir(dirname(destination), { recursive: true }) - await rm(temporary, { force: true }) - try { - for (const part of parts) { - await pipeline( - createReadStream(localPartPath(uploadId, part.partNumber)), - createWriteStream(temporary, { flags: 'a' }) - ) - } - const assembled = await stat(temporary) - if (assembled.size === 0) throw new Error('Assembled upload is empty') - await rename(temporary, destination) - await rm(localPartsDirectory(uploadId), { recursive: true, force: true }) - } catch (error) { - await rm(temporary, { force: true }).catch(() => {}) - throw error - } -} - -function requiredEtag(provider: 's3' | 'gcs', part: CompletedUploadPart): string { - if (!part.etag) throw new Error(`Missing etag for ${provider} part ${part.partNumber}`) - return part.etag -} diff --git a/apps/sim/lib/uploads/multipart-session/service.test.ts b/apps/sim/lib/uploads/multipart-session/service.test.ts deleted file mode 100644 index 3c030a9028b..00000000000 --- a/apps/sim/lib/uploads/multipart-session/service.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { - mockCheckStorageQuotaForBillingContext, - mockInitiateMultipartProviderUpload, - mockResolveStorageBillingContext, -} = vi.hoisted(() => ({ - mockCheckStorageQuotaForBillingContext: vi.fn(), - mockInitiateMultipartProviderUpload: vi.fn(), - mockResolveStorageBillingContext: vi.fn(), -})) - -vi.mock('@/lib/billing/storage', () => ({ - checkStorageQuotaForBillingContext: mockCheckStorageQuotaForBillingContext, - resolveStorageBillingContext: mockResolveStorageBillingContext, -})) - -vi.mock('@/lib/uploads/core/storage-service', () => ({ headObject: vi.fn() })) - -vi.mock('@/lib/uploads/contexts/workspace', () => ({ - generateWorkspaceFileKey: vi.fn( - (workspaceId: string, fileName: string) => `workspace/${workspaceId}/${fileName}` - ), -})) - -vi.mock('@/lib/uploads/multipart-session/provider', () => ({ - abortMultipartProviderUpload: vi.fn(), - completeMultipartProviderUpload: vi.fn(), - getMultipartProviderPartUrls: vi.fn(), - initiateMultipartProviderUpload: mockInitiateMultipartProviderUpload, -})) - -import { - createUploadSession, - getOwnedUploadSession, - verifyUploadSessionToken, -} from '@/lib/uploads/multipart-session/service' - -const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' - -describe('knowledge-document multipart sessions', () => { - beforeEach(() => { - vi.clearAllMocks() - mockResolveStorageBillingContext.mockResolvedValue({ workspaceId: WORKSPACE_ID }) - mockCheckStorageQuotaForBillingContext.mockResolvedValue({ allowed: true }) - mockInitiateMultipartProviderUpload.mockResolvedValue({ - provider: 's3', - providerUploadId: 'provider-upload-1', - }) - }) - - it('binds knowledge ownership and all storage state into the signed token', async () => { - const created = await createUploadSession({ - id: 'upload-1', - workspaceId: WORKSPACE_ID, - userId: 'user-1', - knowledgeBaseId: 'kb-1', - purpose: 'knowledge_document', - fileName: 'guide.pdf', - contentType: 'application/pdf', - fileSize: 1024, - }) - - const verified = verifyUploadSessionToken(created.uploadToken) - expect(verified).toMatchObject({ - id: 'upload-1', - workspaceId: WORKSPACE_ID, - userId: 'user-1', - knowledgeBaseId: 'kb-1', - purpose: 'knowledge_document', - storageContext: 'knowledge-base', - storageProvider: 's3', - providerUploadId: 'provider-upload-1', - fileName: 'guide.pdf', - contentType: 'application/pdf', - fileSize: 1024, - }) - expect(verified.storageKey).toMatch(/^kb\/.*-guide\.pdf$/) - }) - - it.each([ - { userId: 'other-user', knowledgeBaseId: 'kb-1', purpose: 'knowledge_document' as const }, - { userId: 'user-1', knowledgeBaseId: 'kb-2', purpose: 'knowledge_document' as const }, - { userId: 'user-1', knowledgeBaseId: 'kb-1', purpose: 'workspace_file' as const }, - ])('rejects a session whose signed scope does not match $purpose', async (scope) => { - const created = await createUploadSession({ - id: 'upload-1', - workspaceId: WORKSPACE_ID, - userId: 'user-1', - knowledgeBaseId: 'kb-1', - purpose: 'knowledge_document', - fileName: 'guide.pdf', - contentType: 'application/pdf', - fileSize: 1024, - }) - - expect(() => - getOwnedUploadSession({ - uploadId: 'upload-1', - workspaceId: WORKSPACE_ID, - uploadToken: created.uploadToken, - ...scope, - }) - ).toThrow('Upload session not found') - }) - - it('runs the storage quota gate before creating provider state', async () => { - mockCheckStorageQuotaForBillingContext.mockResolvedValue({ - allowed: false, - error: 'Storage limit exceeded', - }) - - await expect( - createUploadSession({ - workspaceId: WORKSPACE_ID, - userId: 'user-1', - knowledgeBaseId: 'kb-1', - purpose: 'knowledge_document', - fileName: 'guide.pdf', - contentType: 'application/pdf', - fileSize: 1024, - }) - ).rejects.toMatchObject({ code: 'payload_too_large' }) - expect(mockInitiateMultipartProviderUpload).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/uploads/multipart-session/service.ts b/apps/sim/lib/uploads/multipart-session/service.ts deleted file mode 100644 index 2157091cb73..00000000000 --- a/apps/sim/lib/uploads/multipart-session/service.ts +++ /dev/null @@ -1,454 +0,0 @@ -import { generateId } from '@sim/utils/id' -import { - checkStorageQuotaForBillingContext, - resolveStorageBillingContext, -} from '@/lib/billing/storage' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { generateKnowledgeBaseFileKey } from '@/lib/uploads/contexts/knowledge-base/knowledge-base-file-manager' -import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace' -import { headObject } from '@/lib/uploads/core/storage-service' -import { signUploadToken, verifyUploadToken } from '@/lib/uploads/core/upload-token' -import { - abortMultipartProviderUpload, - type CompletedUploadPart, - completeMultipartProviderUpload, - getMultipartProviderPartUrls, - initiateMultipartProviderUpload, - type MultipartPartUrl, - type MultipartStorageProvider, -} from '@/lib/uploads/multipart-session/provider' -import { - MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE, - MAX_WORKSPACE_FILE_SIZE, - type StorageContext, -} from '@/lib/uploads/shared/types' -import { sanitizeFileName } from '@/executor/constants' - -export const MULTIPART_SESSION_PART_SIZE = 8 * 1024 * 1024 -export const MULTIPART_SESSION_MAX_PART_URLS = 100 -export const MULTIPART_SESSION_TTL_MS = 24 * 60 * 60 * 1000 - -export type UploadSessionPurpose = 'workspace_file' | 'table_import' | 'knowledge_document' -export type UploadSessionStatus = 'uploading' | 'completed' | 'aborted' - -export interface UploadSessionRecord { - id: string - workspaceId: string - userId: string - knowledgeBaseId: string | null - purpose: UploadSessionPurpose - storageContext: StorageContext - storageKey: string - storageProvider: MultipartStorageProvider - providerUploadId: string | null - fileName: string - contentType: string - fileSize: number - partSize: number - partCount: number - status: UploadSessionStatus - metadata: Record - uploadToken: string - createdAt: Date - expiresAt: Date - completedFileId: string | null - error: string | null - completedAt: Date | null - updatedAt: Date -} - -export class UploadSessionError extends OrchestrationError { - constructor( - code: 'validation' | 'not_found' | 'forbidden' | 'conflict' | 'payload_too_large' | 'internal', - message: string - ) { - super(code, message) - this.name = 'UploadSessionError' - } -} - -interface CreateUploadSessionBaseParams { - id?: string - workspaceId: string - userId: string - fileName: string - contentType: string - fileSize: number - metadata?: Record -} - -type CreateUploadSessionParams = CreateUploadSessionBaseParams & - ( - | { purpose: 'workspace_file' | 'table_import'; knowledgeBaseId?: never } - | { purpose: 'knowledge_document'; knowledgeBaseId: string } - ) - -export async function createUploadSession( - params: CreateUploadSessionParams -): Promise { - validateFile(params) - const id = params.id ?? generateId() - const { storageContext, storageKey } = resolveUploadStorage(params, id) - const partCount = Math.ceil(params.fileSize / MULTIPART_SESSION_PART_SIZE) - - if (params.purpose === 'workspace_file' || params.purpose === 'knowledge_document') { - const billingContext = await resolveStorageBillingContext(params.workspaceId) - const quota = await checkStorageQuotaForBillingContext(billingContext, params.fileSize) - if (!quota.allowed) { - throw new UploadSessionError('payload_too_large', quota.error ?? 'Storage limit exceeded') - } - } - - const initiated = await initiateMultipartProviderUpload({ - key: storageKey, - fileName: params.fileName, - contentType: params.contentType, - fileSize: params.fileSize, - context: storageContext, - localUploadId: id, - }) - const createdAt = new Date() - const expiresAt = new Date(createdAt.getTime() + MULTIPART_SESSION_TTL_MS) - const metadata = params.metadata ?? {} - const uploadToken = signUploadToken( - { - uploadId: id, - key: storageKey, - userId: params.userId, - workspaceId: params.workspaceId, - context: storageContext, - ...(params.purpose === 'knowledge_document' - ? { knowledgeBaseId: params.knowledgeBaseId } - : {}), - fileName: params.fileName, - contentType: params.contentType, - fileSize: params.fileSize, - purpose: params.purpose, - provider: initiated.provider, - providerUploadId: initiated.providerUploadId, - partSize: MULTIPART_SESSION_PART_SIZE, - partCount, - metadata, - createdAt: createdAt.toISOString(), - expiresAt: expiresAt.toISOString(), - }, - MULTIPART_SESSION_TTL_MS / 1000 - ) - - return { - id, - workspaceId: params.workspaceId, - userId: params.userId, - knowledgeBaseId: params.purpose === 'knowledge_document' ? params.knowledgeBaseId : null, - purpose: params.purpose, - storageContext, - storageKey, - storageProvider: initiated.provider, - providerUploadId: initiated.providerUploadId, - fileName: params.fileName, - contentType: params.contentType, - fileSize: params.fileSize, - partSize: MULTIPART_SESSION_PART_SIZE, - partCount, - status: 'uploading', - metadata, - uploadToken, - createdAt, - expiresAt, - completedFileId: null, - error: null, - completedAt: null, - updatedAt: createdAt, - } -} - -export function getOwnedUploadSession(params: { - uploadId: string - workspaceId: string - userId?: string - purpose: UploadSessionPurpose - knowledgeBaseId?: string - uploadToken: string -}): UploadSessionRecord { - const session = verifyUploadSessionToken(params.uploadToken) - if (session.id !== params.uploadId || session.workspaceId !== params.workspaceId) { - throw new UploadSessionError('not_found', 'Upload session not found') - } - if (params.userId && session.userId !== params.userId) { - throw new UploadSessionError('not_found', 'Upload session not found') - } - if (session.purpose !== params.purpose) { - throw new UploadSessionError('not_found', 'Upload session not found') - } - if (params.knowledgeBaseId !== undefined && session.knowledgeBaseId !== params.knowledgeBaseId) { - throw new UploadSessionError('not_found', 'Upload session not found') - } - return session -} - -export function verifyUploadSessionToken(uploadToken: string): UploadSessionRecord { - const verified = verifyUploadToken(uploadToken) - if (!verified.valid) throw new UploadSessionError('forbidden', 'Invalid or expired upload token') - const payload = verified.payload - if ( - !payload.fileName || - !payload.contentType || - typeof payload.fileSize !== 'number' || - !Number.isSafeInteger(payload.fileSize) || - !payload.purpose || - !payload.provider || - typeof payload.partSize !== 'number' || - !Number.isSafeInteger(payload.partSize) || - typeof payload.partCount !== 'number' || - !Number.isSafeInteger(payload.partCount) || - !payload.createdAt || - !payload.expiresAt - ) { - throw new UploadSessionError('forbidden', 'Upload token is not a multipart session token') - } - if ( - payload.context !== 'workspace' && - payload.context !== 'table-import' && - payload.context !== 'knowledge-base' - ) { - throw new UploadSessionError('forbidden', 'Upload token has an invalid storage context') - } - const knowledgeBaseId = payload.knowledgeBaseId?.trim() || null - if ( - (payload.purpose === 'workspace_file' && payload.context !== 'workspace') || - (payload.purpose === 'table_import' && payload.context !== 'table-import') || - (payload.purpose === 'knowledge_document' && - (payload.context !== 'knowledge-base' || !knowledgeBaseId || !payload.key.startsWith('kb/'))) - ) { - throw new UploadSessionError('forbidden', 'Upload token purpose does not match its storage') - } - if (payload.purpose !== 'knowledge_document' && knowledgeBaseId) { - throw new UploadSessionError('forbidden', 'Upload token has unexpected knowledge-base state') - } - const createdAt = new Date(payload.createdAt) - const expiresAt = new Date(payload.expiresAt) - if (!Number.isFinite(createdAt.getTime()) || !Number.isFinite(expiresAt.getTime())) { - throw new UploadSessionError('forbidden', 'Upload token has invalid timestamps') - } - const now = new Date() - return { - id: payload.uploadId, - workspaceId: payload.workspaceId, - userId: payload.userId, - knowledgeBaseId, - purpose: payload.purpose, - storageContext: payload.context, - storageKey: payload.key, - storageProvider: payload.provider, - providerUploadId: payload.providerUploadId ?? null, - fileName: payload.fileName, - contentType: payload.contentType, - fileSize: payload.fileSize, - partSize: payload.partSize, - partCount: payload.partCount, - status: 'uploading', - metadata: payload.metadata ?? {}, - uploadToken, - createdAt, - expiresAt, - completedFileId: null, - error: null, - completedAt: null, - updatedAt: now, - } -} - -export async function createUploadPartUrls(params: { - session: UploadSessionRecord - partNumbers: number[] - localOrigin: string -}): Promise { - assertUploadable(params.session) - const unique = new Set(params.partNumbers) - if (unique.size !== params.partNumbers.length) { - throw new UploadSessionError('validation', 'partNumbers must not contain duplicates') - } - if ( - params.partNumbers.length === 0 || - params.partNumbers.length > MULTIPART_SESSION_MAX_PART_URLS - ) { - throw new UploadSessionError( - 'validation', - `partNumbers must contain between 1 and ${MULTIPART_SESSION_MAX_PART_URLS} entries` - ) - } - for (const partNumber of params.partNumbers) { - if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > params.session.partCount) { - throw new UploadSessionError( - 'validation', - `partNumber must be between 1 and ${params.session.partCount}` - ) - } - } - - return getMultipartProviderPartUrls({ - provider: params.session.storageProvider, - providerUploadId: params.session.providerUploadId, - key: params.session.storageKey, - context: params.session.storageContext, - partNumbers: params.partNumbers, - localUrl: (partNumber) => - `${params.localOrigin}/api/v2/uploads/${params.session.id}/parts/${partNumber}?token=${encodeURIComponent(params.session.uploadToken)}`, - }) -} - -export async function completeUploadSession(params: { - session: UploadSessionRecord - parts: CompletedUploadPart[] - finalize: (session: UploadSessionRecord) => Promise<{ value: T; completedFileId?: string }> -}): Promise<{ session: UploadSessionRecord; value: T; alreadyCompleted: boolean }> { - assertUploadable(params.session) - validateCompletedParts(params.session, params.parts) - - const existingObject = await headObject(params.session.storageKey, params.session.storageContext) - const alreadyCompleted = existingObject?.size === params.session.fileSize - if (existingObject && !alreadyCompleted) { - throw new UploadSessionError( - 'conflict', - `Upload object has ${existingObject.size} bytes; expected ${params.session.fileSize}` - ) - } - if (!alreadyCompleted) { - await completeMultipartProviderUpload({ - provider: params.session.storageProvider, - providerUploadId: params.session.providerUploadId, - uploadId: params.session.id, - key: params.session.storageKey, - contentType: params.session.contentType, - context: params.session.storageContext, - parts: params.parts, - }) - } - - const head = await headObject(params.session.storageKey, params.session.storageContext) - if (!head) throw new Error('Completed upload object not found') - if (head.size !== params.session.fileSize) { - throw new UploadSessionError( - 'validation', - `Uploaded object has ${head.size} bytes; expected ${params.session.fileSize}` - ) - } - - const finalized = await params.finalize(params.session) - const completedAt = new Date() - return { - session: { - ...params.session, - status: 'completed', - completedFileId: finalized.completedFileId ?? null, - completedAt, - updatedAt: completedAt, - }, - value: finalized.value, - alreadyCompleted, - } -} - -export async function abortUploadSession( - session: UploadSessionRecord -): Promise { - assertUploadable(session) - await abortMultipartProviderUpload({ - provider: session.storageProvider, - providerUploadId: session.providerUploadId, - uploadId: session.id, - key: session.storageKey, - context: session.storageContext, - }) - const completedAt = new Date() - return { ...session, status: 'aborted', completedAt, updatedAt: completedAt } -} - -export function expectedUploadPartSize(session: UploadSessionRecord, partNumber: number): number { - if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > session.partCount) { - throw new UploadSessionError('validation', 'Invalid upload part number') - } - if (partNumber < session.partCount) return session.partSize - return session.fileSize - session.partSize * (session.partCount - 1) -} - -function assertUploadable(session: UploadSessionRecord): void { - if (session.status !== 'uploading') { - throw new UploadSessionError('conflict', `Upload session is ${session.status}`) - } - if (session.expiresAt.getTime() <= Date.now()) { - throw new UploadSessionError('conflict', 'Upload session has expired') - } -} - -function validateCompletedParts(session: UploadSessionRecord, parts: CompletedUploadPart[]): void { - if (parts.length !== session.partCount) { - throw new UploadSessionError( - 'validation', - `Expected ${session.partCount} completed parts; received ${parts.length}` - ) - } - const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber) - for (let index = 0; index < sorted.length; index++) { - if (sorted[index].partNumber !== index + 1) { - throw new UploadSessionError( - 'validation', - 'Completed parts must contain every part exactly once' - ) - } - if ( - (session.storageProvider === 's3' || session.storageProvider === 'gcs') && - !sorted[index].etag - ) { - throw new UploadSessionError( - 'validation', - `etag is required for ${session.storageProvider} part ${sorted[index].partNumber}` - ) - } - } -} - -function validateFile(params: CreateUploadSessionParams): void { - if (!params.fileName.trim()) { - throw new UploadSessionError('validation', 'fileName must not be empty') - } - if (!params.contentType.trim()) { - throw new UploadSessionError('validation', 'contentType must not be empty') - } - if (!Number.isSafeInteger(params.fileSize) || params.fileSize < 1) { - throw new UploadSessionError('validation', 'fileSize must be a positive integer') - } - const maximum = - params.purpose === 'knowledge_document' - ? MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE - : MAX_WORKSPACE_FILE_SIZE - if (params.fileSize > maximum) { - throw new UploadSessionError('validation', `File size exceeds maximum of ${maximum} bytes`) - } - if (params.purpose === 'knowledge_document' && !params.knowledgeBaseId.trim()) { - throw new UploadSessionError('validation', 'knowledgeBaseId must not be empty') - } -} - -function resolveUploadStorage( - params: CreateUploadSessionParams, - id: string -): { storageContext: StorageContext; storageKey: string } { - switch (params.purpose) { - case 'workspace_file': - return { - storageContext: 'workspace', - storageKey: generateWorkspaceFileKey(params.workspaceId, params.fileName), - } - case 'table_import': - return { - storageContext: 'table-import', - storageKey: `table-import/${params.workspaceId}/${id}/${sanitizeFileName(params.fileName)}`, - } - case 'knowledge_document': - return { - storageContext: 'knowledge-base', - storageKey: generateKnowledgeBaseFileKey(params.fileName), - } - } -} diff --git a/apps/sim/lib/uploads/providers/blob/client.test.ts b/apps/sim/lib/uploads/providers/blob/client.test.ts index f31cf572108..5b604425474 100644 --- a/apps/sim/lib/uploads/providers/blob/client.test.ts +++ b/apps/sim/lib/uploads/providers/blob/client.test.ts @@ -10,6 +10,9 @@ const { mockDownload, mockDelete, mockDeleteIfExists, + mockBeginCopyFromURL, + mockPollUntilDone, + mockGetProperties, mockGetBlockBlobClient, mockGetContainerClient, mockFromConnectionString, @@ -21,6 +24,9 @@ const { mockDownload: vi.fn(), mockDelete: vi.fn(), mockDeleteIfExists: vi.fn(), + mockBeginCopyFromURL: vi.fn(), + mockPollUntilDone: vi.fn(), + mockGetProperties: vi.fn(), mockGetBlockBlobClient: vi.fn(), mockGetContainerClient: vi.fn(), mockFromConnectionString: vi.fn(), @@ -52,10 +58,14 @@ vi.mock('@/lib/uploads/config', () => ({ import { abortMultipartUpload, + deleteBlobObjectVersion, deleteFromBlob, downloadFromBlob, + getBlobPresignedUploadUrl, getPresignedUrl, + headBlobObject, parseConnectionString, + promoteBlobObject, uploadToBlob, } from '@/lib/uploads/providers/blob/client' import { sanitizeFilenameForMetadata } from '@/lib/uploads/utils/file-utils' @@ -71,6 +81,8 @@ describe('Azure Blob Storage Client', () => { download: mockDownload, delete: mockDelete, deleteIfExists: mockDeleteIfExists, + beginCopyFromURL: mockBeginCopyFromURL, + getProperties: mockGetProperties, url: 'https://test.blob.core.windows.net/container/test-file', }) @@ -85,6 +97,8 @@ describe('Azure Blob Storage Client', () => { mockGenerateBlobSASQueryParameters.mockReturnValue({ toString: () => 'sv=2021-06-08&se=2023-01-01T00%3A00%3A00Z&sr=b&sp=r&sig=test', }) + mockBeginCopyFromURL.mockResolvedValue({ pollUntilDone: mockPollUntilDone }) + mockPollUntilDone.mockResolvedValue({ copyStatus: 'success' }) }) describe('uploadToBlob', () => { @@ -136,6 +150,94 @@ describe('Azure Blob Storage Client', () => { }) }) + describe('staged upload primitives', () => { + const customConfig = { + containerName: 'testcontainer', + accountName: 'testaccount', + accountKey: 'testkey', + connectionString: + 'DefaultEndpointsProtocol=https;AccountName=testaccount;AccountKey=testkey;EndpointSuffix=core.windows.net', + } + + it('signs a PUT with the required blob and metadata headers', async () => { + mockBlobSASPermissionsParse.mockReturnValueOnce('w') + + const result = await getBlobPresignedUploadUrl({ + key: 'upload-sessions/upload-1/file.bin', + contentType: 'application/octet-stream', + metadata: { uploadId: 'upload-1', purpose: 'workspace_file' }, + customConfig, + expiresIn: 600, + }) + + expect(mockBlobSASPermissionsParse).toHaveBeenCalledWith('w') + expect(result).toEqual({ + url: expect.stringContaining('?sv=2021-06-08'), + headers: { + 'Content-Type': 'application/octet-stream', + 'x-ms-blob-type': 'BlockBlob', + 'x-ms-blob-content-type': 'application/octet-stream', + 'x-ms-meta-uploadId': 'upload-1', + 'x-ms-meta-purpose': 'workspace_file', + }, + }) + }) + + it('pins the source ETag and requires an absent promotion destination', async () => { + await promoteBlobObject({ + sourceKey: 'upload-sessions/upload-1/file.bin', + destinationKey: 'workspace/workspace-1/file.bin', + sourceEtag: '"etag-1"', + customConfig, + }) + + expect(mockBeginCopyFromURL).toHaveBeenCalledWith( + 'https://test.blob.core.windows.net/container/test-file', + { + conditions: { ifNoneMatch: '*' }, + sourceConditions: { ifMatch: '"etag-1"' }, + } + ) + expect(mockPollUntilDone).toHaveBeenCalledOnce() + }) + + it('returns only completed copied objects as usable upload identities', async () => { + mockGetProperties.mockResolvedValueOnce({ + contentLength: 3, + contentType: 'application/octet-stream', + metadata: { uploadid: 'upload-1' }, + etag: '"etag-1"', + copyStatus: 'success', + }) + + await expect(headBlobObject('workspace/workspace-1/file.bin', customConfig)).resolves.toEqual( + { + size: 3, + contentType: 'application/octet-stream', + uploadId: 'upload-1', + version: '"etag-1"', + } + ) + + mockGetProperties.mockResolvedValueOnce({ copyStatus: 'pending' }) + await expect(headBlobObject('workspace/workspace-1/file.bin', customConfig)).rejects.toThrow( + 'Blob copy for workspace/workspace-1/file.bin is pending' + ) + }) + + it('deletes staging only when its ETag still matches', async () => { + mockDeleteIfExists.mockResolvedValueOnce({}) + + await deleteBlobObjectVersion({ + key: 'upload-sessions/upload-1/file.bin', + etag: '"etag-1"', + customConfig, + }) + + expect(mockDeleteIfExists).toHaveBeenCalledWith({ conditions: { ifMatch: '"etag-1"' } }) + }) + }) + describe('downloadFromBlob', () => { it('should download a file from Azure Blob Storage', async () => { const testKey = 'test-file-key' diff --git a/apps/sim/lib/uploads/providers/blob/client.ts b/apps/sim/lib/uploads/providers/blob/client.ts index 7f2faf821e3..993eeda8c74 100644 --- a/apps/sim/lib/uploads/providers/blob/client.ts +++ b/apps/sim/lib/uploads/providers/blob/client.ts @@ -257,6 +257,52 @@ export async function getPresignedUrlWithConfig( return `${blockBlobClient.url}?${sasToken}` } +/** Generates a SAS-backed single-object PUT for a caller-selected staging key. */ +export async function getBlobPresignedUploadUrl(params: { + key: string + contentType: string + metadata: Record + customConfig: BlobConfig + expiresIn: number +}): Promise<{ url: string; headers: Record }> { + const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } = + await import('@azure/storage-blob') + const client = await getBlockBlobClientFor(params.key, params.customConfig) + const credentials = params.customConfig.connectionString + ? parseConnectionString(params.customConfig.connectionString) + : { + accountName: params.customConfig.accountName, + accountKey: params.customConfig.accountKey, + } + if (!credentials.accountName || !credentials.accountKey) { + throw new Error('Azure Blob SAS generation requires accountName and accountKey') + } + const startsOn = new Date() + const expiresOn = new Date(startsOn.getTime() + params.expiresIn * 1000) + const sasToken = generateBlobSASQueryParameters( + { + containerName: params.customConfig.containerName, + blobName: params.key, + permissions: BlobSASPermissions.parse('w'), + startsOn, + expiresOn, + }, + new StorageSharedKeyCredential(credentials.accountName, credentials.accountKey) + ).toString() + const metadata = sanitizeStorageMetadata(params.metadata, 8000) + return { + url: `${client.url}?${sasToken}`, + headers: { + 'Content-Type': params.contentType, + 'x-ms-blob-type': 'BlockBlob', + 'x-ms-blob-content-type': params.contentType, + ...Object.fromEntries( + Object.entries(metadata).map(([key, value]) => [`x-ms-meta-${key}`, value]) + ), + }, + } +} + /** * Download a file from Azure Blob Storage * @param key Blob name @@ -392,7 +438,12 @@ export async function downloadFromBlobStream( export async function headBlobObject( key: string, customConfig?: BlobConfig -): Promise<{ size: number; contentType?: string } | null> { +): Promise<{ + size: number + contentType?: string + uploadId?: string + version: string +} | null> { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType let containerName: string @@ -423,9 +474,14 @@ export async function headBlobObject( try { const properties = await blockBlobClient.getProperties() + if (properties.copyStatus && properties.copyStatus !== 'success') { + throw new Error(`Blob copy for ${key} is ${properties.copyStatus}`) + } return { size: properties.contentLength ?? 0, contentType: properties.contentType, + uploadId: readUploadId(properties.metadata), + version: properties.etag ?? '', } } catch (err) { const status = (err as { statusCode?: number }).statusCode @@ -437,6 +493,40 @@ export async function headBlobObject( } } +/** + * Copies one immutable staging version into a destination that must not already exist. + * The asynchronous API supports objects above the synchronous copy operation's 256 MiB limit. + */ +export async function promoteBlobObject(params: { + sourceKey: string + destinationKey: string + sourceEtag: string + customConfig: BlobConfig +}): Promise { + if (!params.sourceEtag) throw new Error('Blob staging object is missing its ETag') + const source = await getBlockBlobClientFor(params.sourceKey, params.customConfig) + const destination = await getBlockBlobClientFor(params.destinationKey, params.customConfig) + const copy = await destination.beginCopyFromURL(source.url, { + conditions: { ifNoneMatch: '*' }, + sourceConditions: { ifMatch: params.sourceEtag }, + }) + const result = await copy.pollUntilDone() + if (result.copyStatus !== 'success') { + throw new Error(`Blob promotion finished with status ${result.copyStatus ?? 'unknown'}`) + } +} + +/** Deletes a staging blob only if it is still the version completion inspected. */ +export async function deleteBlobObjectVersion(params: { + key: string + etag: string + customConfig: BlobConfig +}): Promise { + if (!params.etag) throw new Error('Blob staging object is missing its ETag') + const client = await getBlockBlobClientFor(params.key, params.customConfig) + await client.deleteIfExists({ conditions: { ifMatch: params.etag } }) +} + /** * Delete a file from Azure Blob Storage * @param key Blob name @@ -498,49 +588,13 @@ export function deriveBlobBlockId(partNumber: number): string { export async function initiateMultipartUpload( options: AzureMultipartUploadInit ): Promise<{ uploadId: string; key: string }> { - const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') - const { fileName, contentType, customConfig, customKey } = options - - let blobServiceClient: BlobServiceClientType - let containerName: string - - if (customConfig) { - if (customConfig.connectionString) { - blobServiceClient = BlobServiceClient.fromConnectionString(customConfig.connectionString) - } else if (customConfig.accountName && customConfig.accountKey) { - const credential = new StorageSharedKeyCredential( - customConfig.accountName, - customConfig.accountKey - ) - blobServiceClient = new BlobServiceClient( - `https://${customConfig.accountName}.blob.core.windows.net`, - credential - ) - } else { - throw new Error('Invalid custom blob configuration') - } - containerName = customConfig.containerName - } else { - blobServiceClient = await getBlobServiceClient() - containerName = BLOB_CONFIG.containerName - } + const { fileName, customKey } = options const safeFileName = sanitizeFileName(fileName) const uniqueKey = customKey || `kb/${generateId()}-${safeFileName}` const uploadId = generateId() - const containerClient = blobServiceClient.getContainerClient(containerName) - const blockBlobClient = containerClient.getBlockBlobClient(uniqueKey) - - await blockBlobClient.setMetadata({ - uploadId, - fileName: encodeURIComponent(fileName), - contentType, - uploadStarted: new Date().toISOString(), - multipartUpload: 'true', - }) - return { uploadId, key: uniqueKey, @@ -692,7 +746,8 @@ export async function completeMultipartUpload( key: string, parts: AzureMultipartPart[], customConfig?: BlobConfig, - contentType?: string + contentType?: string, + metadata?: Record ): Promise<{ location: string; path: string; key: string }> { const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob') let blobServiceClient: BlobServiceClientType @@ -731,6 +786,7 @@ export async function completeMultipartUpload( metadata: { multipartUpload: 'completed', uploadCompletedAt: new Date().toISOString(), + ...sanitizeStorageMetadata(metadata ?? {}, 8000), }, }) @@ -754,3 +810,8 @@ export async function completeMultipartUpload( export function abortMultipartUpload(_key: string, _customConfig?: BlobConfig): Promise { return Promise.resolve() } + +function readUploadId(metadata?: Record): string | undefined { + if (!metadata) return undefined + return Object.entries(metadata).find(([key]) => key.toLowerCase() === 'uploadid')?.[1] +} diff --git a/apps/sim/lib/uploads/providers/blob/types.ts b/apps/sim/lib/uploads/providers/blob/types.ts index a24a87e384e..5d0e36e307a 100644 --- a/apps/sim/lib/uploads/providers/blob/types.ts +++ b/apps/sim/lib/uploads/providers/blob/types.ts @@ -15,6 +15,8 @@ export interface AzureMultipartUploadInit { * Caller is responsible for uniqueness and prefix conventions. */ customKey?: string + /** Additional object metadata preserved when the block list is committed. */ + metadata?: Record } export interface AzurePartUploadUrl { diff --git a/apps/sim/lib/uploads/providers/gcs/client.test.ts b/apps/sim/lib/uploads/providers/gcs/client.test.ts index dc0b01d2093..280066feec3 100644 --- a/apps/sim/lib/uploads/providers/gcs/client.test.ts +++ b/apps/sim/lib/uploads/providers/gcs/client.test.ts @@ -21,6 +21,7 @@ const { getMetadata: vi.fn(), delete: vi.fn(), getSignedUrl: vi.fn(), + copy: vi.fn(), } const mockBucket = { file: vi.fn(() => mockFile) } const mockGetAccessToken = vi.fn() @@ -71,6 +72,7 @@ import { abortGcsMultipartUpload, completeGcsMultipartUpload, deleteFromGcs, + deleteGcsObjectVersion, downloadFromGcs, getGcsClient, getGcsMultipartPartUrls, @@ -78,6 +80,7 @@ import { getPresignedUrlWithConfig, headGcsObject, initiateGcsMultipartUpload, + promoteGcsObject, resetGcsClientForTesting, uploadGcsPart, uploadToGcs, @@ -300,11 +303,23 @@ describe('GCS Client', () => { describe('headGcsObject', () => { it('should return size and content type when the object exists', async () => { - mockFile.getMetadata.mockResolvedValueOnce([{ size: '2048', contentType: 'text/csv' }]) + mockFile.getMetadata.mockResolvedValueOnce([ + { + size: '2048', + contentType: 'text/csv', + generation: '42', + metadata: { uploadid: 'upload-1' }, + }, + ]) const result = await headGcsObject('data.csv') - expect(result).toEqual({ size: 2048, contentType: 'text/csv' }) + expect(result).toEqual({ + size: 2048, + contentType: 'text/csv', + uploadId: 'upload-1', + version: '42', + }) }) it('should return null when the object is missing', async () => { @@ -326,6 +341,42 @@ describe('GCS Client', () => { }) }) + describe('staged upload promotion', () => { + it('pins the source generation and requires an absent destination', async () => { + mockFile.copy.mockResolvedValueOnce(undefined) + + await promoteGcsObject({ + sourceKey: 'upload-sessions/upload-1/file.bin', + destinationKey: 'workspace/workspace-1/file.bin', + sourceGeneration: '42', + customConfig: { bucket: 'test-bucket' }, + }) + + expect(mockBucket.file).toHaveBeenCalledWith('upload-sessions/upload-1/file.bin', { + generation: '42', + }) + expect(mockBucket.file).toHaveBeenCalledWith('workspace/workspace-1/file.bin') + expect(mockFile.copy).toHaveBeenCalledWith(mockFile, { + preconditionOpts: { ifGenerationMatch: 0 }, + }) + }) + + it('deletes staging only at the inspected generation', async () => { + mockFile.delete.mockResolvedValueOnce(undefined) + + await deleteGcsObjectVersion({ + key: 'upload-sessions/upload-1/file.bin', + generation: '42', + customConfig: { bucket: 'test-bucket' }, + }) + + expect(mockBucket.file).toHaveBeenCalledWith('upload-sessions/upload-1/file.bin', { + generation: '42', + }) + expect(mockFile.delete).toHaveBeenCalledWith({ ifGenerationMatch: '42' }) + }) + }) + describe('deleteFromGcs', () => { it('should delete a file, ignoring missing objects', async () => { mockFile.delete.mockResolvedValueOnce(undefined) @@ -483,10 +534,10 @@ describe('GCS Client', () => { expect(init.method).toBe('DELETE') }) - it('should swallow abort errors', async () => { + it('should surface abort errors', async () => { mockFetch.mockResolvedValueOnce(new Response('boom', { status: 500, statusText: 'ISE' })) - await expect(abortGcsMultipartUpload('key.csv', 'upload-123')).resolves.toBeUndefined() + await expect(abortGcsMultipartUpload('key.csv', 'upload-123')).rejects.toThrow('500 ISE') }) it('should fail multipart calls when no access token is available', async () => { diff --git a/apps/sim/lib/uploads/providers/gcs/client.ts b/apps/sim/lib/uploads/providers/gcs/client.ts index b4d5194a0d9..29c45559ef7 100644 --- a/apps/sim/lib/uploads/providers/gcs/client.ts +++ b/apps/sim/lib/uploads/providers/gcs/client.ts @@ -328,7 +328,12 @@ export async function downloadFromGcsStream( export async function headGcsObject( key: string, customConfig?: GcsConfig -): Promise<{ size: number; contentType?: string } | null> { +): Promise<{ + size: number + contentType?: string + uploadId?: string + version: string +} | null> { const config = customConfig || { bucket: GCS_CONFIG.bucket } const storage = await getGcsClient() @@ -337,6 +342,8 @@ export async function headGcsObject( return { size: Number(fileMetadata.size) || 0, contentType: fileMetadata.contentType, + uploadId: readUploadId(fileMetadata.metadata as Record | undefined), + version: String(fileMetadata.generation ?? ''), } } catch (error) { const code = (error as { code?: number } | null)?.code @@ -347,6 +354,35 @@ export async function headGcsObject( } } +/** Copies one immutable staging generation into a destination that must not already exist. */ +export async function promoteGcsObject(params: { + sourceKey: string + destinationKey: string + sourceGeneration: string + customConfig: GcsConfig +}): Promise { + if (!params.sourceGeneration) throw new Error('GCS staging object is missing its generation') + const storage = await getGcsClient() + const bucket = storage.bucket(params.customConfig.bucket) + const source = bucket.file(params.sourceKey, { generation: params.sourceGeneration }) + const destination = bucket.file(params.destinationKey) + await source.copy(destination, { preconditionOpts: { ifGenerationMatch: 0 } }) +} + +/** Deletes a staging object only if it is still the generation completion inspected. */ +export async function deleteGcsObjectVersion(params: { + key: string + generation: string + customConfig: GcsConfig +}): Promise { + if (!params.generation) throw new Error('GCS staging object is missing its generation') + const storage = await getGcsClient() + await storage + .bucket(params.customConfig.bucket) + .file(params.key, { generation: params.generation }) + .delete({ ifGenerationMatch: params.generation }) +} + /** * Get the custom metadata stored on a GCS object. */ @@ -439,7 +475,7 @@ async function gcsXmlApiRequest( export async function initiateGcsMultipartUpload( options: GcsMultipartUploadInit ): Promise<{ uploadId: string; key: string }> { - const { fileName, contentType, customConfig, customKey, purpose } = options + const { fileName, contentType, customConfig, customKey, purpose, metadata } = options const config = customConfig || { bucket: GCS_CONFIG.bucket } @@ -452,6 +488,12 @@ export async function initiateGcsMultipartUpload( 'x-goog-meta-originalname': encodeURIComponent(sanitizeFilenameForMetadata(fileName)), 'x-goog-meta-uploadedat': new Date().toISOString(), 'x-goog-meta-purpose': purpose || 'knowledge-base', + ...Object.fromEntries( + Object.entries(sanitizeStorageMetadata(metadata ?? {}, 8000)).map(([key, value]) => [ + `x-goog-meta-${key.toLowerCase()}`, + value, + ]) + ), }, }) @@ -581,9 +623,10 @@ export async function abortGcsMultipartUpload( customConfig?: GcsConfig ): Promise { const config = customConfig || { bucket: GCS_CONFIG.bucket } - try { - await gcsXmlApiRequest('DELETE', config.bucket, key, `uploadId=${encodeURIComponent(uploadId)}`) - } catch (error) { - logger.warn('Error cleaning up GCS multipart upload:', error) - } + await gcsXmlApiRequest('DELETE', config.bucket, key, `uploadId=${encodeURIComponent(uploadId)}`) +} + +function readUploadId(metadata?: Record): string | undefined { + if (!metadata) return undefined + return Object.entries(metadata).find(([key]) => key.toLowerCase() === 'uploadid')?.[1] } diff --git a/apps/sim/lib/uploads/providers/gcs/types.ts b/apps/sim/lib/uploads/providers/gcs/types.ts index 4a54bf3c1d3..3705296edb7 100644 --- a/apps/sim/lib/uploads/providers/gcs/types.ts +++ b/apps/sim/lib/uploads/providers/gcs/types.ts @@ -17,6 +17,8 @@ export interface GcsMultipartUploadInit { * for backwards compatibility. */ purpose?: string + /** Additional object metadata fixed when the multipart upload is initiated. */ + metadata?: Record } export interface GcsPartUploadUrl { diff --git a/apps/sim/lib/uploads/providers/s3/client.test.ts b/apps/sim/lib/uploads/providers/s3/client.test.ts index 75eea9a3dde..0979c5c874a 100644 --- a/apps/sim/lib/uploads/providers/s3/client.test.ts +++ b/apps/sim/lib/uploads/providers/s3/client.test.ts @@ -11,7 +11,9 @@ const { mockS3ClientConstructor, mockPutObjectCommand, mockGetObjectCommand, + mockHeadObjectCommand, mockDeleteObjectCommand, + mockCopyObjectCommand, mockCompleteMultipartUploadCommand, mockGetSignedUrl, mockEnv, @@ -51,7 +53,9 @@ const { ), mockPutObjectCommand: vi.fn().mockImplementation(class {}), mockGetObjectCommand: vi.fn().mockImplementation(class {}), + mockHeadObjectCommand: vi.fn().mockImplementation(class {}), mockDeleteObjectCommand: vi.fn().mockImplementation(class {}), + mockCopyObjectCommand: vi.fn().mockImplementation(class {}), mockCompleteMultipartUploadCommand: vi.fn().mockImplementation(class {}), mockGetSignedUrl: vi.fn(), mockEnv, @@ -62,7 +66,9 @@ vi.mock('@aws-sdk/client-s3', () => ({ S3Client: mockS3ClientConstructor, PutObjectCommand: mockPutObjectCommand, GetObjectCommand: mockGetObjectCommand, + HeadObjectCommand: mockHeadObjectCommand, DeleteObjectCommand: mockDeleteObjectCommand, + CopyObjectCommand: mockCopyObjectCommand, CompleteMultipartUploadCommand: mockCompleteMultipartUploadCommand, })) @@ -97,9 +103,13 @@ vi.mock('@/lib/uploads/config', () => ({ import { completeS3MultipartUpload, deleteFromS3, + deleteS3ObjectVersion, downloadFromS3, getPresignedUrl, getS3Client, + getS3PresignedUploadUrl, + headS3Object, + promoteS3Object, resetS3ClientForTesting, uploadToS3, } from '@/lib/uploads/providers/s3/client' @@ -236,6 +246,89 @@ describe('S3 Client', () => { }) }) + describe('staged upload primitives', () => { + it('signs metadata without returning duplicate x-amz-meta headers', async () => { + mockGetSignedUrl.mockResolvedValueOnce('https://example.com/signed-put') + + const result = await getS3PresignedUploadUrl({ + key: 'upload-sessions/upload-1/file.bin', + contentType: 'application/octet-stream', + fileSize: 3, + metadata: { uploadId: 'upload-1', purpose: 'workspace_file' }, + customConfig: mockS3Config, + expiresIn: 600, + }) + + expect(mockPutObjectCommand).toHaveBeenCalledWith({ + Bucket: 'test-bucket', + Key: 'upload-sessions/upload-1/file.bin', + ContentType: 'application/octet-stream', + ContentLength: 3, + Metadata: { uploadId: 'upload-1', purpose: 'workspace_file' }, + }) + expect(result).toEqual({ + url: 'https://example.com/signed-put', + headers: { + 'Content-Type': 'application/octet-stream', + }, + }) + }) + + it('reads the upload identity and immutable ETag', async () => { + mockSend.mockResolvedValueOnce({ + ContentLength: 3, + ContentType: 'application/octet-stream', + Metadata: { uploadid: 'upload-1' }, + ETag: '"etag-1"', + }) + + await expect( + headS3Object('upload-sessions/upload-1/file.bin', mockS3Config) + ).resolves.toEqual({ + size: 3, + contentType: 'application/octet-stream', + uploadId: 'upload-1', + version: '"etag-1"', + }) + }) + + it('pins the source ETag and requires an absent promotion destination', async () => { + mockSend.mockResolvedValueOnce({}) + + await promoteS3Object({ + sourceKey: 'upload-sessions/upload-1/file.bin', + destinationKey: 'workspace/workspace-1/file.bin', + sourceEtag: '"etag-1"', + customConfig: mockS3Config, + }) + + expect(mockCopyObjectCommand).toHaveBeenCalledWith({ + Bucket: 'test-bucket', + Key: 'workspace/workspace-1/file.bin', + CopySource: 'test-bucket/upload-sessions/upload-1/file.bin', + CopySourceIfMatch: '"etag-1"', + IfNoneMatch: '*', + MetadataDirective: 'COPY', + }) + }) + + it('deletes staging only when its ETag still matches', async () => { + mockSend.mockResolvedValueOnce({}) + + await deleteS3ObjectVersion({ + key: 'upload-sessions/upload-1/file.bin', + etag: '"etag-1"', + customConfig: mockS3Config, + }) + + expect(mockDeleteObjectCommand).toHaveBeenCalledWith({ + Bucket: 'test-bucket', + Key: 'upload-sessions/upload-1/file.bin', + IfMatch: '"etag-1"', + }) + }) + }) + describe('downloadFromS3', () => { it('should download a file from S3', async () => { const mockStream = { diff --git a/apps/sim/lib/uploads/providers/s3/client.ts b/apps/sim/lib/uploads/providers/s3/client.ts index fafe4fc8897..ecbc8879562 100644 --- a/apps/sim/lib/uploads/providers/s3/client.ts +++ b/apps/sim/lib/uploads/providers/s3/client.ts @@ -2,6 +2,7 @@ import type { Readable } from 'node:stream' import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, + CopyObjectCommand, CreateMultipartUploadCommand, DeleteObjectCommand, DeleteObjectsCommand, @@ -167,6 +168,37 @@ export async function getPresignedUrlWithConfig( return getSignedUrl(getS3Client(), command, { expiresIn }) } +/** + * Generates a signed single-object PUT for a caller-selected staging key. + * The AWS presigner hoists `x-amz-meta-*` values into the signed query string, + * so only ordinary transfer headers are returned. Repeating that metadata as + * request headers makes S3 reject the otherwise-valid signature. + */ +export async function getS3PresignedUploadUrl(params: { + key: string + contentType: string + fileSize: number + metadata: Record + customConfig: S3Config + expiresIn: number +}): Promise<{ url: string; headers: Record }> { + const metadata = sanitizeStorageMetadata(params.metadata, 2000) + const command = new PutObjectCommand({ + Bucket: params.customConfig.bucket, + Key: params.key, + ContentType: params.contentType, + ContentLength: params.fileSize, + Metadata: metadata, + }) + const url = await getSignedUrl(getS3Client(), command, { expiresIn: params.expiresIn }) + return { + url, + headers: { + 'Content-Type': params.contentType, + }, + } +} + /** * Download a file from S3 * @param key S3 object key @@ -243,7 +275,12 @@ export async function downloadFromS3Stream( export async function headS3Object( key: string, customConfig?: S3Config -): Promise<{ size: number; contentType?: string } | null> { +): Promise<{ + size: number + contentType?: string + uploadId?: string + version: string +} | null> { const config = customConfig || { bucket: S3_CONFIG.bucket, region: S3_CONFIG.region } try { @@ -253,6 +290,8 @@ export async function headS3Object( return { size: response.ContentLength ?? 0, contentType: response.ContentType, + uploadId: readUploadId(response.Metadata), + version: response.ETag ?? '', } } catch (error) { const code = (error as { name?: string; $metadata?: { httpStatusCode?: number } } | null)?.name @@ -265,6 +304,48 @@ export async function headS3Object( } } +/** + * Copies one immutable staging version into a destination that must not already exist. + */ +export async function promoteS3Object(params: { + sourceKey: string + destinationKey: string + sourceEtag: string + customConfig: S3Config +}): Promise { + if (!params.sourceEtag) throw new Error('S3 staging object is missing its ETag') + const encodedSource = `${params.customConfig.bucket}/${params.sourceKey + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/')}` + await getS3Client().send( + new CopyObjectCommand({ + Bucket: params.customConfig.bucket, + Key: params.destinationKey, + CopySource: encodedSource, + CopySourceIfMatch: params.sourceEtag, + IfNoneMatch: '*', + MetadataDirective: 'COPY', + }) + ) +} + +/** Deletes a staging object only if it is still the version completion inspected. */ +export async function deleteS3ObjectVersion(params: { + key: string + etag: string + customConfig: S3Config +}): Promise { + if (!params.etag) throw new Error('S3 staging object is missing its ETag') + await getS3Client().send( + new DeleteObjectCommand({ + Bucket: params.customConfig.bucket, + Key: params.key, + IfMatch: params.etag, + }) + ) +} + /** * Delete a file from S3 * @param key S3 object key @@ -341,7 +422,7 @@ export async function deleteManyFromS3( export async function initiateS3MultipartUpload( options: S3MultipartUploadInit ): Promise<{ uploadId: string; key: string }> { - const { fileName, contentType, customConfig, customKey, purpose } = options + const { fileName, contentType, customConfig, customKey, purpose, metadata } = options const config = customConfig || { bucket: S3_KB_CONFIG.bucket, region: S3_KB_CONFIG.region } const s3Client = getS3Client() @@ -357,6 +438,7 @@ export async function initiateS3MultipartUpload( originalName: sanitizeFilenameForMetadata(fileName), uploadedAt: new Date().toISOString(), purpose: purpose || 'knowledge-base', + ...sanitizeStorageMetadata(metadata ?? {}, 2000), }, }) @@ -372,6 +454,11 @@ export async function initiateS3MultipartUpload( } } +function readUploadId(metadata?: Record): string | undefined { + if (!metadata) return undefined + return Object.entries(metadata).find(([key]) => key.toLowerCase() === 'uploadid')?.[1] +} + /** * Upload a single multipart part from the server (Body in hand), returning its * `{ PartNumber, ETag }`. The presigned variant ({@link getS3MultipartPartUrls}) diff --git a/apps/sim/lib/uploads/providers/s3/types.ts b/apps/sim/lib/uploads/providers/s3/types.ts index 266a86a862c..0f960e3e4a0 100644 --- a/apps/sim/lib/uploads/providers/s3/types.ts +++ b/apps/sim/lib/uploads/providers/s3/types.ts @@ -18,6 +18,8 @@ export interface S3MultipartUploadInit { * for backwards compatibility. */ purpose?: string + /** Additional object metadata fixed when the multipart upload is initiated. */ + metadata?: Record } export interface S3PartUploadUrl { diff --git a/apps/sim/lib/uploads/server/metadata.ts b/apps/sim/lib/uploads/server/metadata.ts index 94b8317cdc9..48c2e50939b 100644 --- a/apps/sim/lib/uploads/server/metadata.ts +++ b/apps/sim/lib/uploads/server/metadata.ts @@ -114,45 +114,6 @@ export async function insertFileMetadata( } } -/** - * Bulk-insert file metadata rows in a single statement. - * - * Intended for batch upload flows that create many fresh keys at once (e.g. the - * presigned batch route), replacing a fan-out of individual `insertFileMetadata` - * calls. Uses `ON CONFLICT DO NOTHING` on the active-key unique index, so it is - * safe against a concurrent single insert and idempotent for already-present - * active keys. Unlike {@link insertFileMetadata} it does NOT restore - * soft-deleted rows — callers use this only for newly generated keys. - */ -export async function insertFileMetadataMany( - rows: Array & { id?: string }> -): Promise { - if (rows.length === 0) { - return - } - - await db - .insert(workspaceFiles) - .values( - rows.map((row) => ({ - id: row.id || generateId(), - key: row.key, - userId: row.userId, - workspaceId: row.workspaceId || null, - folderId: row.folderId ?? null, - context: row.context, - originalName: row.originalName, - displayName: row.originalName, - contentType: row.contentType, - size: toLegacyWorkspaceFileSize(row.size), - sizeBytes: row.size, - deletedAt: null, - uploadedAt: new Date(), - })) - ) - .onConflictDoNothing() -} - /** * Get file metadata by key with optional context filter */ @@ -255,26 +216,11 @@ export interface KnowledgeBaseFileOwnership { * Record the ownership binding for a single knowledge-base upload. KB file * authorization (`verifyKBFileAccess`) resolves the owning workspace from this * binding, so every KB object must have exactly one. Single source of truth for - * the binding shape across the presigned, batch-presigned, and multipart upload - * paths — keep all callers routed through here so they cannot drift. + * the binding shape for knowledge upload sessions — keep all callers routed + * through here so they cannot drift. */ export async function recordKnowledgeBaseFileOwnership( ownership: KnowledgeBaseFileOwnership ): Promise { await insertFileMetadata({ ...ownership, context: 'knowledge-base' }) } - -/** - * Bulk variant of {@link recordKnowledgeBaseFileOwnership} for batch upload flows. - * Idempotent against the active-key unique index (ON CONFLICT DO NOTHING). - */ -export async function recordKnowledgeBaseFileOwnershipMany( - ownerships: KnowledgeBaseFileOwnership[] -): Promise { - if (ownerships.length === 0) { - return - } - await insertFileMetadataMany( - ownerships.map((ownership) => ({ ...ownership, context: 'knowledge-base' })) - ) -} diff --git a/apps/sim/lib/uploads/shared/types.ts b/apps/sim/lib/uploads/shared/types.ts index d7de7961ee5..b26977f117e 100644 --- a/apps/sim/lib/uploads/shared/types.ts +++ b/apps/sim/lib/uploads/shared/types.ts @@ -1,6 +1,6 @@ /** * Defense-in-depth ceiling on the size of any single workspace file upload. - * Enforced both server-side (presigned route) and client-side (Files tab) so + * Enforced both server-side (upload-session creation) and client-side (Files tab) so * users get fast feedback before bytes are streamed. */ export const MAX_WORKSPACE_FILE_SIZE = 5 * 1024 * 1024 * 1024 diff --git a/apps/sim/lib/uploads/upload-session/README.md b/apps/sim/lib/uploads/upload-session/README.md new file mode 100644 index 00000000000..c29f9a312cc --- /dev/null +++ b/apps/sim/lib/uploads/upload-session/README.md @@ -0,0 +1,27 @@ +# Upload sessions + +Upload sessions use a signed, stateless control-plane token and an immutable staging object. Files +up to and including 50 MiB use one signed `PUT`; larger files use multipart upload. Completion +verifies the staged object's upload ID, byte size, and content type before promoting it to a +create-only final key. + +The `upload-sessions/` prefix is temporary. Production S3 and GCS buckets must expire objects under +that prefix after two days and abort incomplete multipart uploads after two days. Azure containers +must expire committed blobs under that prefix after two days; Azure automatically garbage-collects +uncommitted blocks after seven days. These policies exceed the 24-hour token lifetime, preserve a +retry window, and bound abandoned provider state. Local storage applies the equivalent 25-hour +policy with the bounded cleanup sweep in `cleanup.ts`. The local sweep retains process-local +directory cursors between bounded runs, so a large set of fresh entries cannot indefinitely hide +expired entries later in either directory. + +Local cleanup currently runs opportunistically when that same Sim process creates an upload +session. The repository has no scheduler that safely reaches every process-local filesystem in a +multi-replica self-hosted deployment: an HTTP cron request can land on only one replica, while the +Trigger workers do not own the web replica's disk. Operators using non-shared local disks must +therefore ensure uploads continue to trigger the sweep on each replica or invoke the exported +bounded sweep from their own per-replica maintenance hook. Cloud deployments should use the +provider lifecycle rules above instead. + +Final objects are not covered by the staging lifecycle. Completion retains staging until the +domain finalizer succeeds, then conditionally deletes only the exact staging version it verified. +Abort is also staging-only and must never delete a promoted final object. diff --git a/apps/sim/lib/uploads/upload-session/cleanup.test.ts b/apps/sim/lib/uploads/upload-session/cleanup.test.ts new file mode 100644 index 00000000000..2a9dad064a2 --- /dev/null +++ b/apps/sim/lib/uploads/upload-session/cleanup.test.ts @@ -0,0 +1,96 @@ +/** + * @vitest-environment node + */ +import { mkdir, rm, stat, utimes, writeFile } from 'node:fs/promises' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { testUploadDirectory } = vi.hoisted(() => ({ + testUploadDirectory: `/tmp/sim-upload-session-cleanup-${process.pid}`, +})) + +vi.mock('@/lib/uploads/core/setup.server', () => ({ + UPLOAD_DIR_SERVER: testUploadDirectory, +})) + +import { + LOCAL_UPLOAD_ARTIFACT_TTL_MS, + maybeCleanupLocalUploadArtifacts, + resetLocalUploadCleanupForTesting, + sweepLocalUploadArtifacts, +} from '@/lib/uploads/upload-session/cleanup' + +describe('local upload artifact cleanup', () => { + beforeEach(async () => { + resetLocalUploadCleanupForTesting() + await rm(testUploadDirectory, { recursive: true, force: true }) + await mkdir(testUploadDirectory, { recursive: true }) + }) + + it('removes expired multipart and staging entries while retaining fresh entries', async () => { + const now = Date.UTC(2026, 7, 4, 12) + await createArtifact('.multipart/expired', now - LOCAL_UPLOAD_ARTIFACT_TTL_MS - 1) + await createArtifact('upload-sessions/expired', now - LOCAL_UPLOAD_ARTIFACT_TTL_MS - 1) + await createArtifact('upload-sessions/fresh', now) + + await expect(sweepLocalUploadArtifacts({ now })).resolves.toEqual({ scanned: 3, removed: 2 }) + await expect(stat(`${testUploadDirectory}/.multipart/expired`)).rejects.toMatchObject({ + code: 'ENOENT', + }) + await expect(stat(`${testUploadDirectory}/upload-sessions/expired`)).rejects.toMatchObject({ + code: 'ENOENT', + }) + await expect(stat(`${testUploadDirectory}/upload-sessions/fresh`)).resolves.toBeDefined() + }) + + it('bounds each sweep by the requested entry count', async () => { + const now = Date.UTC(2026, 7, 4, 12) + await createArtifact('.multipart/one', now - LOCAL_UPLOAD_ARTIFACT_TTL_MS - 1) + await createArtifact('.multipart/two', now - LOCAL_UPLOAD_ARTIFACT_TTL_MS - 1) + + const result = await sweepLocalUploadArtifacts({ now, maxEntries: 1 }) + + expect(result).toEqual({ scanned: 1, removed: 1 }) + }) + + it('continues from its directory cursor so old entries cannot starve behind fresh ones', async () => { + const now = Date.UTC(2026, 7, 4, 12) + for (let index = 0; index < 5; index++) { + await createArtifact(`.multipart/fresh-${index}`, now) + } + await createArtifact('.multipart/expired-last', now - LOCAL_UPLOAD_ARTIFACT_TTL_MS - 1) + + const sweeps = [ + await sweepLocalUploadArtifacts({ now, maxEntries: 2 }), + await sweepLocalUploadArtifacts({ now, maxEntries: 2 }), + await sweepLocalUploadArtifacts({ now, maxEntries: 2 }), + ] + + expect(sweeps.reduce((total, result) => total + result.scanned, 0)).toBe(6) + expect(sweeps.reduce((total, result) => total + result.removed, 0)).toBe(1) + await expect(stat(`${testUploadDirectory}/.multipart/expired-last`)).rejects.toMatchObject({ + code: 'ENOENT', + }) + }) + + it('coalesces concurrent cleanup and rate-limits the next sweep', async () => { + const now = Date.UTC(2026, 7, 4, 12) + await createArtifact('.multipart/expired', now - LOCAL_UPLOAD_ARTIFACT_TTL_MS - 1) + + const [first, concurrent] = await Promise.all([ + maybeCleanupLocalUploadArtifacts(now), + maybeCleanupLocalUploadArtifacts(now), + ]) + + expect(first).toEqual({ scanned: 1, removed: 1 }) + expect(concurrent).toEqual(first) + await expect(maybeCleanupLocalUploadArtifacts(now)).resolves.toEqual({ scanned: 0, removed: 0 }) + }) +}) + +async function createArtifact(relativePath: string, modifiedAt: number): Promise { + const path = `${testUploadDirectory}/${relativePath}` + await mkdir(path, { recursive: true }) + await writeFile(`${path}/payload`, 'test') + const time = new Date(modifiedAt) + await utimes(path, time, time) +} diff --git a/apps/sim/lib/uploads/upload-session/cleanup.ts b/apps/sim/lib/uploads/upload-session/cleanup.ts new file mode 100644 index 00000000000..bdb39b75cb9 --- /dev/null +++ b/apps/sim/lib/uploads/upload-session/cleanup.ts @@ -0,0 +1,127 @@ +import type { Dirent } from 'node:fs' +import { opendir, rm, stat } from 'node:fs/promises' +import { join } from 'node:path' +import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' + +export const LOCAL_UPLOAD_CLEANUP_INTERVAL_MS = 15 * 60 * 1000 +export const LOCAL_UPLOAD_ARTIFACT_TTL_MS = 25 * 60 * 60 * 1000 +export const LOCAL_UPLOAD_CLEANUP_MAX_ENTRIES = 200 + +export interface LocalUploadCleanupResult { + scanned: number + removed: number +} + +let activeCleanup: Promise | null = null +let lastCleanupAt = 0 + +const CLEANUP_ROOTS = ['.multipart', 'upload-sessions'] as const + +interface CleanupRootState { + directory: Awaited> | null +} + +const cleanupRootStates: CleanupRootState[] = CLEANUP_ROOTS.map(() => ({ directory: null })) +let nextCleanupRootIndex = 0 + +/** + * Opportunistically removes expired local multipart and staged-PUT state. + * Calls are single-flight and rate-limited; each sweep examines a bounded number of entries. + */ +export function maybeCleanupLocalUploadArtifacts( + now = Date.now() +): Promise { + if (activeCleanup) return activeCleanup + if (now - lastCleanupAt < LOCAL_UPLOAD_CLEANUP_INTERVAL_MS) { + return Promise.resolve({ scanned: 0, removed: 0 }) + } + activeCleanup = sweepLocalUploadArtifacts({ now }).then((result) => { + lastCleanupAt = now + return result + }) + return activeCleanup.finally(() => { + activeCleanup = null + }) +} + +/** Performs one bounded sweep for per-replica maintenance hooks and deterministic tests. */ +export async function sweepLocalUploadArtifacts(params?: { + now?: number + maxEntries?: number +}): Promise { + const now = params?.now ?? Date.now() + const maxEntries = params?.maxEntries ?? LOCAL_UPLOAD_CLEANUP_MAX_ENTRIES + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1) { + throw new Error('maxEntries must be a positive integer') + } + const cutoff = now - LOCAL_UPLOAD_ARTIFACT_TTL_MS + let scanned = 0 + let removed = 0 + const exhaustedRoots = new Set() + + while (scanned < maxEntries) { + const artifact = await readNextCleanupArtifact(exhaustedRoots) + if (!artifact) break + scanned++ + const path = join(artifact.directoryPath, artifact.entry.name) + let file + try { + file = await stat(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue + throw error + } + if (file.mtimeMs > cutoff) continue + await rm(path, { recursive: artifact.entry.isDirectory(), force: true }) + removed++ + } + + return { scanned, removed } +} + +export function resetLocalUploadCleanupForTesting(): void { + for (const state of cleanupRootStates) { + if (!state.directory) continue + try { + state.directory.closeSync() + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ERR_DIR_CLOSED') throw error + } + state.directory = null + } + nextCleanupRootIndex = 0 + activeCleanup = null + lastCleanupAt = 0 +} + +async function readNextCleanupArtifact( + exhaustedRoots: Set +): Promise<{ directoryPath: string; entry: Dirent } | null> { + for (let attempt = 0; attempt < CLEANUP_ROOTS.length; attempt++) { + const rootIndex = nextCleanupRootIndex + nextCleanupRootIndex = (nextCleanupRootIndex + 1) % CLEANUP_ROOTS.length + if (exhaustedRoots.has(rootIndex)) continue + + const directoryPath = join(UPLOAD_DIR_SERVER, CLEANUP_ROOTS[rootIndex]) + const state = cleanupRootStates[rootIndex] + if (!state.directory) { + try { + state.directory = await opendir(directoryPath) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + exhaustedRoots.add(rootIndex) + continue + } + throw error + } + } + + const entry = await state.directory.read() + if (entry) return { directoryPath, entry } + + await state.directory.close() + state.directory = null + exhaustedRoots.add(rootIndex) + } + return null +} diff --git a/apps/sim/lib/uploads/upload-session/provider.test.ts b/apps/sim/lib/uploads/upload-session/provider.test.ts new file mode 100644 index 00000000000..7fa9990ea11 --- /dev/null +++ b/apps/sim/lib/uploads/upload-session/provider.test.ts @@ -0,0 +1,243 @@ +/** + * @vitest-environment node + */ +import { mkdir, readdir, readFile, rm, stat } from 'node:fs/promises' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { testUploadDirectory } = vi.hoisted(() => ({ + testUploadDirectory: `/tmp/sim-upload-session-provider-${process.pid}`, +})) + +vi.mock('@/lib/uploads/core/setup.server', () => ({ + UPLOAD_DIR_SERVER: testUploadDirectory, +})) + +vi.mock('@/lib/uploads/config', () => ({ + USE_BLOB_STORAGE: false, + USE_GCS_STORAGE: false, + USE_S3_STORAGE: false, + getStorageConfig: vi.fn(() => ({})), +})) + +import { deleteFile } from '@/lib/uploads/core/storage-service' +import { + headProviderObject, + LocalUploadBodyError, + promoteProviderObject, + writeLocalMultipartPart, + writeLocalPutObject, +} from '@/lib/uploads/upload-session/provider' + +const CONTEXT = 'workspace' as const +const METADATA = { + uploadId: 'upload-1', + userId: 'user-1', + originalName: 'file.bin', + purpose: 'workspace_file', + workspaceId: 'workspace-1', +} + +describe('local upload-session provider', () => { + beforeEach(async () => { + await rm(testUploadDirectory, { recursive: true, force: true }) + await mkdir(testUploadDirectory, { recursive: true }) + }) + + it('streams an exact-size PUT and persists its object identity', async () => { + await writeLocalPutObject({ + uploadId: 'upload-1', + stagingKey: 'upload-sessions/upload-1/file.bin', + body: byteStream('ab', 'cd'), + expectedSize: 4, + contentType: 'application/octet-stream', + metadata: METADATA, + }) + + await expect(readFile(localPath('upload-sessions/upload-1/file.bin'), 'utf8')).resolves.toBe( + 'abcd' + ) + await expect( + headProviderObject({ + provider: 'local', + key: 'upload-sessions/upload-1/file.bin', + context: CONTEXT, + }) + ).resolves.toMatchObject({ + size: 4, + contentType: 'application/octet-stream', + uploadId: 'upload-1', + version: expect.any(String), + }) + expect(await temporaryFiles('upload-sessions/upload-1')).toEqual([]) + }) + + it('persists an empty PUT object with its identity metadata', async () => { + await writeLocalPutObject({ + uploadId: 'upload-1', + stagingKey: 'upload-sessions/upload-1/empty.md', + body: byteStream(), + expectedSize: 0, + contentType: 'text/markdown', + metadata: METADATA, + }) + + await expect(stat(localPath('upload-sessions/upload-1/empty.md'))).resolves.toMatchObject({ + size: 0, + }) + await expect( + headProviderObject({ + provider: 'local', + key: 'upload-sessions/upload-1/empty.md', + context: CONTEXT, + }) + ).resolves.toMatchObject({ + size: 0, + contentType: 'text/markdown', + uploadId: 'upload-1', + version: expect.any(String), + }) + }) + + it.each([ + { name: 'short', chunks: ['ab'], expectedSize: 3 }, + { name: 'oversized', chunks: ['ab', 'cd'], expectedSize: 3 }, + ])('rejects a $name PUT and removes its temporary files', async ({ chunks, expectedSize }) => { + await expect( + writeLocalPutObject({ + uploadId: 'upload-1', + stagingKey: 'upload-sessions/upload-1/file.bin', + body: byteStream(...chunks), + expectedSize, + contentType: 'application/octet-stream', + metadata: METADATA, + }) + ).rejects.toBeInstanceOf(LocalUploadBodyError) + + await expect( + headProviderObject({ + provider: 'local', + key: 'upload-sessions/upload-1/file.bin', + context: CONTEXT, + }) + ).resolves.toBeNull() + expect(await temporaryFiles('upload-sessions/upload-1')).toEqual([]) + }) + + it('publishes a multipart part atomically after exact-size validation', async () => { + await writeLocalMultipartPart({ + uploadId: 'upload-1', + partNumber: 1, + body: byteStream('abc'), + expectedSize: 3, + }) + + await expect(readFile(localPath('.multipart/upload-1/1.part'), 'utf8')).resolves.toBe('abc') + + await expect( + writeLocalMultipartPart({ + uploadId: 'upload-1', + partNumber: 1, + body: byteStream('x'), + expectedSize: 3, + }) + ).rejects.toBeInstanceOf(LocalUploadBodyError) + + await expect(readFile(localPath('.multipart/upload-1/1.part'), 'utf8')).resolves.toBe('abc') + expect(await temporaryFiles('.multipart/upload-1')).toEqual([]) + }) + + it('promotes only the inspected source version into a new destination', async () => { + const stagingKey = 'upload-sessions/upload-1/file.bin' + await writeLocalPutObject({ + uploadId: 'upload-1', + stagingKey, + body: byteStream('old'), + expectedSize: 3, + contentType: 'application/octet-stream', + metadata: METADATA, + }) + const inspected = await requiredLocalHead(stagingKey) + + await promoteProviderObject({ + provider: 'local', + sourceKey: stagingKey, + destinationKey: 'workspace/workspace-1/file.bin', + sourceVersion: inspected.version, + context: CONTEXT, + }) + + await expect(readFile(localPath('workspace/workspace-1/file.bin'), 'utf8')).resolves.toBe('old') + await expect( + promoteProviderObject({ + provider: 'local', + sourceKey: stagingKey, + destinationKey: 'workspace/workspace-1/file.bin', + sourceVersion: inspected.version, + context: CONTEXT, + }) + ).rejects.toMatchObject({ code: 'EEXIST' }) + + await writeLocalPutObject({ + uploadId: 'upload-1', + stagingKey, + body: byteStream('new'), + expectedSize: 3, + contentType: 'application/octet-stream', + metadata: METADATA, + }) + await expect( + promoteProviderObject({ + provider: 'local', + sourceKey: stagingKey, + destinationKey: 'workspace/workspace-1/changed.bin', + sourceVersion: inspected.version, + context: CONTEXT, + }) + ).rejects.toThrow('Local staging object changed during promotion') + await expect( + headProviderObject({ + provider: 'local', + key: 'workspace/workspace-1/changed.bin', + context: CONTEXT, + }) + ).resolves.toBeNull() + + await deleteFile({ key: 'workspace/workspace-1/file.bin', context: CONTEXT }) + await expect(stat(localPath('workspace/workspace-1/file.bin'))).rejects.toMatchObject({ + code: 'ENOENT', + }) + await expect( + stat(localPath('workspace/workspace-1/file.bin.upload-metadata.json')) + ).rejects.toMatchObject({ code: 'ENOENT' }) + }) +}) + +function byteStream(...chunks: string[]): ReadableStream { + const encoder = new TextEncoder() + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(encoder.encode(chunk)) + controller.close() + }, + }) +} + +function localPath(key: string): string { + return `${testUploadDirectory}/${key}` +} + +async function temporaryFiles(relativeDirectory: string): Promise { + const entries = await readdir(localPath(relativeDirectory)).catch( + (error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return [] + throw error + } + ) + return entries.filter((entry) => entry.startsWith('.')) +} + +async function requiredLocalHead(key: string) { + const head = await headProviderObject({ provider: 'local', key, context: CONTEXT }) + if (!head) throw new Error(`Missing local test object ${key}`) + return head +} diff --git a/apps/sim/lib/uploads/upload-session/provider.ts b/apps/sim/lib/uploads/upload-session/provider.ts new file mode 100644 index 00000000000..f40f74ee921 --- /dev/null +++ b/apps/sim/lib/uploads/upload-session/provider.ts @@ -0,0 +1,754 @@ +import { createReadStream, createWriteStream } from 'node:fs' +import { link, mkdir, readFile, rename, rm, rmdir, stat, unlink, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { pipeline } from 'node:stream/promises' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { + getStorageConfig, + USE_BLOB_STORAGE, + USE_GCS_STORAGE, + USE_S3_STORAGE, +} from '@/lib/uploads/config' +import { UPLOAD_DIR_SERVER } from '@/lib/uploads/core/setup.server' +import { + createBlobConfig, + createGcsConfig, + createS3Config, + LOCAL_UPLOAD_METADATA_SUFFIX, +} from '@/lib/uploads/core/storage-service' +import type { UploadStorageProvider } from '@/lib/uploads/core/upload-token' +import type { StorageContext } from '@/lib/uploads/shared/types' +import { sanitizeFileKey } from '@/lib/uploads/utils/file-utils' + +export type { UploadStorageProvider } from '@/lib/uploads/core/upload-token' + +export interface CompletedUploadPart { + partNumber: number + etag?: string +} + +export interface UploadPartUrl { + partNumber: number + url: string + headers: Record + expiresAt: string +} + +export interface UploadObjectHead { + size: number + contentType: string + uploadId: string + version: string +} + +interface LocalUploadMetadata { + uploadId: string + contentType: string + metadata: Record +} + +export class LocalUploadBodyError extends Error { + constructor(message: string) { + super(message) + this.name = 'LocalUploadBodyError' + } +} + +export function uploadStorageProvider(): UploadStorageProvider { + if (USE_BLOB_STORAGE) return 'blob' + if (USE_S3_STORAGE) return 's3' + if (USE_GCS_STORAGE) return 'gcs' + return 'local' +} + +export async function initiateMultipartProviderUpload(params: { + stagingKey: string + fileName: string + contentType: string + fileSize: number + context: StorageContext + uploadId: string + metadata: Record +}): Promise<{ provider: UploadStorageProvider; providerUploadId: string | null }> { + const provider = uploadStorageProvider() + const config = getStorageConfig(params.context) + const metadata = { ...params.metadata, uploadId: params.uploadId } + + if (provider === 's3') { + const { initiateS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') + const result = await initiateS3MultipartUpload({ + fileName: params.fileName, + contentType: params.contentType, + fileSize: params.fileSize, + customConfig: createS3Config(config), + customKey: params.stagingKey, + purpose: params.context, + metadata, + }) + return { provider, providerUploadId: result.uploadId } + } + if (provider === 'blob') { + const { initiateMultipartUpload } = await import('@/lib/uploads/providers/blob/client') + const result = await initiateMultipartUpload({ + fileName: params.fileName, + contentType: params.contentType, + fileSize: params.fileSize, + customConfig: createBlobConfig(config), + customKey: params.stagingKey, + metadata, + }) + return { provider, providerUploadId: result.uploadId } + } + if (provider === 'gcs') { + const { initiateGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') + const result = await initiateGcsMultipartUpload({ + fileName: params.fileName, + contentType: params.contentType, + fileSize: params.fileSize, + customConfig: createGcsConfig(config), + customKey: params.stagingKey, + purpose: params.context, + metadata, + }) + return { provider, providerUploadId: result.uploadId } + } + + await mkdir(localPartsDirectory(params.uploadId), { recursive: true }) + return { provider, providerUploadId: null } +} + +export async function createPutProviderTransfer(params: { + provider: UploadStorageProvider + stagingKey: string + contentType: string + fileSize: number + context: StorageContext + uploadId: string + uploadToken: string + localOrigin?: string + expiresAt: Date + metadata: Record +}): Promise<{ method: 'put'; url: string; headers: Record }> { + const expiresIn = Math.floor((params.expiresAt.getTime() - Date.now()) / 1000) + if (expiresIn < 1) throw new Error('Cannot sign an expired PUT upload session') + + if (params.provider === 'local') { + if (!params.localOrigin) throw new Error('localOrigin is required for local PUT uploads') + const origin = new URL(params.localOrigin) + const url = new URL(`/api/v2/uploads/${encodeURIComponent(params.uploadId)}`, origin) + return { + method: 'put', + url: url.toString(), + headers: { + 'Content-Type': params.contentType, + 'upload-token': params.uploadToken, + }, + } + } + + const config = getStorageConfig(params.context) + const metadata = { ...params.metadata, uploadId: params.uploadId } + if (params.provider === 's3') { + const { getS3PresignedUploadUrl } = await import('@/lib/uploads/providers/s3/client') + const transfer = await getS3PresignedUploadUrl({ + key: params.stagingKey, + contentType: params.contentType, + fileSize: params.fileSize, + metadata, + customConfig: createS3Config(config), + expiresIn, + }) + return { method: 'put', ...transfer } + } + if (params.provider === 'blob') { + const { getBlobPresignedUploadUrl } = await import('@/lib/uploads/providers/blob/client') + const transfer = await getBlobPresignedUploadUrl({ + key: params.stagingKey, + contentType: params.contentType, + metadata, + customConfig: createBlobConfig(config), + expiresIn, + }) + return { method: 'put', ...transfer } + } + const { getGcsPresignedUploadUrl } = await import('@/lib/uploads/providers/gcs/client') + const transfer = await getGcsPresignedUploadUrl( + params.stagingKey, + params.contentType, + metadata, + createGcsConfig(config), + expiresIn + ) + return { method: 'put', url: transfer.url, headers: transfer.signedHeaders } +} + +export async function getMultipartProviderPartUrls(params: { + provider: UploadStorageProvider + providerUploadId: string | null + stagingKey: string + context: StorageContext + partNumbers: number[] + localUrl: (partNumber: number) => string +}): Promise { + const expiresAt = new Date(Date.now() + 60 * 60 * 1000).toISOString() + if (params.provider === 'local') { + return params.partNumbers.map((partNumber) => ({ + partNumber, + url: params.localUrl(partNumber), + headers: { 'Content-Type': 'application/octet-stream' }, + expiresAt, + })) + } + if (!params.providerUploadId) throw new Error(`Missing ${params.provider} multipart upload id`) + const config = getStorageConfig(params.context) + + if (params.provider === 's3') { + const { getS3MultipartPartUrls } = await import('@/lib/uploads/providers/s3/client') + const urls = await getS3MultipartPartUrls( + params.stagingKey, + params.providerUploadId, + params.partNumbers, + createS3Config(config) + ) + return urls.map(({ partNumber, url }) => ({ + partNumber, + url, + headers: { 'Content-Type': 'application/octet-stream' }, + expiresAt, + })) + } + if (params.provider === 'blob') { + const { getMultipartPartUrls } = await import('@/lib/uploads/providers/blob/client') + const urls = await getMultipartPartUrls( + params.stagingKey, + params.partNumbers, + createBlobConfig(config) + ) + return urls.map(({ partNumber, url }) => ({ + partNumber, + url, + headers: { 'Content-Type': 'application/octet-stream' }, + expiresAt, + })) + } + const { getGcsMultipartPartUrls } = await import('@/lib/uploads/providers/gcs/client') + const urls = await getGcsMultipartPartUrls( + params.stagingKey, + params.providerUploadId, + params.partNumbers, + createGcsConfig(config) + ) + return urls.map(({ partNumber, url }) => ({ + partNumber, + url, + headers: { 'Content-Type': 'application/octet-stream' }, + expiresAt, + })) +} + +export async function completeMultipartProviderUpload(params: { + provider: UploadStorageProvider + providerUploadId: string | null + uploadId: string + stagingKey: string + contentType: string + context: StorageContext + parts: CompletedUploadPart[] + metadata: Record +}): Promise { + if (params.provider === 'local') { + await assembleLocalParts( + params.uploadId, + params.stagingKey, + params.parts, + params.contentType, + params.metadata + ) + return + } + if (!params.providerUploadId) throw new Error(`Missing ${params.provider} multipart upload id`) + const config = getStorageConfig(params.context) + if (params.provider === 's3') { + const { completeS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') + await completeS3MultipartUpload( + params.stagingKey, + params.providerUploadId, + params.parts.map((part) => ({ + PartNumber: part.partNumber, + ETag: requiredEtag('s3', part), + })), + createS3Config(config) + ) + return + } + if (params.provider === 'blob') { + const { completeMultipartUpload, deriveBlobBlockId } = await import( + '@/lib/uploads/providers/blob/client' + ) + await completeMultipartUpload( + params.stagingKey, + params.parts.map((part) => ({ + partNumber: part.partNumber, + blockId: deriveBlobBlockId(part.partNumber), + })), + createBlobConfig(config), + params.contentType, + { ...params.metadata, uploadId: params.uploadId } + ) + return + } + const { completeGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') + await completeGcsMultipartUpload( + params.stagingKey, + params.providerUploadId, + params.parts.map((part) => ({ + PartNumber: part.partNumber, + ETag: requiredEtag('gcs', part), + })), + createGcsConfig(config) + ) +} + +export async function headProviderObject(params: { + provider: UploadStorageProvider + key: string + context: StorageContext +}): Promise { + if (params.provider === 'local') return headLocalObject(params.key) + const config = getStorageConfig(params.context) + const head = + params.provider === 's3' + ? await import('@/lib/uploads/providers/s3/client').then(({ headS3Object }) => + headS3Object(params.key, createS3Config(config)) + ) + : params.provider === 'blob' + ? await import('@/lib/uploads/providers/blob/client').then(({ headBlobObject }) => + headBlobObject(params.key, createBlobConfig(config)) + ) + : await import('@/lib/uploads/providers/gcs/client').then(({ headGcsObject }) => + headGcsObject(params.key, createGcsConfig(config)) + ) + if (!head) return null + if (!head.contentType || !head.uploadId || !head.version) { + throw new Error(`Upload object ${params.key} is missing required provider metadata`) + } + return { + size: head.size, + contentType: head.contentType, + uploadId: head.uploadId, + version: head.version, + } +} + +export async function promoteProviderObject(params: { + provider: UploadStorageProvider + sourceKey: string + destinationKey: string + sourceVersion: string + context: StorageContext +}): Promise { + if (params.provider === 'local') { + await promoteLocalObject(params.sourceKey, params.destinationKey, params.sourceVersion) + return + } + const config = getStorageConfig(params.context) + if (params.provider === 's3') { + const { promoteS3Object } = await import('@/lib/uploads/providers/s3/client') + await promoteS3Object({ + sourceKey: params.sourceKey, + destinationKey: params.destinationKey, + sourceEtag: params.sourceVersion, + customConfig: createS3Config(config), + }) + return + } + if (params.provider === 'blob') { + const { promoteBlobObject } = await import('@/lib/uploads/providers/blob/client') + await promoteBlobObject({ + sourceKey: params.sourceKey, + destinationKey: params.destinationKey, + sourceEtag: params.sourceVersion, + customConfig: createBlobConfig(config), + }) + return + } + const { promoteGcsObject } = await import('@/lib/uploads/providers/gcs/client') + await promoteGcsObject({ + sourceKey: params.sourceKey, + destinationKey: params.destinationKey, + sourceGeneration: params.sourceVersion, + customConfig: createGcsConfig(config), + }) +} + +export async function deleteProviderObjectVersion(params: { + provider: UploadStorageProvider + key: string + version: string + context: StorageContext +}): Promise { + if (params.provider === 'local') { + await deleteLocalObjectVersion(params.key, params.version) + return + } + const config = getStorageConfig(params.context) + if (params.provider === 's3') { + const { deleteS3ObjectVersion } = await import('@/lib/uploads/providers/s3/client') + await deleteS3ObjectVersion({ + key: params.key, + etag: params.version, + customConfig: createS3Config(config), + }) + return + } + if (params.provider === 'blob') { + const { deleteBlobObjectVersion } = await import('@/lib/uploads/providers/blob/client') + await deleteBlobObjectVersion({ + key: params.key, + etag: params.version, + customConfig: createBlobConfig(config), + }) + return + } + const { deleteGcsObjectVersion } = await import('@/lib/uploads/providers/gcs/client') + await deleteGcsObjectVersion({ + key: params.key, + generation: params.version, + customConfig: createGcsConfig(config), + }) +} + +export async function abortProviderUpload(params: { + provider: UploadStorageProvider + method: 'put' | 'multipart' + providerUploadId: string | null + uploadId: string + stagingKey: string + context: StorageContext +}): Promise { + if (params.provider === 'local') { + await rm(localPartsDirectory(params.uploadId), { recursive: true, force: true }) + await rm(localUploadDirectory(params.uploadId), { recursive: true, force: true }) + return + } + + const config = getStorageConfig(params.context) + if (params.method === 'multipart') { + if (!params.providerUploadId) { + throw new Error(`Missing ${params.provider} multipart upload id`) + } + if (params.provider === 's3') { + const { abortS3MultipartUpload } = await import('@/lib/uploads/providers/s3/client') + await abortS3MultipartUpload( + params.stagingKey, + params.providerUploadId, + createS3Config(config) + ) + } else if (params.provider === 'blob') { + const { abortMultipartUpload } = await import('@/lib/uploads/providers/blob/client') + await abortMultipartUpload(params.stagingKey, createBlobConfig(config)) + } else { + const { abortGcsMultipartUpload } = await import('@/lib/uploads/providers/gcs/client') + await abortGcsMultipartUpload( + params.stagingKey, + params.providerUploadId, + createGcsConfig(config) + ) + } + } + + if (params.provider === 's3') { + const { deleteFromS3 } = await import('@/lib/uploads/providers/s3/client') + await deleteFromS3(params.stagingKey, createS3Config(config)) + } else if (params.provider === 'blob') { + const { deleteFromBlob } = await import('@/lib/uploads/providers/blob/client') + await deleteFromBlob(params.stagingKey, createBlobConfig(config)) + } else { + const { deleteFromGcs } = await import('@/lib/uploads/providers/gcs/client') + await deleteFromGcs(params.stagingKey, createGcsConfig(config)) + } +} + +export async function writeLocalPutObject(params: { + uploadId: string + stagingKey: string + body: ReadableStream + expectedSize: number + contentType: string + metadata: Record +}): Promise { + assertLocalStagingKey(params.stagingKey, params.uploadId) + const { Readable, Transform } = await import('node:stream') + const directory = localUploadDirectory(params.uploadId) + const destination = localObjectPath(params.stagingKey) + const temporary = join(directory, `.put-${generateId()}`) + const temporaryMetadata = `${temporary}${LOCAL_UPLOAD_METADATA_SUFFIX}` + await mkdir(dirname(destination), { recursive: true }) + let bytes = 0 + const counter = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + bytes += chunk.length + if (bytes > params.expectedSize) { + callback(new LocalUploadBodyError(`Upload exceeds ${params.expectedSize} bytes`)) + return + } + callback(null, chunk) + }, + }) + + try { + await pipeline( + Readable.fromWeb(params.body as Parameters[0]), + counter, + createWriteStream(temporary, { flags: 'wx' }) + ) + if (bytes !== params.expectedSize) { + throw new LocalUploadBodyError(`Upload has ${bytes} bytes; expected ${params.expectedSize}`) + } + await writeLocalMetadata(temporaryMetadata, { + uploadId: params.uploadId, + contentType: params.contentType, + metadata: { ...params.metadata, uploadId: params.uploadId }, + }) + await rename(temporary, destination) + try { + await rename(temporaryMetadata, localMetadataPath(params.stagingKey)) + } catch (error) { + await rm(destination, { force: true }) + throw error + } + } catch (error) { + await Promise.allSettled([ + rm(temporary, { force: true }), + rm(temporaryMetadata, { force: true }), + ]) + if (error instanceof LocalUploadBodyError) throw error + throw new Error(getErrorMessage(error, 'Failed to store PUT upload'), { cause: error }) + } +} + +export async function writeLocalMultipartPart(params: { + uploadId: string + partNumber: number + body: ReadableStream + expectedSize: number +}): Promise { + const { Readable, Transform } = await import('node:stream') + const directory = localPartsDirectory(params.uploadId) + await mkdir(directory, { recursive: true }) + const destination = localPartPath(params.uploadId, params.partNumber) + const temporary = join(directory, `.${params.partNumber}-${generateId()}.part`) + let bytes = 0 + const counter = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + bytes += chunk.length + if (bytes > params.expectedSize) { + callback( + new LocalUploadBodyError(`Part ${params.partNumber} exceeds ${params.expectedSize} bytes`) + ) + return + } + callback(null, chunk) + }, + }) + try { + await pipeline( + Readable.fromWeb(params.body as Parameters[0]), + counter, + createWriteStream(temporary, { flags: 'wx' }) + ) + if (bytes !== params.expectedSize) { + throw new LocalUploadBodyError( + `Part ${params.partNumber} has ${bytes} bytes; expected ${params.expectedSize}` + ) + } + await rename(temporary, destination) + } catch (error) { + await rm(temporary, { force: true }).catch(() => {}) + if (error instanceof LocalUploadBodyError) throw error + throw new Error(getErrorMessage(error, `Failed to store part ${params.partNumber}`), { + cause: error, + }) + } +} + +function localPartsDirectory(uploadId: string): string { + return join(UPLOAD_DIR_SERVER, '.multipart', uploadId) +} + +function localUploadDirectory(uploadId: string): string { + return join(UPLOAD_DIR_SERVER, 'upload-sessions', uploadId) +} + +function localPartPath(uploadId: string, partNumber: number): string { + return join(localPartsDirectory(uploadId), `${partNumber}.part`) +} + +function localObjectPath(key: string): string { + return join(UPLOAD_DIR_SERVER, sanitizeFileKey(key)) +} + +function localMetadataPath(key: string): string { + return `${localObjectPath(key)}${LOCAL_UPLOAD_METADATA_SUFFIX}` +} + +async function assembleLocalParts( + uploadId: string, + stagingKey: string, + parts: CompletedUploadPart[], + contentType: string, + metadata: Record +): Promise { + assertLocalStagingKey(stagingKey, uploadId) + const destination = localObjectPath(stagingKey) + const temporary = join(localUploadDirectory(uploadId), `.multipart-${generateId()}`) + const temporaryMetadata = `${temporary}${LOCAL_UPLOAD_METADATA_SUFFIX}` + await mkdir(dirname(destination), { recursive: true }) + try { + for (const part of parts) { + await pipeline( + createReadStream(localPartPath(uploadId, part.partNumber)), + createWriteStream(temporary, { flags: 'a' }) + ) + } + await writeLocalMetadata(temporaryMetadata, { + uploadId, + contentType, + metadata: { ...metadata, uploadId }, + }) + await rename(temporary, destination) + try { + await rename(temporaryMetadata, localMetadataPath(stagingKey)) + } catch (error) { + await rm(destination, { force: true }) + throw error + } + await rm(localPartsDirectory(uploadId), { recursive: true, force: true }) + } catch (error) { + await Promise.allSettled([ + rm(temporary, { force: true }), + rm(temporaryMetadata, { force: true }), + ]) + throw error + } +} + +async function headLocalObject(key: string): Promise { + const path = localObjectPath(key) + let file: Awaited> + try { + file = await stat(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null + throw error + } + const metadata = await readLocalMetadata(localMetadataPath(key)) + return { + size: file.size, + contentType: metadata.contentType, + uploadId: metadata.uploadId, + version: localVersion(file), + } +} + +async function promoteLocalObject( + sourceKey: string, + destinationKey: string, + sourceVersion: string +): Promise { + const source = localObjectPath(sourceKey) + const sourceMetadata = localMetadataPath(sourceKey) + const destination = localObjectPath(destinationKey) + const destinationMetadata = localMetadataPath(destinationKey) + const metadata = await readLocalMetadata(sourceMetadata) + await mkdir(dirname(destination), { recursive: true }) + + let createdMetadata = false + try { + await link(sourceMetadata, destinationMetadata) + createdMetadata = true + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + const existing = await readLocalMetadata(destinationMetadata) + if (existing.uploadId !== metadata.uploadId) throw error + } + + try { + await link(source, destination) + } catch (error) { + if (createdMetadata) await rm(destinationMetadata, { force: true }) + throw error + } + + const destinationStat = await stat(destination) + if (localVersion(destinationStat) !== sourceVersion) { + await Promise.allSettled([ + rm(destination, { force: true }), + ...(createdMetadata ? [rm(destinationMetadata, { force: true })] : []), + ]) + throw new Error('Local staging object changed during promotion') + } +} + +async function deleteLocalObjectVersion(key: string, version: string): Promise { + const path = localObjectPath(key) + let current: Awaited> + try { + current = await stat(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return + throw error + } + if (localVersion(current) !== version) return + await unlink(path) + await rm(localMetadataPath(key), { force: true }) + await rmdir(dirname(path)).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOTEMPTY' && error.code !== 'ENOENT') throw error + }) +} + +async function writeLocalMetadata(path: string, metadata: LocalUploadMetadata): Promise { + await writeFile(path, JSON.stringify(metadata), { encoding: 'utf8', flag: 'wx' }) +} + +async function readLocalMetadata(path: string): Promise { + const parsed: unknown = JSON.parse(await readFile(path, 'utf8')) + if ( + typeof parsed !== 'object' || + parsed === null || + Array.isArray(parsed) || + typeof (parsed as Record).uploadId !== 'string' || + typeof (parsed as Record).contentType !== 'string' || + typeof (parsed as Record).metadata !== 'object' || + (parsed as Record).metadata === null || + Array.isArray((parsed as Record).metadata) + ) { + throw new Error(`Invalid local upload metadata at ${path}`) + } + const record = parsed as Record + const metadata = record.metadata as Record + if (Object.values(metadata).some((value) => typeof value !== 'string')) { + throw new Error(`Invalid local upload metadata at ${path}`) + } + return { + uploadId: record.uploadId as string, + contentType: record.contentType as string, + metadata: metadata as Record, + } +} + +function localVersion(file: Awaited>): string { + return `${file.dev}:${file.ino}:${file.size}:${file.mtimeMs}` +} + +function assertLocalStagingKey(stagingKey: string, uploadId: string): void { + if (!stagingKey.startsWith(`upload-sessions/${uploadId}/`)) { + throw new Error('Local staging key does not belong to this upload') + } +} + +function requiredEtag(provider: 's3' | 'gcs', part: CompletedUploadPart): string { + if (!part.etag) throw new Error(`Missing etag for ${provider} part ${part.partNumber}`) + return part.etag +} diff --git a/apps/sim/lib/uploads/upload-session/service.test.ts b/apps/sim/lib/uploads/upload-session/service.test.ts new file mode 100644 index 00000000000..d228c24980a --- /dev/null +++ b/apps/sim/lib/uploads/upload-session/service.test.ts @@ -0,0 +1,432 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockCheckStorageQuota, + mockCompleteMultipart, + mockCreatePutTransfer, + mockDeleteObjectVersion, + mockHeadObject, + mockInitiateMultipart, + mockPromoteObject, + mockResolveBillingContext, +} = vi.hoisted(() => ({ + mockCheckStorageQuota: vi.fn(), + mockCompleteMultipart: vi.fn(), + mockCreatePutTransfer: vi.fn(), + mockDeleteObjectVersion: vi.fn(), + mockHeadObject: vi.fn(), + mockInitiateMultipart: vi.fn(), + mockPromoteObject: vi.fn(), + mockResolveBillingContext: vi.fn(), +})) + +vi.mock('@/lib/billing/storage', () => ({ + checkStorageQuotaForBillingContext: mockCheckStorageQuota, + resolveStorageBillingContext: mockResolveBillingContext, +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + generateWorkspaceFileKey: vi.fn( + (workspaceId: string, fileName: string) => `workspace/${workspaceId}/final-${fileName}` + ), +})) + +vi.mock('@/lib/uploads/upload-session/cleanup', () => ({ + maybeCleanupLocalUploadArtifacts: vi.fn().mockResolvedValue({ scanned: 0, removed: 0 }), +})) + +vi.mock('@/lib/uploads/upload-session/provider', () => ({ + abortProviderUpload: vi.fn(), + completeMultipartProviderUpload: mockCompleteMultipart, + createPutProviderTransfer: mockCreatePutTransfer, + deleteProviderObjectVersion: mockDeleteObjectVersion, + getMultipartProviderPartUrls: vi.fn(), + headProviderObject: mockHeadObject, + initiateMultipartProviderUpload: mockInitiateMultipart, + promoteProviderObject: mockPromoteObject, + uploadStorageProvider: vi.fn(() => 's3'), +})) + +import { + MAX_WORKSPACE_FILE_SIZE, + MAX_WORKSPACE_FORMDATA_FILE_SIZE, +} from '@/lib/uploads/shared/types' +import { + completeUploadSession, + createUploadSession, + UPLOAD_SESSION_PUT_MAX_BYTES, + validateUploadCompletion, + verifyUploadSessionToken, +} from '@/lib/uploads/upload-session/service' + +const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' + +describe('upload sessions', () => { + beforeEach(() => { + vi.clearAllMocks() + mockResolveBillingContext.mockResolvedValue({ workspaceId: WORKSPACE_ID }) + mockCheckStorageQuota.mockResolvedValue({ allowed: true }) + mockCreatePutTransfer.mockResolvedValue({ + method: 'put', + url: 'https://storage.example/upload', + headers: { 'Content-Type': 'application/octet-stream' }, + }) + mockInitiateMultipart.mockResolvedValue({ + provider: 's3', + providerUploadId: 'provider-upload-1', + }) + }) + + it('selects PUT at exactly 50 MiB', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES) + + expect(created.transfer.method).toBe('put') + expect(created.method).toBe('put') + expect(created.partSize).toBeNull() + expect(created.partCount).toBeNull() + expect(mockInitiateMultipart).not.toHaveBeenCalled() + }) + + it('creates a PUT session for an empty workspace file', async () => { + const created = await createWorkspaceUpload(0) + + expect(created).toMatchObject({ + purpose: 'workspace_file', + fileSize: 0, + method: 'put', + transfer: { method: 'put' }, + }) + expect(verifyUploadSessionToken(created.uploadToken)).toMatchObject({ + purpose: 'workspace_file', + fileSize: 0, + method: 'put', + }) + }) + + it('rejects an empty upload for non-workspace-file purposes', async () => { + await expect( + createUploadSession({ + id: 'empty-attachment', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + purpose: 'mothership_attachment', + fileName: 'empty.txt', + contentType: 'text/plain', + fileSize: 0, + }) + ).rejects.toThrow('fileSize must be a positive integer') + }) + + it('rejects a negative workspace-file size', async () => { + await expect(createWorkspaceUpload(-1)).rejects.toThrow( + 'fileSize must be a non-negative integer' + ) + }) + + it('selects multipart at 50 MiB plus one byte', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES + 1) + + expect(created.transfer).toMatchObject({ method: 'multipart', partCount: 7 }) + expect(created.method).toBe('multipart') + expect(mockInitiateMultipart).toHaveBeenCalledOnce() + }) + + it('binds purpose scope, staging, destination, method, and identity into the token', async () => { + const created = await createUploadSession({ + id: 'upload-1', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + knowledgeBaseId: 'kb-1', + purpose: 'knowledge_document', + fileName: 'guide.pdf', + contentType: 'application/pdf', + fileSize: 1024, + }) + + expect(verifyUploadSessionToken(created.uploadToken)).toMatchObject({ + id: 'upload-1', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + knowledgeBaseId: 'kb-1', + purpose: 'knowledge_document', + method: 'put', + storageContext: 'knowledge-base', + storageProvider: 's3', + stagingKey: 'upload-sessions/upload-1/guide.pdf', + fileName: 'guide.pdf', + contentType: 'application/pdf', + fileSize: 1024, + }) + }) + + it('quota-gates durable files while exempting retention-scoped attachments', async () => { + await createWorkspaceUpload(1024) + expect(mockResolveBillingContext).toHaveBeenCalledOnce() + expect(mockCheckStorageQuota).toHaveBeenCalledOnce() + + await createUploadSession({ + id: 'execution-upload', + workspaceId: WORKSPACE_ID, + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + purpose: 'execution_attachment', + fileName: 'result.txt', + contentType: 'text/plain', + fileSize: 1024, + }) + expect(mockResolveBillingContext).toHaveBeenCalledOnce() + expect(mockCheckStorageQuota).toHaveBeenCalledOnce() + + await createUploadSession({ + id: 'mothership-upload', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + purpose: 'mothership_attachment', + fileName: 'prompt.txt', + contentType: 'text/plain', + fileSize: 1024, + }) + expect(mockResolveBillingContext).toHaveBeenCalledOnce() + expect(mockCheckStorageQuota).toHaveBeenCalledOnce() + }) + + it('preserves the 5 GiB mothership limit while bounding execution attachments at 100 MiB', async () => { + await expect( + createUploadSession({ + id: 'mothership-upload', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + purpose: 'mothership_attachment', + fileName: 'archive.zip', + contentType: 'application/zip', + fileSize: MAX_WORKSPACE_FILE_SIZE, + }) + ).resolves.toMatchObject({ + method: 'multipart', + transfer: { method: 'multipart', partCount: 640 }, + }) + + await expect( + createUploadSession({ + id: 'oversized-mothership-upload', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + purpose: 'mothership_attachment', + fileName: 'archive.zip', + contentType: 'application/zip', + fileSize: MAX_WORKSPACE_FILE_SIZE + 1, + }) + ).rejects.toThrow(`File size exceeds maximum of ${MAX_WORKSPACE_FILE_SIZE} bytes`) + + await expect( + createUploadSession({ + id: 'execution-upload', + workspaceId: WORKSPACE_ID, + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + purpose: 'execution_attachment', + fileName: 'result.txt', + contentType: 'text/plain', + fileSize: MAX_WORKSPACE_FORMDATA_FILE_SIZE + 1, + }) + ).rejects.toThrow(`File size exceeds maximum of ${MAX_WORKSPACE_FORMDATA_FILE_SIZE} bytes`) + }) + + it('validates PUT completion input independently of finalization', async () => { + const created = await createWorkspaceUpload(1024) + + expect(validateUploadCompletion(created, {})).toEqual([]) + expect(() => validateUploadCompletion(created, { parts: [] })).toThrow( + 'PUT completion must not include parts' + ) + }) + + it('requires every multipart part and cloud ETag before finalization or replay', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES + 1) + const parts = Array.from({ length: created.partCount ?? 0 }, (_, index) => ({ + partNumber: index + 1, + etag: `etag-${index + 1}`, + })) + + expect(validateUploadCompletion(created, { parts })).toBe(parts) + expect(() => validateUploadCompletion(created, {})).toThrow( + 'Multipart completion requires parts' + ) + expect(() => + validateUploadCompletion(created, { parts: parts.map(({ partNumber }) => ({ partNumber })) }) + ).toThrow('etag is required for s3 part 1') + }) + + it('resumes after multipart assembly without consuming the provider upload twice', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES + 1) + const identity = objectIdentity(created.id, created.fileSize, created.contentType) + mockHeadObject + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + + await expect( + completeUploadSession({ + session: created, + completion: { parts: completedParts(created.partCount) }, + finalize: async () => ({ value: 'file-1', completedFileId: 'file-1' }), + }) + ).resolves.toMatchObject({ value: 'file-1', alreadyCompleted: false }) + + expect(mockCompleteMultipart).not.toHaveBeenCalled() + expect(mockPromoteObject).toHaveBeenCalledOnce() + }) + + it('completes multipart at staging when no assembled object exists yet', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES + 1) + const identity = objectIdentity(created.id, created.fileSize, created.contentType) + mockHeadObject + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + + await completeUploadSession({ + session: created, + completion: { parts: completedParts(created.partCount) }, + finalize: async () => ({ value: 'file-1' }), + }) + + expect(mockCompleteMultipart).toHaveBeenCalledOnce() + expect(mockCompleteMultipart).toHaveBeenCalledWith( + expect.objectContaining({ stagingKey: created.stagingKey }) + ) + }) + + it('recovers when another completion consumes the provider upload concurrently', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES + 1) + const identity = objectIdentity(created.id, created.fileSize, created.contentType) + mockCompleteMultipart.mockRejectedValueOnce(new Error('NoSuchUpload')) + mockHeadObject + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + + await expect( + completeUploadSession({ + session: created, + completion: { parts: completedParts(created.partCount) }, + finalize: async () => ({ value: 'file-1' }), + }) + ).resolves.toMatchObject({ value: 'file-1', alreadyCompleted: false }) + + expect(mockCompleteMultipart).toHaveBeenCalledOnce() + expect(mockPromoteObject).toHaveBeenCalledOnce() + }) + + it('preserves the provider completion error when no staged object was created', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES + 1) + mockCompleteMultipart.mockRejectedValueOnce(new Error('NoSuchUpload')) + mockHeadObject + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null) + + await expect( + completeUploadSession({ + session: created, + completion: { parts: completedParts(created.partCount) }, + finalize: async () => ({ value: 'file-1' }), + }) + ).rejects.toThrow('NoSuchUpload') + + expect(mockPromoteObject).not.toHaveBeenCalled() + }) + + it('rejects a mismatched staged multipart object before provider completion', async () => { + const created = await createWorkspaceUpload(UPLOAD_SESSION_PUT_MAX_BYTES + 1) + mockHeadObject.mockResolvedValueOnce(null).mockResolvedValueOnce({ + ...objectIdentity(created.id, created.fileSize, created.contentType), + uploadId: 'another-upload', + }) + + await expect( + completeUploadSession({ + session: created, + completion: { parts: completedParts(created.partCount) }, + finalize: async () => ({ value: 'file-1' }), + }) + ).rejects.toThrow('Uploaded object belongs to another upload') + + expect(mockCompleteMultipart).not.toHaveBeenCalled() + expect(mockPromoteObject).not.toHaveBeenCalled() + }) + + it('retains staging when finalization fails after promotion', async () => { + const created = await createWorkspaceUpload(1024) + const identity = objectIdentity(created.id, created.fileSize, created.contentType) + mockHeadObject + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(identity) + .mockResolvedValueOnce(identity) + + await expect( + completeUploadSession({ + session: created, + completion: {}, + finalize: async () => { + throw new Error('database unavailable') + }, + }) + ).rejects.toThrow('database unavailable') + + expect(mockPromoteObject).toHaveBeenCalledOnce() + expect(mockDeleteObjectVersion).not.toHaveBeenCalled() + }) + + it('retries finalization from an exact final object, then removes staging conditionally', async () => { + const created = await createWorkspaceUpload(1024) + const identity = objectIdentity(created.id, created.fileSize, created.contentType) + mockHeadObject.mockResolvedValueOnce(identity).mockResolvedValueOnce(identity) + + await expect( + completeUploadSession({ + session: created, + completion: {}, + finalize: async () => ({ value: 'file-1', completedFileId: 'file-1' }), + }) + ).resolves.toMatchObject({ value: 'file-1', alreadyCompleted: true }) + + expect(mockPromoteObject).not.toHaveBeenCalled() + expect(mockDeleteObjectVersion).toHaveBeenCalledWith( + expect.objectContaining({ key: created.stagingKey, version: 'version-1' }) + ) + }) +}) + +async function createWorkspaceUpload(fileSize: number) { + return createUploadSession({ + id: 'upload-1', + workspaceId: WORKSPACE_ID, + userId: 'user-1', + purpose: 'workspace_file', + fileName: 'file.bin', + contentType: 'application/octet-stream', + fileSize, + }) +} + +function objectIdentity(uploadId: string, size: number, contentType: string) { + return { uploadId, size, contentType, version: 'version-1' } +} + +function completedParts(partCount: number | null) { + return Array.from({ length: partCount ?? 0 }, (_, index) => ({ + partNumber: index + 1, + etag: `etag-${index + 1}`, + })) +} diff --git a/apps/sim/lib/uploads/upload-session/service.ts b/apps/sim/lib/uploads/upload-session/service.ts new file mode 100644 index 00000000000..048ff658c11 --- /dev/null +++ b/apps/sim/lib/uploads/upload-session/service.ts @@ -0,0 +1,792 @@ +import { generateId } from '@sim/utils/id' +import { + checkStorageQuotaForBillingContext, + resolveStorageBillingContext, +} from '@/lib/billing/storage' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateExecutionFileKey } from '@/lib/uploads/contexts/execution/utils' +import { generateKnowledgeBaseFileKey } from '@/lib/uploads/contexts/knowledge-base/knowledge-base-file-manager' +import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace' +import { + signUploadToken, + type UploadSessionPurpose, + type UploadStorageProvider, + type UploadTokenPayload, + type UploadTransferMethod, + verifyUploadToken, +} from '@/lib/uploads/core/upload-token' +import { + MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE, + MAX_WORKSPACE_FILE_SIZE, + MAX_WORKSPACE_FORMDATA_FILE_SIZE, + type StorageContext, +} from '@/lib/uploads/shared/types' +import { maybeCleanupLocalUploadArtifacts } from '@/lib/uploads/upload-session/cleanup' +import { + abortProviderUpload, + type CompletedUploadPart, + completeMultipartProviderUpload, + createPutProviderTransfer, + deleteProviderObjectVersion, + getMultipartProviderPartUrls, + headProviderObject, + initiateMultipartProviderUpload, + promoteProviderObject, + type UploadPartUrl, + uploadStorageProvider, +} from '@/lib/uploads/upload-session/provider' +import { sanitizeFileName } from '@/executor/constants' + +export const UPLOAD_SESSION_PUT_MAX_BYTES = 50 * 1024 * 1024 +export const UPLOAD_SESSION_PART_SIZE = 8 * 1024 * 1024 +export const UPLOAD_SESSION_MAX_PART_URLS = 100 +export const UPLOAD_SESSION_TTL_MS = 24 * 60 * 60 * 1000 +export const UPLOAD_SESSION_ASSET_MAX_BYTES = 5 * 1024 * 1024 + +export type { UploadSessionPurpose, UploadTransferMethod } + +export type UploadSessionStatus = 'uploading' | 'completed' | 'aborted' + +export type UploadSessionTransfer = + | { method: 'put'; url: string; headers: Record } + | { method: 'multipart'; partSize: number; partCount: number } + +export interface UploadSessionRecord { + id: string + workspaceId: string | null + userId: string + knowledgeBaseId: string | null + workflowId: string | null + executionId: string | null + purpose: UploadSessionPurpose + method: UploadTransferMethod + storageContext: StorageContext + /** Canonical destination key retained for existing domain finalizers. */ + storageKey: string + finalKey: string + stagingKey: string + storageProvider: UploadStorageProvider + providerUploadId: string | null + fileName: string + contentType: string + fileSize: number + partSize: number | null + partCount: number | null + status: UploadSessionStatus + metadata: Record + uploadToken: string + createdAt: Date + expiresAt: Date + completedFileId: string | null + error: string | null + completedAt: Date | null + updatedAt: Date +} + +export interface CreatedUploadSession extends UploadSessionRecord { + transfer: UploadSessionTransfer +} + +export type UploadCompletion = { parts?: never } | { parts: CompletedUploadPart[] } + +export class UploadSessionError extends OrchestrationError { + constructor( + code: 'validation' | 'not_found' | 'forbidden' | 'conflict' | 'payload_too_large' | 'internal', + message: string + ) { + super(code, message) + this.name = 'UploadSessionError' + } +} + +interface CreateUploadSessionBaseParams { + id?: string + userId: string + fileName: string + contentType: string + fileSize: number + metadata?: Record + localOrigin?: string +} + +export type CreateUploadSessionParams = CreateUploadSessionBaseParams & + ( + | { purpose: 'workspace_file' | 'table_import'; workspaceId: string } + | { purpose: 'knowledge_document'; workspaceId: string; knowledgeBaseId: string } + | { purpose: 'profile_picture'; workspaceId?: null } + | { purpose: 'workspace_logo' | 'mothership_attachment'; workspaceId: string } + | { + purpose: 'execution_attachment' + workspaceId: string + workflowId: string + executionId: string + } + ) + +export async function createUploadSession( + params: CreateUploadSessionParams +): Promise { + validateFile(params) + const id = params.id ?? generateId() + const workspaceId = params.purpose === 'profile_picture' ? null : params.workspaceId + const { storageContext, finalKey } = resolveUploadStorage(params, id) + const stagingKey = `upload-sessions/${id}/${sanitizeFileName(params.fileName)}` + const method: UploadTransferMethod = + params.fileSize <= UPLOAD_SESSION_PUT_MAX_BYTES ? 'put' : 'multipart' + const partSize = method === 'multipart' ? UPLOAD_SESSION_PART_SIZE : null + const partCount = + method === 'multipart' ? Math.ceil(params.fileSize / UPLOAD_SESSION_PART_SIZE) : null + + if (requiresStorageQuota(params.purpose)) { + if (!workspaceId) throw new Error(`${params.purpose} upload is missing workspaceId`) + const billingContext = await resolveStorageBillingContext(workspaceId) + const quota = await checkStorageQuotaForBillingContext(billingContext, params.fileSize) + if (!quota.allowed) { + throw new UploadSessionError('payload_too_large', quota.error ?? 'Storage limit exceeded') + } + } + + const provider = uploadStorageProvider() + if (provider === 'local') await maybeCleanupLocalUploadArtifacts() + const objectMetadata = uploadSessionObjectMetadata({ + id, + userId: params.userId, + workspaceId, + purpose: params.purpose, + fileName: params.fileName, + knowledgeBaseId: params.purpose === 'knowledge_document' ? params.knowledgeBaseId : null, + workflowId: params.purpose === 'execution_attachment' ? params.workflowId : null, + executionId: params.purpose === 'execution_attachment' ? params.executionId : null, + }) + const initiated = + method === 'multipart' + ? await initiateMultipartProviderUpload({ + stagingKey, + fileName: params.fileName, + contentType: params.contentType, + fileSize: params.fileSize, + context: storageContext, + uploadId: id, + metadata: objectMetadata, + }) + : { provider, providerUploadId: null } + if (initiated.provider !== provider) { + throw new Error('Storage provider changed while creating upload session') + } + + const createdAt = new Date() + const expiresAt = new Date(createdAt.getTime() + UPLOAD_SESSION_TTL_MS) + const metadata = params.metadata ?? {} + const tokenPayload = createUploadTokenPayload({ + params, + id, + workspaceId, + storageContext, + finalKey, + stagingKey, + provider, + providerUploadId: initiated.providerUploadId, + method, + partSize, + partCount, + metadata, + createdAt, + expiresAt, + }) + const uploadToken = signUploadToken(tokenPayload) + const transfer: UploadSessionTransfer = + method === 'put' + ? await createPutProviderTransfer({ + provider, + stagingKey, + contentType: params.contentType, + fileSize: params.fileSize, + context: storageContext, + uploadId: id, + uploadToken, + localOrigin: params.localOrigin, + expiresAt, + metadata: objectMetadata, + }) + : { method, partSize: requireNumber(partSize), partCount: requireNumber(partCount) } + + return sessionFromPayload(tokenPayload, uploadToken, transfer) +} + +export function getOwnedUploadSession(params: { + uploadId: string + uploadToken: string + userId?: string + workspaceId?: string | null + purpose?: UploadSessionPurpose + knowledgeBaseId?: string + workflowId?: string + executionId?: string +}): UploadSessionRecord { + const session = verifyUploadSessionToken(params.uploadToken) + if (session.id !== params.uploadId) throw uploadNotFound() + if (params.userId !== undefined && session.userId !== params.userId) throw uploadNotFound() + if (params.workspaceId !== undefined && session.workspaceId !== params.workspaceId) { + throw uploadNotFound() + } + if (params.purpose !== undefined && session.purpose !== params.purpose) throw uploadNotFound() + if (params.knowledgeBaseId !== undefined && session.knowledgeBaseId !== params.knowledgeBaseId) { + throw uploadNotFound() + } + if (params.workflowId !== undefined && session.workflowId !== params.workflowId) { + throw uploadNotFound() + } + if (params.executionId !== undefined && session.executionId !== params.executionId) { + throw uploadNotFound() + } + return session +} + +export function verifyUploadSessionToken(uploadToken: string): UploadSessionRecord { + const verified = verifyUploadToken(uploadToken) + if (!verified.valid) throw new UploadSessionError('forbidden', 'Invalid or expired upload token') + return sessionFromPayload(verified.payload, uploadToken) +} + +export async function createUploadPartUrls(params: { + session: UploadSessionRecord + partNumbers: number[] + localOrigin: string +}): Promise { + assertUploadable(params.session) + if (params.session.method !== 'multipart' || !params.session.partCount) { + throw new UploadSessionError('conflict', 'PUT upload sessions do not have multipart parts') + } + const unique = new Set(params.partNumbers) + if (unique.size !== params.partNumbers.length) { + throw new UploadSessionError('validation', 'partNumbers must not contain duplicates') + } + if (params.partNumbers.length === 0 || params.partNumbers.length > UPLOAD_SESSION_MAX_PART_URLS) { + throw new UploadSessionError( + 'validation', + `partNumbers must contain between 1 and ${UPLOAD_SESSION_MAX_PART_URLS} entries` + ) + } + for (const partNumber of params.partNumbers) { + if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > params.session.partCount) { + throw new UploadSessionError( + 'validation', + `partNumber must be between 1 and ${params.session.partCount}` + ) + } + } + + return getMultipartProviderPartUrls({ + provider: params.session.storageProvider, + providerUploadId: params.session.providerUploadId, + stagingKey: params.session.stagingKey, + context: params.session.storageContext, + partNumbers: params.partNumbers, + localUrl: (partNumber) => + `${params.localOrigin}/api/v2/uploads/${params.session.id}/parts/${partNumber}?token=${encodeURIComponent(params.session.uploadToken)}`, + }) +} + +export async function completeUploadSession(params: { + session: UploadSessionRecord + completion: UploadCompletion + finalize: (session: UploadSessionRecord) => Promise<{ value: T; completedFileId?: string }> +}): Promise<{ session: UploadSessionRecord; value: T; alreadyCompleted: boolean }> { + assertUploadable(params.session) + const parts = validateUploadCompletion(params.session, params.completion) + let existingFinal = await headProviderObject({ + provider: params.session.storageProvider, + key: params.session.finalKey, + context: params.session.storageContext, + }) + const alreadyCompleted = existingFinal !== null + if (existingFinal) assertObjectIdentity(params.session, existingFinal, 'Final') + + if (!existingFinal) { + let staging = await headProviderObject({ + provider: params.session.storageProvider, + key: params.session.stagingKey, + context: params.session.storageContext, + }) + if (staging) assertObjectIdentity(params.session, staging, 'Uploaded') + + if (!staging && params.session.method === 'multipart') { + try { + await completeMultipartProviderUpload({ + provider: params.session.storageProvider, + providerUploadId: params.session.providerUploadId, + uploadId: params.session.id, + stagingKey: params.session.stagingKey, + contentType: params.session.contentType, + context: params.session.storageContext, + parts, + metadata: uploadSessionObjectMetadata(params.session), + }) + } catch (completionError) { + staging = await headProviderObject({ + provider: params.session.storageProvider, + key: params.session.stagingKey, + context: params.session.storageContext, + }) + if (!staging) throw completionError + assertObjectIdentity(params.session, staging, 'Uploaded') + } + } + + if (!staging) { + staging = await headProviderObject({ + provider: params.session.storageProvider, + key: params.session.stagingKey, + context: params.session.storageContext, + }) + } + + if (!staging) throw new UploadSessionError('conflict', 'Uploaded staging object not found') + assertObjectIdentity(params.session, staging, 'Uploaded') + + try { + await promoteProviderObject({ + provider: params.session.storageProvider, + sourceKey: params.session.stagingKey, + destinationKey: params.session.finalKey, + sourceVersion: staging.version, + context: params.session.storageContext, + }) + } catch (error) { + existingFinal = await headProviderObject({ + provider: params.session.storageProvider, + key: params.session.finalKey, + context: params.session.storageContext, + }) + if (!existingFinal) throw error + assertObjectIdentity(params.session, existingFinal, 'Final') + } + + const promoted = await headProviderObject({ + provider: params.session.storageProvider, + key: params.session.finalKey, + context: params.session.storageContext, + }) + if (!promoted) throw new Error('Promoted upload object not found') + assertObjectIdentity(params.session, promoted, 'Promoted') + } + + const finalized = await params.finalize(params.session) + await cleanupStagingObject(params.session) + const completedAt = new Date() + return { + session: { + ...params.session, + status: 'completed', + completedFileId: finalized.completedFileId ?? null, + completedAt, + updatedAt: completedAt, + }, + value: finalized.value, + alreadyCompleted, + } +} + +export async function abortUploadSession( + session: UploadSessionRecord +): Promise { + assertUploadable(session) + await abortProviderUpload({ + provider: session.storageProvider, + method: session.method, + providerUploadId: session.providerUploadId, + uploadId: session.id, + stagingKey: session.stagingKey, + context: session.storageContext, + }) + const completedAt = new Date() + return { ...session, status: 'aborted', completedAt, updatedAt: completedAt } +} + +export function expectedUploadPartSize(session: UploadSessionRecord, partNumber: number): number { + if (session.method !== 'multipart' || !session.partSize || !session.partCount) { + throw new UploadSessionError('conflict', 'PUT upload sessions do not have multipart parts') + } + if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > session.partCount) { + throw new UploadSessionError('validation', 'Invalid upload part number') + } + if (partNumber < session.partCount) return session.partSize + return session.fileSize - session.partSize * (session.partCount - 1) +} + +export function uploadSessionObjectMetadata( + session: Pick< + UploadSessionRecord, + | 'id' + | 'userId' + | 'workspaceId' + | 'purpose' + | 'fileName' + | 'knowledgeBaseId' + | 'workflowId' + | 'executionId' + > +): Record { + return { + uploadId: session.id, + userId: session.userId, + originalName: session.fileName, + purpose: session.purpose, + ...(session.workspaceId ? { workspaceId: session.workspaceId } : {}), + ...(session.knowledgeBaseId ? { knowledgeBaseId: session.knowledgeBaseId } : {}), + ...(session.workflowId ? { workflowId: session.workflowId } : {}), + ...(session.executionId ? { executionId: session.executionId } : {}), + } +} + +function sessionFromPayload( + payload: UploadTokenPayload, + uploadToken: string, + transfer: UploadSessionTransfer +): CreatedUploadSession +function sessionFromPayload( + payload: UploadTokenPayload, + uploadToken: string, + transfer?: undefined +): UploadSessionRecord +function sessionFromPayload( + payload: UploadTokenPayload, + uploadToken: string, + transfer?: UploadSessionTransfer +): CreatedUploadSession | UploadSessionRecord { + const createdAt = new Date(payload.createdAt) + const expiresAt = new Date(payload.expiresAt) + const session: UploadSessionRecord = { + id: payload.uploadId, + workspaceId: payload.workspaceId, + userId: payload.actorId, + knowledgeBaseId: payload.purpose === 'knowledge_document' ? payload.knowledgeBaseId : null, + workflowId: payload.purpose === 'execution_attachment' ? payload.workflowId : null, + executionId: payload.purpose === 'execution_attachment' ? payload.executionId : null, + purpose: payload.purpose, + method: payload.method, + storageContext: payload.context, + storageKey: payload.finalKey, + finalKey: payload.finalKey, + stagingKey: payload.stagingKey, + storageProvider: payload.provider, + providerUploadId: payload.providerUploadId, + fileName: payload.fileName, + contentType: payload.contentType, + fileSize: payload.fileSize, + partSize: payload.method === 'multipart' ? payload.partSize : null, + partCount: payload.method === 'multipart' ? payload.partCount : null, + status: 'uploading', + metadata: payload.metadata, + uploadToken, + createdAt, + expiresAt, + completedFileId: null, + error: null, + completedAt: null, + updatedAt: new Date(), + } + return transfer ? { ...session, transfer } : session +} + +function createUploadTokenPayload(params: { + params: CreateUploadSessionParams + id: string + workspaceId: string | null + storageContext: StorageContext + finalKey: string + stagingKey: string + provider: UploadStorageProvider + providerUploadId: string | null + method: UploadTransferMethod + partSize: number | null + partCount: number | null + metadata: Record + createdAt: Date + expiresAt: Date +}): UploadTokenPayload { + const base = { + uploadId: params.id, + actorId: params.params.userId, + finalKey: params.finalKey, + stagingKey: params.stagingKey, + provider: params.provider, + fileName: params.params.fileName, + contentType: params.params.contentType, + fileSize: params.params.fileSize, + metadata: params.metadata, + createdAt: params.createdAt.toISOString(), + expiresAt: params.expiresAt.toISOString(), + } + const transfer = + params.method === 'put' + ? ({ method: 'put', providerUploadId: null } as const) + : ({ + method: 'multipart', + providerUploadId: params.providerUploadId, + partSize: requireNumber(params.partSize), + partCount: requireNumber(params.partCount), + } as const) + + switch (params.params.purpose) { + case 'workspace_file': + return { + ...base, + ...transfer, + purpose: params.params.purpose, + workspaceId: params.params.workspaceId, + context: 'workspace', + } + case 'table_import': + return { + ...base, + ...transfer, + purpose: params.params.purpose, + workspaceId: params.params.workspaceId, + context: 'table-import', + } + case 'knowledge_document': + return { + ...base, + ...transfer, + purpose: params.params.purpose, + workspaceId: params.params.workspaceId, + context: 'knowledge-base', + knowledgeBaseId: params.params.knowledgeBaseId, + } + case 'profile_picture': + return { + ...base, + ...transfer, + purpose: params.params.purpose, + workspaceId: null, + context: 'profile-pictures', + } + case 'workspace_logo': + return { + ...base, + ...transfer, + purpose: params.params.purpose, + workspaceId: params.params.workspaceId, + context: 'workspace-logos', + } + case 'mothership_attachment': + return { + ...base, + ...transfer, + purpose: params.params.purpose, + workspaceId: params.params.workspaceId, + context: 'mothership', + } + case 'execution_attachment': + return { + ...base, + ...transfer, + purpose: params.params.purpose, + workspaceId: params.params.workspaceId, + context: 'execution', + workflowId: params.params.workflowId, + executionId: params.params.executionId, + } + } +} + +function assertUploadable(session: UploadSessionRecord): void { + if (session.status !== 'uploading') { + throw new UploadSessionError('conflict', `Upload session is ${session.status}`) + } + if (session.expiresAt.getTime() <= Date.now()) { + throw new UploadSessionError('conflict', 'Upload session has expired') + } +} + +/** + * Validates method-specific completion input before any idempotent replay shortcut is taken. + * Callers that can return an already-completed resource must run this first as well. + */ +export function validateUploadCompletion( + session: UploadSessionRecord, + completion: UploadCompletion +): CompletedUploadPart[] { + if (session.method === 'put') { + if ('parts' in completion) { + throw new UploadSessionError('validation', 'PUT completion must not include parts') + } + return [] + } + if (!('parts' in completion) || !completion.parts) { + throw new UploadSessionError('validation', 'Multipart completion requires parts') + } + validateCompletedParts(session, completion.parts) + return completion.parts +} + +function validateCompletedParts(session: UploadSessionRecord, parts: CompletedUploadPart[]): void { + if (!session.partCount) throw new Error('Multipart upload is missing partCount') + if (parts.length !== session.partCount) { + throw new UploadSessionError( + 'validation', + `Expected ${session.partCount} completed parts; received ${parts.length}` + ) + } + const sorted = [...parts].sort((a, b) => a.partNumber - b.partNumber) + for (let index = 0; index < sorted.length; index++) { + if (sorted[index].partNumber !== index + 1) { + throw new UploadSessionError( + 'validation', + 'Completed parts must contain every part exactly once' + ) + } + if ( + (session.storageProvider === 's3' || session.storageProvider === 'gcs') && + !sorted[index].etag + ) { + throw new UploadSessionError( + 'validation', + `etag is required for ${session.storageProvider} part ${sorted[index].partNumber}` + ) + } + } +} + +function assertObjectIdentity( + session: UploadSessionRecord, + object: { size: number; contentType: string; uploadId: string }, + label: string +): void { + if (object.uploadId !== session.id) { + throw new UploadSessionError('conflict', `${label} object belongs to another upload`) + } + if (object.size !== session.fileSize) { + throw new UploadSessionError( + 'conflict', + `${label} object has ${object.size} bytes; expected ${session.fileSize}` + ) + } + if (object.contentType !== session.contentType) { + throw new UploadSessionError( + 'conflict', + `${label} object has content type ${object.contentType}; expected ${session.contentType}` + ) + } +} + +async function cleanupStagingObject(session: UploadSessionRecord): Promise { + const staging = await headProviderObject({ + provider: session.storageProvider, + key: session.stagingKey, + context: session.storageContext, + }) + if (!staging) return + assertObjectIdentity(session, staging, 'Staging') + await deleteProviderObjectVersion({ + provider: session.storageProvider, + key: session.stagingKey, + version: staging.version, + context: session.storageContext, + }) +} + +function validateFile(params: CreateUploadSessionParams): void { + if (!params.fileName.trim()) { + throw new UploadSessionError('validation', 'fileName must not be empty') + } + if (!params.contentType.trim()) { + throw new UploadSessionError('validation', 'contentType must not be empty') + } + const minimum = params.purpose === 'workspace_file' ? 0 : 1 + if (!Number.isSafeInteger(params.fileSize) || params.fileSize < minimum) { + const range = minimum === 0 ? 'a non-negative integer' : 'a positive integer' + throw new UploadSessionError('validation', `fileSize must be ${range}`) + } + const maximum = maximumFileSize(params.purpose) + if (params.fileSize > maximum) { + throw new UploadSessionError('validation', `File size exceeds maximum of ${maximum} bytes`) + } + if (params.purpose !== 'profile_picture' && !params.workspaceId.trim()) { + throw new UploadSessionError('validation', 'workspaceId must not be empty') + } + if (params.purpose === 'knowledge_document' && !params.knowledgeBaseId.trim()) { + throw new UploadSessionError('validation', 'knowledgeBaseId must not be empty') + } + if ( + params.purpose === 'execution_attachment' && + (!params.workflowId.trim() || !params.executionId.trim()) + ) { + throw new UploadSessionError('validation', 'workflowId and executionId must not be empty') + } +} + +function maximumFileSize(purpose: UploadSessionPurpose): number { + if (purpose === 'knowledge_document') return MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE + if (purpose === 'profile_picture' || purpose === 'workspace_logo') { + return UPLOAD_SESSION_ASSET_MAX_BYTES + } + if (purpose === 'execution_attachment') { + return MAX_WORKSPACE_FORMDATA_FILE_SIZE + } + return MAX_WORKSPACE_FILE_SIZE +} + +function requiresStorageQuota(purpose: UploadSessionPurpose): boolean { + return purpose === 'workspace_file' || purpose === 'knowledge_document' +} + +function resolveUploadStorage( + params: CreateUploadSessionParams, + id: string +): { storageContext: StorageContext; finalKey: string } { + switch (params.purpose) { + case 'workspace_file': + return { + storageContext: 'workspace', + finalKey: generateWorkspaceFileKey(params.workspaceId, params.fileName), + } + case 'table_import': + return { + storageContext: 'table-import', + finalKey: `table-import/${params.workspaceId}/${id}/${sanitizeFileName(params.fileName)}`, + } + case 'knowledge_document': + return { + storageContext: 'knowledge-base', + finalKey: generateKnowledgeBaseFileKey(params.fileName), + } + case 'profile_picture': + return { + storageContext: 'profile-pictures', + finalKey: `profile-pictures/${id}-${sanitizeFileName(params.fileName)}`, + } + case 'workspace_logo': + return { + storageContext: 'workspace-logos', + finalKey: `workspace-logos/${params.workspaceId}/${id}-${sanitizeFileName(params.fileName)}`, + } + case 'mothership_attachment': + return { + storageContext: 'mothership', + finalKey: generateWorkspaceFileKey(params.workspaceId, params.fileName), + } + case 'execution_attachment': + return { + storageContext: 'execution', + finalKey: generateExecutionFileKey( + { + workspaceId: params.workspaceId, + workflowId: params.workflowId, + executionId: params.executionId, + }, + `${id}-${params.fileName}` + ), + } + } +} + +function uploadNotFound(): UploadSessionError { + return new UploadSessionError('not_found', 'Upload session not found') +} + +function requireNumber(value: number | null): number { + if (value === null) throw new Error('Multipart upload geometry is missing') + return value +} diff --git a/apps/sim/lib/workspace-files/orchestration/content.ts b/apps/sim/lib/workspace-files/orchestration/content.ts index dacb48be300..f9703370d8e 100644 --- a/apps/sim/lib/workspace-files/orchestration/content.ts +++ b/apps/sim/lib/workspace-files/orchestration/content.ts @@ -17,6 +17,9 @@ const logger = createLogger('WorkspaceFileContentOrchestration') /** Ceiling on a single content replace, independent of the workspace quota. */ export const MAX_WORKSPACE_FILE_CONTENT_BYTES = 50 * 1024 * 1024 +/** JSON-body ceiling with room for a 50 MiB file's base64 expansion and envelope. */ +export const MAX_WORKSPACE_FILE_INLINE_BODY_BYTES = 70 * 1024 * 1024 + export interface PerformUpdateWorkspaceFileContentParams { workspaceId: string fileId: string diff --git a/apps/sim/lib/workspace-files/orchestration/create.test.ts b/apps/sim/lib/workspace-files/orchestration/create.test.ts new file mode 100644 index 00000000000..1e093ccac65 --- /dev/null +++ b/apps/sim/lib/workspace-files/orchestration/create.test.ts @@ -0,0 +1,212 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCaptureServerEvent, mockRecordAudit, mockUploadWorkspaceFile } = vi.hoisted(() => ({ + mockCaptureServerEvent: vi.fn(), + mockRecordAudit: vi.fn(), + mockUploadWorkspaceFile: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { FILE_UPLOADED: 'file.uploaded' }, + AuditResourceType: { FILE: 'file' }, + recordAudit: mockRecordAudit, +})) + +vi.mock('@/lib/uploads/contexts/workspace', () => ({ + FileConflictError: class FileConflictError extends Error { + constructor(name: string) { + super(`A file named "${name}" already exists in this workspace`) + this.name = 'FileConflictError' + } + }, + uploadWorkspaceFile: mockUploadWorkspaceFile, +})) + +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mockCaptureServerEvent })) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { FileConflictError } from '@/lib/uploads/contexts/workspace' +import { + MAX_WORKSPACE_FILE_CONTENT_BYTES, + performCreateWorkspaceFile, +} from '@/lib/workspace-files/orchestration' + +const WORKSPACE_ID = 'workspace-1' +const USER_ID = 'user-1' +const CREATED_FILE = { + id: 'wf_created', + workspaceId: WORKSPACE_ID, + name: 'untitled.md', + key: 'workspace/workspace-1/untitled.md', + path: '/api/files/serve/untitled.md', + url: '/api/files/serve/untitled.md', + size: 0, + type: 'text/markdown', + uploadedBy: USER_ID, + folderId: null, + folderPath: null, + deletedAt: null, + uploadedAt: new Date('2026-08-04T00:00:00.000Z'), + updatedAt: new Date('2026-08-04T00:00:00.000Z'), + context: 'workspace' as const, +} + +describe('performCreateWorkspaceFile', () => { + beforeEach(() => { + vi.clearAllMocks() + mockUploadWorkspaceFile.mockResolvedValue(CREATED_FILE) + }) + + it('creates an empty file with exact-name conflict semantics and returns its canonical record', async () => { + const request = new Request('https://sim.ai', { headers: { 'user-agent': 'test' } }) + + const result = await performCreateWorkspaceFile({ + workspaceId: WORKSPACE_ID, + userId: USER_ID, + actorName: 'Test User', + actorEmail: 'test@sim.ai', + name: 'untitled.md', + contentType: 'text/markdown', + folderId: null, + request, + }) + + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + WORKSPACE_ID, + USER_ID, + expect.objectContaining({ length: 0 }), + 'untitled.md', + 'text/markdown', + { folderId: null, exactName: true } + ) + expect(result).toEqual({ success: true, file: CREATED_FILE }) + expect(mockRecordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: USER_ID, + actorName: 'Test User', + actorEmail: 'test@sim.ai', + workspaceId: WORKSPACE_ID, + resourceId: CREATED_FILE.id, + resourceName: CREATED_FILE.name, + metadata: { fileSize: 0, fileType: 'text/markdown' }, + request, + }) + ) + expect(mockCaptureServerEvent).toHaveBeenCalledWith( + USER_ID, + 'file_uploaded', + { workspace_id: WORKSPACE_ID, file_type: 'text/markdown' }, + { groups: { workspace: WORKSPACE_ID } } + ) + }) + + it('preserves initialized content, folder, and content type', async () => { + const content = Buffer.from('# Ready\n') + const file = { + ...CREATED_FILE, + name: 'ready.md', + folderId: 'folder-1', + folderPath: 'Docs', + size: content.length, + } + mockUploadWorkspaceFile.mockResolvedValue(file) + + const result = await performCreateWorkspaceFile({ + workspaceId: WORKSPACE_ID, + userId: USER_ID, + name: 'ready.md', + contentType: 'text/markdown; charset=utf-8', + folderId: 'folder-1', + content, + exactName: false, + }) + + expect(mockUploadWorkspaceFile).toHaveBeenCalledWith( + WORKSPACE_ID, + USER_ID, + content, + 'ready.md', + 'text/markdown; charset=utf-8', + { folderId: 'folder-1', exactName: false } + ) + expect(result).toEqual({ success: true, file }) + }) + + it('rejects decoded content above the content-update limit before storage I/O', async () => { + const result = await performCreateWorkspaceFile({ + workspaceId: WORKSPACE_ID, + userId: USER_ID, + name: 'too-large.md', + contentType: 'text/markdown', + content: Buffer.alloc(MAX_WORKSPACE_FILE_CONTENT_BYTES + 1), + }) + + expect(result).toEqual({ + success: false, + error: 'File size exceeds 50MB limit', + errorCode: 'payload_too_large', + }) + expect(mockUploadWorkspaceFile).not.toHaveBeenCalled() + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) + + it('classifies an exact-name collision as conflict and records no audit', async () => { + mockUploadWorkspaceFile.mockRejectedValue(new FileConflictError('untitled.md')) + + const result = await performCreateWorkspaceFile({ + workspaceId: WORKSPACE_ID, + userId: USER_ID, + name: 'untitled.md', + contentType: 'text/markdown', + }) + + expect(result).toEqual({ + success: false, + error: 'A file named "untitled.md" already exists in this workspace', + errorCode: 'conflict', + }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) + + it('preserves classified folder failures and keeps unexpected faults internal', async () => { + mockUploadWorkspaceFile.mockRejectedValueOnce( + new OrchestrationError('not_found', 'Target folder not found') + ) + + const missingFolder = await performCreateWorkspaceFile({ + workspaceId: WORKSPACE_ID, + userId: USER_ID, + name: 'untitled.md', + contentType: 'text/markdown', + folderId: 'missing', + }) + + expect(missingFolder).toEqual({ + success: false, + error: 'Target folder not found', + errorCode: 'not_found', + }) + + mockUploadWorkspaceFile.mockRejectedValueOnce(new Error('connection terminated')) + + const unexpected = await performCreateWorkspaceFile({ + workspaceId: WORKSPACE_ID, + userId: USER_ID, + name: 'untitled.md', + contentType: 'text/markdown', + }) + + expect(unexpected).toEqual({ + success: false, + error: 'connection terminated', + errorCode: 'internal', + }) + expect(mockRecordAudit).not.toHaveBeenCalled() + expect(mockCaptureServerEvent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/workspace-files/orchestration/create.ts b/apps/sim/lib/workspace-files/orchestration/create.ts new file mode 100644 index 00000000000..c1f70cb0d46 --- /dev/null +++ b/apps/sim/lib/workspace-files/orchestration/create.ts @@ -0,0 +1,120 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { createLogger } from '@sim/logger' +import { getPostgresErrorCode, toError } from '@sim/utils/errors' +import { + asOrchestrationError, + type OrchestrationErrorCode, + type OrchestrationRequestContext, +} from '@/lib/core/orchestration/types' +import { captureServerEvent } from '@/lib/posthog/server' +import { + FileConflictError, + uploadWorkspaceFile, + type WorkspaceFileRecord, +} from '@/lib/uploads/contexts/workspace' +import { MAX_WORKSPACE_FILE_CONTENT_BYTES } from '@/lib/workspace-files/orchestration/content' + +const logger = createLogger('WorkspaceFileCreateOrchestration') + +export interface PerformCreateWorkspaceFileParams { + workspaceId: string + userId: string + name: string + contentType: string + folderId?: string | null + content?: Buffer + exactName?: boolean + actorName?: string + actorEmail?: string + request?: OrchestrationRequestContext +} + +export interface PerformCreateWorkspaceFileResult { + success: boolean + error?: string + errorCode?: OrchestrationErrorCode + file?: WorkspaceFileRecord +} + +/** + * Creates a workspace file from server-held bytes and returns its canonical record. + * + * Exact-name mode keeps this operation suitable for public create surfaces: a + * live sibling with the requested name is a conflict instead of silently + * producing a suffixed copy. Uploads remain the storage and accounting + * primitive, including for an empty buffer. + */ +export async function performCreateWorkspaceFile( + params: PerformCreateWorkspaceFileParams +): Promise { + const { + workspaceId, + userId, + name, + contentType, + folderId, + content = Buffer.alloc(0), + exactName = true, + actorName, + actorEmail, + request, + } = params + + if (content.length > MAX_WORKSPACE_FILE_CONTENT_BYTES) { + return { + success: false, + error: `File size exceeds ${MAX_WORKSPACE_FILE_CONTENT_BYTES / 1024 / 1024}MB limit`, + errorCode: 'payload_too_large', + } + } + + try { + const file = await uploadWorkspaceFile(workspaceId, userId, content, name, contentType, { + folderId, + exactName, + }) + + logger.info('Created workspace file', { + workspaceId, + fileId: file.id, + folderId: file.folderId, + size: file.size, + }) + + recordAudit({ + workspaceId, + actorId: userId, + actorName, + actorEmail, + action: AuditAction.FILE_UPLOADED, + resourceType: AuditResourceType.FILE, + resourceId: file.id, + resourceName: file.name, + description: `Uploaded file "${file.name}"`, + metadata: { fileSize: file.size, fileType: file.type }, + request, + }) + + captureServerEvent( + userId, + 'file_uploaded', + { workspace_id: workspaceId, file_type: file.type }, + { groups: { workspace: workspaceId } } + ) + + return { success: true, file } + } catch (error) { + logger.error('Failed to create workspace file', { error, workspaceId, folderId }) + + if (error instanceof FileConflictError || getPostgresErrorCode(error) === '23505') { + return { success: false, error: toError(error).message, errorCode: 'conflict' } + } + + const classified = asOrchestrationError(error) + if (classified) { + return { success: false, error: classified.message, errorCode: classified.code } + } + + return { success: false, error: toError(error).message, errorCode: 'internal' } + } +} diff --git a/apps/sim/lib/workspace-files/orchestration/index.ts b/apps/sim/lib/workspace-files/orchestration/index.ts index 1940e6c165c..166870b86b6 100644 --- a/apps/sim/lib/workspace-files/orchestration/index.ts +++ b/apps/sim/lib/workspace-files/orchestration/index.ts @@ -1,9 +1,15 @@ export { MAX_WORKSPACE_FILE_CONTENT_BYTES, + MAX_WORKSPACE_FILE_INLINE_BODY_BYTES, type PerformUpdateWorkspaceFileContentParams, type PerformUpdateWorkspaceFileContentResult, performUpdateWorkspaceFileContent, } from './content' +export { + type PerformCreateWorkspaceFileParams, + type PerformCreateWorkspaceFileResult, + performCreateWorkspaceFile, +} from './create' export { type PerformCreateWorkspaceFileFolderParams, type PerformCreateWorkspaceFileFolderResult, diff --git a/apps/sim/proxy.test.ts b/apps/sim/proxy.test.ts index 2ee2476e2c4..83627d42a5a 100644 --- a/apps/sim/proxy.test.ts +++ b/apps/sim/proxy.test.ts @@ -121,7 +121,7 @@ describe('resolveApiCorsPolicy', () => { }) it('returns default policy with APP_URL and credentials for other API routes', () => { - const policy = resolveApiCorsPolicy(makeRequest('/api/files/upload')) + const policy = resolveApiCorsPolicy(makeRequest('/api/files/uploads')) expect(policy).toEqual({ origin: 'https://app.sim.test', credentials: true, @@ -137,7 +137,7 @@ describe('resolveApiCorsPolicy', () => { '/api/chat/abc', '/api/workflows/wf/execute', '/api/v2/workflows/wf/execute', - '/api/files/upload', + '/api/files/uploads', ] for (const path of paths) { const policy = resolveApiCorsPolicy(makeRequest(path)) diff --git a/packages/testing/src/mocks/storage-service.mock.ts b/packages/testing/src/mocks/storage-service.mock.ts index effa9666f11..a61c5a381fd 100644 --- a/packages/testing/src/mocks/storage-service.mock.ts +++ b/packages/testing/src/mocks/storage-service.mock.ts @@ -20,7 +20,6 @@ export const storageServiceMockFns = { mockDeleteFile: vi.fn(), mockHeadObject: vi.fn(), mockGeneratePresignedUploadUrl: vi.fn(), - mockGenerateBatchPresignedUploadUrls: vi.fn(), mockGeneratePresignedDownloadUrl: vi.fn(), mockHasCloudStorage: vi.fn(() => false), mockGetS3InfoForKey: vi.fn(), @@ -40,7 +39,6 @@ export const storageServiceMock = { deleteFile: storageServiceMockFns.mockDeleteFile, headObject: storageServiceMockFns.mockHeadObject, generatePresignedUploadUrl: storageServiceMockFns.mockGeneratePresignedUploadUrl, - generateBatchPresignedUploadUrls: storageServiceMockFns.mockGenerateBatchPresignedUploadUrls, generatePresignedDownloadUrl: storageServiceMockFns.mockGeneratePresignedDownloadUrl, hasCloudStorage: storageServiceMockFns.mockHasCloudStorage, getS3InfoForKey: storageServiceMockFns.mockGetS3InfoForKey,