From 9e23e542a154a958f0bf8109f9dd5cef87c2b614 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Mon, 8 Jun 2026 19:51:27 +1000 Subject: [PATCH] test(fal): add speech/transcription usage coverage; drop e2e billing test Follow-up to #723 addressing CodeRabbit review feedback. - Add unit coverage for `result.usage.unitsBilled` on the fal speech and transcription adapters (billed + unbilled cases), mirroring the existing audio/image/video adapter tests. - Remove the e2e `fal-billable-units` spec, its `/api/fal-billable-units` route, and the hand-rolled `/fal-queue` aimock mount. aimock has no seam to stamp the `x-fal-billable-units` response header the feature reads, so any e2e test required manipulating `fetch` to redirect fal's hardcoded `queue.fal.run` URLs. The billed-units behavior is now covered by unit tests across all five fal adapters instead. - Drop the now-orphaned `@tanstack/ai-fal` dependency from the e2e app and regenerate the route tree. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/ai-fal/tests/speech-adapter.test.ts | 58 +++++++++++++ .../tests/transcription-adapter.test.ts | 46 ++++++++++ pnpm-lock.yaml | 3 - testing/e2e/global-setup.ts | 85 ------------------- testing/e2e/package.json | 1 - testing/e2e/src/routeTree.gen.ts | 21 ----- .../e2e/src/routes/api.fal-billable-units.ts | 70 --------------- testing/e2e/tests/fal-billable-units.spec.ts | 40 --------- 8 files changed, 104 insertions(+), 220 deletions(-) delete mode 100644 testing/e2e/src/routes/api.fal-billable-units.ts delete mode 100644 testing/e2e/tests/fal-billable-units.spec.ts diff --git a/packages/ai-fal/tests/speech-adapter.test.ts b/packages/ai-fal/tests/speech-adapter.test.ts index f172400a5..2edafe4f9 100644 --- a/packages/ai-fal/tests/speech-adapter.test.ts +++ b/packages/ai-fal/tests/speech-adapter.test.ts @@ -2,6 +2,18 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { generateSpeech } from '@tanstack/ai' import { falSpeech } from '../src/adapters/speech' +import { recordBillableUnitsFromResponse } from '../src/utils/billing' + +function seedBillableUnits(requestId: string, units: string) { + recordBillableUnitsFromResponse( + new Response(null, { + headers: { + 'x-fal-request-id': requestId, + 'x-fal-billable-units': units, + }, + }), + ) +} // Declare mocks at module level let mockSubscribe: any @@ -303,4 +315,50 @@ describe('Fal Speech Adapter', () => { // URL has no extension and no content-type — default to wav. expect(result.format).toBe('wav') }) + + it('surfaces fal billable units as usage', async () => { + seedBillableUnits('req-billed-speech', '3') + mockSubscribe.mockResolvedValueOnce({ + data: { + audio: { + url: 'https://fal.media/files/billed.wav', + content_type: 'audio/wav', + }, + }, + requestId: 'req-billed-speech', + }) + + const result = await generateSpeech({ + adapter: createAdapter(), + text: 'billed speech', + modelOptions: { audio_url: REFERENCE_AUDIO }, + }) + + expect(result.usage).toEqual({ + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + unitsBilled: 3, + }) + }) + + it('omits usage when fal does not report billable units', async () => { + mockSubscribe.mockResolvedValueOnce({ + data: { + audio: { + url: 'https://fal.media/files/unbilled.wav', + content_type: 'audio/wav', + }, + }, + requestId: 'req-unbilled-speech', + }) + + const result = await generateSpeech({ + adapter: createAdapter(), + text: 'unbilled speech', + modelOptions: { audio_url: REFERENCE_AUDIO }, + }) + + expect(result.usage).toBeUndefined() + }) }) diff --git a/packages/ai-fal/tests/transcription-adapter.test.ts b/packages/ai-fal/tests/transcription-adapter.test.ts index c88fddfec..7b969b2af 100644 --- a/packages/ai-fal/tests/transcription-adapter.test.ts +++ b/packages/ai-fal/tests/transcription-adapter.test.ts @@ -2,6 +2,18 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { generateTranscription } from '@tanstack/ai' import { falTranscription } from '../src/adapters/transcription' +import { recordBillableUnitsFromResponse } from '../src/utils/billing' + +function seedBillableUnits(requestId: string, units: string) { + recordBillableUnitsFromResponse( + new Response(null, { + headers: { + 'x-fal-request-id': requestId, + 'x-fal-billable-units': units, + }, + }), + ) +} // Declare mocks at module level let mockSubscribe: any @@ -292,4 +304,38 @@ describe('Fal Transcription Adapter', () => { expect(result.segments).toHaveLength(1) expect(result.segments![0]!.text).toBe('Only.') }) + + it('surfaces fal billable units as usage', async () => { + seedBillableUnits('req-billed-transcription', '1.5') + mockSubscribe.mockResolvedValueOnce({ + data: { text: 'Billed transcription.' }, + requestId: 'req-billed-transcription', + }) + + const result = await generateTranscription({ + adapter: createAdapter(), + audio: 'https://example.com/audio.mp3', + }) + + expect(result.usage).toEqual({ + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + unitsBilled: 1.5, + }) + }) + + it('omits usage when fal does not report billable units', async () => { + mockSubscribe.mockResolvedValueOnce({ + data: { text: 'Unbilled transcription.' }, + requestId: 'req-unbilled-transcription', + }) + + const result = await generateTranscription({ + adapter: createAdapter(), + audio: 'https://example.com/audio.mp3', + }) + + expect(result.usage).toBeUndefined() + }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f403d64a..ab84c8703 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1862,9 +1862,6 @@ importers: '@tanstack/ai-elevenlabs': specifier: workspace:* version: link:../../packages/ai-elevenlabs - '@tanstack/ai-fal': - specifier: workspace:* - version: link:../../packages/ai-fal '@tanstack/ai-gemini': specifier: workspace:* version: link:../../packages/ai-gemini diff --git a/testing/e2e/global-setup.ts b/testing/e2e/global-setup.ts index 2af2b5bd0..f869df01a 100644 --- a/testing/e2e/global-setup.ts +++ b/testing/e2e/global-setup.ts @@ -66,14 +66,6 @@ export default async function globalSetup() { // `promptTokensDetails.cachedTokens` / `completionTokensDetails.reasoningTokens`. mock.mount('/openai-usage-details', openaiUsageDetailsMount()) - // fal billable-units capture. aimock doesn't model fal's queue protocol - // (submit → poll status → fetch result) or its `x-fal-billable-units` / - // `x-fal-request-id` result headers, so this mount hand-rolls the three queue - // round-trips and stamps the billing headers on the result fetch. The - // companion api.fal-billable-units route redirects fal's hardcoded - // queue.fal.run URLs here and asserts the units reach `result.usage`. - mock.mount('/fal-queue', falQueueMount()) - await mock.start() console.log(`[aimock] started on port 4010`) ;(globalThis as any).__aimock = mock @@ -474,83 +466,6 @@ function openaiUsageDetailsMount(): Mountable { } } -/** - * Request id and billed quantity the fal queue mount reports. Exported-by-value - * to the companion route/spec via the literal below — kept in one place so the - * assertion and the mock can't drift. - */ -const FAL_E2E_REQUEST_ID = 'fal-req-e2e' -const FAL_E2E_BILLABLE_UNITS = '4' - -/** - * Mimics fal's queue protocol for a single image generation: - * POST /{appId} → submit, returns request_id - * GET /{appId}/requests/{id}/status → poll, returns COMPLETED - * GET /{appId}/requests/{id} → result, returns the image payload - * with `x-fal-request-id` and - * `x-fal-billable-units` headers - * The billing fetch installed by @tanstack/ai-fal reads those headers off the - * result fetch and the adapter surfaces them as `result.usage.unitsBilled`. - */ -function falQueueMount(): Mountable { - return { - async handleRequest( - req: http.IncomingMessage, - res: http.ServerResponse, - // Mount prefix (/fal-queue) is stripped; pathname is `/{appId}/...`. - pathname: string, - ): Promise { - const isResultPath = - req.method === 'GET' && /\/requests\/[^/]+$/.test(pathname) - const isStatusPath = req.method === 'GET' && pathname.endsWith('/status') - const isSubmitPath = - req.method === 'POST' && !pathname.includes('/requests/') - - if (isSubmitPath) { - await drainBody(req) - res.statusCode = 200 - res.setHeader('Content-Type', 'application/json') - res.end( - JSON.stringify({ - request_id: FAL_E2E_REQUEST_ID, - status: 'IN_QUEUE', - }), - ) - return true - } - - if (isStatusPath) { - res.statusCode = 200 - res.setHeader('Content-Type', 'application/json') - res.end( - JSON.stringify({ - status: 'COMPLETED', - request_id: FAL_E2E_REQUEST_ID, - }), - ) - return true - } - - if (isResultPath) { - res.statusCode = 200 - res.setHeader('Content-Type', 'application/json') - // The two headers the feature hangs on: the billed quantity, and the - // request id the adapter correlates it against. - res.setHeader('x-fal-request-id', FAL_E2E_REQUEST_ID) - res.setHeader('x-fal-billable-units', FAL_E2E_BILLABLE_UNITS) - res.end( - JSON.stringify({ - images: [{ url: 'https://fal.media/files/e2e-billed.png' }], - }), - ) - return true - } - - return false - }, - } -} - function buildToolPlusServerToolEvents(): Array> { const messageId = 'msg_bug_604' const model = 'claude-sonnet-4-5' diff --git a/testing/e2e/package.json b/testing/e2e/package.json index bbf9c9203..af0e4ad99 100644 --- a/testing/e2e/package.json +++ b/testing/e2e/package.json @@ -20,7 +20,6 @@ "@tanstack/ai-anthropic": "workspace:*", "@tanstack/ai-client": "workspace:*", "@tanstack/ai-elevenlabs": "workspace:*", - "@tanstack/ai-fal": "workspace:*", "@tanstack/ai-gemini": "workspace:*", "@tanstack/ai-grok": "workspace:*", "@tanstack/ai-groq": "workspace:*", diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index b58d9e55e..165f14d4d 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -37,7 +37,6 @@ import { Route as ApiMcpServerRouteImport } from './routes/api.mcp-server' import { Route as ApiMcpManagedTestRouteImport } from './routes/api.mcp-managed-test' import { Route as ApiMcpLifecycleTestRouteImport } from './routes/api.mcp-lifecycle-test' import { Route as ApiImageRouteImport } from './routes/api.image' -import { Route as ApiFalBillableUnitsRouteImport } from './routes/api.fal-billable-units' import { Route as ApiChatRouteImport } from './routes/api.chat' import { Route as ApiAudioRouteImport } from './routes/api.audio' import { Route as ApiArktypeToolWireRouteImport } from './routes/api.arktype-tool-wire' @@ -193,11 +192,6 @@ const ApiImageRoute = ApiImageRouteImport.update({ path: '/api/image', getParentRoute: () => rootRouteImport, } as any) -const ApiFalBillableUnitsRoute = ApiFalBillableUnitsRouteImport.update({ - id: '/api/fal-billable-units', - path: '/api/fal-billable-units', - getParentRoute: () => rootRouteImport, -} as any) const ApiChatRoute = ApiChatRouteImport.update({ id: '/api/chat', path: '/api/chat', @@ -271,7 +265,6 @@ export interface FileRoutesByFullPath { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute - '/api/fal-billable-units': typeof ApiFalBillableUnitsRoute '/api/image': typeof ApiImageRouteWithChildren '/api/mcp-lifecycle-test': typeof ApiMcpLifecycleTestRoute '/api/mcp-managed-test': typeof ApiMcpManagedTestRoute @@ -313,7 +306,6 @@ export interface FileRoutesByTo { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute - '/api/fal-billable-units': typeof ApiFalBillableUnitsRoute '/api/image': typeof ApiImageRouteWithChildren '/api/mcp-lifecycle-test': typeof ApiMcpLifecycleTestRoute '/api/mcp-managed-test': typeof ApiMcpManagedTestRoute @@ -356,7 +348,6 @@ export interface FileRoutesById { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute - '/api/fal-billable-units': typeof ApiFalBillableUnitsRoute '/api/image': typeof ApiImageRouteWithChildren '/api/mcp-lifecycle-test': typeof ApiMcpLifecycleTestRoute '/api/mcp-managed-test': typeof ApiMcpManagedTestRoute @@ -400,7 +391,6 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' - | '/api/fal-billable-units' | '/api/image' | '/api/mcp-lifecycle-test' | '/api/mcp-managed-test' @@ -442,7 +432,6 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' - | '/api/fal-billable-units' | '/api/image' | '/api/mcp-lifecycle-test' | '/api/mcp-managed-test' @@ -484,7 +473,6 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' - | '/api/fal-billable-units' | '/api/image' | '/api/mcp-lifecycle-test' | '/api/mcp-managed-test' @@ -527,7 +515,6 @@ export interface RootRouteChildren { ApiArktypeToolWireRoute: typeof ApiArktypeToolWireRoute ApiAudioRoute: typeof ApiAudioRouteWithChildren ApiChatRoute: typeof ApiChatRoute - ApiFalBillableUnitsRoute: typeof ApiFalBillableUnitsRoute ApiImageRoute: typeof ApiImageRouteWithChildren ApiMcpLifecycleTestRoute: typeof ApiMcpLifecycleTestRoute ApiMcpManagedTestRoute: typeof ApiMcpManagedTestRoute @@ -746,13 +733,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiImageRouteImport parentRoute: typeof rootRouteImport } - '/api/fal-billable-units': { - id: '/api/fal-billable-units' - path: '/api/fal-billable-units' - fullPath: '/api/fal-billable-units' - preLoaderRoute: typeof ApiFalBillableUnitsRouteImport - parentRoute: typeof rootRouteImport - } '/api/chat': { id: '/api/chat' path: '/api/chat' @@ -908,7 +888,6 @@ const rootRouteChildren: RootRouteChildren = { ApiArktypeToolWireRoute: ApiArktypeToolWireRoute, ApiAudioRoute: ApiAudioRouteWithChildren, ApiChatRoute: ApiChatRoute, - ApiFalBillableUnitsRoute: ApiFalBillableUnitsRoute, ApiImageRoute: ApiImageRouteWithChildren, ApiMcpLifecycleTestRoute: ApiMcpLifecycleTestRoute, ApiMcpManagedTestRoute: ApiMcpManagedTestRoute, diff --git a/testing/e2e/src/routes/api.fal-billable-units.ts b/testing/e2e/src/routes/api.fal-billable-units.ts deleted file mode 100644 index d271b5c10..000000000 --- a/testing/e2e/src/routes/api.fal-billable-units.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { createFileRoute } from '@tanstack/react-router' -import { generateImage } from '@tanstack/ai' -import { falImage } from '@tanstack/ai-fal' - -const LLMOCK_DEFAULT_BASE = process.env.LLMOCK_URL || 'http://127.0.0.1:4010' - -/** fal hardcodes its queue endpoint; we redirect these to the aimock mount. */ -const FAL_QUEUE_PREFIX = 'https://queue.fal.run/' - -/** - * Drives the fal image adapter against the `/fal-queue` aimock mount, which - * stamps `x-fal-billable-units` on the result fetch. The companion spec asserts - * those units reach `result.usage.unitsBilled` — proving the adapter's billing - * capture forwards fal's real billed quantity. - * - * fal's queue URLs are not configurable, so we pass a per-request `fetch` to the - * adapter that rewrites `queue.fal.run` requests to the mock. This is scoped to - * the adapter instance (no global mutation), so concurrent requests can't - * interfere with each other. Non-fal requests pass through untouched. - */ -export const Route = createFileRoute('/api/fal-billable-units')({ - server: { - handlers: { - POST: async () => { - const mockBase = `${LLMOCK_DEFAULT_BASE}/fal-queue/` - const redirectFetch = (( - input: RequestInfo | URL, - init?: RequestInit, - ) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.href - : input.url - if (url.startsWith(FAL_QUEUE_PREFIX)) { - return fetch(mockBase + url.slice(FAL_QUEUE_PREFIX.length), init) - } - return fetch(input, init) - }) as typeof fetch - - try { - const adapter = falImage('fal-ai/flux/dev', { - apiKey: 'fal-e2e-dummy', - fetch: redirectFetch, - }) - const result = await generateImage({ - adapter, - prompt: 'a billed image', - }) - return new Response( - JSON.stringify({ ok: true, usage: result.usage }), - { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }, - ) - } catch (error) { - return new Response( - JSON.stringify({ - ok: false, - error: error instanceof Error ? error.message : String(error), - }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, - ) - } - }, - }, - }, -}) diff --git a/testing/e2e/tests/fal-billable-units.spec.ts b/testing/e2e/tests/fal-billable-units.spec.ts deleted file mode 100644 index 026072e43..000000000 --- a/testing/e2e/tests/fal-billable-units.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { test, expect } from './fixtures' - -/** - * Verifies that fal's `x-fal-billable-units` result header reaches - * `result.usage.unitsBilled`. The `/api/fal-billable-units` route drives the - * fal image adapter against the `/fal-queue` aimock mount, which mimics fal's - * queue protocol (submit → poll → result) and stamps the billing headers on the - * result fetch. The adapter's `config.fetch` reads those headers and surfaces - * the real billed quantity on the generation result — the end-to-end proof that - * media-generation spend is recoverable through the SDK without an app-side - * `fetch` interceptor. - */ -test.describe('fal — billable units', () => { - test('x-fal-billable-units reaches result.usage.unitsBilled', async ({ - request, - }) => { - const res = await request.post('/api/fal-billable-units') - expect(res.ok()).toBe(true) - - const { ok, usage, error } = (await res.json()) as { - ok: boolean - error?: string - usage?: { - promptTokens?: number - completionTokens?: number - totalTokens?: number - unitsBilled?: number - } - } - - expect(error ?? null).toBeNull() - expect(ok).toBe(true) - expect(usage).toMatchObject({ - promptTokens: 0, - completionTokens: 0, - totalTokens: 0, - unitsBilled: 4, - }) - }) -})