Skip to content

Commit 3b36d27

Browse files
AdirAmsalemAdirAmsalem
authored
feat(realtime): cap capture FPS at 30 via ideal/max constraint (#141)
## Summary Realtime models now request capture FPS via an `{ ideal: 30, max: 30 }` constraint instead of a hardcoded per-model rate (was 20 for Lucy 2.1, 22 for Lucy Restyle 2). The camera delivers its native rate (typically 24/25/30) and the server resamples to whatever the model needs. ## Why - Decouples the SDK from server-side FPS support — bumping a model's supported rate no longer requires a client release. - Avoids client-side frame duplication / dropping when the camera's native rate doesn't match the model's rate. Resampling now happens once, on the server, where it can be done correctly with PTS-aware logic. - Keeps a steady frame supply ahead of inference so the GPU isn't waiting on the next frame. ## User-facing impact None for the documented pattern — `frameRate: model.fps` in `getUserMedia` still compiles and works, because `MediaTrackConstraints["frameRate"]` natively accepts both `number` and constraint-object forms: ```ts const model = models.realtime("lucy-2.1"); const stream = await navigator.mediaDevices.getUserMedia({ video: { frameRate: model.fps, width: model.width, height: model.height }, }); ``` Video/image model `fps` stays a `number` (it's output-rate metadata, not a capture constraint); the type narrowing is enforced at the `models.video(...)` / `models.image(...)` getter boundary so consumers of those registries see no change. ## Test plan - [x] `pnpm typecheck` passes - [x] `pnpm test` — 199/199 unit tests pass (added regression guard that video/image `fps` stays typed as `number`) - [ ] `pnpm test:e2e:realtime` against staging across all realtime models - [ ] Manual browser check: `track.getSettings().frameRate` is ≤ 30 across cameras with native 24/25/30/60 rates <!-- CURSOR_SUMMARY --> --- > [!NOTE] > **Medium Risk** > Changes the public `models.realtime(...).fps` shape from a number to a constraint object, which may break consumers doing numeric operations or passing it to APIs expecting a scalar. Runtime behavior changes capture/mirroring FPS handling, so regressions would show up as altered stream cadence or compatibility issues in edge browsers. > > **Overview** > **Realtime model FPS is now represented as a capture constraint object** rather than a fixed per-model number. The realtime registry entries switch to `fps: { ideal: 30, max: 30 }`, and `modelDefinitionSchema`/`ModelDefinition` are updated to accept either a number or constraint-object form. > > Adds `resolveFpsNumber()` to derive a scalar FPS when needed, and uses it in `realtime/client.ts` when creating mirrored streams. Tests are updated to assert the new realtime `fps` shape, add coverage for `resolveFpsNumber`, and ensure `models.video(...)`/`models.image(...)` still expose `fps` as a `number`; the realtime E2E synthetic stream now captures at 30 FPS explicitly. > > <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 49342d1. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup> <!-- /CURSOR_SUMMARY -->
1 parent da6e227 commit 3b36d27

4 files changed

Lines changed: 82 additions & 31 deletions

File tree

‎packages/sdk/src/realtime/client.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
import { z } from "zod";
2-
import { type CustomModelDefinition, type ModelDefinition, modelDefinitionSchema } from "../shared/model";
2+
import {
3+
type CustomModelDefinition,
4+
type ModelDefinition,
5+
modelDefinitionSchema,
6+
resolveFpsNumber,
7+
} from "../shared/model";
38
import { modelStateSchema } from "../shared/types";
49
import { classifyWebrtcError, type DecartSDKError } from "../utils/errors";
510
import type { Logger } from "../utils/logger";
@@ -153,7 +158,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
153158
try {
154159
const firstVideoTrack = inputStream.getVideoTracks?.()[0];
155160
if (firstVideoTrack && (mirror === true || shouldMirrorTrack(firstVideoTrack))) {
156-
mirroredStream = createMirroredStream(inputStream, { fps: options.model.fps });
161+
mirroredStream = createMirroredStream(inputStream, { fps: resolveFpsNumber(options.model.fps) });
157162
inputStream = mirroredStream.stream;
158163
} else if (mirror === true && !firstVideoTrack) {
159164
logger.warn("mirror: true requested but no video track was found on the input stream");

‎packages/sdk/src/shared/model.ts‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -273,16 +273,23 @@ export const modelInputSchemas = {
273273

274274
export type ModelInputSchemas = typeof modelInputSchemas;
275275

276+
export type ModelFps = number | { max?: number; min?: number; ideal?: number; exact?: number };
277+
276278
export type ModelDefinition<T extends Model = Model> = {
277279
name: T;
278280
urlPath: string;
279281
queueUrlPath?: string;
280-
fps: number;
282+
fps: ModelFps;
281283
width: number;
282284
height: number;
283285
inputSchema: T extends keyof ModelInputSchemas ? ModelInputSchemas[T] : z.ZodTypeAny;
284286
};
285287

288+
export function resolveFpsNumber(fps: ModelFps, fallback = 30): number {
289+
if (typeof fps === "number") return fps;
290+
return fps.ideal ?? fps.max ?? fps.exact ?? fps.min ?? fallback;
291+
}
292+
286293
/**
287294
* A model definition with an arbitrary (non-registry) model name.
288295
* Use this when providing your own model configuration.
@@ -297,20 +304,28 @@ export type CustomModelDefinition = Omit<ModelDefinition, "name" | "inputSchema"
297304
* Only image models support the sync/process API.
298305
* Requires `queueUrlPath` to distinguish from realtime definitions of the same model name.
299306
*/
300-
export type ImageModelDefinition = ModelDefinition<ImageModels> & { queueUrlPath: string };
307+
export type ImageModelDefinition = ModelDefinition<ImageModels> & { fps: number; queueUrlPath: string };
301308

302309
/**
303310
* Type alias for model definitions that support queue processing.
304311
* Only video models support the queue API.
305312
* Requires `queueUrlPath` to distinguish from realtime definitions of the same model name.
306313
*/
307-
export type VideoModelDefinition = ModelDefinition<VideoModels> & { queueUrlPath: string };
314+
export type VideoModelDefinition = ModelDefinition<VideoModels> & { fps: number; queueUrlPath: string };
308315

309316
export const modelDefinitionSchema = z.object({
310317
name: z.string(),
311318
urlPath: z.string(),
312319
queueUrlPath: z.string().optional(),
313-
fps: z.number().min(1),
320+
fps: z.union([
321+
z.number().min(1),
322+
z.object({
323+
max: z.number().min(1).optional(),
324+
min: z.number().min(1).optional(),
325+
ideal: z.number().min(1).optional(),
326+
exact: z.number().min(1).optional(),
327+
}),
328+
]),
314329
width: z.number().min(1),
315330
height: z.number().min(1),
316331
inputSchema: z.any().optional(),
@@ -322,31 +337,31 @@ const _models = {
322337
"lucy-2.1": {
323338
urlPath: "/v1/stream",
324339
name: "lucy-2.1" as const,
325-
fps: 20,
340+
fps: { ideal: 30, max: 30 },
326341
width: 1088,
327342
height: 624,
328343
inputSchema: z.object({}),
329344
},
330345
"lucy-2.1-vton": {
331346
urlPath: "/v1/stream",
332347
name: "lucy-2.1-vton" as const,
333-
fps: 20,
348+
fps: { ideal: 30, max: 30 },
334349
width: 1088,
335350
height: 624,
336351
inputSchema: z.object({}),
337352
},
338353
"lucy-vton-2": {
339354
urlPath: "/v1/stream",
340355
name: "lucy-vton-2" as const,
341-
fps: 20,
356+
fps: { ideal: 30, max: 30 },
342357
width: 1088,
343358
height: 624,
344359
inputSchema: z.object({}),
345360
},
346361
"lucy-restyle-2": {
347362
urlPath: "/v1/stream",
348363
name: "lucy-restyle-2" as const,
349-
fps: 22,
364+
fps: { ideal: 30, max: 30 },
350365
width: 1280,
351366
height: 704,
352367
inputSchema: z.object({}),
@@ -355,7 +370,7 @@ const _models = {
355370
"lucy-latest": {
356371
urlPath: "/v1/stream",
357372
name: "lucy-latest" as const,
358-
fps: 20,
373+
fps: { ideal: 30, max: 30 },
359374
width: 1088,
360375
height: 624,
361376
inputSchema: z.object({}),
@@ -364,15 +379,15 @@ const _models = {
364379
"lucy-vton-latest": {
365380
urlPath: "/v1/stream",
366381
name: "lucy-vton-latest" as const,
367-
fps: 20,
382+
fps: { ideal: 30, max: 30 },
368383
width: 1088,
369384
height: 624,
370385
inputSchema: z.object({}),
371386
},
372387
"lucy-restyle-latest": {
373388
urlPath: "/v1/stream",
374389
name: "lucy-restyle-latest" as const,
375-
fps: 22,
390+
fps: { ideal: 30, max: 30 },
376391
width: 1280,
377392
height: 704,
378393
inputSchema: z.object({}),
@@ -381,23 +396,23 @@ const _models = {
381396
mirage_v2: {
382397
urlPath: "/v1/stream",
383398
name: "mirage_v2" as const,
384-
fps: 22,
399+
fps: { ideal: 30, max: 30 },
385400
width: 1280,
386401
height: 704,
387402
inputSchema: z.object({}),
388403
},
389404
"lucy-vton": {
390405
urlPath: "/v1/stream",
391406
name: "lucy-vton" as const,
392-
fps: 20,
407+
fps: { ideal: 30, max: 30 },
393408
width: 1088,
394409
height: 624,
395410
inputSchema: z.object({}),
396411
},
397412
"lucy-2.1-vton-2": {
398413
urlPath: "/v1/stream",
399414
name: "lucy-2.1-vton-2" as const,
400-
fps: 20,
415+
fps: { ideal: 30, max: 30 },
401416
width: 1088,
402417
height: 624,
403418
inputSchema: z.object({}),
@@ -616,26 +631,26 @@ export const models = {
616631
* - `"lucy-vton-2"` - Virtual try-on 2 video editing
617632
* - `"lucy-restyle-2"` - Video restyling
618633
*/
619-
video: <T extends VideoModels>(model: T): ModelDefinition<T> & { queueUrlPath: string } => {
634+
video: <T extends VideoModels>(model: T): ModelDefinition<T> & { fps: number; queueUrlPath: string } => {
620635
warnDeprecated(model);
621636
const modelDefinition = _models.video[model];
622637
if (!modelDefinition) {
623638
throw createModelNotFoundError(model);
624639
}
625-
return modelDefinition as ModelDefinition<T> & { queueUrlPath: string };
640+
return modelDefinition as ModelDefinition<T> & { fps: number; queueUrlPath: string };
626641
},
627642
/**
628643
* Get an image model identifier.
629644
*
630645
* Available options:
631646
* - `"lucy-image-2"` - Image-to-image editing
632647
*/
633-
image: <T extends ImageModels>(model: T): ModelDefinition<T> & { queueUrlPath: string } => {
648+
image: <T extends ImageModels>(model: T): ModelDefinition<T> & { fps: number; queueUrlPath: string } => {
634649
warnDeprecated(model);
635650
const modelDefinition = _models.image[model];
636651
if (!modelDefinition) {
637652
throw createModelNotFoundError(model);
638653
}
639-
return modelDefinition as ModelDefinition<T> & { queueUrlPath: string };
654+
return modelDefinition as ModelDefinition<T> & { fps: number; queueUrlPath: string };
640655
},
641656
};

‎packages/sdk/tests/e2e-realtime.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ declare const __DECART_API_KEY__: string;
33
import { createDecartClient, type DecartSDKError, models, type RealTimeModels } from "@decartai/sdk";
44
import { beforeAll, describe, expect, it } from "vitest";
55

6-
function createSyntheticStream(fps: number, width: number, height: number): MediaStream {
6+
function createSyntheticStream(width: number, height: number): MediaStream {
77
const canvas = document.createElement("canvas");
88
canvas.width = width;
99
canvas.height = height;
10-
return canvas.captureStream(fps);
10+
return canvas.captureStream(30);
1111
}
1212

1313
const REALTIME_MODELS: RealTimeModels[] = [
@@ -37,7 +37,7 @@ describe.concurrent("Realtime E2E Tests", { timeout: TIMEOUT, retry: 2 }, () =>
3737
for (const modelName of REALTIME_MODELS) {
3838
it(modelName, async () => {
3939
const model = models.realtime(modelName);
40-
const stream = createSyntheticStream(model.fps, model.width, model.height);
40+
const stream = createSyntheticStream(model.width, model.height);
4141

4242
let remoteStreamReceived = false;
4343

‎packages/sdk/tests/unit.test.ts‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import {
2727
imageModels,
2828
modelSchema,
2929
realtimeModels,
30+
resolveFpsNumber,
3031
videoModels,
3132
} from "../src/shared/model.js";
3233

@@ -1159,7 +1160,7 @@ describe("Lucy 2.1 realtime", () => {
11591160

11601161
it("has correct fps", () => {
11611162
const lucyModel = models.realtime("lucy-2.1");
1162-
expect(lucyModel.fps).toBe(20);
1163+
expect(lucyModel.fps).toEqual({ ideal: 30, max: 30 });
11631164
});
11641165

11651166
it("is recognized as a realtime model", () => {
@@ -1168,6 +1169,36 @@ describe("Lucy 2.1 realtime", () => {
11681169
});
11691170
});
11701171

1172+
describe("resolveFpsNumber", () => {
1173+
it("returns the number for a scalar fps value", () => {
1174+
expect(resolveFpsNumber(25)).toBe(25);
1175+
});
1176+
1177+
it("prefers ideal, then max, then exact, then min", () => {
1178+
expect(resolveFpsNumber({ ideal: 30, max: 60 })).toBe(30);
1179+
expect(resolveFpsNumber({ max: 30 })).toBe(30);
1180+
expect(resolveFpsNumber({ exact: 24 })).toBe(24);
1181+
expect(resolveFpsNumber({ min: 15 })).toBe(15);
1182+
});
1183+
1184+
it("falls back to 30 when the constraint object is empty", () => {
1185+
expect(resolveFpsNumber({})).toBe(30);
1186+
});
1187+
1188+
it("resolves the realtime model default to 30 for canvas.captureStream-style consumers", () => {
1189+
expect(resolveFpsNumber(models.realtime("lucy-2.1").fps)).toBe(30);
1190+
});
1191+
});
1192+
1193+
describe("Model fps types", () => {
1194+
it("video and image model fps stays typed as number", () => {
1195+
// Compile-time check: arithmetic on .fps must work for video/image models without narrowing.
1196+
const videoFps: number = models.video("lucy-clip").fps;
1197+
const imageFps: number = models.image("lucy-image-2").fps;
1198+
expect(videoFps + imageFps).toBeGreaterThan(0);
1199+
});
1200+
});
1201+
11711202
describe("WebRTCConnection", () => {
11721203
describe("setImageBase64", () => {
11731204
it("rejects immediately when WebSocket is not open", async () => {
@@ -3596,7 +3627,7 @@ describe("Canonical Model Names", () => {
35963627
const model = models.realtime("lucy-2.1");
35973628
expect(model.name).toBe("lucy-2.1");
35983629
expect(model.urlPath).toBe("/v1/stream");
3599-
expect(model.fps).toBe(20);
3630+
expect(model.fps).toEqual({ ideal: 30, max: 30 });
36003631
expect(model.width).toBe(1088);
36013632
expect(model.height).toBe(624);
36023633
});
@@ -3605,7 +3636,7 @@ describe("Canonical Model Names", () => {
36053636
const model = models.realtime("lucy-2.1-vton");
36063637
expect(model.name).toBe("lucy-2.1-vton");
36073638
expect(model.urlPath).toBe("/v1/stream");
3608-
expect(model.fps).toBe(20);
3639+
expect(model.fps).toEqual({ ideal: 30, max: 30 });
36093640
expect(model.width).toBe(1088);
36103641
expect(model.height).toBe(624);
36113642
});
@@ -3614,15 +3645,15 @@ describe("Canonical Model Names", () => {
36143645
const model = models.realtime("lucy-vton-2");
36153646
expect(model.name).toBe("lucy-vton-2");
36163647
expect(model.urlPath).toBe("/v1/stream");
3617-
expect(model.fps).toBe(20);
3648+
expect(model.fps).toEqual({ ideal: 30, max: 30 });
36183649
expect(model.width).toBe(1088);
36193650
expect(model.height).toBe(624);
36203651
});
36213652

36223653
it("lucy-restyle-2 canonical name works", () => {
36233654
const model = models.realtime("lucy-restyle-2");
36243655
expect(model.name).toBe("lucy-restyle-2");
3625-
expect(model.fps).toBe(22);
3656+
expect(model.fps).toEqual({ ideal: 30, max: 30 });
36263657
});
36273658
});
36283659

@@ -3682,7 +3713,7 @@ describe("Canonical Model Names", () => {
36823713
const model = models.realtime("lucy-latest");
36833714
expect(model.name).toBe("lucy-latest");
36843715
expect(model.urlPath).toBe("/v1/stream");
3685-
expect(model.fps).toBe(20);
3716+
expect(model.fps).toEqual({ ideal: 30, max: 30 });
36863717
expect(model.width).toBe(1088);
36873718
expect(model.height).toBe(624);
36883719
});
@@ -3691,7 +3722,7 @@ describe("Canonical Model Names", () => {
36913722
const model = models.realtime("lucy-vton-latest");
36923723
expect(model.name).toBe("lucy-vton-latest");
36933724
expect(model.urlPath).toBe("/v1/stream");
3694-
expect(model.fps).toBe(20);
3725+
expect(model.fps).toEqual({ ideal: 30, max: 30 });
36953726
expect(model.width).toBe(1088);
36963727
expect(model.height).toBe(624);
36973728
});
@@ -3700,7 +3731,7 @@ describe("Canonical Model Names", () => {
37003731
const model = models.realtime("lucy-restyle-latest");
37013732
expect(model.name).toBe("lucy-restyle-latest");
37023733
expect(model.urlPath).toBe("/v1/stream");
3703-
expect(model.fps).toBe(22);
3734+
expect(model.fps).toEqual({ ideal: 30, max: 30 });
37043735
expect(model.width).toBe(1280);
37053736
expect(model.height).toBe(704);
37063737
});

0 commit comments

Comments
 (0)