From a6b50d8d69ac8fb7962bc643090d83e052f60f9f Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Tue, 8 Apr 2025 16:27:43 +0900 Subject: [PATCH 1/7] fix --- packages/cdk/lambda/utils/models.ts | 38 +++ packages/common/src/application/model.ts | 1 + packages/types/src/video.d.ts | 1 + .../web/public/locales/translation/en.yaml | 1 + .../web/public/locales/translation/ja.yaml | 1 + packages/web/src/pages/GenerateVideoPage.tsx | 262 ++++++++++++------ 6 files changed, 214 insertions(+), 90 deletions(-) diff --git a/packages/cdk/lambda/utils/models.ts b/packages/cdk/lambda/utils/models.ts index e85e852fa..5bbf47efe 100644 --- a/packages/cdk/lambda/utils/models.ts +++ b/packages/cdk/lambda/utils/models.ts @@ -719,6 +719,41 @@ const createBodyVideoNovaReel = (params: GenerateVideoParams) => { }; }; +const createBodyVideoNovaReelV11 = (params: GenerateVideoParams) => { + if (params.taskType === 'TEXT_VIDEO') { + return { + taskType: 'TEXT_VIDEO', + textToVideoParams: { + text: params.prompt, + images: params.images, + }, + videoGenerationConfig: { + durationSeconds: params.durationSeconds, + fps: params.fps, + dimension: params.dimension, + seed: params.seed, + }, + }; + } else if (params.taskType === 'MULTI_SHOT_AUTOMATED') { + return { + taskType: 'MULTI_SHOT_AUTOMATED', + multiShotAutomatedParams: { + text: params.prompt, + }, + videoGenerationConfig: { + durationSeconds: params.durationSeconds, + fps: params.fps, + dimension: params.dimension, + seed: params.seed, + }, + }; + } else if (params.taskType === 'MULTI_SHOT_MANUAL') { + throw new Error('Not implemented yet'); + } else { + throw new Error(`Unknown task type ${params.taskType}`); + } +}; + const createBodyVideoLumaRayV2 = (params: GenerateVideoParams) => { return { prompt: params.prompt, @@ -1261,6 +1296,9 @@ export const BEDROCK_VIDEO_GEN_MODELS: { 'amazon.nova-reel-v1:0': { createBodyVideo: createBodyVideoNovaReel, }, + 'amazon.nova-reel-v1:1': { + createBodyVideo: createBodyVideoNovaReelV11, + }, 'luma.ray-v2:0': { createBodyVideo: createBodyVideoLumaRayV2, }, diff --git a/packages/common/src/application/model.ts b/packages/common/src/application/model.ts index 2b1b6b790..1302c423f 100644 --- a/packages/common/src/application/model.ts +++ b/packages/common/src/application/model.ts @@ -161,6 +161,7 @@ export const modelFeatureFlags: Record = { // === Video === 'amazon.nova-reel-v1:0': MODEL_FEATURE.VIDEO_GEN, + 'amazon.nova-reel-v1:1': MODEL_FEATURE.VIDEO_GEN, 'luma.ray-v2:0': MODEL_FEATURE.VIDEO_GEN, // === Embedding === diff --git a/packages/types/src/video.d.ts b/packages/types/src/video.d.ts index 1575312a3..ce7a1c9f2 100644 --- a/packages/types/src/video.d.ts +++ b/packages/types/src/video.d.ts @@ -1,6 +1,7 @@ import { PrimaryKey } from './base'; export type GenerateVideoParams = { + taskType?: string; prompt: string; durationSeconds: number; dimension?: string; diff --git a/packages/web/public/locales/translation/en.yaml b/packages/web/public/locales/translation/en.yaml index 9bf06e701..89bd55f66 100644 --- a/packages/web/public/locales/translation/en.yaml +++ b/packages/web/public/locales/translation/en.yaml @@ -1086,6 +1086,7 @@ video: play: Play prompt: Prompt status: Status + taskType: Task Type upload: Upload uploadImage: Upload Image uploadImageHelp: Specify the image used in the first frame of the video. diff --git a/packages/web/public/locales/translation/ja.yaml b/packages/web/public/locales/translation/ja.yaml index a4d2fa8b2..d3d03e684 100644 --- a/packages/web/public/locales/translation/ja.yaml +++ b/packages/web/public/locales/translation/ja.yaml @@ -949,6 +949,7 @@ video: play: 再生 prompt: プロンプト status: ステータス + taskType: タスクタイプ upload: アップロード uploadImage: 画像アップロード uploadImageHelp: 動画の 1 フレーム目に使われる画像を指定できます diff --git a/packages/web/src/pages/GenerateVideoPage.tsx b/packages/web/src/pages/GenerateVideoPage.tsx index 508e4dc04..ddf320dee 100644 --- a/packages/web/src/pages/GenerateVideoPage.tsx +++ b/packages/web/src/pages/GenerateVideoPage.tsx @@ -28,34 +28,77 @@ import queryString from 'query-string'; import { toast } from 'sonner'; import { useTranslation } from 'react-i18next'; -// eslint-disable-next-line @typescript-eslint/no-explicit-any -const MODEL_PARAMS: Record = { - 'amazon.nova-reel-v1:0': { - dimension: ['1280x720'], - durationSeconds: [6], - fps: [24], - seed: 0, - // type of the images below: FileLimit & { label: string } - images: { - accept: { - image: ['.jpg', '.jpeg', '.png'], +const TASK_TYPES = (modelId: string): string[] => { + if (modelId === 'amazon.nova-reel-v1:1') { + return ['TEXT_VIDEO', 'MULTI_SHOT_AUTOMATED']; + } + + return []; +}; + +const MODEL_PARAMS = ( + modelId: string, + taskType: string + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): Record => { + if (modelId === 'amazon.nova-reel-v1:0') { + return { + dimension: ['1280x720'], + durationSeconds: [6], + fps: [24], + seed: 0, + images: { + accept: { + image: ['.jpg', '.jpeg', '.png'], + }, + maxImageFileCount: 1, + maxImageFileSizeMB: 10, + strictImageDimensions: [{ width: 1280, height: 720 }], }, - maxImageFileCount: 1, - maxImageFileSizeMB: 10, - strictImageDimensions: [{ width: 1280, height: 720 }], - }, - }, - 'luma.ray-v2:0': { - resolution: ['540p', '720p'], - durationSeconds: [5, 9], - aspectRatio: ['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21'], - loop: false, - }, + }; + } else if (modelId === 'amazon.nova-reel-v1:1') { + if (taskType === 'TEXT_VIDEO') { + return { + dimension: ['1280x720'], + durationSeconds: [6], + fps: [24], + seed: 0, + images: { + accept: { + image: ['.jpg', '.jpeg', '.png'], + }, + maxImageFileCount: 1, + maxImageFileSizeMB: 10, + strictImageDimensions: [{ width: 1280, height: 720 }], + }, + }; + } else if (taskType === 'MULTI_SHOT_AUTOMATED') { + return { + dimension: ['1280x720'], + durationSeconds: [ + 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, 96, 102, 108, + 114, 120, + ], + fps: [24], + seed: 0, + }; + } + } else if (modelId === 'luma.ray-v2:0') { + return { + resolution: ['540p', '720p'], + durationSeconds: [5, 9], + aspectRatio: ['1:1', '16:9', '9:16', '4:3', '3:4', '21:9', '9:21'], + loop: false, + }; + } + + return {}; }; type StateType = Omit, 'images'> & { videoGenModelId: string; setVideoGenModelId: (s: string) => void; + setTaskType: (s: string) => void; setPrompt: (s: string) => void; setDimension: (s: string) => void; setDurationSeconds: (n: number) => void; @@ -70,6 +113,7 @@ type StateType = Omit, 'images'> & { const useGenerateVideoPageState = create((set) => { const INIT_STATE = { videoGenModelId: '', + taskType: '', prompt: '', dimension: '', durationSeconds: 0, @@ -86,6 +130,11 @@ const useGenerateVideoPageState = create((set) => { videoGenModelId: s, })); }, + setTaskType: (s: string) => { + set(() => ({ + taskType: s, + })); + }, setPrompt: (s: string) => { set(() => ({ prompt: s, @@ -137,6 +186,8 @@ const GenerateVideoPage: React.FC = () => { const { videoGenModelId, setVideoGenModelId, + taskType, + setTaskType, prompt, setPrompt, dimension, @@ -188,16 +239,18 @@ const GenerateVideoPage: React.FC = () => { const onChangeFiles = useCallback( (e: React.ChangeEvent) => { const files = e.target.files; - const fileLimit = MODEL_PARAMS[videoGenModelId]!.images; - const accept = - MODEL_PARAMS[videoGenModelId].images?.accept?.image?.join(','); + const fileLimit = MODEL_PARAMS(videoGenModelId, taskType).images; + const accept = MODEL_PARAMS( + videoGenModelId, + taskType + ).images?.accept?.image?.join(','); if (files) { // Reflect the files and upload uploadFiles(Array.from(files), fileLimit, accept); } }, - [videoGenModelId, uploadFiles] + [videoGenModelId, uploadFiles, taskType] ); useEffect(() => { @@ -222,37 +275,45 @@ const GenerateVideoPage: React.FC = () => { if (videoGenModelId === '') { setVideoGenModelId(videoGenModelIds[0]); } else { - if (MODEL_PARAMS[videoGenModelId].dimension) { - setDimension(MODEL_PARAMS[videoGenModelId].dimension[0]); + if (!TASK_TYPES(videoGenModelId).includes(taskType)) { + setTaskType(TASK_TYPES(videoGenModelId)[0] ?? ''); } - if (MODEL_PARAMS[videoGenModelId].durationSeconds) { - setDurationSeconds(MODEL_PARAMS[videoGenModelId].durationSeconds[0]!); + if (MODEL_PARAMS(videoGenModelId, taskType).dimension) { + setDimension(MODEL_PARAMS(videoGenModelId, taskType).dimension[0]); } - if (MODEL_PARAMS[videoGenModelId].fps) { - setFps(MODEL_PARAMS[videoGenModelId].fps[0]!); + if (MODEL_PARAMS(videoGenModelId, taskType).durationSeconds) { + setDurationSeconds( + MODEL_PARAMS(videoGenModelId, taskType).durationSeconds[0] + ); } - if (MODEL_PARAMS[videoGenModelId].seed !== undefined) { - setSeed(MODEL_PARAMS[videoGenModelId].seed); + if (MODEL_PARAMS(videoGenModelId, taskType).fps) { + setFps(MODEL_PARAMS(videoGenModelId, taskType).fps[0]!); } - if (MODEL_PARAMS[videoGenModelId].resolution) { - setResolution(MODEL_PARAMS[videoGenModelId].resolution[0]!); + if (MODEL_PARAMS(videoGenModelId, taskType).seed !== undefined) { + setSeed(MODEL_PARAMS(videoGenModelId, taskType).seed); } - if (MODEL_PARAMS[videoGenModelId].aspectRatio) { - setAspectRatio(MODEL_PARAMS[videoGenModelId].aspectRatio[0]!); + if (MODEL_PARAMS(videoGenModelId, taskType).resolution) { + setResolution(MODEL_PARAMS(videoGenModelId, taskType).resolution[0]!); } - if (MODEL_PARAMS[videoGenModelId].loop !== undefined) { - setLoop(MODEL_PARAMS[videoGenModelId].loop); + if (MODEL_PARAMS(videoGenModelId, taskType).aspectRatio) { + setAspectRatio(MODEL_PARAMS(videoGenModelId, taskType).aspectRatio[0]!); + } + + if (MODEL_PARAMS(videoGenModelId, taskType).loop !== undefined) { + setLoop(MODEL_PARAMS(videoGenModelId, taskType).loop); } } }, [ videoGenModelId, + taskType, setVideoGenModelId, + setTaskType, setDimension, setDurationSeconds, setFps, @@ -268,6 +329,7 @@ const GenerateVideoPage: React.FC = () => { try { const params: GenerateVideoParams = { + taskType, prompt, durationSeconds, dimension, @@ -277,7 +339,8 @@ const GenerateVideoPage: React.FC = () => { aspectRatio, loop, images: - MODEL_PARAMS[videoGenModelId].images && uploadedFiles.length > 0 + MODEL_PARAMS(videoGenModelId, taskType).images && + uploadedFiles.length > 0 ? uploadedFiles.map((f) => { return { format: f.base64EncodedData!.includes('data:image/png') @@ -291,8 +354,6 @@ const GenerateVideoPage: React.FC = () => { : undefined, }; - console.log(params); - await generate( params, videoGenModels.find((m) => m.modelId === videoGenModelId) @@ -311,6 +372,7 @@ const GenerateVideoPage: React.FC = () => { setIsGenerating(false); }, [ + taskType, generate, prompt, durationSeconds, @@ -393,7 +455,19 @@ const GenerateVideoPage: React.FC = () => { fullWidth /> - {MODEL_PARAMS[videoGenModelId] && ( + {TASK_TYPES(videoGenModelId).length > 0 && ( +