diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 400993d..4c08ac1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: python-version: ${{ inputs.python-version || '3.12' }} - run: | if [ -d "packages" ]; then - uv run mypy -p celeste && uv run mypy tests/ && uv run mypy packages/image-generation packages/text-generation + uv run mypy -p celeste && uv run mypy tests/ && uv run mypy packages/image-generation packages/text-generation packages/video-generation else uv run mypy -p celeste && uv run mypy tests/ fi diff --git a/Makefile b/Makefile index 97e2f4b..0034405 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ format: # Type checking (fail fast on any error) typecheck: - @uv run mypy -p celeste && uv run mypy tests/ && uv run mypy packages/image-generation packages/text-generation + @uv run mypy -p celeste && uv run mypy tests/ && uv run mypy packages/image-generation packages/text-generation packages/video-generation # Testing test: diff --git a/packages/image-generation/pyproject.toml b/packages/image-generation/pyproject.toml index f146397..7fb3971 100644 --- a/packages/image-generation/pyproject.toml +++ b/packages/image-generation/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "celeste-image-generation" -version = "0.2.2" -description = "Type-safe image generation for Celeste AI. Unified interface for OpenAI, Google, ByteDance, and more" +version = "0.2.7" +description = "Image generation package for Celeste AI. Unified interface for all providers" authors = [{name = "Kamilbenkirane", email = "kamil@withceleste.ai"}] readme = "README.md" license = {text = "Apache-2.0"} diff --git a/packages/image-generation/src/celeste_image_generation/streaming.py b/packages/image-generation/src/celeste_image_generation/streaming.py index 5574a57..fda1bfe 100644 --- a/packages/image-generation/src/celeste_image_generation/streaming.py +++ b/packages/image-generation/src/celeste_image_generation/streaming.py @@ -17,7 +17,7 @@ class ImageGenerationStream( ): """Streaming for image generation.""" - def _parse_output( + def _parse_output( # type: ignore[override] self, chunks: list[ImageGenerationChunk], **parameters: Unpack[ImageGenerationParameters], diff --git a/packages/text-generation/pyproject.toml b/packages/text-generation/pyproject.toml index 335dda8..81b1001 100644 --- a/packages/text-generation/pyproject.toml +++ b/packages/text-generation/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "celeste-text-generation" -version = "0.2.2" -description = "Type-safe text generation for Celeste AI. Unified interface for OpenAI, Anthropic, Google, Mistral, Cohere, and more" +version = "0.2.7" +description = "Text generation package for Celeste AI. Unified interface for all providers" authors = [{name = "Kamilbenkirane", email = "kamil@withceleste.ai"}] readme = "README.md" license = {text = "Apache-2.0"} diff --git a/packages/text-generation/src/celeste_text_generation/streaming.py b/packages/text-generation/src/celeste_text_generation/streaming.py index 0b610cb..085422c 100644 --- a/packages/text-generation/src/celeste_text_generation/streaming.py +++ b/packages/text-generation/src/celeste_text_generation/streaming.py @@ -21,7 +21,7 @@ class TextGenerationStream( def _parse_chunk(self, event: dict[str, Any]) -> TextGenerationChunk | None: """Parse SSE event into Chunk (provider-specific).""" - def _parse_output( + def _parse_output( # type: ignore[override] self, chunks: list[TextGenerationChunk], **parameters: Unpack[TextGenerationParameters], diff --git a/packages/video-generation/README.md b/packages/video-generation/README.md new file mode 100644 index 0000000..7e56c9e --- /dev/null +++ b/packages/video-generation/README.md @@ -0,0 +1,79 @@ +
+ +# Celeste Logo Celeste Video Generation + +**Video Generation capability for Celeste AI** + +[![Python](https://img.shields.io/badge/Python-3.12+-blue?style=for-the-badge)](https://www.python.org/) +[![License](https://img.shields.io/badge/License-Apache_2.0-red?style=for-the-badge)](../../LICENSE) + +[Quick Start](#-quick-start) • [Documentation](https://withceleste.ai/docs) • [Request Provider](https://github.com/withceleste/celeste-python/issues/new) + +
+ +--- + +## 🚀 Quick Start + +```python +from celeste import create_client, Capability, Provider + +client = create_client( + capability=Capability.VIDEO_GENERATION, + provider=Provider.OPENAI, +) + +response = await client.generate(prompt="A cinematic video of a sunset over mountains") +print(response.content) +``` + +**Install:** +```bash +uv add "celeste-ai[video-generation]" +``` + +--- + +## Supported Providers + + +
+ +ByteDance +OpenAI +Google + + +**Missing a provider?** [Request it](https://github.com/withceleste/celeste-python/issues/new) – ⚡ **we ship fast**. + +
+ +--- + +**Streaming**: ❌ Not Supported + +**Parameters**: See [API Documentation](https://withceleste.ai/docs/api) for full parameter reference. + +--- + +## 🤝 Contributing + +See [CONTRIBUTING.md](../../CONTRIBUTING.md) for guidelines. + +**Request a provider:** [GitHub Issues](https://github.com/withceleste/celeste-python/issues/new) + +--- + +## 📄 License + +Apache 2.0 License – see [LICENSE](../../LICENSE) for details. + +--- + +
+ +**[Get Started](https://withceleste.ai/docs/quickstart)** • **[Documentation](https://withceleste.ai/docs)** • **[GitHub](https://github.com/withceleste/celeste-python)** + +Made with ❤️ by developers tired of framework lock-in + +
diff --git a/packages/video-generation/pyproject.toml b/packages/video-generation/pyproject.toml new file mode 100644 index 0000000..51d802b --- /dev/null +++ b/packages/video-generation/pyproject.toml @@ -0,0 +1,42 @@ +[project] +name = "celeste-video-generation" +version = "0.2.7" +description = "Video generation package for Celeste AI. Unified interface for all providers" +authors = [{name = "Kamilbenkirane", email = "kamil@withceleste.ai"}] +readme = "README.md" +license = {text = "Apache-2.0"} +requires-python = ">=3.12" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Operating System :: OS Independent", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Typing :: Typed", +] +keywords = ["ai", "video-generation", "sora", "runway", "openai", "google", "video-ai"] +dependencies = [ + "pillow>=10.0.0", +] + +[project.urls] +Homepage = "https://withceleste.ai" +Documentation = "https://withceleste.ai/docs" +Repository = "https://github.com/withceleste/celeste-python" +Issues = "https://github.com/withceleste/celeste-python/issues" + +[tool.uv.sources] +celeste-ai = { workspace = true } + +[project.entry-points."celeste.packages"] +video-generation = "celeste_video_generation:register_package" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/celeste_video_generation"] diff --git a/packages/video-generation/src/celeste_video_generation/__init__.py b/packages/video-generation/src/celeste_video_generation/__init__.py new file mode 100644 index 0000000..4caa4a5 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/__init__.py @@ -0,0 +1,29 @@ +"""Celeste video generation capability.""" + + +def register_package() -> None: + """Register video generation package (client and models).""" + from celeste.client import register_client + from celeste.core import Capability + from celeste.models import register_models + from celeste_video_generation.models import MODELS + from celeste_video_generation.providers import PROVIDERS + + for provider, client_class in PROVIDERS: + register_client(Capability.VIDEO_GENERATION, provider, client_class) + + register_models(MODELS, capability=Capability.VIDEO_GENERATION) + + +from celeste_video_generation.io import ( # noqa: E402 + VideoGenerationInput, + VideoGenerationOutput, + VideoGenerationUsage, +) + +__all__ = [ + "VideoGenerationInput", + "VideoGenerationOutput", + "VideoGenerationUsage", + "register_package", +] diff --git a/packages/video-generation/src/celeste_video_generation/client.py b/packages/video-generation/src/celeste_video_generation/client.py new file mode 100644 index 0000000..89fc8bb --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/client.py @@ -0,0 +1,64 @@ +"""Base client for video generation.""" + +from abc import abstractmethod +from typing import Any, Unpack + +import httpx + +from celeste.artifacts import VideoArtifact +from celeste.client import Client +from celeste_video_generation.io import ( + VideoGenerationInput, + VideoGenerationOutput, + VideoGenerationUsage, +) +from celeste_video_generation.parameters import VideoGenerationParameters + + +class VideoGenerationClient( + Client[VideoGenerationInput, VideoGenerationOutput, VideoGenerationParameters] +): + """Client for video generation operations.""" + + @abstractmethod + def _init_request(self, inputs: VideoGenerationInput) -> dict[str, Any]: + """Initialize provider-specific request structure.""" + + @abstractmethod + def _parse_usage(self, response_data: dict[str, Any]) -> VideoGenerationUsage: + """Parse usage information from provider response.""" + + @abstractmethod + def _parse_content( + self, + response_data: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> VideoArtifact: + """Parse content from provider response.""" + + def _create_inputs( + self, + prompt: str, + **parameters: Unpack[VideoGenerationParameters], + ) -> VideoGenerationInput: + """Map positional arguments to Input type.""" + return VideoGenerationInput(prompt=prompt) + + @classmethod + def _output_class(cls) -> type[VideoGenerationOutput]: + """Return the Output class for this client.""" + return VideoGenerationOutput + + def _build_metadata(self, response_data: dict[str, Any]) -> dict[str, Any]: + """Build metadata dictionary from response data.""" + metadata = super()._build_metadata(response_data) + metadata["raw_response"] = response_data + return metadata + + @abstractmethod + async def _make_request( + self, + request_body: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> httpx.Response: + """Make HTTP request(s) and return response object.""" diff --git a/packages/video-generation/src/celeste_video_generation/io.py b/packages/video-generation/src/celeste_video_generation/io.py new file mode 100644 index 0000000..d39006a --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/io.py @@ -0,0 +1,33 @@ +"""Input and output types for video generation.""" + +from celeste.artifacts import VideoArtifact +from celeste.io import Input, Output, Usage + + +class VideoGenerationInput(Input): + """Input for video generation operations.""" + + prompt: str + + +class VideoGenerationUsage(Usage): + """Video generation usage metrics. + + All fields optional since providers vary. + """ + + total_tokens: int | None = None + billing_units: float | None = None + + +class VideoGenerationOutput(Output[VideoArtifact]): + """Output with VideoArtifact content.""" + + pass + + +__all__ = [ + "VideoGenerationInput", + "VideoGenerationOutput", + "VideoGenerationUsage", +] diff --git a/packages/video-generation/src/celeste_video_generation/models.py b/packages/video-generation/src/celeste_video_generation/models.py new file mode 100644 index 0000000..c3c6a24 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/models.py @@ -0,0 +1,14 @@ +"""Model definitions for video generation.""" + +from celeste import Model +from celeste_video_generation.providers.bytedance.models import ( + MODELS as BYTEDANCE_MODELS, +) +from celeste_video_generation.providers.google.models import MODELS as GOOGLE_MODELS +from celeste_video_generation.providers.openai.models import MODELS as OPENAI_MODELS + +MODELS: list[Model] = [ + *BYTEDANCE_MODELS, + *GOOGLE_MODELS, + *OPENAI_MODELS, +] diff --git a/packages/video-generation/src/celeste_video_generation/parameters.py b/packages/video-generation/src/celeste_video_generation/parameters.py new file mode 100644 index 0000000..c52f893 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/parameters.py @@ -0,0 +1,28 @@ +"""Parameters for video generation.""" + +from enum import StrEnum + +from celeste.artifacts import ImageArtifact +from celeste.parameters import Parameters + + +class VideoGenerationParameter(StrEnum): + """Unified parameter names for video generation capability.""" + + ASPECT_RATIO = "aspect_ratio" + RESOLUTION = "resolution" + DURATION = "duration" + REFERENCE_IMAGES = "reference_images" + FIRST_FRAME = "first_frame" + LAST_FRAME = "last_frame" + + +class VideoGenerationParameters(Parameters): + """Parameters for video generation.""" + + aspect_ratio: str | None + resolution: str | None + duration: int | None + reference_images: list[ImageArtifact] | None + first_frame: ImageArtifact | None + last_frame: ImageArtifact | None diff --git a/packages/video-generation/src/celeste_video_generation/providers/__init__.py b/packages/video-generation/src/celeste_video_generation/providers/__init__.py new file mode 100644 index 0000000..84006e1 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/__init__.py @@ -0,0 +1,28 @@ +"""Provider implementations for video generation.""" + +from celeste import Client, Provider + +__all__ = ["PROVIDERS"] + + +def _get_providers() -> list[tuple[Provider, type[Client]]]: + """Lazy-load providers.""" + # Import clients directly from .client modules to avoid __init__.py imports + from celeste_video_generation.providers.bytedance.client import ( + ByteDanceVideoGenerationClient, + ) + from celeste_video_generation.providers.google.client import ( + GoogleVideoGenerationClient, + ) + from celeste_video_generation.providers.openai.client import ( + OpenAIVideoGenerationClient, + ) + + return [ + (Provider.BYTEDANCE, ByteDanceVideoGenerationClient), + (Provider.GOOGLE, GoogleVideoGenerationClient), + (Provider.OPENAI, OpenAIVideoGenerationClient), + ] + + +PROVIDERS: list[tuple[Provider, type[Client]]] = _get_providers() diff --git a/packages/video-generation/src/celeste_video_generation/providers/bytedance/README.md b/packages/video-generation/src/celeste_video_generation/providers/bytedance/README.md new file mode 100644 index 0000000..cdeb1be --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/bytedance/README.md @@ -0,0 +1,13 @@ +# ByteDance Video Generation Provider + +## Credentials + +**Environment variable:** `BYTEDANCE_API_KEY` + +**Setup:** +1. Register at [console.byteplus.com](https://console.byteplus.com) +2. Activate model in ModelArk section +3. Generate API key with video generation permissions +4. Set environment variable: `export BYTEDANCE_API_KEY="your-key"` + +**Note:** Models must be activated in BytePlus console before use. If you get a 404 error, activate the model or use an Endpoint ID (`ep-xxx`) instead of Model ID. diff --git a/packages/video-generation/src/celeste_video_generation/providers/bytedance/__init__.py b/packages/video-generation/src/celeste_video_generation/providers/bytedance/__init__.py new file mode 100644 index 0000000..31a9eb8 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/bytedance/__init__.py @@ -0,0 +1,11 @@ +"""ByteDance provider for video generation.""" + +from celeste.core import Provider + +from .client import ByteDanceVideoGenerationClient + +__all__ = ["PROVIDERS", "ByteDanceVideoGenerationClient"] + +PROVIDERS: list[tuple[Provider, type[ByteDanceVideoGenerationClient]]] = [ + (Provider.BYTEDANCE, ByteDanceVideoGenerationClient), +] diff --git a/packages/video-generation/src/celeste_video_generation/providers/bytedance/client.py b/packages/video-generation/src/celeste_video_generation/providers/bytedance/client.py new file mode 100644 index 0000000..322c137 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/bytedance/client.py @@ -0,0 +1,245 @@ +"""ByteDance client implementation for video generation.""" + +import asyncio +import base64 +import json +import logging +import time +from typing import Any, Unpack + +import httpx + +from celeste.artifacts import ImageArtifact, VideoArtifact +from celeste.mime_types import ApplicationMimeType, VideoMimeType +from celeste.parameters import ParameterMapper +from celeste_video_generation.client import VideoGenerationClient +from celeste_video_generation.io import ( + VideoGenerationInput, + VideoGenerationUsage, +) +from celeste_video_generation.parameters import VideoGenerationParameters + +from . import config +from .parameters import BYTEDANCE_PARAMETER_MAPPERS + +logger = logging.getLogger(__name__) + + +class ByteDanceVideoGenerationClient(VideoGenerationClient): + """ByteDance client for video generation.""" + + @classmethod + def parameter_mappers(cls) -> list[ParameterMapper]: + return BYTEDANCE_PARAMETER_MAPPERS + + def _validate_artifacts( + self, + inputs: VideoGenerationInput, + **parameters: Unpack[VideoGenerationParameters], + ) -> tuple[VideoGenerationInput, dict[str, Any]]: + """Validate and prepare artifacts for ByteDance API.""" + + # Helper function to convert ImageArtifact to base64 data URI + def convert_to_data_url(img: ImageArtifact) -> ImageArtifact: + if img.url: + return img + elif img.data: + file_data = img.data + elif img.path: + with open(img.path, "rb") as f: + file_data = f.read() + else: + msg = "ImageArtifact must have url, data, or path" + raise ValueError(msg) + + base64_data = base64.b64encode(file_data).decode("utf-8") + mime_type = img.mime_type.value if img.mime_type else "image/jpeg" + + return ImageArtifact( + url=f"data:{mime_type};base64,{base64_data}", + mime_type=img.mime_type, + metadata=img.metadata, + ) + + reference_images = parameters.get("reference_images") + if reference_images: + converted_images = [convert_to_data_url(img) for img in reference_images] + parameters["reference_images"] = converted_images + + first_frame = parameters.get("first_frame") + if first_frame: + parameters["first_frame"] = convert_to_data_url(first_frame) + + last_frame = parameters.get("last_frame") + if last_frame: + parameters["last_frame"] = convert_to_data_url(last_frame) + + return inputs, dict(parameters) + + def _add_image_content_item( + self, + content: list[dict[str, Any]], + artifact: ImageArtifact | VideoArtifact, + role: str, + artifact_type: str, + ) -> None: + """Add image content item to content array.""" + if artifact.url: + content.append( + { + "type": "image_url", + "image_url": { + "url": artifact.url, + }, + "role": role, + } + ) + elif artifact.data: + logger.warning( + f"ByteDance requires {artifact_type} URL, not base64 data. Upload {artifact_type} first." + ) + elif artifact.path: + logger.warning( + f"ByteDance requires {artifact_type} URL, not file path. Upload {artifact_type} first." + ) + + def _init_request(self, inputs: VideoGenerationInput) -> dict[str, Any]: + """Initialize request from BytePlus ModelArk API format.""" + content: list[dict[str, Any]] = [ + { + "type": "text", + "text": inputs.prompt, + } + ] + + request: dict[str, Any] = { + "model": self.model.id, + "content": content, + } + + return request + + def _parse_usage(self, response_data: dict[str, Any]) -> VideoGenerationUsage: + """Parse usage from response.""" + usage_data = response_data.get("usage", {}) + total_tokens = usage_data.get("total_tokens") + + return VideoGenerationUsage( + total_tokens=total_tokens, + ) + + def _parse_content( + self, + response_data: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> VideoArtifact: + """Parse content from response.""" + content = response_data.get("content") + if not isinstance(content, dict): + msg = f"No content field in ByteDance response. Available keys: {list(response_data.keys())}" + raise ValueError(msg) + + video_url = content.get("video_url") + if not video_url: + msg = f"No video_url in content field. Available content keys: {list(content.keys())}" + raise ValueError(msg) + + return VideoArtifact( + url=video_url, + mime_type=VideoMimeType.MP4, + ) + + def _build_metadata(self, response_data: dict[str, Any]) -> dict[str, Any]: + """Build metadata dictionary from response data.""" + content_fields = {"content"} + filtered_data = { + k: v for k, v in response_data.items() if k not in content_fields + } + metadata = super()._build_metadata(filtered_data) + + task_id = response_data.get("id") + if task_id: + metadata["task_id"] = task_id + + status = response_data.get("status") + if status: + metadata["status"] = status + + content = response_data.get("content") + if isinstance(content, dict): + last_frame_url = content.get("last_frame_url") + if last_frame_url: + metadata["last_frame_url"] = last_frame_url + + return metadata + + async def _make_request( + self, + request_body: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> httpx.Response: + """Make HTTP request with async polling.""" + headers = { + config.AUTH_HEADER_NAME: f"{config.AUTH_HEADER_PREFIX}{self.api_key.get_secret_value()}", + "Content-Type": ApplicationMimeType.JSON, + } + + logger.debug("Submitting video generation task to ByteDance") + submit_response = await self.http_client.post( + f"{config.BASE_URL}{config.ENDPOINT}", + headers=headers, + json_body=request_body, + ) + self._handle_error_response(submit_response) + submit_data = submit_response.json() + + task_id = submit_data["id"] + logger.info(f"ByteDance task submitted: {task_id}") + + start_time = time.time() + polling_interval = config.DEFAULT_POLLING_INTERVAL + + # Wait before first poll (consistent with Google pattern) + await asyncio.sleep(polling_interval) + + while True: + elapsed = time.time() - start_time + if elapsed > config.MAX_POLLING_TIMEOUT: + msg = f"ByteDance task {task_id} timed out after {elapsed:.0f}s" + raise TimeoutError(msg) + + status_url = f"{config.BASE_URL}{config.STATUS_ENDPOINT_TEMPLATE.format(task_id=task_id)}" + logger.debug(f"Polling ByteDance task status: {task_id}") + + status_response = await self.http_client.get( + status_url, + headers=headers, + ) + self._handle_error_response(status_response) + status_data = status_response.json() + + status = status_data.get("status") + logger.debug(f"ByteDance task {task_id} status: {status}") + + if status == config.STATUS_SUCCEEDED: + logger.info(f"ByteDance task {task_id} completed in {elapsed:.0f}s") + return httpx.Response( + 200, + content=json.dumps(status_data).encode(), + headers={"Content-Type": ApplicationMimeType.JSON}, + ) + + if status in [config.STATUS_FAILED, config.STATUS_CANCELED]: + error = status_data.get("error", {}) + error_msg = ( + error.get("message", "Unknown error") + if isinstance(error, dict) + else "Unknown error" + ) + msg = f"ByteDance task {task_id} failed: {error_msg}" + raise ValueError(msg) + + await asyncio.sleep(polling_interval) + + +__all__ = ["ByteDanceVideoGenerationClient"] diff --git a/packages/video-generation/src/celeste_video_generation/providers/bytedance/config.py b/packages/video-generation/src/celeste_video_generation/providers/bytedance/config.py new file mode 100644 index 0000000..2fabcca --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/bytedance/config.py @@ -0,0 +1,19 @@ +"""ByteDance provider configuration.""" + +# HTTP Configuration +BASE_URL = "https://ark.ap-southeast.bytepluses.com/api/v3" +ENDPOINT = "/contents/generations/tasks" +STATUS_ENDPOINT_TEMPLATE = "/contents/generations/tasks/{task_id}" + +# Authentication +AUTH_HEADER_NAME = "Authorization" +AUTH_HEADER_PREFIX = "Bearer " + +# Polling Configuration +DEFAULT_POLLING_INTERVAL = 5 # seconds +MAX_POLLING_TIMEOUT = 300 # 5 minutes + +# Status Constants +STATUS_SUCCEEDED = "succeeded" +STATUS_FAILED = "failed" +STATUS_CANCELED = "canceled" diff --git a/packages/video-generation/src/celeste_video_generation/providers/bytedance/models.py b/packages/video-generation/src/celeste_video_generation/providers/bytedance/models.py new file mode 100644 index 0000000..e5c7eb7 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/bytedance/models.py @@ -0,0 +1,99 @@ +"""ByteDance models for video generation. + +Model IDs use lowercase format with version suffixes (e.g., seedance-1-0-pro-250528). +Console display names differ from API model IDs. +""" + +from celeste import Capability, Model, Provider +from celeste.constraints import Choice, ImageConstraint, ImagesConstraint, Range +from celeste.mime_types import ImageMimeType +from celeste_video_generation.parameters import VideoGenerationParameter + +# Supported MIME types for ByteDance image parameters +BYTEDANCE_SUPPORTED_MIME_TYPES = [ + ImageMimeType.JPEG, + ImageMimeType.PNG, + ImageMimeType.WEBP, + ImageMimeType.BMP, + ImageMimeType.TIFF, + ImageMimeType.GIF, +] + +MODELS: list[Model] = [ + Model( + id="seedance-1-0-lite-t2v-250428", + provider=Provider.BYTEDANCE, + capabilities={Capability.VIDEO_GENERATION}, + display_name="Seedance 1.0 Lite (Text-to-Video)", + parameter_constraints={ + VideoGenerationParameter.DURATION: Range(min=2, max=12), + VideoGenerationParameter.RESOLUTION: Choice( + options=["480p", "720p", "1080p"] + ), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + VideoGenerationParameter.LAST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + }, + ), + Model( + id="seedance-1-0-lite-i2v-250428", + provider=Provider.BYTEDANCE, + capabilities={Capability.VIDEO_GENERATION}, + display_name="Seedance 1.0 Lite (Image-to-Video)", + parameter_constraints={ + VideoGenerationParameter.DURATION: Range(min=2, max=12), + VideoGenerationParameter.RESOLUTION: Choice( + options=["480p", "720p", "1080p"] + ), + VideoGenerationParameter.REFERENCE_IMAGES: ImagesConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + max_count=4, + ), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + VideoGenerationParameter.LAST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + }, + ), + Model( + id="seedance-1-0-pro-250528", + provider=Provider.BYTEDANCE, + capabilities={Capability.VIDEO_GENERATION}, + display_name="Seedance 1.0 Pro", + parameter_constraints={ + VideoGenerationParameter.DURATION: Range(min=2, max=12), + VideoGenerationParameter.RESOLUTION: Choice( + options=["480p", "720p", "1080p"] + ), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + VideoGenerationParameter.LAST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + }, + ), + Model( + id="seedance-1-0-pro-fast-251015", + provider=Provider.BYTEDANCE, + capabilities={Capability.VIDEO_GENERATION}, + display_name="Seedance 1.0 Pro Fast", + parameter_constraints={ + VideoGenerationParameter.DURATION: Range(min=2, max=12), + VideoGenerationParameter.RESOLUTION: Choice( + options=["480p", "720p", "1080p"] + ), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + VideoGenerationParameter.LAST_FRAME: ImageConstraint( + supported_mime_types=BYTEDANCE_SUPPORTED_MIME_TYPES, + ), + }, + ), +] diff --git a/packages/video-generation/src/celeste_video_generation/providers/bytedance/parameters.py b/packages/video-generation/src/celeste_video_generation/providers/bytedance/parameters.py new file mode 100644 index 0000000..5ee19a8 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/bytedance/parameters.py @@ -0,0 +1,192 @@ +"""ByteDance parameter mappers.""" + +from typing import Any + +from celeste import Model +from celeste.exceptions import ValidationError +from celeste.parameters import ParameterMapper +from celeste_video_generation.parameters import VideoGenerationParameter + + +class DurationMapper(ParameterMapper): + """Map duration parameter to BytePlus ModelArk text prompt format. + + BytePlus ModelArk API expects parameters embedded in the text prompt: + "prompt text --duration 5" instead of {"duration": 5}. + + All Seedance models support duration: 2-12 seconds. + """ + + name = VideoGenerationParameter.DURATION + + def map( + self, request: dict[str, Any], value: object, model: Model + ) -> dict[str, Any]: + """Append --duration parameter to text prompt in content array. + + Args: + request: The request dictionary with content array. + value: The duration value in seconds (integer). + model: The model being used (provides constraints). + + Returns: + Modified request with duration appended to text prompt. + """ + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # BytePlus ModelArk uses content array with text type + # Parameters must be embedded in the text field: "prompt --duration 5" + content = request.get("content", []) + for item in content: + if item.get("type") == "text": + text = item.get("text", "") + # Append duration parameter to text prompt + item["text"] = f"{text} --duration {validated_value}" + break + + return request + + +class ResolutionMapper(ParameterMapper): + """Map resolution parameter to BytePlus ModelArk text prompt format. + + BytePlus ModelArk API expects parameters embedded in the text prompt: + "prompt text --resolution 720p" instead of {"resolution": "720p"}. + + Supported values: + - All models: 480p, 720p, 1080p + """ + + name = VideoGenerationParameter.RESOLUTION + + def map( + self, request: dict[str, Any], value: object, model: Model + ) -> dict[str, Any]: + """Append --resolution parameter to text prompt in content array. + + Args: + request: The request dictionary with content array. + value: The resolution value (e.g., "720p"). + model: The model being used (provides constraints). + + Returns: + Modified request with resolution appended to text prompt. + """ + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # BytePlus ModelArk uses content array with text type + # Parameters must be embedded in the text field: "prompt --resolution 720p" + content = request.get("content", []) + for item in content: + if item.get("type") == "text": + text = item.get("text", "") + # Append resolution parameter to text prompt + item["text"] = f"{text} --resolution {validated_value}" + break + + return request + + +class ReferenceImagesMapper(ParameterMapper): + """Map reference_images parameter to BytePlus ModelArk content array format.""" + + name = VideoGenerationParameter.REFERENCE_IMAGES + + def map( + self, request: dict[str, Any], value: object, model: Model + ) -> dict[str, Any]: + """Add reference images to content array with reference_image role.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + content = request.setdefault("content", []) + for img in validated_value: + content.append( + { + "type": "image_url", + "image_url": { + "url": img.url, + }, + "role": "reference_image", + } + ) + + return request + + +class FirstFrameMapper(ParameterMapper): + """Map first_frame parameter to BytePlus ModelArk content array format.""" + + name = VideoGenerationParameter.FIRST_FRAME + + def map( + self, request: dict[str, Any], value: object, model: Model + ) -> dict[str, Any]: + """Add first frame image to content array with first_frame role.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + if not validated_value.url: + msg = "ByteDance requires image URL (including data URIs) for first_frame. ImageArtifact must have url, data, or path" + raise ValidationError(msg) + + content = request.setdefault("content", []) + content.append( + { + "type": "image_url", + "image_url": { + "url": validated_value.url, + }, + "role": "first_frame", + } + ) + + return request + + +class LastFrameMapper(ParameterMapper): + """Map last_frame parameter to BytePlus ModelArk content array format.""" + + name = VideoGenerationParameter.LAST_FRAME + + def map( + self, request: dict[str, Any], value: object, model: Model + ) -> dict[str, Any]: + """Add last frame image to content array with last_frame role.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + if not validated_value.url: + msg = "ByteDance requires image URL (including data URIs) for last_frame. ImageArtifact must have url, data, or path" + raise ValidationError(msg) + + content = request.setdefault("content", []) + content.append( + { + "type": "image_url", + "image_url": { + "url": validated_value.url, + }, + "role": "last_frame", + } + ) + + return request + + +BYTEDANCE_PARAMETER_MAPPERS: list[ParameterMapper] = [ + DurationMapper(), + ResolutionMapper(), + ReferenceImagesMapper(), + FirstFrameMapper(), + LastFrameMapper(), +] + +__all__ = ["BYTEDANCE_PARAMETER_MAPPERS"] diff --git a/packages/video-generation/src/celeste_video_generation/providers/google/__init__.py b/packages/video-generation/src/celeste_video_generation/providers/google/__init__.py new file mode 100644 index 0000000..0616a9f --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/google/__init__.py @@ -0,0 +1,10 @@ +"""Google provider for video generation.""" + +from celeste.core import Provider +from celeste_video_generation.providers.google.client import GoogleVideoGenerationClient + +__all__ = ["PROVIDERS", "GoogleVideoGenerationClient"] + +PROVIDERS: list[tuple[Provider, type[GoogleVideoGenerationClient]]] = [ + (Provider.GOOGLE, GoogleVideoGenerationClient), +] diff --git a/packages/video-generation/src/celeste_video_generation/providers/google/client.py b/packages/video-generation/src/celeste_video_generation/providers/google/client.py new file mode 100644 index 0000000..4e17ddf --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/google/client.py @@ -0,0 +1,216 @@ +"""Google provider client for video generation.""" + +import asyncio +import base64 +import json +import logging +from typing import Any, Unpack + +import httpx + +from celeste.artifacts import ImageArtifact, VideoArtifact +from celeste.mime_types import ApplicationMimeType, VideoMimeType +from celeste.parameters import ParameterMapper +from celeste_video_generation.client import VideoGenerationClient +from celeste_video_generation.io import ( + VideoGenerationInput, + VideoGenerationUsage, +) +from celeste_video_generation.parameters import VideoGenerationParameters +from celeste_video_generation.providers.google import config +from celeste_video_generation.providers.google.parameters import ( + GOOGLE_PARAMETER_MAPPERS, +) + +logger = logging.getLogger(__name__) + + +class GoogleVideoGenerationClient(VideoGenerationClient): + """Google client for video generation.""" + + @classmethod + def parameter_mappers(cls) -> list[ParameterMapper]: + return GOOGLE_PARAMETER_MAPPERS + + def _validate_artifacts( + self, + inputs: VideoGenerationInput, + **parameters: Unpack[VideoGenerationParameters], + ) -> tuple[VideoGenerationInput, dict[str, Any]]: + """Validate and prepare artifacts for Google Veo API.""" + + def convert_to_base64_uri(img: ImageArtifact) -> ImageArtifact: + if img.data: + file_data = img.data + elif img.path: + with open(img.path, "rb") as f: + file_data = f.read() + else: + msg = "ImageArtifact must have data or path" + raise ValueError(msg) + + base64_data = base64.b64encode(file_data).decode("utf-8") + mime_type = img.mime_type.value if img.mime_type else "image/jpeg" + + return ImageArtifact( + url=f"data:image/{mime_type.split('/')[-1]};base64,{base64_data}", + mime_type=img.mime_type, + metadata=img.metadata, + ) + + reference_images = parameters.get("reference_images") + if reference_images: + converted_images = [convert_to_base64_uri(img) for img in reference_images] + parameters["reference_images"] = converted_images + + first_frame = parameters.get("first_frame") + if first_frame: + parameters["first_frame"] = convert_to_base64_uri(first_frame) + + last_frame = parameters.get("last_frame") + if last_frame: + parameters["last_frame"] = convert_to_base64_uri(last_frame) + + return inputs, dict(parameters) + + def _init_request(self, inputs: VideoGenerationInput) -> dict[str, Any]: + """Initialize request from Google API format.""" + instance: dict[str, Any] = {"prompt": inputs.prompt} + + request: dict[str, Any] = {"instances": [instance]} + request["parameters"] = {} + + return request + + def _parse_usage(self, response_data: dict[str, Any]) -> VideoGenerationUsage: + """Parse usage from response.""" + return VideoGenerationUsage() + + def _parse_content( + self, + response_data: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> VideoArtifact: + """Parse content from response.""" + try: + generate_response = response_data.get("response", {}).get( + "generateVideoResponse", {} + ) + generated_samples = generate_response.get("generatedSamples", []) + if not generated_samples: + msg = "No generated samples in response" + raise ValueError(msg) + + video_data = generated_samples[0].get("video", {}) + uri = video_data.get("uri") + if not uri: + msg = "No video URI in response" + raise ValueError(msg) + + video_artifact = VideoArtifact(url=uri) + + transformed = self._transform_output(video_artifact, **parameters) + if isinstance(transformed, VideoArtifact): + return transformed + return video_artifact + except (KeyError, IndexError) as e: + msg = f"Invalid response structure: {e}" + raise ValueError(msg) from e + + async def _make_request( + self, + request_body: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> httpx.Response: + """Make HTTP request with async polling for Google video generation.""" + model_id = self.model.id + endpoint = config.GENERATE_ENDPOINT.format(model_id=model_id) + url = f"{config.BASE_URL}{endpoint}" + + headers = { + "x-goog-api-key": self.api_key.get_secret_value(), + "Content-Type": ApplicationMimeType.JSON, + } + + logger.info(f"Initiating video generation with model {model_id}") + response = await self.http_client.post( + url, + headers=headers, + json_body=request_body, + timeout=config.DEFAULT_TIMEOUT, + ) + + self._handle_error_response(response) + operation_data = response.json() + + operation_name = operation_data["name"] + logger.info(f"Video generation started: {operation_name}") + + poll_url = f"{config.BASE_URL}{config.POLL_ENDPOINT.format(operation_name=operation_name)}" + poll_headers = {"x-goog-api-key": self.api_key.get_secret_value()} + + while True: + await asyncio.sleep(config.POLL_INTERVAL) + logger.debug(f"Polling operation status: {operation_name}") + + poll_response = await self.http_client.get( + poll_url, + headers=poll_headers, + timeout=config.DEFAULT_TIMEOUT, + ) + + self._handle_error_response(poll_response) + operation_data = poll_response.json() + + if operation_data.get("done"): + if "error" in operation_data: + error = operation_data["error"] + error_msg = error.get("message", "Unknown error") + error_code = error.get("code", "UNKNOWN") + msg = f"Video generation failed: {error_code} - {error_msg}" + raise ValueError(msg) + + logger.info(f"Video generation completed: {operation_name}") + break + + return httpx.Response( + 200, + content=json.dumps(operation_data).encode(), + headers={"Content-Type": ApplicationMimeType.JSON}, + ) + + async def download_content(self, artifact: VideoArtifact) -> VideoArtifact: + """Download video content from URI. + + Google-specific method. Google Veo returns gs:// URIs that require + downloading with API key authentication. Other providers return video + content directly in the response. + """ + if not artifact.url: + msg = "VideoArtifact has no URL to download from" + raise ValueError(msg) + + download_url = artifact.url + if download_url.startswith("gs://"): + download_url = download_url.replace("gs://", config.STORAGE_BASE_URL, 1) + + logger.info(f"Downloading video from: {download_url}") + + headers = {"x-goog-api-key": self.api_key.get_secret_value()} + + response = await self.http_client.get( + download_url, + headers=headers, + timeout=config.DEFAULT_TIMEOUT, + follow_redirects=True, + ) + + self._handle_error_response(response) + video_data = response.content + + return VideoArtifact( + url=artifact.url, # Keep original URI + data=video_data, + mime_type=VideoMimeType.MP4, # Default to MP4 for videos + metadata=artifact.metadata, + ) diff --git a/packages/video-generation/src/celeste_video_generation/providers/google/config.py b/packages/video-generation/src/celeste_video_generation/providers/google/config.py new file mode 100644 index 0000000..a6828d9 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/google/config.py @@ -0,0 +1,8 @@ +"""Google provider configuration.""" + +BASE_URL = "https://generativelanguage.googleapis.com/v1beta" +GENERATE_ENDPOINT = "/models/{model_id}:predictLongRunning" +POLL_ENDPOINT = "/{operation_name}" +POLL_INTERVAL = 10 # seconds +DEFAULT_TIMEOUT = 300.0 # 5 minutes for long-running operations +STORAGE_BASE_URL = "https://storage.googleapis.com/" diff --git a/packages/video-generation/src/celeste_video_generation/providers/google/models.py b/packages/video-generation/src/celeste_video_generation/providers/google/models.py new file mode 100644 index 0000000..c2023ae --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/google/models.py @@ -0,0 +1,78 @@ +"""Google models.""" + +from celeste import Model, Provider +from celeste.constraints import Choice, ImageConstraint, ImagesConstraint +from celeste.mime_types import ImageMimeType +from celeste_video_generation.parameters import VideoGenerationParameter + +# Supported MIME types for all Veo models +VEO_SUPPORTED_MIME_TYPES = [ + ImageMimeType.JPEG, + ImageMimeType.PNG, + ImageMimeType.WEBP, +] + +MODELS: list[Model] = [ + Model( + id="veo-3.0-generate-001", + provider=Provider.GOOGLE, + display_name="Veo 3", + parameter_constraints={ + VideoGenerationParameter.ASPECT_RATIO: Choice(options=["16:9", "9:16"]), + VideoGenerationParameter.RESOLUTION: Choice(options=["720p"]), + VideoGenerationParameter.DURATION: Choice(options=[4, 6, 8]), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=VEO_SUPPORTED_MIME_TYPES, + ), + }, + ), + Model( + id="veo-3.0-fast-generate-001", + provider=Provider.GOOGLE, + display_name="Veo 3 Fast", + parameter_constraints={ + VideoGenerationParameter.ASPECT_RATIO: Choice(options=["16:9", "9:16"]), + VideoGenerationParameter.RESOLUTION: Choice(options=["720p"]), + VideoGenerationParameter.DURATION: Choice(options=[4, 6, 8]), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=VEO_SUPPORTED_MIME_TYPES + ), + }, + ), + Model( + id="veo-3.1-generate-preview", + provider=Provider.GOOGLE, + display_name="Veo 3.1 (Preview)", + parameter_constraints={ + VideoGenerationParameter.ASPECT_RATIO: Choice(options=["16:9", "9:16"]), + VideoGenerationParameter.RESOLUTION: Choice(options=["720p", "1080p"]), + VideoGenerationParameter.DURATION: Choice(options=[4, 6, 8]), + VideoGenerationParameter.REFERENCE_IMAGES: ImagesConstraint( + supported_mime_types=VEO_SUPPORTED_MIME_TYPES, + max_count=3, + ), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=VEO_SUPPORTED_MIME_TYPES, + ), + VideoGenerationParameter.LAST_FRAME: ImageConstraint( + supported_mime_types=VEO_SUPPORTED_MIME_TYPES, + ), + }, + ), + Model( + id="veo-3.1-fast-generate-preview", + provider=Provider.GOOGLE, + display_name="Veo 3.1 Fast (Preview)", + parameter_constraints={ + VideoGenerationParameter.ASPECT_RATIO: Choice(options=["16:9", "9:16"]), + VideoGenerationParameter.RESOLUTION: Choice(options=["720p", "1080p"]), + VideoGenerationParameter.DURATION: Choice(options=[4, 6, 8]), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=VEO_SUPPORTED_MIME_TYPES, + ), + VideoGenerationParameter.LAST_FRAME: ImageConstraint( + supported_mime_types=VEO_SUPPORTED_MIME_TYPES, + ), + }, + ), +] diff --git a/packages/video-generation/src/celeste_video_generation/providers/google/parameters.py b/packages/video-generation/src/celeste_video_generation/providers/google/parameters.py new file mode 100644 index 0000000..48c17d7 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/google/parameters.py @@ -0,0 +1,201 @@ +"""Google parameter mappers.""" + +from typing import Any + +from celeste.exceptions import ValidationError +from celeste.models import Model +from celeste.parameters import ParameterMapper +from celeste_video_generation.parameters import VideoGenerationParameter + + +class AspectRatioMapper(ParameterMapper): + """Map aspect_ratio parameter to Google API format.""" + + name = VideoGenerationParameter.ASPECT_RATIO + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform aspect_ratio into provider request.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Transform to provider-specific request format + request.setdefault("parameters", {})["aspectRatio"] = validated_value + return request + + +class ResolutionMapper(ParameterMapper): + """Map resolution parameter to Google API format.""" + + name = VideoGenerationParameter.RESOLUTION + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform resolution into provider request.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Transform to provider-specific request format + request.setdefault("parameters", {})["resolution"] = validated_value + return request + + +class DurationSecondsMapper(ParameterMapper): + """Map duration parameter to Google API format.""" + + name = VideoGenerationParameter.DURATION + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform duration into provider request.""" + # Coerce to integer if string provided (for backward compatibility) + if isinstance(value, str): + value = int(value) + + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Transform to provider-specific request format (API expects integer) + request.setdefault("parameters", {})["durationSeconds"] = validated_value + return request + + +class ReferenceImagesMapper(ParameterMapper): + """Map reference_images parameter to Google API format.""" + + name = VideoGenerationParameter.REFERENCE_IMAGES + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform reference_images into provider request.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + reference_images = [] + for img in validated_value: + ref_image: dict[str, Any] = { + "image": {}, + "referenceType": "asset", + } + + # Check if URL is base64 data URI + if img.url and img.url.startswith("data:image/"): + # Extract base64 data from data URI + header, encoded = img.url.split(",", 1) + mime_type = header.split(":")[1].split(";")[0] + ref_image["image"]["bytesBase64Encoded"] = encoded + ref_image["image"]["mimeType"] = mime_type + else: + msg = "ImageArtifact must have data or path for reference images (base64 encoding required)" + raise ValidationError(msg) + + reference_images.append(ref_image) + + request.setdefault("instances", [{}])[0]["referenceImages"] = reference_images + return request + + +class FirstFrameMapper(ParameterMapper): + """Map first_frame parameter to Google API format.""" + + name = VideoGenerationParameter.FIRST_FRAME + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform first_frame into provider request.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Check if URL is base64 data URI + if validated_value.url and validated_value.url.startswith("data:image/"): + # Extract base64 data from data URI + header, encoded = validated_value.url.split(",", 1) + mime_type = header.split(":")[1].split(";")[0] + + # Set image in instances[0].image + request.setdefault("instances", [{}])[0]["image"] = { + "bytesBase64Encoded": encoded, + "mimeType": mime_type, + } + else: + msg = "ImageArtifact must have data or path for first_frame (base64 encoding required)" + raise ValidationError(msg) + + return request + + +class LastFrameMapper(ParameterMapper): + """Map last_frame parameter to Google API format.""" + + name = VideoGenerationParameter.LAST_FRAME + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform last_frame into provider request.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Check if first_frame (image) exists - lastFrame requires image per API docs + instances = request.get("instances", [{}]) + if not instances or "image" not in instances[0]: + msg = "last_frame requires first_frame to be provided" + raise ValidationError(msg) + + # Check if URL is base64 data URI + if validated_value.url and validated_value.url.startswith("data:image/"): + # Extract base64 data from data URI + header, encoded = validated_value.url.split(",", 1) + mime_type = header.split(":")[1].split(";")[0] + + # Set lastFrame in instances[0] to match image structure + request.setdefault("instances", [{}])[0]["lastFrame"] = { + "bytesBase64Encoded": encoded, + "mimeType": mime_type, + } + else: + msg = "ImageArtifact must have data or path for last_frame (base64 encoding required)" + raise ValidationError(msg) + + return request + + +GOOGLE_PARAMETER_MAPPERS: list[ParameterMapper] = [ + AspectRatioMapper(), + ResolutionMapper(), + DurationSecondsMapper(), + ReferenceImagesMapper(), + FirstFrameMapper(), + LastFrameMapper(), +] + +__all__ = ["GOOGLE_PARAMETER_MAPPERS"] diff --git a/packages/video-generation/src/celeste_video_generation/providers/openai/__init__.py b/packages/video-generation/src/celeste_video_generation/providers/openai/__init__.py new file mode 100644 index 0000000..c951d3d --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/openai/__init__.py @@ -0,0 +1,10 @@ +"""OpenAI provider for video generation.""" + +from celeste.core import Provider +from celeste_video_generation.providers.openai.client import OpenAIVideoGenerationClient + +__all__ = ["PROVIDERS", "OpenAIVideoGenerationClient"] + +PROVIDERS: list[tuple[Provider, type[OpenAIVideoGenerationClient]]] = [ + (Provider.OPENAI, OpenAIVideoGenerationClient), +] diff --git a/packages/video-generation/src/celeste_video_generation/providers/openai/client.py b/packages/video-generation/src/celeste_video_generation/providers/openai/client.py new file mode 100644 index 0000000..48700b6 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/openai/client.py @@ -0,0 +1,246 @@ +"""OpenAI client implementation.""" + +import asyncio +import base64 +import io +import json +import logging +from typing import Any, Unpack + +import httpx +from PIL import Image + +from celeste.artifacts import ImageArtifact, VideoArtifact +from celeste.exceptions import ValidationError +from celeste.mime_types import ApplicationMimeType, VideoMimeType +from celeste.parameters import ParameterMapper +from celeste_video_generation.client import VideoGenerationClient +from celeste_video_generation.io import ( + VideoGenerationInput, + VideoGenerationUsage, +) +from celeste_video_generation.parameters import VideoGenerationParameters + +from . import config +from .parameters import OPENAI_PARAMETER_MAPPERS + +logger = logging.getLogger(__name__) + + +class OpenAIVideoGenerationClient(VideoGenerationClient): + """OpenAI client for video generation.""" + + @classmethod + def parameter_mappers(cls) -> list[ParameterMapper]: + return OPENAI_PARAMETER_MAPPERS + + def _init_request(self, inputs: VideoGenerationInput) -> dict[str, Any]: + """Initialize request from OpenAI API format.""" + request = { + "prompt": inputs.prompt, + "model": self.model.id, + } + + return request + + def _build_request( + self, + inputs: VideoGenerationInput, + **parameters: Unpack[VideoGenerationParameters], + ) -> dict[str, Any]: + """Build request with parameter mapping and size derivation.""" + request = super()._build_request(inputs, **parameters) + + aspect_ratio = parameters.get("aspect_ratio") + resolution = parameters.get("resolution") + + if bool(aspect_ratio) != bool(resolution): + msg = ( + "Both aspect_ratio and resolution must be specified together. " + f"Got aspect_ratio={aspect_ratio!r}, resolution={resolution!r}" + ) + raise ValidationError(msg) + + if aspect_ratio and resolution: + ASPECT_RATIO_MAP = { + ("16:9", "720p"): "1280x720", + ("9:16", "720p"): "720x1280", + } + + size = ASPECT_RATIO_MAP.get((aspect_ratio, resolution)) + if size: + request["size"] = size + + return request + + def _parse_usage(self, response_data: dict[str, Any]) -> VideoGenerationUsage: + """Parse usage from response.""" + seconds = response_data.get("seconds") + return VideoGenerationUsage( + billing_units=float(seconds) if seconds else None, + ) + + def _parse_content( + self, + response_data: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> VideoArtifact: + """Parse content from response.""" + video_data_b64 = response_data["video_data"] + video_data = base64.b64decode(video_data_b64) + return VideoArtifact( + data=video_data, + mime_type=VideoMimeType.MP4, + ) + + async def _prepare_multipart_request( + self, + request_body: dict[str, Any], + ) -> tuple[dict[str, tuple[str, bytes, str]], dict[str, str]]: + """Prepare multipart form data from request_body with input_reference.""" + size = request_body.get("size", "720x1280") + + input_reference = request_body.pop("input_reference", None) + if input_reference is None: + return {}, {} + + if not isinstance(input_reference, ImageArtifact): + msg = f"input_reference must be ImageArtifact, got {type(input_reference).__name__}" + raise ValueError(msg) + + if input_reference.data: + image_data = input_reference.data + elif input_reference.path: + with open(input_reference.path, "rb") as f: + image_data = f.read() + else: + msg = "ImageArtifact must have data or path for input_reference" + raise ValueError(msg) + + img = Image.open(io.BytesIO(image_data)) + actual_size = f"{img.width}x{img.height}" + if actual_size != size: + msg = ( + f"Image dimensions ({actual_size}) must match video size ({size}). " + f"Please resize your image to {size} before uploading." + ) + raise ValueError(msg) + + mime_type = ( + input_reference.mime_type.value + if input_reference.mime_type + else "image/jpeg" + ) + + files = { + "input_reference": ("image.jpg", image_data, mime_type), + } + + data = { + k: str(v) if isinstance(v, (str, int, float)) else json.dumps(v) + for k, v in request_body.items() + } + + return files, data + + async def _make_request( + self, + request_body: dict[str, Any], + **parameters: Unpack[VideoGenerationParameters], + ) -> httpx.Response: + """Make HTTP request with async polling for OpenAI video generation.""" + headers = { + config.AUTH_HEADER_NAME: f"{config.AUTH_HEADER_PREFIX}{self.api_key.get_secret_value()}", + } + + files, data = await self._prepare_multipart_request(request_body.copy()) + + if files: + logger.info("Sending multipart request to OpenAI with input_reference") + response = await self.http_client.post_multipart( + f"{config.BASE_URL}{config.ENDPOINT}", + headers=headers, + files=files, + data=data, + ) + else: + logger.info(f"Sending request to OpenAI: {request_body}") + response = await self.http_client.post( + f"{config.BASE_URL}{config.ENDPOINT}", + headers=headers, + json_body=request_body, + ) + self._handle_error_response(response) + video_obj = response.json() + + video_id = video_obj["id"] + logger.info(f"Created video job: {video_id}") + + for _ in range(config.MAX_POLLS): + status_response = await self.http_client.get( + f"{config.BASE_URL}{config.ENDPOINT}/{video_id}", + headers=headers, + ) + self._handle_error_response(status_response) + video_obj = status_response.json() + + status = video_obj["status"] + progress = video_obj.get("progress", 0) + + logger.info(f"Video {video_id}: {status} ({progress}%)") + + if status == config.STATUS_COMPLETED: + break + elif status == config.STATUS_FAILED: + error = video_obj.get("error", {}) + msg = ( + f"Video generation failed: {error.get('message', 'Unknown error')}" + ) + raise RuntimeError(msg) + + await asyncio.sleep(config.POLL_INTERVAL) + else: + msg = f"Video generation timeout after {config.MAX_POLLS * config.POLL_INTERVAL} seconds" + raise TimeoutError(msg) + + content_response = await self.http_client.get( + f"{config.BASE_URL}{config.ENDPOINT}/{video_id}{config.CONTENT_ENDPOINT_SUFFIX}", + headers=headers, + ) + self._handle_error_response(content_response) + video_data = content_response.content + + response_data = { + "video_data": base64.b64encode(video_data).decode("utf-8"), + "model": video_obj.get("model", self.model.id), + "video_id": video_id, + "seconds": video_obj.get("seconds"), + "size": video_obj.get("size"), + "created_at": video_obj.get("created_at"), + "completed_at": video_obj.get("completed_at"), + "expires_at": video_obj.get("expires_at"), + } + + return httpx.Response( + 200, + content=json.dumps(response_data).encode(), + headers={"Content-Type": ApplicationMimeType.JSON}, + ) + + def _build_metadata(self, response_data: dict[str, Any]) -> dict[str, Any]: + """Build metadata from response data.""" + content_fields = {"video_data"} + filtered_data = { + k: v for k, v in response_data.items() if k not in content_fields + } + metadata = super()._build_metadata(filtered_data) + metadata["video_id"] = response_data.get("video_id") + metadata["seconds"] = response_data.get("seconds") + metadata["size"] = response_data.get("size") + metadata["created_at"] = response_data.get("created_at") + metadata["completed_at"] = response_data.get("completed_at") + metadata["expires_at"] = response_data.get("expires_at") + return metadata + + +__all__ = ["OpenAIVideoGenerationClient"] diff --git a/packages/video-generation/src/celeste_video_generation/providers/openai/config.py b/packages/video-generation/src/celeste_video_generation/providers/openai/config.py new file mode 100644 index 0000000..eaed356 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/openai/config.py @@ -0,0 +1,18 @@ +"""OpenAI provider configuration.""" + +# HTTP Configuration +BASE_URL = "https://api.openai.com" +ENDPOINT = "/v1/videos" +CONTENT_ENDPOINT_SUFFIX = "/content" + +# Authentication +AUTH_HEADER_NAME = "Authorization" +AUTH_HEADER_PREFIX = "Bearer " + +# Polling Configuration +MAX_POLLS = 60 +POLL_INTERVAL = 5 # seconds + +# Status Constants +STATUS_COMPLETED = "completed" +STATUS_FAILED = "failed" diff --git a/packages/video-generation/src/celeste_video_generation/providers/openai/models.py b/packages/video-generation/src/celeste_video_generation/providers/openai/models.py new file mode 100644 index 0000000..714899a --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/openai/models.py @@ -0,0 +1,36 @@ +"""OpenAI models.""" + +from celeste import Model, Provider +from celeste.constraints import Choice, ImageConstraint +from celeste.mime_types import ImageMimeType +from celeste_video_generation.parameters import VideoGenerationParameter + +MODELS: list[Model] = [ + Model( + id="sora-2", + provider=Provider.OPENAI, + display_name="Sora 2", + parameter_constraints={ + VideoGenerationParameter.DURATION: Choice(options=["4", "8", "12"]), + VideoGenerationParameter.ASPECT_RATIO: Choice(options=["16:9", "9:16"]), + VideoGenerationParameter.RESOLUTION: Choice(options=["720p"]), + }, + ), + Model( + id="sora-2-pro", + provider=Provider.OPENAI, + display_name="Sora 2 Pro", + parameter_constraints={ + VideoGenerationParameter.DURATION: Choice(options=["4", "8", "12"]), + VideoGenerationParameter.ASPECT_RATIO: Choice(options=["16:9", "9:16"]), + VideoGenerationParameter.RESOLUTION: Choice(options=["720p"]), + VideoGenerationParameter.FIRST_FRAME: ImageConstraint( + supported_mime_types=[ + ImageMimeType.JPEG, + ImageMimeType.PNG, + ImageMimeType.WEBP, + ], + ), + }, + ), +] diff --git a/packages/video-generation/src/celeste_video_generation/providers/openai/parameters.py b/packages/video-generation/src/celeste_video_generation/providers/openai/parameters.py new file mode 100644 index 0000000..f957819 --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/providers/openai/parameters.py @@ -0,0 +1,116 @@ +"""OpenAI parameter mappers.""" + +from typing import Any + +from celeste.models import Model +from celeste.parameters import ParameterMapper +from celeste_video_generation.parameters import VideoGenerationParameter + + +class AspectRatioMapper(ParameterMapper): + """Validate aspect_ratio parameter. + + Validation only - size derivation happens in client. + """ + + name = VideoGenerationParameter.ASPECT_RATIO + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Validate aspect_ratio parameter.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Validate but don't transform (size derivation happens in client) + return request + + +class ResolutionMapper(ParameterMapper): + """Validate resolution parameter. + + Validation only - size derivation happens in client. + """ + + name = VideoGenerationParameter.RESOLUTION + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Validate resolution parameter.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Validate but don't transform (size derivation happens in client) + return request + + +class DurationSecondsMapper(ParameterMapper): + """Map duration parameter to OpenAI API format. + + Converts user-facing int to API-required string. + """ + + name = VideoGenerationParameter.DURATION + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform duration into provider request.""" + # Coerce int to string (user provides int, API expects string) + if isinstance(value, int): + value = str(value) + + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + # Transform to provider-specific request format (top-level field) + request["seconds"] = validated_value + return request + + +class FirstFrameMapper(ParameterMapper): + """Map first_frame parameter to OpenAI API format. + + OpenAI Sora's input_reference acts as the first frame of the video. + Image must match target video resolution. + Note: OpenAI uses multipart/form-data for file uploads. + """ + + name = VideoGenerationParameter.FIRST_FRAME + + def map( + self, + request: dict[str, Any], + value: object, + model: Model, + ) -> dict[str, Any]: + """Transform first_frame into provider request.""" + validated_value = self._validate_value(value, model) + if validated_value is None: + return request + + request["input_reference"] = validated_value + return request + + +OPENAI_PARAMETER_MAPPERS: list[ParameterMapper] = [ + AspectRatioMapper(), + ResolutionMapper(), + DurationSecondsMapper(), + FirstFrameMapper(), +] + +__all__ = ["OPENAI_PARAMETER_MAPPERS"] diff --git a/packages/video-generation/src/celeste_video_generation/py.typed b/packages/video-generation/src/celeste_video_generation/py.typed new file mode 100644 index 0000000..321d0ae --- /dev/null +++ b/packages/video-generation/src/celeste_video_generation/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 - this package supports type checking diff --git a/packages/video-generation/tests/integration_tests/test_video_generation/__init__.py b/packages/video-generation/tests/integration_tests/test_video_generation/__init__.py new file mode 100644 index 0000000..c6eb386 --- /dev/null +++ b/packages/video-generation/tests/integration_tests/test_video_generation/__init__.py @@ -0,0 +1 @@ +"""Integration tests for video generation.""" diff --git a/packages/video-generation/tests/integration_tests/test_video_generation/test_generate.py b/packages/video-generation/tests/integration_tests/test_video_generation/test_generate.py new file mode 100644 index 0000000..90e59de --- /dev/null +++ b/packages/video-generation/tests/integration_tests/test_video_generation/test_generate.py @@ -0,0 +1,73 @@ +"""Integration tests for video generation across all providers.""" + +import pytest + +from celeste import Capability, Provider, create_client + + +@pytest.mark.parametrize( + ("provider", "model", "parameters"), + [ + ( + Provider.OPENAI, + "sora-2", + {"duration": "4", "aspect_ratio": "16:9", "resolution": "720p"}, + ), + ( + Provider.GOOGLE, + "veo-3.0-fast-generate-001", + {"duration": 4, "resolution": "720p"}, + ), + ( + Provider.BYTEDANCE, + "seedance-1-0-lite-t2v-250428", + {"duration": 2, "resolution": "480p"}, + ), + ], +) +@pytest.mark.integration +@pytest.mark.asyncio +async def test_generate(provider: Provider, model: str, parameters: dict) -> None: + """Test video generation with prompt parameter across all providers. + + This test demonstrates that the unified API works identically across + all providers using the same code - proving the abstraction value. + Uses cheapest models with minimum duration and lowest resolution to minimize costs. + """ + # Import here to avoid circular import during pytest collection + from celeste_video_generation import ( + VideoGenerationOutput, + VideoGenerationUsage, + ) + + from celeste.artifacts import VideoArtifact + + # Arrange + client = create_client( + capability=Capability.VIDEO_GENERATION, + provider=provider, + ) + prompt = "A cinematic video of a sunset over mountains" + + # Act + response = await client.generate( + prompt=prompt, + model=model, + **parameters, + ) + + # Assert + assert isinstance(response, VideoGenerationOutput), ( + f"Expected VideoGenerationOutput, got {type(response)}" + ) + assert isinstance(response.content, VideoArtifact), ( + f"Expected VideoArtifact content, got {type(response.content)}" + ) + assert response.content.has_content, ( + f"VideoArtifact has no content (url/data/path): {response.content}" + ) + + # Validate usage metrics + assert isinstance(response.usage, VideoGenerationUsage), ( + f"Expected VideoGenerationUsage, got {type(response.usage)}" + ) diff --git a/pyproject.toml b/pyproject.toml index 1036efc..ee0f9ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "celeste-ai" -version = "0.2.6" +version = "0.2.7" description = "Open source, type-safe primitives for multi-modal AI. All capabilities, all providers, one interface" authors = [{name = "Kamilbenkirane", email = "kamil@withceleste.ai"}] readme = "README.md" @@ -33,13 +33,15 @@ Repository = "https://github.com/withceleste/celeste-python" Issues = "https://github.com/withceleste/celeste-python/issues" [project.optional-dependencies] -text-generation = ["celeste-text-generation>=0.2.2"] -image-generation = ["celeste-image-generation>=0.2.2"] +text-generation = ["celeste-text-generation>=0.2.7"] +image-generation = ["celeste-image-generation>=0.2.7"] +video-generation = ["celeste-video-generation>=0.2.7"] image-intelligence = ["celeste-image-intelligence>=0.2.1"] speech-generation = ["celeste-speech-generation>=0.2.3"] all = [ - "celeste-text-generation>=0.2.2", - "celeste-image-generation>=0.2.2", + "celeste-text-generation>=0.2.7", + "celeste-image-generation>=0.2.7", + "celeste-video-generation>=0.2.7", "celeste-image-intelligence>=0.2.1", "celeste-speech-generation>=0.2.3", ] @@ -64,6 +66,7 @@ members = ["packages/*"] [tool.uv.sources] celeste-text-generation = { workspace = true } celeste-image-generation = { workspace = true } +celeste-video-generation = { workspace = true } [build-system] requires = ["hatchling"] @@ -81,6 +84,7 @@ pythonpath = [ "src", "packages/text-generation/src", "packages/image-generation/src", + "packages/video-generation/src", ] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", @@ -163,17 +167,12 @@ ignore_missing_imports = true [[tool.mypy.overrides]] module = [ - "celeste_text_generation.*", "celeste_text_generation.client", "celeste_text_generation.providers.*", -] -disable_error_code = ["override", "return-value", "arg-type", "call-arg", "assignment", "no-any-return"] - -[[tool.mypy.overrides]] -module = [ - "celeste_image_generation.*", "celeste_image_generation.client", "celeste_image_generation.providers.*", + "celeste_video_generation.client", + "celeste_video_generation.providers.*", ] disable_error_code = ["override", "return-value", "arg-type", "call-arg", "assignment", "no-any-return"]