Add multimodal model support (STT/TTS/image) + fix UTF-8 server crash

CMakeLists.txt

@@ -2,6 +2,10 @@ cmake_minimum_required(VERSION 3.15)
 project(arbiterAI VERSION 0.1.13 LANGUAGES C CXX)
 
 option(ARBITERAI_ENABLE_LLAMA "Enable local llama.cpp model runtime support" ON)
+option(ARBITERAI_ENABLE_WHISPER "Enable local whisper.cpp speech-to-text support" OFF)
+option(ARBITERAI_ENABLE_STABLE_DIFFUSION "Enable local stable-diffusion.cpp image generation support" OFF)
+option(ARBITERAI_ENABLE_VIBEVOICE "Enable local vibevoice.cpp text-to-speech support" OFF)
+option(ARBITERAI_ENABLE_ORPHEUS "Enable local Orpheus TTS via a dlopen'd chatllm.cpp engine library" OFF)
 
 # Read llama-cpp build number from our custom vcpkg port
 if(ARBITERAI_ENABLE_LLAMA)
@@ -51,6 +55,18 @@ find_package(cxxopts CONFIG REQUIRED)
 if(ARBITERAI_ENABLE_LLAMA)
     find_package(llama CONFIG REQUIRED)
 endif()
+if(ARBITERAI_ENABLE_WHISPER)
+    find_package(whisper CONFIG REQUIRED)
+endif()
+if(ARBITERAI_ENABLE_STABLE_DIFFUSION)
+    find_package(stable-diffusion CONFIG REQUIRED)
+    find_package(ZLIB REQUIRED)
+endif()
+if(ARBITERAI_ENABLE_VIBEVOICE)
+    # vibevoice.cpp installs a lib + header but no package config.
+    find_path(VIBEVOICE_INCLUDE_DIR NAMES vibevoice_capi.h REQUIRED)
+    find_library(VIBEVOICE_LIBRARY NAMES vibevoice REQUIRED)
+endif()
 find_package(libgit2 CONFIG REQUIRED)
 find_package(Threads REQUIRED)
 find_package(spdlog CONFIG REQUIRED)
@@ -108,6 +124,35 @@ if(ARBITERAI_ENABLE_LLAMA)
     )
 endif()
 
+if(ARBITERAI_ENABLE_WHISPER)
+    list(APPEND arbiterai_src
+        ./src/arbiterAI/providers/whisper.h
+        ./src/arbiterAI/providers/whisper.cpp
+    )
+endif()
+
+if(ARBITERAI_ENABLE_STABLE_DIFFUSION)
+    list(APPEND arbiterai_src
+        ./src/arbiterAI/providers/stableDiffusion.h
+        ./src/arbiterAI/providers/stableDiffusion.cpp
+    )
+endif()
+
+if(ARBITERAI_ENABLE_VIBEVOICE)
+    list(APPEND arbiterai_src
+        ./src/arbiterAI/providers/vibevoice.h
+        ./src/arbiterAI/providers/vibevoice.cpp
+    )
+endif()
+
+if(ARBITERAI_ENABLE_ORPHEUS)
+    # dlopen-based: needs only libdl (already linked via CMAKE_DL_LIBS).
+    list(APPEND arbiterai_src
+        ./src/arbiterAI/providers/orpheus.h
+        ./src/arbiterAI/providers/orpheus.cpp
+    )
+endif()
+
 # Add library
 add_library(arbiterai
     ${arbiterai_src}
@@ -130,11 +175,32 @@ if(ARBITERAI_ENABLE_LLAMA)
     target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_LLAMA=1)
 endif()
 
+if(ARBITERAI_ENABLE_WHISPER)
+    target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_WHISPER=1)
+endif()
+
+if(ARBITERAI_ENABLE_STABLE_DIFFUSION)
+    target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_STABLE_DIFFUSION=1)
+endif()
+
+if(ARBITERAI_ENABLE_VIBEVOICE)
+    target_include_directories(arbiterai PRIVATE ${VIBEVOICE_INCLUDE_DIR})
+    target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_VIBEVOICE=1)
+endif()
+
+if(ARBITERAI_ENABLE_ORPHEUS)
+    target_compile_definitions(arbiterai PUBLIC ARBITERAI_ENABLE_ORPHEUS=1)
+endif()
+
 target_link_libraries(arbiterai
     PUBLIC
         cpr::cpr
         nlohmann_json::nlohmann_json
         $<$<BOOL:${ARBITERAI_ENABLE_LLAMA}>:llama>
+        $<$<BOOL:${ARBITERAI_ENABLE_WHISPER}>:whisper>
+        $<$<BOOL:${ARBITERAI_ENABLE_STABLE_DIFFUSION}>:stable-diffusion>
+        $<$<BOOL:${ARBITERAI_ENABLE_STABLE_DIFFUSION}>:ZLIB::ZLIB>
+        $<$<BOOL:${ARBITERAI_ENABLE_VIBEVOICE}>:${VIBEVOICE_LIBRARY}>
         libgit2::libgit2package
     PRIVATE
         spdlog::spdlog
@@ -170,6 +236,30 @@ target_link_libraries(arbiterai
             tests/inferenceSchedulerTests.cpp
         )
     endif()
+
+    if(ARBITERAI_ENABLE_WHISPER)
+        target_sources(arbiterai_tests PRIVATE
+            tests/whisperProviderTests.cpp
+        )
+    endif()
+
+    if(ARBITERAI_ENABLE_STABLE_DIFFUSION)
+        target_sources(arbiterai_tests PRIVATE
+            tests/stableDiffusionProviderTests.cpp
+        )
+    endif()
+
+    if(ARBITERAI_ENABLE_VIBEVOICE)
+        target_sources(arbiterai_tests PRIVATE
+            tests/vibevoiceProviderTests.cpp
+        )
+    endif()
+
+    if(ARBITERAI_ENABLE_ORPHEUS)
+        target_sources(arbiterai_tests PRIVATE
+            tests/orpheusProviderTests.cpp
+        )
+    endif()
 
     target_link_libraries(arbiterai_tests
         PRIVATE

docs/feature_request.md

@@ -2,4 +2,16 @@
 - Allow for configurable devices that can be used and well as vram limits
 - Add support to just unload a model or all models
 - Need configuration for listening host/port, default 9010
-- Allow model config to be inject through the api, add support to the server so a client can add a model config.
+- Allow model config to be inject through the api, add support to the server so a client can add a model config.
+- [DONE] Wire ModelDownloader/StorageManager for whisper and stable-diffusion providers so weights auto-download from an injected config's variants[].download.url on load (same as llama GGUFs) — today they only resolve a local file_path.
+  → `BaseProvider::resolveDownloadableModelFile()` downloads every configured variant file (incl. multi-file bundles like vibevoice tts+tokenizer+voice) to the StorageManager models dir when absent; whisper/sd/vibevoice/orpheus delegate to it.
+- [DONE] Record modality-aware speed telemetry (STT: audio-seconds/realtime-factor; image: images and steps/sec) into InferenceStats and expose on /api/stats/history keyed by (model, variant, hardware), so clients can benchmark/normalize non-LLM speed.
+  → InferenceStats gained modality/imagesGenerated/imageSteps/audioSeconds/audioCharacters/realtimeFactor/stepsPerSecond/cost, computed at dispatch and emitted per-entry on /api/stats/history (client groups by model+variant+hardware).
+- [DONE] Include the computed audio duration in the /v1/audio/transcriptions response body.
+  → `duration` is included in the JSON response.
+- [DONE] Extend model_config schema with optional whisper_options / sd_options blocks for engine-specific knobs (language, steps, sampler, vae, etc.) without loosening additionalProperties.
+  → schema + ModelInfo.whisperOptions/sdOptions parsing; whisper applies language/translate defaults, sd applies steps/width/height/cfg_scale defaults.
+- [OPEN] See docs/tasks/multimodal_benchmark_integration.md for the full benchmark-integration contract.
+  → The speed-telemetry foundation (above) is in place; the full benchmark runner/endpoints per that contract remain.
+- [DONE] BUG (server-wide crash): invalid UTF-8 in a model's generated output aborts the whole server. When a generation clips a multi-byte UTF-8 character at a token/length boundary (or emits a llama.cpp byte-fallback token), arbiterResponse.text is not valid UTF-8; serializing it with nlohmann::json's default (strict) handler throws json.exception.type_error.316 ("incomplete UTF-8 string; last byte: 0x9E"). The exception is uncaught → terminate() → SIGABRT (code=dumped, status=6/ABRT), taking down every in-flight request. Reproduced twice on ai-minisforum with model qwythos-9b-claude-mythos-5-1m during HumanEval (2026-07-18 10:32 and 11:26 EDT). The model text enters JSON at src/server/routes.cpp:2117 (messageJson["content"]=arbiterResponse.text) and dies at the response .dump(). Fix: (1) serialize every response body that carries model text with the replace error handler — j.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) — on both the non-streaming path and the streaming deltas (routes.cpp ~1804/1847), and wrap serialization in try/catch so no model output can ever abort the process; (2) root cause — buffer incomplete multi-byte UTF-8 sequences across token boundaries in the llama provider's detokenization so partial characters are never emitted.
+  → (1) `safeDump()` (error_handler_t::replace + try/catch) now serializes all model-text response bodies (streaming deltas + non-streaming + transcription/image). (2) `Utf8StreamBuffer` in the llama provider's two generation loops holds an incomplete multi-byte tail until the next token completes it (verified: café/emoji boundary cases).

examples/multimodal_models.json

@@ -0,0 +1,90 @@
+{
+    "schema_version": "1.1.0",
+    "models": [
+        {
+            "model": "mock-transcribe",
+            "provider": "mock",
+            "mode": "transcription",
+            "ranking": 1,
+            "pricing": {
+                "audio_input_cost_per_second": 0.0001
+            }
+        },
+        {
+            "model": "mock-speech",
+            "provider": "mock",
+            "mode": "speech",
+            "ranking": 1,
+            "pricing": {
+                "audio_output_cost_per_character": 0.00003
+            }
+        },
+        {
+            "model": "mock-image",
+            "provider": "mock",
+            "mode": "image",
+            "ranking": 1,
+            "pricing": {
+                "image_cost": 0.04
+            }
+        },
+        {
+            "model": "whisper-1",
+            "provider": "openai",
+            "mode": "transcription",
+            "api_base": "https://api.openai.com/v1",
+            "ranking": 50,
+            "pricing": {
+                "audio_input_cost_per_second": 0.0001
+            }
+        },
+        {
+            "model": "tts-1",
+            "provider": "openai",
+            "mode": "speech",
+            "api_base": "https://api.openai.com/v1",
+            "ranking": 50,
+            "pricing": {
+                "audio_output_cost_per_character": 0.000015
+            }
+        },
+        {
+            "model": "dall-e-3",
+            "provider": "openai",
+            "mode": "image",
+            "api_base": "https://api.openai.com/v1",
+            "ranking": 50,
+            "pricing": {
+                "image_cost": 0.04
+            }
+        },
+        {
+            "model": "whisper-base-en",
+            "provider": "whisper",
+            "mode": "transcription",
+            "ranking": 50,
+            "file_path": "/srv/models/whisper/ggml-base.en.bin"
+        },
+        {
+            "model": "sd-v1-5",
+            "provider": "stable-diffusion",
+            "mode": "image",
+            "ranking": 50,
+            "file_path": "/srv/models/sd/v1-5-pruned-emaonly.safetensors"
+        },
+        {
+            "model": "vibevoice-0.5b",
+            "provider": "vibevoice",
+            "mode": "speech",
+            "ranking": 50,
+            "file_path": "/srv/models/vibevoice/vibevoice-realtime-0.5B-q8_0.gguf"
+        },
+        {
+            "model": "orpheus-3b",
+            "provider": "orpheus",
+            "mode": "speech",
+            "ranking": 40,
+            "file_path": "/srv/models/orpheus/orpheus-3b-q8_0.bin"
+        }
+    ]
+}

schemas/model_config.schema.json

@@ -30,7 +30,7 @@
           "provider": {
             "type": "string",
             "description": "Model provider/service",
-            "enum": ["anthropic", "llama", "openai", "deepseek", "openrouter", "mock"]
+            "enum": ["anthropic", "llama", "openai", "deepseek", "openrouter", "mock", "whisper", "stable-diffusion", "vibevoice", "orpheus"]
           },
           "ranking": {
             "type": "integer",
@@ -41,7 +41,8 @@
           },
           "mode": {
             "type": "string",
-            "description": "Operation mode",
+            "description": "Model modality. Routes requests to the matching endpoint/method: chat completions, embeddings, speech-to-text (transcription), text-to-speech (speech), or image generation (image).",
+            "enum": ["chat", "embedding", "transcription", "speech", "image"],
             "default": "chat"
           },
           "api_base": {
@@ -125,7 +126,7 @@
           },
           "pricing": {
             "type": "object",
-            "description": "Token pricing information",
+            "description": "Pricing information. Token costs apply to chat/embedding models; the modality-specific costs apply to image/audio models.",
             "properties": {
               "prompt_token_cost": {
                 "type": "number",
@@ -134,6 +135,21 @@
               "completion_token_cost": {
                 "type": "number",
                 "minimum": 0
+              },
+              "image_cost": {
+                "type": "number",
+                "description": "Cost per generated image (mode: image)",
+                "minimum": 0
+              },
+              "audio_input_cost_per_second": {
+                "type": "number",
+                "description": "Cost per second of transcribed audio (mode: transcription)",
+                "minimum": 0
+              },
+              "audio_output_cost_per_character": {
+                "type": "number",
+                "description": "Cost per character of synthesized speech (mode: speech)",
+                "minimum": 0
               }
             }
           },
@@ -305,6 +321,56 @@
             "description": "Output format produced by the model. When set (e.g. 'harmony'), the server converts the model's native output to standard OpenAI API format so clients don't need to understand the model's native format.",
             "enum": ["", "harmony"],
             "default": ""
+          },
+          "whisper_options": {
+            "type": "object",
+            "description": "Engine-specific defaults for the whisper (transcription) provider. Applied when the request omits the corresponding field.",
+            "properties": {
+              "language": {
+                "type": "string",
+                "description": "Default language hint (ISO-639-1, or 'auto' to detect)"
+              },
+              "translate": {
+                "type": "boolean",
+                "description": "Translate to English instead of transcribing in the source language"
+              }
+            },
+            "additionalProperties": false
+          },
+          "sd_options": {
+            "type": "object",
+            "description": "Engine-specific defaults for the stable-diffusion (image) provider. Applied when the request omits the corresponding field.",
+            "properties": {
+              "steps": {
+                "type": "integer",
+                "description": "Default number of diffusion sample steps",
+                "minimum": 1
+              },
+              "cfg_scale": {
+                "type": "number",
+                "description": "Default classifier-free guidance scale",
+                "minimum": 0
+              },
+              "sampler": {
+                "type": "string",
+                "description": "Sampling method (e.g. euler_a, dpm++2m, dpm++2mv2)"
+              },
+              "width": {
+                "type": "integer",
+                "description": "Default image width in pixels",
+                "minimum": 64
+              },
+              "height": {
+                "type": "integer",
+                "description": "Default image height in pixels",
+                "minimum": 64
+              },
+              "vae_path": {
+                "type": "string",
+                "description": "Optional path to an external VAE file"
+              }
+            },
+            "additionalProperties": false
           }
         }
       }