diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Filters/LlmConfigFilter.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Filters/LlmConfigFilter.cs new file mode 100644 index 000000000..89ec04103 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Filters/LlmConfigFilter.cs @@ -0,0 +1,13 @@ +using BotSharp.Abstraction.MLTasks.Settings; + +namespace BotSharp.Abstraction.MLTasks.Filters; + +public class LlmConfigFilter +{ + public List? Providers { get; set; } + public List? ModelIds { get; set; } + public List? ModelNames { get; set; } + public List? ModelTypes { get; set; } + public List? ModelCapabilities { get; set; } + public bool? MultiModal { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index f70a9496e..da8a8f534 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -1,12 +1,13 @@ +using BotSharp.Abstraction.MLTasks.Filters; using BotSharp.Abstraction.MLTasks.Settings; namespace BotSharp.Abstraction.MLTasks; public interface ILlmProviderService { - LlmModelSetting GetSetting(string provider, string model); + LlmModelSetting? GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, bool imageGenerate = false); + LlmModelSetting? GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, IEnumerable? capabilities = null); List GetProviderModels(string provider); - List GetLlmConfigs(LlmConfigOptions? options = null); + List GetLlmConfigs(LlmConfigFilter? filter = null); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs deleted file mode 100644 index eae417a41..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmConfigOptions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks.Settings; - -public class LlmConfigOptions -{ - public LlmModelType? Type { get; set; } - public bool? MultiModal { get; set; } - public bool? ImageGeneration { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 3249b9eb4..ff2c1d3e6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -31,17 +31,13 @@ public class LlmModelSetting public string ApiKey { get; set; } = null!; public string? Endpoint { get; set; } public LlmModelType Type { get; set; } = LlmModelType.Chat; + public List Capabilities { get; set; } = []; /// - /// If true, allow sending images/vidoes to this model + /// If true, allow sending images/videos to this model /// public bool MultiModal { get; set; } - /// - /// If true, allow generating images - /// - public bool ImageGeneration { get; set; } - /// /// Settings for embedding /// @@ -173,10 +169,29 @@ public class LlmCostSetting public enum LlmModelType { + All = 0, Text = 1, Chat = 2, Image = 3, Embedding = 4, Audio = 5, Realtime = 6, + Web = 7 } + +public enum LlmModelCapability +{ + All = 0, + Text = 1, + Chat = 2, + ImageReading = 3, + ImageGeneration = 4, + ImageEdit = 5, + ImageVariation = 6, + Embedding = 7, + AudioTranscription = 8, + AudioGeneration = 9, + Realtime = 10, + WebSearch = 11, + PdfReading = 12 +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/ObjectExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/ObjectExtensions.cs index bed3b71ab..a36516c8a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/ObjectExtensions.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/ObjectExtensions.cs @@ -6,7 +6,7 @@ namespace BotSharp.Abstraction.Utilities; public static class ObjectExtensions { - private static readonly JsonSerializerOptions DefaultJsonOptions = new() + private static readonly JsonSerializerOptions _defaultJsonOptions = new() { PropertyNameCaseInsensitive = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -14,28 +14,55 @@ public static class ObjectExtensions ReferenceHandler = ReferenceHandler.IgnoreCycles }; - public static T? DeepClone(this T? obj, Action? modifier = null, BotSharpOptions? options = null) where T : class + public static T? DeepClone(this T? inputObj, Action? modifier = null, BotSharpOptions? options = null) where T : class { - if (obj == null) + if (inputObj == null) { return null; } try { - var json = JsonSerializer.Serialize(obj, options?.JsonSerializerOptions ?? DefaultJsonOptions); - var newObj = JsonSerializer.Deserialize(json, options?.JsonSerializerOptions ?? DefaultJsonOptions); - if (modifier != null && newObj != null) + var json = JsonSerializer.Serialize(inputObj, options?.JsonSerializerOptions ?? _defaultJsonOptions); + var outputObj = JsonSerializer.Deserialize(json, options?.JsonSerializerOptions ?? _defaultJsonOptions); + if (modifier != null && outputObj != null) { - modifier(newObj); + modifier(outputObj); } - return newObj; + return outputObj; } catch (Exception ex) { - Console.WriteLine($"DeepClone Error in {nameof(DeepClone)}: {ex}"); + Console.WriteLine($"DeepClone Error in {nameof(DeepClone)} for {typeof(T).Name}: {ex}"); return null; } } + + public static TOutput? DeepClone(this TInput? inputObj, Action? modifier = null, BotSharpOptions? options = null) + where TInput : class + where TOutput : class + { + if (inputObj == null) + { + return null; + } + + try + { + var json = JsonSerializer.Serialize(inputObj, options?.JsonSerializerOptions ?? _defaultJsonOptions); + var outputObj = JsonSerializer.Deserialize(json, options?.JsonSerializerOptions ?? _defaultJsonOptions); + if (modifier != null && outputObj != null) + { + modifier(outputObj); + } + + return outputObj; + } + catch (Exception ex) + { + Console.WriteLine($"DeepClone Error in {nameof(DeepClone)} for {typeof(TInput).Name} and {typeof(TOutput).Name}: {ex}"); + return null; + } + } } diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabWatcher.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabWatcher.cs index 288458430..ccd89cf45 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabWatcher.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabWatcher.cs @@ -36,7 +36,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await RunCronChecker(scope.ServiceProvider); await Task.Delay(1000, stoppingToken); }); - if (isLocked == false) + + if (!isLocked) { await Task.Delay(1000, stoppingToken); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs b/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs index a8bafdd25..cadeb3b47 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Hooks/BasicAgentHook.cs @@ -70,12 +70,18 @@ public override void OnAgentUtilityLoaded(Agent agent) foreach (var utility in innerUtilities) { var isVisible = agentService.RenderVisibility(utility.VisibilityExpression, renderDict); - if (!isVisible || utility.Items.IsNullOrEmpty()) continue; + if (!isVisible || utility.Items.IsNullOrEmpty()) + { + continue; + } foreach (var item in utility.Items) { isVisible = agentService.RenderVisibility(item.VisibilityExpression, renderDict); - if (!isVisible) continue; + if (!isVisible) + { + continue; + } if (item.FunctionName?.StartsWith(UTIL_PREFIX) == true) { diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index 55980e963..83e83ef5a 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -155,7 +155,7 @@ public bool RenderVisibility(string? visibilityExpression, IDictionary(); - var copy = new Dictionary(dict); + var copy = dict != null ? new Dictionary(dict) : []; var result = render.Render(visibilityExpression, new Dictionary { { "states", copy } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 60645bcf2..b5d989298 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -5,7 +5,8 @@ namespace BotSharp.Core.Infrastructures; public class CompletionProvider { - public static object GetCompletion(IServiceProvider services, + public static object GetCompletion( + IServiceProvider services, string? provider = null, string? model = null, AgentLlmConfig? agentConfig = null) @@ -42,7 +43,8 @@ public static object GetCompletion(IServiceProvider services, } } - public static IChatCompletion GetChatCompletion(IServiceProvider services, + public static IChatCompletion GetChatCompletion( + IServiceProvider services, string? provider = null, string? model = null, string? modelId = null, @@ -66,7 +68,8 @@ public static IChatCompletion GetChatCompletion(IServiceProvider services, return completer; } - public static ITextCompletion GetTextCompletion(IServiceProvider services, + public static ITextCompletion GetTextCompletion( + IServiceProvider services, string? provider = null, string? model = null, AgentLlmConfig? agentConfig = null) @@ -86,15 +89,16 @@ public static ITextCompletion GetTextCompletion(IServiceProvider services, return completer; } - public static IImageCompletion GetImageCompletion(IServiceProvider services, + public static IImageCompletion GetImageCompletion( + IServiceProvider services, string? provider = null, string? model = null, string? modelId = null, - bool imageGenerate = false) + IEnumerable? capabilities = null) { var completions = services.GetServices(); (provider, model) = GetProviderAndModel(services, provider: provider, - model: model, modelId: modelId, imageGenerate: imageGenerate); + model: model, modelId: modelId, capabilities: capabilities); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) @@ -107,7 +111,8 @@ public static IImageCompletion GetImageCompletion(IServiceProvider services, return completer; } - public static ITextEmbedding GetTextEmbedding(IServiceProvider services, + public static ITextEmbedding GetTextEmbedding( + IServiceProvider services, string? provider = null, string? model = null) { @@ -166,7 +171,8 @@ public static IAudioSynthesis GetAudioSynthesizer( return completer; } - public static IRealTimeCompletion GetRealTimeCompletion(IServiceProvider services, + public static IRealTimeCompletion GetRealTimeCompletion( + IServiceProvider services, string? provider = null, string? model = null, string? modelId = null, @@ -176,7 +182,7 @@ public static IRealTimeCompletion GetRealTimeCompletion(IServiceProvider service var completions = services.GetServices(); (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, multiModal: multiModal, - modelType: LlmModelType.Realtime, + modelType: LlmModelType.Realtime, agentConfig: agentConfig); var completer = completions.FirstOrDefault(x => x.Provider == provider); @@ -190,13 +196,14 @@ public static IRealTimeCompletion GetRealTimeCompletion(IServiceProvider service return completer; } - private static (string, string) GetProviderAndModel(IServiceProvider services, + private static (string, string) GetProviderAndModel( + IServiceProvider services, string? provider = null, string? model = null, string? modelId = null, bool? multiModal = null, LlmModelType? modelType = null, - bool imageGenerate = false, + IEnumerable? capabilities = null, AgentLlmConfig? agentConfig = null) { var agentSetting = services.GetRequiredService(); @@ -220,9 +227,9 @@ private static (string, string) GetProviderAndModel(IServiceProvider services, var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; var llmProviderService = services.GetRequiredService(); model = llmProviderService.GetProviderModel(provider, modelIdentity, - multiModal: multiModal, + multiModal: multiModal, modelType: modelType, - imageGenerate: imageGenerate)?.Name; + capabilities: capabilities)?.Name; } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index 95cfd9c04..3445d3c00 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.MLTasks.Filters; using BotSharp.Abstraction.MLTasks.Settings; using BotSharp.Abstraction.Settings; @@ -40,14 +41,12 @@ public List GetProviderModels(string provider) { var settingService = _services.GetRequiredService(); return settingService.Bind>($"LlmProviders") - .FirstOrDefault(x => x.Provider.Equals(provider)) - ?.Models ?? new List(); + .FirstOrDefault(x => x.Provider.Equals(provider))?.Models ?? []; } - public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, bool imageGenerate = false) + public LlmModelSetting? GetProviderModel(string provider, string id, bool? multiModal = null, LlmModelType? modelType = null, IEnumerable? capabilities = null) { - var models = GetProviderModels(provider) - .Where(x => x.Id == id); + var models = GetProviderModels(provider).Where(x => x.Id == id); if (multiModal.HasValue) { @@ -59,7 +58,15 @@ public LlmModelSetting GetProviderModel(string provider, string id, bool? multiM models = models.Where(x => x.Type == modelType.Value); } - models = models.Where(x => x.ImageGeneration == imageGenerate); + if (capabilities != null) + { + models = models.Where(x => x.Capabilities != null && capabilities.Any(y => x.Capabilities.Contains(y))); + } + + if (models.IsNullOrEmpty()) + { + return null; + } var random = new Random(); var index = random.Next(0, models.Count()); @@ -72,14 +79,14 @@ public LlmModelSetting GetProviderModel(string provider, string id, bool? multiM var settings = _services.GetRequiredService>(); var providerSetting = settings.FirstOrDefault(p => p.Provider.Equals(provider, StringComparison.CurrentCultureIgnoreCase)); + if (providerSetting == null) { _logger.LogError($"Can't find provider settings for {provider}"); return null; } - var modelSetting = providerSetting.Models.FirstOrDefault(m => - m.Name.Equals(model, StringComparison.CurrentCultureIgnoreCase)); + var modelSetting = providerSetting.Models.FirstOrDefault(m => m.Name.Equals(model, StringComparison.CurrentCultureIgnoreCase)); if (modelSetting == null) { _logger.LogError($"Can't find model settings for {provider}.{model}"); @@ -95,42 +102,67 @@ public LlmModelSetting GetProviderModel(string provider, string id, bool? multiM m.Group.Equals(modelSetting.Group, StringComparison.CurrentCultureIgnoreCase)) .ToList(); - // pick one model randomly - var random = new Random(); - var index = random.Next(0, models.Count()); - modelSetting = models.ElementAt(index); + if (!models.IsNullOrEmpty()) + { + // pick one model randomly + var random = new Random(); + var index = random.Next(0, models.Count()); + modelSetting = models.ElementAt(index); + } } return modelSetting; } - public List GetLlmConfigs(LlmConfigOptions? options = null) + public List GetLlmConfigs(LlmConfigFilter? filter = null) { var settingService = _services.GetRequiredService(); var providers = settingService.Bind>($"LlmProviders"); var configs = new List(); + var comparer = StringComparer.OrdinalIgnoreCase; + + if (providers.IsNullOrEmpty()) + { + return configs; + } - if (providers.IsNullOrEmpty()) return configs; + if (filter == null) + { + return providers ?? []; + } - if (options == null) return providers ?? []; + if (filter.Providers != null) + { + providers = providers.Where(x => filter.Providers.Contains(x.Provider, comparer)).ToList(); + } foreach (var provider in providers) { - var models = provider.Models ?? []; - if (options.Type.HasValue) + IEnumerable models = provider.Models ?? []; + if (filter.ModelTypes != null) + { + models = models.Where(x => filter.ModelTypes.Contains(x.Type)); + } + + if (filter.ModelIds != null) + { + models = models.Where(x => filter.ModelIds.Contains(x.Id, comparer)); + } + + if (filter.ModelNames != null) { - models = models.Where(x => x.Type == options.Type.Value).ToList(); + models = models.Where(x => filter.ModelNames.Contains(x.Name, comparer)); } - if (options.MultiModal.HasValue) + if (filter.ModelCapabilities != null) { - models = models.Where(x => x.MultiModal == options.MultiModal.Value).ToList(); + models = models.Where(x => x.Capabilities != null && filter.ModelCapabilities.Any(y => x.Capabilities.Contains(y))); } - if (options.ImageGeneration.HasValue) + if (filter.MultiModal.HasValue) { - models = models.Where(x => x.ImageGeneration == options.ImageGeneration.Value).ToList(); + models = models.Where(x => x.MultiModal == filter.MultiModal.Value); } if (models.IsNullOrEmpty()) @@ -138,7 +170,7 @@ public List GetLlmConfigs(LlmConfigOptions? options = null) continue; } - provider.Models = models; + provider.Models = models.ToList(); configs.Add(provider); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 6b87bec2f..6c2b335c1 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.MLTasks.Filters; using BotSharp.Abstraction.MLTasks.Settings; namespace BotSharp.OpenAPI.Controllers; @@ -9,6 +10,7 @@ public class LlmProviderController : ControllerBase { private readonly IServiceProvider _services; private readonly ILlmProviderService _llmProvider; + public LlmProviderController(IServiceProvider services, ILlmProviderService llmProvider) { _services = services; @@ -22,16 +24,16 @@ public IEnumerable GetLlmProviders() } [HttpGet("/llm-provider/{provider}/models")] - public IEnumerable GetLlmProviderModels([FromRoute] string provider) + public IEnumerable GetLlmProviderModels([FromRoute] string provider, [FromQuery] LlmModelType modelType = LlmModelType.Chat) { var list = _llmProvider.GetProviderModels(provider); - return list.Where(x => x.Type == LlmModelType.Chat); + return list.Where(x => x.Type == modelType); } [HttpGet("/llm-configs")] - public List GetLlmConfigs([FromQuery] LlmConfigOptions options) + public List GetLlmConfigs([FromQuery] LlmConfigFilter filter) { - var configs = _llmProvider.GetLlmConfigs(options); + var configs = _llmProvider.GetLlmConfigs(filter); return configs; } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs index fcd2be3a4..957bce442 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailReaderFn.cs @@ -68,7 +68,7 @@ public async Task Execute(RoleDialogModel message) var llmProviderService = _services.GetRequiredService(); var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); var model = llmProviderService.GetProviderModel(provider: provider ?? "openai", id: "gpt-4o"); - var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); + var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model?.Name); var convService = _services.GetRequiredService(); var conversationId = convService.ConversationId; var dialogs = convService.GetDialogHistory(fromBreakpoint: false); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Sequential/SequentialPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/Sequential/SequentialPlanner.cs index 94b4a6873..a3c7e9464 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Sequential/SequentialPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Sequential/SequentialPlanner.cs @@ -171,7 +171,7 @@ public async Task GetDecomposedStepAsync(Agent router, string me // chat completion var completion = CompletionProvider.GetChatCompletion(_services, provider: "openai", - model: model.Name); + model: model?.Name); int retryCount = 0; while (retryCount < 2) diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 9283d9be4..00d27465c 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -50,7 +50,11 @@ "Name": "gpt-35-turbo", "Version": "1106", "ApiKey": "", - "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/" + "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/", + "Type": "chat", + "Capabilities": [ + "Chat" + ] }, { "Name": "gpt-35-turbo-instruct", @@ -58,6 +62,9 @@ "ApiKey": "", "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/", "Type": "text", + "Capabilities": [ + "Text" + ], "Cost": { "TextInputCost": 0.0015, "CachedTextInputCost": 0, @@ -74,7 +81,10 @@ "Models": [ { "Name": "llama-2-7b-guanaco-qlora.Q2_K.gguf", - "Type": "chat" + "Type": "chat", + "Capabilities": [ + "Chat" + ] } ] }, @@ -83,11 +93,17 @@ "Models": [ { "Name": "mistralai/Mistral-7B-v0.1", - "Type": "text" + "Type": "text", + "Capabilities": [ + "Text" + ] }, { "Name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "Type": "text" + "Type": "text", + "Capabilities": [ + "Text" + ] } ] }, @@ -97,6 +113,9 @@ { "Name": "gpt-35-turbo", "Type": "chat", + "Capabilities": [ + "Chat" + ], "Cost": { "TextInputCost": 0.0015, "CachedTextInputCost": 0, @@ -114,6 +133,9 @@ { "Name": "chatglm3_6b", "Type": "chat", + "Capabilities": [ + "Chat" + ], "Cost": { "TextInputCost": 0.0015, "CachedTextInputCost": 0, @@ -135,6 +157,10 @@ "ApiKey": "", "Type": "chat", "MultiModal": true, + "Capabilities": [ + "Chat", + "ImageReading" + ], "Cost": { "TextInputCost": 0.00015, "CachedTextInputCost": 0, @@ -151,6 +177,10 @@ "ApiKey": "", "Type": "chat", "MultiModal": true, + "Capabilities": [ + "Chat", + "ImageReading" + ], "Cost": { "TextInputCost": 0.0025, "CachedTextInputCost": 0, @@ -165,9 +195,11 @@ "Name": "gpt-4o-mini-realtime-preview-2024-12-17", "Version": "2024-12-17", "ApiKey": "", - "Type": "chat", + "Type": "realtime", "MultiModal": true, - "RealTime": true, + "Capabilities": [ + "Realtime" + ], "Cost": { "TextInputCost": 0.0025, "CachedTextInputCost": 0, @@ -183,6 +215,9 @@ "Version": "3-small", "ApiKey": "", "Type": "embedding", + "Capabilities": [ + "Embedding" + ], "Embedding": { "Dimension": 1536 } @@ -194,7 +229,11 @@ "ApiKey": "", "Endpoint": "", "Type": "image", - "ImageGeneration": true, + "Capabilities": [ + "ImageGeneration", + "ImageEdit", + "ImageVariation" + ], "Image": { "Generation": { "Size": { @@ -246,7 +285,9 @@ "Version": "dall-e-3", "ApiKey": "", "Type": "image", - "ImageGeneration": true, + "Capabilities": [ + "ImageGeneration" + ], "Image": { "Generation": { "Size": { @@ -282,7 +323,10 @@ "Version": "gpt-image-1", "ApiKey": "", "Type": "image", - "ImageGeneration": true, + "Capabilities": [ + "ImageGeneration", + "ImageEdit" + ], "Image": { "Generation": { "Size": { @@ -328,7 +372,10 @@ "Version": "gpt-image-1-mini", "ApiKey": "", "Type": "image", - "ImageGeneration": true, + "Capabilities": [ + "ImageGeneration", + "ImageEdit" + ], "Image": { "Generation": { "Size": { @@ -378,6 +425,10 @@ "ApiKey": "", "Endpoint": "https://api.deepseek.com/v1/", "Type": "chat", + "Capabilities": [ + "Chat", + "ImageReading" + ], "Cost": { "TextInputCost": 0.0015, "CachedTextInputCost": 0, @@ -397,6 +448,11 @@ "ApiKey": "", "Type": "chat", "MultiModal": true, + "Capabilities": [ + "Chat", + "ImageReading", + "PdfReading" + ], "Cost": { "TextInputCost": 0.0015, "CachedTextInputCost": 0, @@ -411,6 +467,9 @@ "ApiKey": "", "Type": "realtime", "MultiModal": true, + "Capabilities": [ + "Realtime" + ], "Cost": { "TextInputCost": 0.0015, "CachedTextInputCost": 0,