diff --git a/BotSharp.sln b/BotSharp.sln index f68ce1c60..e992d26ad 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -147,6 +147,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandle EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ImageHandler", "src\Plugins\BotSharp.Plugin.ImageHandler\BotSharp.Plugin.ImageHandler.csproj", "{242F2D93-FCCE-4982-8075-F3052ECCA92C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -619,6 +621,14 @@ Global {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.ActiveCfg = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.Build.0 = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.Build.0 = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.ActiveCfg = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -690,6 +700,7 @@ Global {B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F} + {242F2D93-FCCE-4982-8075-F3052ECCA92C} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index a2f2a17f2..e7b054d79 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Options; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; @@ -65,4 +66,13 @@ public interface IAgentService Task> GetUserAgents(string userId); PluginDef GetPlugin(string agentId); + + Task> GetAgentCodeScripts(string agentId, AgentCodeScriptFilter? filter = null) + => Task.FromResult(new List()); + + Task GetAgentCodeScript(string agentId, string scriptName, string scriptType = AgentCodeScriptType.Src) + => Task.FromResult(string.Empty); + + Task UpdateAgentCodeScripts(string agentId, List codeScripts, AgentCodeScriptUpdateOptions? options = null) + => Task.FromResult(false); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs index 75e3aa93b..6ed259a3f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs @@ -41,4 +41,49 @@ public class AgentLlmConfig [JsonPropertyName("reasoning_effort_level")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ReasoningEffortLevel { get; set; } + + /// + /// Image generation config + /// + [JsonPropertyName("image_generation")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmImageGenerationConfig? ImageGeneration { get; set; } + + /// + /// Image edit config + /// + [JsonPropertyName("image_edit")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmImageEditConfig? ImageEdit { get; set; } + + /// + /// Audio transcription config + /// + [JsonPropertyName("audio_transcription")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmAudioTranscriptionConfig? AudioTranscription { get; set; } + + /// + /// Realtime config + /// + [JsonPropertyName("realtime")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmRealtimeConfig? Realtime { get; set; } +} + + +public class LlmImageGenerationConfig : LlmProviderModel +{ +} + +public class LlmImageEditConfig : LlmProviderModel +{ } + +public class LlmAudioTranscriptionConfig : LlmProviderModel +{ +} + +public class LlmRealtimeConfig : LlmProviderModel +{ +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs new file mode 100644 index 000000000..4b5e9e957 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.Repositories.Models; + +namespace BotSharp.Abstraction.Agents.Options; + +public class AgentCodeScriptUpdateOptions : AgentCodeScriptDbUpdateOptions +{ + public bool DeleteIfNotIncluded { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index da9caecf3..90cc44889 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -141,7 +141,7 @@ public class RoleDialogModel : ITrackableMessage public bool IsFromAssistant => Role == AgentRole.Assistant || Role == AgentRole.Model; [JsonIgnore(Condition = JsonIgnoreCondition.Always)] - public string RoleContent + public string LlmContent { get { @@ -150,10 +150,6 @@ public string RoleContent { text = !string.IsNullOrWhiteSpace(Payload) ? Payload : Content; } - else - { - text = !string.IsNullOrWhiteSpace(Content) ? Content : RichContent?.Message?.Text; - } return text; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs index 60f6464ad..a47584b44 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs @@ -8,6 +8,9 @@ public class ProgramCodeTemplateMessage : IRichMessage, ITemplateMessage [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; + [JsonPropertyName("code_script")] + public string? CodeScript { get; set; } + [JsonPropertyName("template_type")] public virtual string TemplateType { get; set; } = TemplateTypeEnum.ProgramCode; diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs index 1d88942c4..899f6e7ab 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Models; -public class LlmConfigBase : LlmBase +public class LlmConfigBase : LlmProviderModel { /// /// Llm maximum output tokens @@ -13,15 +13,13 @@ public class LlmConfigBase : LlmBase public string? ReasoningEffortLevel { get; set; } } -public class LlmBase +public class LlmProviderModel { - /// - /// Llm provider - /// - public string? LlmProvider { get; set; } + [JsonPropertyName("provider")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Provider { get; set; } - /// - /// Llm model - /// - public string? LlmModel { get; set; } + [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Model { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index f2ebe6a8f..63ca3da3e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Roles.Models; using BotSharp.Abstraction.Shared; using BotSharp.Abstraction.Statistics.Enums; @@ -111,7 +112,7 @@ List GetAgentCodeScripts(string agentId, AgentCodeScriptFilter? => throw new NotImplementedException(); string? GetAgentCodeScript(string agentId, string scriptName, string scriptType = AgentCodeScriptType.Src) => throw new NotImplementedException(); - bool UpdateAgentCodeScripts(string agentId, List scripts) + bool UpdateAgentCodeScripts(string agentId, List scripts, AgentCodeScriptDbUpdateOptions? options = null) => throw new NotImplementedException(); bool BulkInsertAgentCodeScripts(string agentId, List scripts) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/AgentCodeScriptDbUpdateOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/AgentCodeScriptDbUpdateOptions.cs new file mode 100644 index 000000000..9217445b9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/AgentCodeScriptDbUpdateOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Repositories.Models; + +public class AgentCodeScriptDbUpdateOptions +{ + public bool IsUpsert { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index b34f5fba4..b0a088ccc 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Models; @@ -10,7 +11,8 @@ namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly RealtimeModelSettings _settings; private RealtimeHubConnection _conn; public RealtimeHubConnection HubConn => _conn; @@ -18,10 +20,14 @@ public class RealtimeHub : IRealtimeHub private IRealTimeCompletion _completer; public IRealTimeCompletion Completer => _completer; - public RealtimeHub(IServiceProvider services, ILogger logger) + public RealtimeHub( + IServiceProvider services, + ILogger logger, + RealtimeModelSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task ConnectToModel( @@ -43,10 +49,10 @@ public async Task ConnectToModel( routing.Context.SetDialogs(dialogs); routing.Context.SetMessageId(_conn.ConversationId, Guid.Empty.ToString()); - var states = _services.GetRequiredService(); - var settings = _services.GetRequiredService(); + var (provider, model) = GetLlmProviderModel(agent); - _completer = _services.GetServices().First(x => x.Provider == settings.Provider); + _completer = _services.GetServices().First(x => x.Provider == provider); + _completer.SetModelName(model); _completer.SetOptions(options); await _completer.Connect( @@ -156,7 +162,7 @@ await HookEmitter.Emit(_services, async hook => await hook.OnRouti }, onInterruptionDetected: async () => { - if (settings.InterruptResponse) + if (_settings.InterruptResponse) { // Reset states _conn.ResetResponseState(); @@ -179,4 +185,27 @@ public RealtimeHubConnection SetHubConnection(string conversationId) return _conn; } + + private (string, string) GetLlmProviderModel(Agent agent) + { + var provider = agent?.LlmConfig?.Realtime?.Provider; + var model = agent?.LlmConfig?.Realtime?.Model; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = _settings.Provider; + model = _settings.Model; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-realtime"; + return (provider, model); + } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CodeScripts.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CodeScripts.cs new file mode 100644 index 000000000..5c7c8bc00 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CodeScripts.cs @@ -0,0 +1,46 @@ +using BotSharp.Abstraction.Agents.Options; + +namespace BotSharp.Core.Agents.Services; + +public partial class AgentService +{ + public async Task> GetAgentCodeScripts(string agentId, AgentCodeScriptFilter? filter = null) + { + var db = _services.GetRequiredService(); + var scripts = db.GetAgentCodeScripts(agentId, filter); + return await Task.FromResult(scripts); + } + + public async Task GetAgentCodeScript(string agentId, string scriptName, string scriptType = AgentCodeScriptType.Src) + { + var db = _services.GetRequiredService(); + var script = db.GetAgentCodeScript(agentId, scriptName, scriptType); + return await Task.FromResult(script); + } + + public async Task UpdateAgentCodeScripts(string agentId, List codeScripts, AgentCodeScriptUpdateOptions? options = null) + { + if (string.IsNullOrWhiteSpace(agentId) || codeScripts.IsNullOrEmpty()) + { + return false; + } + + var db = _services.GetRequiredService(); + + var toDeleteScripts = new List(); + if (options?.DeleteIfNotIncluded == true) + { + var curDbScripts = await GetAgentCodeScripts(agentId); + var codePaths = codeScripts.Select(x => x.CodePath).ToList(); + toDeleteScripts = curDbScripts.Where(x => !codePaths.Contains(x.CodePath)).ToList(); + } + + var updateResult = db.UpdateAgentCodeScripts(agentId, codeScripts, options); + if (!toDeleteScripts.IsNullOrEmpty()) + { + db.DeleteAgentCodeScripts(agentId, toDeleteScripts); + } + + return updateResult; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index a08bdca5c..a4cf2d88b 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -141,8 +141,8 @@ private async Task> SelectFiles(IEnumerable> SelectFiles(IEnumerable(); var state = _services.GetRequiredService(); - var db = _services.GetRequiredService(); var hooks = _services.GetHooks(agent.Id); var codeProvider = codeOptions?.CodeInterpretProvider ?? "botsharp-py-interpreter"; @@ -187,7 +186,7 @@ await hook.OnResponseGenerated(new InstructResponseModel } // Get code script - var codeScript = db.GetAgentCodeScript(agent.Id, scriptName, scriptType: AgentCodeScriptType.Src); + var codeScript = await agentService.GetAgentCodeScript(agent.Id, scriptName, scriptType: AgentCodeScriptType.Src); if (string.IsNullOrWhiteSpace(codeScript)) { #if DEBUG @@ -249,7 +248,7 @@ await hook.OnResponseGenerated(new InstructResponseModel Provider = codeInterpreter.Provider, Model = string.Empty, TemplateName = scriptName, - UserMessage = string.Empty, + UserMessage = message.Content, SystemInstruction = string.Empty, CompletionText = response.Text }); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs index 764287ddf..c332d147c 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Repositories.Models; using System.IO; namespace BotSharp.Core.Repository; @@ -73,7 +74,7 @@ public List GetAgentCodeScripts(string agentId, AgentCodeScript return string.Empty; } - public bool UpdateAgentCodeScripts(string agentId, List scripts) + public bool UpdateAgentCodeScripts(string agentId, List scripts, AgentCodeScriptDbUpdateOptions? options = null) { if (string.IsNullOrWhiteSpace(agentId) || scripts.IsNullOrEmpty()) { @@ -95,7 +96,10 @@ public bool UpdateAgentCodeScripts(string agentId, List scripts } var file = Path.Combine(dir, script.Name); - File.WriteAllText(file, script.Content); + if (options?.IsUpsert == true || File.Exists(file)) + { + File.WriteAllText(file, script.Content); + } } return true; diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 0d24bc41f..fcb639101 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -140,11 +140,11 @@ public Task GetChatCompletionsStreamingAsync(Agent agent, List< { if (message.Role == AgentRole.User) { - messages.Add(new Message(RoleType.User, message.RoleContent)); + messages.Add(new Message(RoleType.User, message.LlmContent)); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new Message(RoleType.Assistant, message.RoleContent)); + messages.Add(new Message(RoleType.Assistant, message.LlmContent)); } else if (message.Role == AgentRole.Function) { @@ -170,7 +170,7 @@ public Task GetChatCompletionsStreamingAsync(Agent agent, List< new ToolResultContent() { ToolUseId = message.ToolCallId, - Content = [new TextContent() { Text = message.RoleContent }] + Content = [new TextContent() { Text = message.LlmContent }] } } }); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs index 7306bb2a2..4dd1ada3d 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/ReadAudioFn.cs @@ -34,9 +34,12 @@ public ReadAudioFn( public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var agentService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); var routingCtx = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + var wholeDialogs = routingCtx.GetDialogs(); if (wholeDialogs.IsNullOrEmpty()) { @@ -44,7 +47,7 @@ public async Task Execute(RoleDialogModel message) } var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetAudioTranscription(dialogs); + var response = await GetAudioTranscription(agent, dialogs); message.Content = response; dialogs.ForEach(x => x.Files = null); return true; @@ -54,7 +57,7 @@ private List AssembleFiles(string convId, List { if (dialogs.IsNullOrEmpty()) { - return new List(); + return []; } var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); @@ -85,13 +88,13 @@ private List AssembleFiles(string convId, List return dialogs; } - private async Task GetAudioTranscription(List dialogs) + private async Task GetAudioTranscription(Agent agent, List dialogs) { - var audioCompletion = PrepareModel(); + var audioCompletion = PrepareModel(agent); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).LastOrDefault(); var transcripts = new List(); - if (dialog != null) + if (dialog?.Files != null) { foreach (var file in dialog.Files) { @@ -129,9 +132,9 @@ private async Task GetAudioTranscription(List dialogs) return string.Join("\r\n\r\n", transcripts); } - private IAudioTranscription PrepareModel() + private IAudioTranscription PrepareModel(Agent agent) { - var (provider, model) = GetLlmProviderModel(); + var (provider, model) = GetLlmProviderModel(agent); return CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model); } @@ -142,21 +145,18 @@ private bool VerifyAudioFileType(string fileName) || !string.IsNullOrEmpty(FileUtility.GetFileContentType(fileName)); } - private (string, string) GetLlmProviderModel() + private (string, string) GetLlmProviderModel(Agent agent) { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("audio_read_llm_provider"); - var model = state.GetState("audio_read_llm_provider"); + var provider = agent?.LlmConfig?.AudioTranscription?.Provider; + var model = agent?.LlmConfig?.AudioTranscription?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = _settings?.Audio?.Reading?.LlmProvider; - model = _settings?.Audio?.Reading?.LlmModel; + provider = _settings?.Audio?.Reading?.Provider; + model = _settings?.Audio?.Reading?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index b6353b057..8c4c1af02 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -13,7 +13,7 @@ public class AudioSettings public AudioReadSettings? Reading { get; set; } } -public class AudioReadSettings : LlmBase +public class AudioReadSettings : LlmProviderModel { } #endregion \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index b7efb11ff..dc9a0fbc5 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -432,11 +432,11 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; @@ -448,7 +448,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs index 1cac2c81d..71efda50d 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -12,7 +12,6 @@ public class PlotChartFn : IFunctionCallback public string Name => "util-chart-plot_chart"; public string Indication => "Plotting chart"; - public PlotChartFn( IServiceProvider services, ILogger logger, @@ -35,8 +34,8 @@ public async Task Execute(RoleDialogModel message) var inst = GetChartPlotInstruction(message.CurrentAgentId); var innerAgent = new Agent { - Id = agent.Id, - Name = agent.Name, + Id = agent?.Id ?? BuiltInAgentId.AIProgrammer, + Name = agent?.Name ?? "AI Programmer", Instruction = inst, LlmConfig = GetLlmConfig(), TemplateDict = new Dictionary @@ -84,6 +83,7 @@ public async Task Execute(RoleDialogModel message) Message = new ProgramCodeTemplateMessage { Text = ret?.JsCode ?? string.Empty, + CodeScript = ret?.JsCode, Language = "javascript" } }; @@ -132,16 +132,16 @@ private string GetChartPlotInstruction(string agentId) private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-5"; + var provider = _settings.ChartPlot?.Provider; + var model = _settings.ChartPlot?.Model; - var state = _services.GetRequiredService(); - provider = state.GetState("chart_plot_llm_provider") - .IfNullOrEmptyAs(_settings.ChartPlot?.LlmProvider) - .IfNullOrEmptyAs(provider); - model = state.GetState("chart_plot_llm_model") - .IfNullOrEmptyAs(_settings.ChartPlot?.LlmModel) - .IfNullOrEmptyAs(model); + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5"; return (provider, model); } @@ -151,10 +151,6 @@ private AgentLlmConfig GetLlmConfig() var maxOutputTokens = _settings?.ChartPlot?.MaxOutputTokens ?? 8192; var reasoningEffortLevel = _settings?.ChartPlot?.ReasoningEffortLevel ?? "minimal"; - var state = _services.GetRequiredService(); - maxOutputTokens = int.TryParse(state.GetState("chart_plot_max_output_tokens"), out var tokens) ? tokens : maxOutputTokens; - reasoningEffortLevel = state.GetState("chart_plot_reasoning_effort_level").IfNullOrEmptyAs(reasoningEffortLevel); - return new AgentLlmConfig { MaxOutputTokens = maxOutputTokens, diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index eb143ebb3..6349b1ed0 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -399,11 +399,11 @@ public void SetModelName(string model) ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; @@ -415,7 +415,7 @@ public void SetModelName(string model) } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 8352a8482..e38233c3a 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -76,8 +76,8 @@ private async Task> GetConversationFiles() IsIncludeBotFiles = true, IsAttachFiles = true, MessageLimit = convSettings?.FileSelect?.MessageLimit, - LlmProvider = convSettings?.FileSelect?.LlmProvider, - LlmModel = convSettings?.FileSelect?.LlmModel, + Provider = convSettings?.FileSelect?.Provider, + Model = convSettings?.FileSelect?.Model, MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 4eb83e01a..410616b6b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -11,51 +11,23 @@ - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - PreserveNewest PreserveNewest - - PreserveNewest - - - PreserveNewest - PreserveNewest - - - - diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs index eb2b4e2ba..21a3e87b0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs @@ -2,8 +2,5 @@ namespace BotSharp.Plugin.FileHandler.Enums; public class UtilityName { - public const string ImageGenerator = "image-generator"; - public const string ImageReader = "image-reader"; - public const string ImageEditor = "image-editor"; public const string PdfReader = "pdf-reader"; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs index 4a46667dc..51fa33f1a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.FileHandler.Converters; using BotSharp.Plugin.FileHandler.Hooks; using Microsoft.Extensions.Configuration; @@ -8,8 +7,8 @@ public class FileHandlerPlugin : IBotSharpPlugin { public string Id => "65be5aee-48df-4ff8-a50a-05c8bcd2a793"; public string Name => "File Handler"; - public string Description => "AI reads files, such as image, pdf, excel"; - public string IconUrl => "https://lirp.cdn-website.com/6f8d6d8a/dms3rep/multi/opt/API_Icon-640w.png"; + public string Description => "AI handles files."; + public string IconUrl => "https://cdn-icons-png.flaticon.com/512/2567/2567656.png"; public string[] AgentIds => []; public void RegisterDI(IServiceCollection services, IConfiguration config) @@ -21,7 +20,5 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) }); services.AddScoped(); - services.AddScoped(); } - } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index a2c2f4e5c..c0f9236ee 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -29,20 +29,15 @@ public ReadPdfFn( public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agentService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); var routingCtx = _services.GetRequiredService(); - var agentService = _services.GetRequiredService(); - - Agent? fromAgent = null; - if (!string.IsNullOrEmpty(message.CurrentAgentId)) - { - fromAgent = await agentService.GetAgent(message.CurrentAgentId); - } - + + var fromAgent = await agentService.GetAgent(message.CurrentAgentId); var agent = new Agent { - Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "Utility Assistant", + Id = fromAgent?.Id ?? BuiltInAgentId.FileAssistant, + Name = fromAgent?.Name ?? "File Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", LlmConfig = fromAgent?.LlmConfig ?? new() }; @@ -140,27 +135,16 @@ private async Task GetChatCompletion(Agent agent, List private (string, string) GetLlmProviderModel() { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("pdf_read_llm_provider"); - var model = state.GetState("pdf_read_llm_model"); - - if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) - { - return (provider, model); - } - - provider = _settings?.Pdf?.Reading?.LlmProvider; - model = _settings?.Pdf?.Reading?.LlmModel; + var provider = _settings?.Pdf?.Reading?.Provider; + var model = _settings?.Pdf?.Reading?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = "openai"; - model = "gpt-5-mini"; + provider = "google-ai"; + model = "gemini-2.0-flash"; return (provider, model); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs index f45504abb..a8f60af3f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs @@ -2,51 +2,12 @@ namespace BotSharp.Plugin.FileHandler.Hooks; public class FileHandlerUtilityHook : IAgentUtilityHook { - private const string READ_IMAGE_FN = "util-file-read_image"; private const string READ_PDF_FN = "util-file-read_pdf"; - private const string GENERATE_IMAGE_FN = "util-file-generate_image"; - private const string EDIT_IMAGE_FN = "util-file-edit_image"; public void AddUtilities(List utilities) { var items = new List { - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageGenerator, - Items = [ - new UtilityItem - { - FunctionName = GENERATE_IMAGE_FN, - TemplateName = $"{GENERATE_IMAGE_FN}.fn" - } - ] - }, - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageReader, - Items = [ - new UtilityItem - { - FunctionName = READ_IMAGE_FN, - TemplateName = $"{READ_IMAGE_FN}.fn" - } - ] - }, - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageEditor, - Items = [ - new UtilityItem - { - FunctionName = EDIT_IMAGE_FN, - TemplateName = $"{EDIT_IMAGE_FN}.fn" - } - ] - }, new AgentUtility { Category = "file", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index 0e11a9ee6..efbe8d114 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -4,41 +4,16 @@ namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { - public ImageSettings? Image { get; set; } public PdfSettings? Pdf { get; set; } } -#region Image -public class ImageSettings -{ - public ImageReadSettings? Reading { get; set; } - public ImageGenerationSettings? Generation { get; set; } - public ImageEditSettings? Edit { get; set; } -} - -public class ImageReadSettings : LlmBase -{ - public string? ImageDetailLevel { get; set; } -} - -public class ImageGenerationSettings : LlmBase -{ - -} - -public class ImageEditSettings : LlmBase -{ - public SettingBase? ImageConverter { get; set; } -} -#endregion - #region Pdf public class PdfSettings { public PdfReadSettings? Reading { get; set; } } -public class PdfReadSettings : LlmBase +public class PdfReadSettings : LlmProviderModel { public bool ConvertToImage { get; set; } public string? ImageDetailLevel { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 1d2b4ef48..60e0ef025 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -263,17 +263,17 @@ public void SetModelName(string model) Name = message.FunctionName, Response = new JsonObject() { - ["result"] = message.RoleContent ?? string.Empty + ["result"] = message.LlmContent ?? string.Empty } } } ], AgentRole.Function)); - convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.RoleContent}"); + convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.LlmContent}"); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var contentParts = new List { new() { Text = text } }; if (allowMultiModal && !message.Files.IsNullOrEmpty()) @@ -285,7 +285,7 @@ public void SetModelName(string model) } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var contentParts = new List { new() { Text = text } }; if (allowMultiModal && !message.Files.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index 8dfaa695b..bec65e177 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -108,7 +108,7 @@ public async Task GetChatCompletions(Agent agent, List new PalmChatMessage(c.RoleContent, c.Role == AgentRole.User ? "user" : "AI")) + var messages = conversations.Select(c => new PalmChatMessage(c.LlmContent, c.Role == AgentRole.User ? "user" : "AI")) .ToList(); if (!functions.IsNullOrEmpty()) @@ -124,8 +124,8 @@ public async Task GetChatCompletions(Agent agent, List {dialog.RoleContent}\r\n" : - $"{dialog.Role}: {dialog.RoleContent}\r\n"; + $"{dialog.Role}: {dialog.FunctionName} => {dialog.LlmContent}\r\n" : + $"{dialog.Role}: {dialog.LlmContent}\r\n"; } prompt += "\r\n\r\n" + router.Templates.FirstOrDefault(x => x.Name == "response_with_function").Content; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 802088bdf..180d18c3f 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -18,11 +18,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private string _model = GoogleAIModels.Gemini2FlashLive001; private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly GoogleAiSettings _settings; + private List renderedInstructions = []; private LlmRealtimeSession _session; - private readonly GoogleAiSettings _settings; + private RealtimeOptions? _realtimeOptions; private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; private readonly JsonSerializerOptions _jsonOptions = new() @@ -50,12 +52,12 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public GoogleRealTimeProvider( IServiceProvider services, - GoogleAiSettings settings, - ILogger logger) + ILogger logger, + GoogleAiSettings settings) { - _settings = settings; _services = services; _logger = logger; + _settings = settings; } public async Task Connect( @@ -84,7 +86,7 @@ public async Task Connect( var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); - _model = realtimeModelSettings.Model; + _model ??= realtimeModelSettings.Model; var modelSettings = settingsService.GetSetting(Provider, _model); Reset(); @@ -422,7 +424,7 @@ public void SetModelName(string model) public void SetOptions(RealtimeOptions? options) { - + _realtimeOptions = options; } #region Private methods @@ -565,25 +567,25 @@ await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) Name = message.FunctionName ?? string.Empty, Response = new JsonObject() { - ["result"] = message.RoleContent ?? string.Empty + ["result"] = message.LlmContent ?? string.Empty } } } ], AgentRole.Function)); convPrompts.Add( - $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.RoleContent}"); + $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.LlmContent}"); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; contents.Add(new Content(text, AgentRole.User)); convPrompts.Add($"{AgentRole.User}: {text}"); } else if (message.Role == AgentRole.Assistant) { - contents.Add(new Content(message.RoleContent, AgentRole.Model)); - convPrompts.Add($"{AgentRole.Assistant}: {message.RoleContent}"); + contents.Add(new Content(message.LlmContent, AgentRole.Model)); + convPrompts.Add($"{AgentRole.Assistant}: {message.LlmContent}"); } } diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj new file mode 100644 index 000000000..fc2f33bdf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj @@ -0,0 +1,42 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs similarity index 86% rename from src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs index 6b4ec47cb..d058f335b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs @@ -3,22 +3,22 @@ using SixLabors.ImageSharp.Formats.Png; using System.IO; -namespace BotSharp.Plugin.FileHandler.Converters; +namespace BotSharp.Plugin.ImageHandler.Converters; -public class FileHandlerImageConverter : IImageConverter +public class ImageHandlerImageConverter : IImageConverter { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; - public FileHandlerImageConverter( + public ImageHandlerImageConverter( IServiceProvider services, - ILogger logger) + ILogger logger) { _services = services; _logger = logger; } - public string Provider => "file-handler"; + public string Provider => "image-handler"; public async Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) { diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs new file mode 100644 index 000000000..c79ee9365 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.ImageHandler.Enums; + +public class UtilityName +{ + public const string ImageGenerator = "image-generator"; + public const string ImageReader = "image-reader"; + public const string ImageEditor = "image-editor"; +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs similarity index 70% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs index 9d7521ac2..317727710 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs @@ -1,6 +1,6 @@ using BotSharp.Abstraction.Conversations.Settings; -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class EditImageFn : IFunctionCallback { @@ -9,16 +9,15 @@ public class EditImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; - private Agent _agent; private string _conversationId; private string _messageId; public EditImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -29,11 +28,15 @@ public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); var descrpition = args?.UserRequest ?? string.Empty; + + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + await Init(message); SetImageOptions(); var image = await SelectImage(descrpition); - var response = await GetImageEditGeneration(message, descrpition, image); + var response = await GetImageEdit(agent, message, descrpition, image); message.Content = response; message.StopCompletion = true; return true; @@ -41,10 +44,8 @@ public async Task Execute(RoleDialogModel message) private async Task Init(RoleDialogModel message) { - var agentService = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - _agent = await agentService.GetAgent(message.CurrentAgentId); _conversationId = convService.ConversationId; _messageId = message.MessageId; } @@ -68,15 +69,15 @@ private void SetImageOptions() IsAttachFiles = true, ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], MessageLimit = convSettings?.FileSelect?.MessageLimit, - LlmProvider = convSettings?.FileSelect?.LlmProvider, - LlmModel = convSettings?.FileSelect?.LlmModel, + Provider = convSettings?.FileSelect?.Provider, + Model = convSettings?.FileSelect?.Model, MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); return selecteds?.FirstOrDefault(); } - private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) + private async Task GetImageEdit(Agent agent, RoleDialogModel message, string description, MessageFileModel? image) { if (image == null) { @@ -85,15 +86,10 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin try { - var (provider, model) = GetLlmProviderModel(); + var (provider, model) = GetLlmProviderModel(agent); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant" - }; var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); @@ -112,7 +108,7 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin return response.Content; } - return await GetImageEditResponse(description, defaultContent: null); + return await GetImageEditResponse(agent, description); } catch (Exception ex) { @@ -122,45 +118,23 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin } } - private async Task GetImageEditResponse(string description, string? defaultContent) + private async Task GetImageEditResponse(Agent agent, string description) { - if (defaultContent != null) - { - return defaultContent; - } - - var llmConfig = _agent.LlmConfig; - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", - LlmConfig = new AgentLlmConfig - { - Provider = llmConfig?.Provider ?? "openai", - Model = llmConfig?.Model ?? "gpt-5-mini", - MaxOutputTokens = llmConfig?.MaxOutputTokens, - ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel - } - }; - return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); } - private (string, string) GetLlmProviderModel() + private (string, string) GetLlmProviderModel(Agent agent) { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("image_edit_llm_provider"); - var model = state.GetState("image_edit_llm_provider"); + var provider = agent?.LlmConfig?.ImageEdit?.Provider; + var model = agent?.LlmConfig?.ImageEdit?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = _settings?.Image?.Edit?.LlmProvider; - model = _settings?.Image?.Edit?.LlmModel; + provider = _settings?.Edit?.Provider; + model = _settings?.Edit?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -196,7 +170,7 @@ private IEnumerable SaveGeneratedImage(ImageGeneration? image) private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) { - var provider = _settings?.Image?.Edit?.ImageConverter?.Provider; + var provider = _settings?.Edit?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); if (converter == null) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs similarity index 62% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs index 132b18617..ccb3e43cc 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class GenerateImageFn : IFunctionCallback { @@ -7,16 +7,15 @@ public class GenerateImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; - private Agent _agent; private string _conversationId; private string _messageId; public GenerateImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -30,9 +29,9 @@ public async Task Execute(RoleDialogModel message) SetImageOptions(); var agentService = _services.GetRequiredService(); - _agent = await agentService.GetAgent(message.CurrentAgentId); + var currentAgent = await agentService.GetAgent(message.CurrentAgentId); - var response = await GetImageGeneration(message, args?.ImageDescription); + var response = await GetImageGeneration(currentAgent, message, args?.ImageDescription); message.Content = response; message.StopCompletion = true; return true; @@ -53,18 +52,11 @@ private void SetImageOptions() state.SetState("image_response_format", "bytes"); } - private async Task GetImageGeneration(RoleDialogModel message, string? description) + private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) { try { - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", - Instruction = description - }; - - var (provider, model) = GetLlmProviderModel(); + var (provider, model) = GetLlmProviderModel(agent); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); @@ -76,7 +68,7 @@ private async Task GetImageGeneration(RoleDialogModel message, string? d return result.Content; } - return await GetImageGenerationResponse(description, defaultContent: null); + return await GetImageGenerationResponse(agent, description); } catch (Exception ex) { @@ -86,45 +78,23 @@ private async Task GetImageGeneration(RoleDialogModel message, string? d } } - private async Task GetImageGenerationResponse(string description, string? defaultContent) + private async Task GetImageGenerationResponse(Agent agent, string description) { - if (defaultContent != null) - { - return defaultContent; - } - - var llmConfig = _agent.LlmConfig; - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", - LlmConfig = new AgentLlmConfig - { - Provider = llmConfig?.Provider ?? "openai", - Model = llmConfig?.Model ?? "gpt-5-mini", - MaxOutputTokens = llmConfig?.MaxOutputTokens, - ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel - } - }; - return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); } - private (string, string) GetLlmProviderModel() + private (string, string) GetLlmProviderModel(Agent agent) { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("image_generate_llm_provider"); - var model = state.GetState("image_generate_llm_model"); + var provider = agent?.LlmConfig?.ImageGeneration?.Provider; + var model = agent?.LlmConfig?.ImageGeneration?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = _settings?.Image?.Generation?.LlmProvider; - model = _settings?.Image?.Generation?.LlmModel; + provider = _settings?.Generation?.Provider; + model = _settings?.Generation?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs similarity index 79% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs index 639f3004a..9a42f47d4 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs @@ -1,6 +1,6 @@ using BotSharp.Abstraction.Routing; -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class ReadImageFn : IFunctionCallback { @@ -9,7 +9,7 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; private readonly IEnumerable _imageContentTypes = new List { @@ -20,7 +20,7 @@ public class ReadImageFn : IFunctionCallback public ReadImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -30,20 +30,15 @@ public ReadImageFn( public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agentService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); var routingCtx = _services.GetRequiredService(); - var agentService = _services.GetRequiredService(); - - Agent? fromAgent = null; - if (!string.IsNullOrEmpty(message.CurrentAgentId)) - { - fromAgent = await agentService.GetAgent(message.CurrentAgentId); - } - + + var fromAgent = await agentService.GetAgent(message.CurrentAgentId); var agent = new Agent { - Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "Utility Assistant", + Id = fromAgent?.Id ?? BuiltInAgentId.FileAssistant, + Name = fromAgent?.Name ?? "File Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", LlmConfig = fromAgent?.LlmConfig ?? new() }; @@ -106,7 +101,7 @@ private List AssembleFiles(string conversationId, IEnumerable x?.Trim()) .Where(x => !string.IsNullOrWhiteSpace(x)) - .Select(x => new BotSharpFile { FileUrl = x }).ToList(); + .Select(x => new BotSharpFile { FileUrl = x }).ToList(); lastDialog.Files.AddRange(addnFiles); } @@ -133,19 +128,8 @@ private async Task GetChatCompletion(Agent agent, List private (string, string) GetLlmProviderModel() { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("image_read_llm_provider"); - var model = state.GetState("image_read_llm_model"); - - if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) - { - return (provider, model); - } - - provider = _settings?.Image?.Reading?.LlmProvider; - model = _settings?.Image?.Reading?.LlmModel; + var provider = _settings?.Reading?.Provider; + var model = _settings?.Reading?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -161,14 +145,13 @@ private async Task GetChatCompletion(Agent agent, List private void SetImageDetailLevel() { var state = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var key = "chat_image_detail_level"; var level = state.GetState(key); - if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Image?.Reading?.ImageDetailLevel)) + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(_settings.Reading?.ImageDetailLevel)) { - state.SetState(key, fileSettings.Image.Reading.ImageDetailLevel); + state.SetState(key, _settings.Reading.ImageDetailLevel); } } } diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs new file mode 100644 index 000000000..3c49743c7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs @@ -0,0 +1,31 @@ +namespace BotSharp.Plugin.ImageHandler.Helpers; + +internal static class AiResponseHelper +{ + internal static string GetDefaultResponse(IEnumerable files) + { + if (files.IsNullOrEmpty()) + { + return $"No image is generated."; + } + + if (files.Count() > 1) + { + return $"Here are the images you asked for: {string.Join(", ", files)}"; + } + + return $"Here is the image you asked for: {string.Join(", ", files)}"; + } + + internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description) + { + var text = $"Please generate a user-friendly response from the following description to " + + $"inform user that you have completed the required image: {description}"; + + var provider = agent?.LlmConfig?.Provider ?? "openai"; + var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; + var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs new file mode 100644 index 000000000..36853aa84 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs @@ -0,0 +1,53 @@ +namespace BotSharp.Plugin.ImageHandler.Hooks; + +public class ImageHandlerUtilityHook : IAgentUtilityHook +{ + private const string READ_IMAGE_FN = "util-file-read_image"; + private const string GENERATE_IMAGE_FN = "util-file-generate_image"; + private const string EDIT_IMAGE_FN = "util-file-edit_image"; + + public void AddUtilities(List utilities) + { + var items = new List + { + new AgentUtility + { + Category = "file", + Name = UtilityName.ImageReader, + Items = [ + new UtilityItem + { + FunctionName = READ_IMAGE_FN, + TemplateName = $"{READ_IMAGE_FN}.fn" + } + ] + }, + new AgentUtility + { + Category = "file", + Name = UtilityName.ImageGenerator, + Items = [ + new UtilityItem + { + FunctionName = GENERATE_IMAGE_FN, + TemplateName = $"{GENERATE_IMAGE_FN}.fn" + } + ] + }, + new AgentUtility + { + Category = "file", + Name = UtilityName.ImageEditor, + Items = [ + new UtilityItem + { + FunctionName = EDIT_IMAGE_FN, + TemplateName = $"{EDIT_IMAGE_FN}.fn" + } + ] + } + }; + + utilities.AddRange(items); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs new file mode 100644 index 000000000..1a6268345 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs @@ -0,0 +1,26 @@ +using BotSharp.Plugin.ImageHandler.Converters; +using BotSharp.Plugin.ImageHandler.Hooks; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Plugin.ImageHandler; + +public class ImageHandlerPlugin : IBotSharpPlugin +{ + public string Id => "bac8bbf3-da91-4c92-98d8-db14d68e75ae"; + public string Name => "Image Handler"; + public string Description => "AI handles images."; + public string IconUrl => "https://cdn-icons-png.flaticon.com/512/8002/8002135.png"; + public string[] AgentIds => []; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("ImageHandler"); + }); + + services.AddScoped(); + services.AddScoped(); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs new file mode 100644 index 000000000..59eb16be6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ImageHandler.LlmContexts; + +public class LlmContextIn +{ + [JsonPropertyName("user_request")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserRequest { get; set; } + + [JsonPropertyName("image_description")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImageDescription { get; set; } + + [JsonPropertyName("image_urls")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? ImageUrls { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 000000000..822b49364 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ImageHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("selected_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Selected { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs new file mode 100644 index 000000000..81a11c8d5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.ImageHandler.Settings; + +public class ImageHandlerSettings +{ + public ImageReadSettings? Reading { get; set; } + public ImageGenerationSettings? Generation { get; set; } + public ImageEditSettings? Edit { get; set; } +} + +public class ImageReadSettings : LlmProviderModel +{ + public string? ImageDetailLevel { get; set; } +} + +public class ImageGenerationSettings : LlmProviderModel +{ + +} + +public class ImageEditSettings : LlmProviderModel +{ + public SettingBase? ImageConverter { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs new file mode 100644 index 000000000..88290b6fd --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs @@ -0,0 +1,34 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Linq; +global using System.Text.Json; +global using System.Net.Mime; +global using System.Threading.Tasks; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Converters; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Abstraction.Settings; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Options; +global using BotSharp.Core.Infrastructures; +global using BotSharp.Plugin.ImageHandler.Enums; +global using BotSharp.Plugin.ImageHandler.Settings; +global using BotSharp.Plugin.ImageHandler.LlmContexts; +global using BotSharp.Plugin.ImageHandler.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json similarity index 100% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json similarity index 100% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json similarity index 100% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index ea6f75fdf..a24633939 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -42,7 +42,7 @@ public async Task GetChatCompletions(Agent agent, List $"{x.Role}: {x.RoleContent}")).Trim(); + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.LlmContent}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; var llama = _services.GetRequiredService(); @@ -118,7 +118,7 @@ public async Task GetChatCompletionsAsync(Agent agent, Func onMessageReceived, Func onFunctionExecuting) { - var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.RoleContent}")).Trim(); + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.LlmContent}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; var state = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs index a7a4adddf..a748ed2a7 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs @@ -41,7 +41,7 @@ public async Task GetChatCompletions(Agent agent, List new Message(c.RoleContent, c.Role == AgentRole.User ? MessageRole.Human : MessageRole.Ai)).ToList(); + .Select(c => new Message(c.LlmContent, c.Role == AgentRole.User ? MessageRole.Human : MessageRole.Ai)).ToList(); var response = await model.GenerateAsync(new ChatRequest { Messages = messages }, _settings); diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index 2505eed1e..38c3b398d 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -111,8 +111,8 @@ private string PrepareOptions(Agent agent, List conversations, foreach (var message in samples) { messages.Add(message.Role == AgentRole.User ? - new MessageItem("user", message.RoleContent) : - new MessageItem("assistant", message.RoleContent)); + new MessageItem("user", message.LlmContent) : + new MessageItem("assistant", message.LlmContent)); } foreach (var function in functions) @@ -129,13 +129,13 @@ private string PrepareOptions(Agent agent, List conversations, } else if (message.Role == "user") { - var userMessage = new MessageItem("user",message.RoleContent); + var userMessage = new MessageItem("user",message.LlmContent); messages.Add(userMessage); } else if (message.Role == "assistant") { - messages.Add(new MessageItem("assistant", message.RoleContent)); + messages.Add(new MessageItem("assistant", message.LlmContent)); } } diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 43f5134a0..888fe341d 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -109,16 +109,16 @@ public async Task GetChatCompletions(Agent agent, List>(x.FunctionArgs ?? "{}")), - new FunctionResultContent(x.FunctionName, x.RoleContent) + new FunctionResultContent(x.FunctionName, x.LlmContent) ])); } else if (x.Role == AgentRole.System || x.Role == AgentRole.Assistant) { - messages.Add(new(x.Role == AgentRole.System ? ChatRole.System : ChatRole.Assistant, x.RoleContent)); + messages.Add(new(x.Role == AgentRole.System ? ChatRole.System : ChatRole.Assistant, x.LlmContent)); } else if (x.Role == AgentRole.User) { - List contents = [new TextContent(x.RoleContent)]; + List contents = [new TextContent(x.LlmContent)]; if (allowMultiModal) { foreach (var file in x.Files) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 161842650..7a2390267 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -26,7 +26,7 @@ public class AgentDocument : MongoBase public List Labels { get; set; } public List RoutingRules { get; set; } public List Rules { get; set; } - public AgentLlmConfigMongoElement? LlmConfig { get; set; } + public AgentLlmConfigMongoModel? LlmConfig { get; set; } public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs deleted file mode 100644 index 63bc3b20b..000000000 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs +++ /dev/null @@ -1,44 +0,0 @@ -using BotSharp.Abstraction.Agents.Models; - -namespace BotSharp.Plugin.MongoStorage.Models; - -[BsonIgnoreExtraElements(Inherited = true)] -public class AgentLlmConfigMongoElement -{ - public string? Provider { get; set; } - public string? Model { get; set; } - public bool IsInherit { get; set; } - public int MaxRecursionDepth { get; set; } - public int? MaxOutputTokens { get; set; } - public string? ReasoningEffortLevel { get; set; } - - public static AgentLlmConfigMongoElement? ToMongoElement(AgentLlmConfig? config) - { - if (config == null) return null; - - return new AgentLlmConfigMongoElement - { - Provider = config.Provider, - Model = config.Model, - IsInherit = config.IsInherit, - MaxRecursionDepth = config.MaxRecursionDepth, - MaxOutputTokens = config.MaxOutputTokens, - ReasoningEffortLevel = config.ReasoningEffortLevel - }; - } - - public static AgentLlmConfig? ToDomainElement(AgentLlmConfigMongoElement? config) - { - if (config == null) return null; - - return new AgentLlmConfig - { - Provider = config.Provider, - Model = config.Model, - IsInherit = config.IsInherit, - MaxRecursionDepth = config.MaxRecursionDepth, - MaxOutputTokens = config.MaxOutputTokens, - ReasoningEffortLevel = config.ReasoningEffortLevel - }; - } -} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoModel.cs new file mode 100644 index 000000000..165a590a1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoModel.cs @@ -0,0 +1,193 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +[BsonIgnoreExtraElements(Inherited = true)] +public class AgentLlmConfigMongoModel +{ + public string? Provider { get; set; } + public string? Model { get; set; } + public bool IsInherit { get; set; } + public int MaxRecursionDepth { get; set; } + public int? MaxOutputTokens { get; set; } + public string? ReasoningEffortLevel { get; set; } + + public LlmImageGenerationConfigMongoModel? ImageGeneration { get; set; } + public LlmImageEditConfigMongoModel? ImageEdit { get; set; } + public LlmAudioTranscriptionConfigMongoModel? AudioTranscription { get; set; } + public LlmRealtimeConfigMongoModel? Realtime { get; set; } + + + + public static AgentLlmConfigMongoModel? ToMongoElement(AgentLlmConfig? config) + { + if (config == null) + { + return null; + } + + return new AgentLlmConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, + MaxOutputTokens = config.MaxOutputTokens, + ReasoningEffortLevel = config.ReasoningEffortLevel, + ImageGeneration = LlmImageGenerationConfigMongoModel.ToMongoModel(config.ImageGeneration), + ImageEdit = LlmImageEditConfigMongoModel.ToMongoModel(config.ImageEdit), + AudioTranscription = LlmAudioTranscriptionConfigMongoModel.ToMongoModel(config.AudioTranscription), + Realtime = LlmRealtimeConfigMongoModel.ToMongoModel(config.Realtime) + }; + } + + public static AgentLlmConfig? ToDomainElement(AgentLlmConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new AgentLlmConfig + { + Provider = config.Provider, + Model = config.Model, + IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, + MaxOutputTokens = config.MaxOutputTokens, + ReasoningEffortLevel = config.ReasoningEffortLevel, + ImageGeneration = LlmImageGenerationConfigMongoModel.ToDomainModel(config.ImageGeneration), + ImageEdit = LlmImageEditConfigMongoModel.ToDomainModel(config.ImageEdit), + AudioTranscription = LlmAudioTranscriptionConfigMongoModel.ToDomainModel(config.AudioTranscription), + Realtime = LlmRealtimeConfigMongoModel.ToDomainModel(config.Realtime) + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmImageGenerationConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmImageGenerationConfig? ToDomainModel(LlmImageGenerationConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmImageGenerationConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmImageGenerationConfigMongoModel? ToMongoModel(LlmImageGenerationConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmImageGenerationConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmImageEditConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmImageEditConfig? ToDomainModel(LlmImageEditConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmImageEditConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmImageEditConfigMongoModel? ToMongoModel(LlmImageEditConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmImageEditConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmAudioTranscriptionConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmAudioTranscriptionConfig? ToDomainModel(LlmAudioTranscriptionConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmAudioTranscriptionConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmAudioTranscriptionConfigMongoModel? ToMongoModel(LlmAudioTranscriptionConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmAudioTranscriptionConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmRealtimeConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmRealtimeConfig? ToDomainModel(LlmRealtimeConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmRealtimeConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmRealtimeConfigMongoModel? ToMongoModel(LlmRealtimeConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmRealtimeConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/LlmProviderModelMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/LlmProviderModelMongoModel.cs new file mode 100644 index 000000000..ae5e6b202 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/LlmProviderModelMongoModel.cs @@ -0,0 +1,38 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmProviderModelMongoModel +{ + public string? Provider { get; set; } + public string? Model { get; set; } + + public static LlmProviderModel? ToDomainModel(LlmProviderModelMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmProviderModel + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmProviderModelMongoModel? ToMongoModel(LlmProviderModel? config) + { + if (config == null) + { + return null; + } + + return new LlmProviderModelMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 76cadd324..044a52b13 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -333,7 +333,7 @@ private void UpdateAgentRules(string agentId, List rules) private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) { - var llmConfig = AgentLlmConfigMongoElement.ToMongoElement(config); + var llmConfig = AgentLlmConfigMongoModel.ToMongoElement(config); var filter = Builders.Filter.Eq(x => x.Id, agentId); var update = Builders.Update .Set(x => x.LlmConfig, llmConfig) @@ -377,7 +377,7 @@ private void UpdateAgentAllFields(Agent agent) .Set(x => x.McpTools, agent.McpTools.Select(u => AgentMcpToolMongoElement.ToMongoElement(u)).ToList()) .Set(x => x.KnowledgeBases, agent.KnowledgeBases.Select(u => AgentKnowledgeBaseMongoElement.ToMongoElement(u)).ToList()) .Set(x => x.Rules, agent.Rules.Select(e => AgentRuleMongoElement.ToMongoElement(e)).ToList()) - .Set(x => x.LlmConfig, AgentLlmConfigMongoElement.ToMongoElement(agent.LlmConfig)) + .Set(x => x.LlmConfig, AgentLlmConfigMongoModel.ToMongoElement(agent.LlmConfig)) .Set(x => x.IsPublic, agent.IsPublic) .Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -550,7 +550,7 @@ public void BulkInsertAgents(List agents) MaxMessageCount = x.MaxMessageCount, Profiles = x.Profiles ?? [], Labels = x.Labels ?? [], - LlmConfig = AgentLlmConfigMongoElement.ToMongoElement(x.LlmConfig), + LlmConfig = AgentLlmConfigMongoModel.ToMongoElement(x.LlmConfig), ChannelInstructions = x.ChannelInstructions?.Select(i => ChannelInstructionMongoElement.ToMongoElement(i))?.ToList() ?? [], Templates = x.Templates?.Select(t => AgentTemplateMongoElement.ToMongoElement(t))?.ToList() ?? [], Functions = x.Functions?.Select(f => FunctionDefMongoElement.ToMongoElement(f))?.ToList() ?? [], @@ -651,7 +651,7 @@ private Agent TransformAgentDocument(AgentDocument? agentDoc) Profiles = agentDoc.Profiles ?? [], Labels = agentDoc.Labels ?? [], MaxMessageCount = agentDoc.MaxMessageCount, - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig), + LlmConfig = AgentLlmConfigMongoModel.ToDomainElement(agentDoc.LlmConfig), ChannelInstructions = agentDoc.ChannelInstructions?.Select(i => ChannelInstructionMongoElement.ToDomainElement(i))?.ToList() ?? [], Templates = agentDoc.Templates?.Select(t => AgentTemplateMongoElement.ToDomainElement(t))?.ToList() ?? [], Functions = agentDoc.Functions?.Select(f => FunctionDefMongoElement.ToDomainElement(f)).ToList() ?? [], diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs index 3d551b7d0..8e8a8154d 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -55,7 +56,7 @@ public List GetAgentCodeScripts(string agentId, AgentCodeScript return found?.Content; } - public bool UpdateAgentCodeScripts(string agentId, List scripts) + public bool UpdateAgentCodeScripts(string agentId, List scripts, AgentCodeScriptDbUpdateOptions? options = null) { if (string.IsNullOrWhiteSpace(agentId) || scripts.IsNullOrEmpty()) { @@ -73,7 +74,7 @@ public bool UpdateAgentCodeScripts(string agentId, List scripts }), Builders.Update.Set(y => y.Content, x.Content) .Set(x => x.UpdatedTime, DateTime.UtcNow) - )) + ) { IsUpsert = options?.IsUpsert ?? false }) .ToList(); var result = _dc.AgentCodeScripts.BulkWrite(ops, new BulkWriteOptions { IsOrdered = false }); diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 3812a5916..7ab149e26 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -402,11 +402,11 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; @@ -418,7 +418,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index e6b614e67..379941b40 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -67,7 +67,7 @@ public async Task Connect( var settingsService = _services.GetRequiredService(); var realtimeSettings = _services.GetRequiredService(); - _model = realtimeSettings.Model; + _model ??= realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); _session = new LlmRealtimeSession(_services, new ChatSessionOptions @@ -621,15 +621,15 @@ private async Task OnUserAudioTranscriptionCompleted(RealtimeHu ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - messages.Add(new UserChatMessage(message.RoleContent)); + messages.Add(new UserChatMessage(message.LlmContent)); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.RoleContent)); + messages.Add(new AssistantChatMessage(message.LlmContent)); } } diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs index 0a69b7a8e..8dc9e7d50 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs @@ -75,7 +75,8 @@ public async Task Execute(RoleDialogModel message) Recipient = new Recipient { Id = convService.ConversationId }, Message = new ProgramCodeTemplateMessage { - Text = ret.PythonCode ?? string.Empty, + Text = result, + CodeScript = ret.PythonCode, Language = "python" } }; @@ -115,6 +116,7 @@ public async Task Execute(RoleDialogModel message) dynamic stringIO = io.StringIO(); sys.stdout = stringIO; sys.stderr = stringIO; + sys.argv = new PyList(); // Set global items using var globals = new PyDict(); @@ -186,16 +188,16 @@ private string GetPyCodeInterpreterInstruction(string agentId) private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-5"; + var provider = _settings.CodeGeneration?.Provider; + var model = _settings.CodeGeneration?.Model; - var state = _services.GetRequiredService(); - provider = state.GetState("py_intepreter_llm_provider") - .IfNullOrEmptyAs(_settings.CodeGeneration?.LlmProvider) - .IfNullOrEmptyAs(provider); - model = state.GetState("py_intepreter_llm_model") - .IfNullOrEmptyAs(_settings.CodeGeneration?.LlmModel) - .IfNullOrEmptyAs(model); + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5"; return (provider, model); } @@ -205,10 +207,6 @@ private AgentLlmConfig GetLlmConfig() var maxOutputTokens = _settings?.CodeGeneration?.MaxOutputTokens ?? 8192; var reasoningEffortLevel = _settings?.CodeGeneration?.ReasoningEffortLevel ?? "minimal"; - var state = _services.GetRequiredService(); - maxOutputTokens = int.TryParse(state.GetState("py_intepreter_max_output_tokens"), out var tokens) ? tokens : maxOutputTokens; - reasoningEffortLevel = state.GetState("py_intepreter_reasoning_effort_level").IfNullOrEmptyAs(reasoningEffortLevel); - return new AgentLlmConfig { MaxOutputTokens = maxOutputTokens, diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 93b90d13f..5f0405c17 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -269,13 +269,13 @@ public void SetModelName(string model) } else if (message.Role == "user") { - var userMessage = ChatMessage.FromUser(message.RoleContent); + var userMessage = ChatMessage.FromUser(message.LlmContent); messages.Add(userMessage); } else if (message.Role == "assistant") { - messages.Add(ChatMessage.FromAssistant(message.RoleContent)); + messages.Add(ChatMessage.FromAssistant(message.LlmContent)); } } diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index e5b90ddde..5a7c6eb7b 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -37,6 +37,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 1304313c9..276f853e7 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -301,8 +301,8 @@ "MinTimeSecondsBetweenMessages": 2 }, "FileSelect": { - "LlmProvider": "openai", - "LlmModel": "gpt-5-mini", + "Provider": "openai", + "Model": "gpt-5-mini", "MaxOutputTokens": 8192, "ReasoningEffortLevel": "low", "MessageLimit": 50 @@ -431,28 +431,10 @@ }, "FileHandler": { - "Image": { - "Reading": { - "LlmProvider": "openai", - "LlmModel": "gpt-5-mini", - "ImageDetailLevel": "auto" - }, - "Generation": { - "LlmProvider": "openai", - "LlmModel": "gpt-image-1" - }, - "Edit": { - "LlmProvider": "openai", - "LlmModel": "gpt-image-1", - "ImageConverter": { - "Provider": "file-handler" - } - } - }, "Pdf": { "Reading": { - "LlmProvider": "google-ai", - "LlmModel": "gemini-2.0-flash", + "Provider": "google-ai", + "Model": "gemini-2.0-flash", "ConvertToImage": false, "ImageDetailLevel": "auto", "ImageConverter": { @@ -462,11 +444,30 @@ } }, + "ImageHandler": { + "Reading": { + "Provider": "openai", + "Model": "gpt-5-mini", + "ImageDetailLevel": "auto" + }, + "Generation": { + "Provider": "openai", + "Model": "gpt-image-1" + }, + "Edit": { + "Provider": "openai", + "Model": "gpt-image-1", + "ImageConverter": { + "Provider": "image-handler" + } + } + }, + "AudioHandler": { "Audio": { "Reading": { - "LlmProvider": "openai", - "LlmModel": "gpt-4o-mini-transcribe" + "Provider": "openai", + "Model": "gpt-4o-mini-transcribe" } } }, @@ -485,8 +486,8 @@ "ChartHandler": { "ChartPlot": { - "LlmProvider": "openai", - "LlmModel": "gpt-5", + "Provider": "openai", + "Model": "gpt-5", "MaxOutputTokens": 8192, "ReasoningEffortLevel": "minimal", "MessageLimit": 50 @@ -569,8 +570,8 @@ "InstallLocation": "C:/Users/xxx/AppData/Local/Programs/Python/Python313/python313.dll", "PythonVersion": "3.13.3", "CodeGeneration": { - "LlmProvider": "openai", - "LlmModel": "gpt-5", + "Provider": "openai", + "Model": "gpt-5", "MaxOutputTokens": 8192, "ReasoningEffortLevel": "minimal", "MessageLimit": 50 @@ -621,6 +622,7 @@ "BotSharp.Plugin.MetaGLM", "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", + "BotSharp.Plugin.ImageHandler", "BotSharp.Plugin.EmailHandler", "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.ChartHandler",