From 53d85ec54cc594460d1b7c496b9b53f40aa1e7f6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 9 Oct 2025 16:25:26 -0500 Subject: [PATCH 01/11] init --- BotSharp.sln | 11 ++++ .../FileInstructService.SelectFile.cs | 4 +- .../BotSharp.Plugin.FileHandler.csproj | 24 --------- .../Enums/UtilityName.cs | 3 -- .../FileHandlerPlugin.cs | 2 - .../Hooks/FileHandlerUtilityHook.cs | 39 -------------- .../Settings/FileHandlerSettings.cs | 25 --------- .../templates/util-file-edit_image.fn.liquid | 1 - .../util-file-generate_image.fn.liquid | 2 - .../templates/util-file-read_image.fn.liquid | 2 - .../BotSharp.Plugin.ImageHandler.csproj | 42 +++++++++++++++ .../Converters/ImageHandlerImageConverter.cs} | 12 ++--- .../Enums/UtilityName.cs | 8 +++ .../Functions/EditImageFn.cs | 22 ++++---- .../Functions/GenerateImageFn.cs | 20 +++---- .../Functions/ReadImageFn.cs | 21 ++++---- .../Helpers/AiResponseHelper.cs | 31 +++++++++++ .../Hooks/ImageHandlerUtilityHook.cs | 53 +++++++++++++++++++ .../ImageHandlerPlugin.cs | 27 ++++++++++ .../LlmContexts/LlmContextIn.cs | 18 +++++++ .../LlmContexts/LlmContextOut.cs | 10 ++++ .../Settings/ImageHandlerSettings.cs | 25 +++++++++ .../BotSharp.Plugin.ImageHandler/Using.cs | 34 ++++++++++++ .../functions/util-image-edit_image.json} | 2 +- .../functions/util-image-generate_image.json} | 2 +- .../functions/util-image-read_image.json} | 2 +- .../templates/util-image-edit_image.fn.liquid | 1 + .../util-image-generate_image.fn.liquid | 2 + .../templates/util-image-read_image.fn.liquid | 2 + src/WebStarter/WebStarter.csproj | 1 + src/WebStarter/appsettings.json | 1 + 31 files changed, 308 insertions(+), 141 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj rename src/Plugins/{BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs => BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs} (86%) create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs rename src/Plugins/{BotSharp.Plugin.FileHandler => BotSharp.Plugin.ImageHandler}/Functions/EditImageFn.cs (91%) rename src/Plugins/{BotSharp.Plugin.FileHandler => BotSharp.Plugin.ImageHandler}/Functions/GenerateImageFn.cs (89%) rename src/Plugins/{BotSharp.Plugin.FileHandler => BotSharp.Plugin.ImageHandler}/Functions/ReadImageFn.cs (89%) create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs rename src/Plugins/{BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json => BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json} (91%) rename src/Plugins/{BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json => BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json} (90%) rename src/Plugins/{BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json => BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json} (96%) create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid diff --git a/BotSharp.sln b/BotSharp.sln index f68ce1c60..e992d26ad 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -147,6 +147,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ChartHandle EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ExcelHandler", "src\Plugins\BotSharp.Plugin.ExcelHandler\BotSharp.Plugin.ExcelHandler.csproj", "{FC63C875-E880-D8BB-B8B5-978AB7B62983}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ImageHandler", "src\Plugins\BotSharp.Plugin.ImageHandler\BotSharp.Plugin.ImageHandler.csproj", "{242F2D93-FCCE-4982-8075-F3052ECCA92C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -619,6 +621,14 @@ Global {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|Any CPU.Build.0 = Release|Any CPU {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.ActiveCfg = Release|Any CPU {FC63C875-E880-D8BB-B8B5-978AB7B62983}.Release|x64.Build.0 = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.ActiveCfg = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Debug|x64.Build.0 = Debug|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|Any CPU.Build.0 = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.ActiveCfg = Release|Any CPU + {242F2D93-FCCE-4982-8075-F3052ECCA92C}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -690,6 +700,7 @@ Global {B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {0428DEAA-E4FE-4259-A6D8-6EDD1A9D0702} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F} + {242F2D93-FCCE-4982-8075-F3052ECCA92C} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 3b0b724a7..355df34d4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -137,8 +137,8 @@ private async Task> SelectFiles(IEnumerable - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - PreserveNewest PreserveNewest - - PreserveNewest - - - PreserveNewest - PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs index eb2b4e2ba..21a3e87b0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Enums/UtilityName.cs @@ -2,8 +2,5 @@ namespace BotSharp.Plugin.FileHandler.Enums; public class UtilityName { - public const string ImageGenerator = "image-generator"; - public const string ImageReader = "image-reader"; - public const string ImageEditor = "image-editor"; public const string PdfReader = "pdf-reader"; } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs index 4a46667dc..a8ecac6dc 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs @@ -1,4 +1,3 @@ -using BotSharp.Plugin.FileHandler.Converters; using BotSharp.Plugin.FileHandler.Hooks; using Microsoft.Extensions.Configuration; @@ -21,7 +20,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) }); services.AddScoped(); - services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs index f45504abb..a8f60af3f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Hooks/FileHandlerUtilityHook.cs @@ -2,51 +2,12 @@ namespace BotSharp.Plugin.FileHandler.Hooks; public class FileHandlerUtilityHook : IAgentUtilityHook { - private const string READ_IMAGE_FN = "util-file-read_image"; private const string READ_PDF_FN = "util-file-read_pdf"; - private const string GENERATE_IMAGE_FN = "util-file-generate_image"; - private const string EDIT_IMAGE_FN = "util-file-edit_image"; public void AddUtilities(List utilities) { var items = new List { - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageGenerator, - Items = [ - new UtilityItem - { - FunctionName = GENERATE_IMAGE_FN, - TemplateName = $"{GENERATE_IMAGE_FN}.fn" - } - ] - }, - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageReader, - Items = [ - new UtilityItem - { - FunctionName = READ_IMAGE_FN, - TemplateName = $"{READ_IMAGE_FN}.fn" - } - ] - }, - new AgentUtility - { - Category = "file", - Name = UtilityName.ImageEditor, - Items = [ - new UtilityItem - { - FunctionName = EDIT_IMAGE_FN, - TemplateName = $"{EDIT_IMAGE_FN}.fn" - } - ] - }, new AgentUtility { Category = "file", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index 0e11a9ee6..698896596 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -4,34 +4,9 @@ namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { - public ImageSettings? Image { get; set; } public PdfSettings? Pdf { get; set; } } -#region Image -public class ImageSettings -{ - public ImageReadSettings? Reading { get; set; } - public ImageGenerationSettings? Generation { get; set; } - public ImageEditSettings? Edit { get; set; } -} - -public class ImageReadSettings : LlmBase -{ - public string? ImageDetailLevel { get; set; } -} - -public class ImageGenerationSettings : LlmBase -{ - -} - -public class ImageEditSettings : LlmBase -{ - public SettingBase? ImageConverter { get; set; } -} -#endregion - #region Pdf public class PdfSettings { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid deleted file mode 100644 index 54b722c91..000000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid +++ /dev/null @@ -1 +0,0 @@ -Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid deleted file mode 100644 index fcbae06a8..000000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -** Please call util-file-generate_image if user wants you to provide or generate an image or picture. -** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid deleted file mode 100644 index c245d21dc..000000000 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -Please call function util-file-read_image if user wants to describe an image or images. -You can also call function util-file-read_image to access the image or images that user uploaded. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj new file mode 100644 index 000000000..ee344a0fb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj @@ -0,0 +1,42 @@ + + + + $(TargetFramework) + enable + $(LangVersion) + $(BotSharpVersion) + $(GeneratePackageOnBuild) + $(GenerateDocumentationFile) + $(SolutionDir)packages + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs similarity index 86% rename from src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs index 6b4ec47cb..d058f335b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Converters/FileHandlerImageConverter.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Converters/ImageHandlerImageConverter.cs @@ -3,22 +3,22 @@ using SixLabors.ImageSharp.Formats.Png; using System.IO; -namespace BotSharp.Plugin.FileHandler.Converters; +namespace BotSharp.Plugin.ImageHandler.Converters; -public class FileHandlerImageConverter : IImageConverter +public class ImageHandlerImageConverter : IImageConverter { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; - public FileHandlerImageConverter( + public ImageHandlerImageConverter( IServiceProvider services, - ILogger logger) + ILogger logger) { _services = services; _logger = logger; } - public string Provider => "file-handler"; + public string Provider => "image-handler"; public async Task ConvertImage(BinaryData binary, ImageConvertOptions? options = null) { diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs new file mode 100644 index 000000000..c79ee9365 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Enums/UtilityName.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Plugin.ImageHandler.Enums; + +public class UtilityName +{ + public const string ImageGenerator = "image-generator"; + public const string ImageReader = "image-reader"; + public const string ImageEditor = "image-editor"; +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs index 670261ac7..e493f1307 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs @@ -1,15 +1,15 @@ using BotSharp.Abstraction.Conversations.Settings; -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class EditImageFn : IFunctionCallback { - public string Name => "util-file-edit_image"; + public string Name => "util-image-edit_image"; public string Indication => "Editing image"; private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; private Agent _agent; private string _conversationId; @@ -18,7 +18,7 @@ public class EditImageFn : IFunctionCallback public EditImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -91,8 +91,8 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin var dialog = RoleDialogModel.From(message, AgentRole.User, text); var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant" + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant" }; var fileStorage = _services.GetRequiredService(); @@ -132,8 +132,8 @@ private async Task GetImageEditResponse(string description, string? defa var llmConfig = _agent.LlmConfig; var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant", LlmConfig = new AgentLlmConfig { Provider = llmConfig?.Provider ?? "openai", @@ -159,8 +159,8 @@ private async Task GetImageEditResponse(string description, string? defa return (provider, model); } - provider = _settings?.Image?.Edit?.LlmProvider; - model = _settings?.Image?.Edit?.LlmModel; + provider = _settings?.Edit?.LlmProvider; + model = _settings?.Edit?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -196,7 +196,7 @@ private IEnumerable SaveGeneratedImage(ImageGeneration? image) private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) { - var provider = _settings?.Image?.Edit?.ImageConverter?.Provider; + var provider = _settings?.Edit?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); if (converter == null) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs index 33fa8e81f..cd733260b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs @@ -1,13 +1,13 @@ -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class GenerateImageFn : IFunctionCallback { - public string Name => "util-file-generate_image"; + public string Name => "util-image-generate_image"; public string Indication => "Generating image"; private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; private Agent _agent; private string _conversationId; @@ -16,7 +16,7 @@ public class GenerateImageFn : IFunctionCallback public GenerateImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -59,8 +59,8 @@ private async Task GetImageGeneration(RoleDialogModel message, string? d { var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant", Instruction = description }; @@ -96,8 +96,8 @@ private async Task GetImageGenerationResponse(string description, string var llmConfig = _agent.LlmConfig; var agent = new Agent { - Id = _agent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = _agent?.Name ?? "Utility Assistant", + Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, + Name = _agent?.Name ?? "File Assistant", LlmConfig = new AgentLlmConfig { Provider = llmConfig?.Provider ?? "openai", @@ -123,8 +123,8 @@ private async Task GetImageGenerationResponse(string description, string return (provider, model); } - provider = _settings?.Image?.Generation?.LlmProvider; - model = _settings?.Image?.Generation?.LlmModel; + provider = _settings?.Generation?.LlmProvider; + model = _settings?.Generation?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs rename to src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs index 639f3004a..98ddc7e2c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs @@ -1,15 +1,15 @@ using BotSharp.Abstraction.Routing; -namespace BotSharp.Plugin.FileHandler.Functions; +namespace BotSharp.Plugin.ImageHandler.Functions; public class ReadImageFn : IFunctionCallback { - public string Name => "util-file-read_image"; + public string Name => "util-image-read_image"; public string Indication => "Reading images"; private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly FileHandlerSettings _settings; + private readonly ImageHandlerSettings _settings; private readonly IEnumerable _imageContentTypes = new List { @@ -20,7 +20,7 @@ public class ReadImageFn : IFunctionCallback public ReadImageFn( IServiceProvider services, ILogger logger, - FileHandlerSettings settings) + ImageHandlerSettings settings) { _services = services; _logger = logger; @@ -42,8 +42,8 @@ public async Task Execute(RoleDialogModel message) var agent = new Agent { - Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "Utility Assistant", + Id = fromAgent?.Id ?? BuiltInAgentId.FileAssistant, + Name = fromAgent?.Name ?? "File Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the image(s).", LlmConfig = fromAgent?.LlmConfig ?? new() }; @@ -144,8 +144,8 @@ private async Task GetChatCompletion(Agent agent, List return (provider, model); } - provider = _settings?.Image?.Reading?.LlmProvider; - model = _settings?.Image?.Reading?.LlmModel; + provider = _settings?.Reading?.LlmProvider; + model = _settings?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -161,14 +161,13 @@ private async Task GetChatCompletion(Agent agent, List private void SetImageDetailLevel() { var state = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var key = "chat_image_detail_level"; var level = state.GetState(key); - if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(fileSettings.Image?.Reading?.ImageDetailLevel)) + if (string.IsNullOrWhiteSpace(level) && !string.IsNullOrWhiteSpace(_settings.Reading?.ImageDetailLevel)) { - state.SetState(key, fileSettings.Image.Reading.ImageDetailLevel); + state.SetState(key, _settings.Reading.ImageDetailLevel); } } } diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs new file mode 100644 index 000000000..3c49743c7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Helpers/AiResponseHelper.cs @@ -0,0 +1,31 @@ +namespace BotSharp.Plugin.ImageHandler.Helpers; + +internal static class AiResponseHelper +{ + internal static string GetDefaultResponse(IEnumerable files) + { + if (files.IsNullOrEmpty()) + { + return $"No image is generated."; + } + + if (files.Count() > 1) + { + return $"Here are the images you asked for: {string.Join(", ", files)}"; + } + + return $"Here is the image you asked for: {string.Join(", ", files)}"; + } + + internal static async Task GetImageGenerationResponse(IServiceProvider services, Agent agent, string description) + { + var text = $"Please generate a user-friendly response from the following description to " + + $"inform user that you have completed the required image: {description}"; + + var provider = agent?.LlmConfig?.Provider ?? "openai"; + var model = agent?.LlmConfig?.Model ?? "gpt-4o-mini"; + var completion = CompletionProvider.GetChatCompletion(services, provider: provider, model: model); + var response = await completion.GetChatCompletions(agent, [new RoleDialogModel(AgentRole.User, text)]); + return response.Content; + } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs new file mode 100644 index 000000000..825fc0272 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs @@ -0,0 +1,53 @@ +namespace BotSharp.Plugin.ImageHandler.Hooks; + +public class ImageHandlerUtilityHook : IAgentUtilityHook +{ + private const string READ_IMAGE_FN = "util-file-read_image"; + private const string GENERATE_IMAGE_FN = "util-file-generate_image"; + private const string EDIT_IMAGE_FN = "util-file-edit_image"; + + public void AddUtilities(List utilities) + { + var items = new List + { + new AgentUtility + { + Category = "image", + Name = UtilityName.ImageGenerator, + Items = [ + new UtilityItem + { + FunctionName = GENERATE_IMAGE_FN, + TemplateName = $"{GENERATE_IMAGE_FN}.fn" + } + ] + }, + new AgentUtility + { + Category = "image", + Name = UtilityName.ImageReader, + Items = [ + new UtilityItem + { + FunctionName = READ_IMAGE_FN, + TemplateName = $"{READ_IMAGE_FN}.fn" + } + ] + }, + new AgentUtility + { + Category = "image", + Name = UtilityName.ImageEditor, + Items = [ + new UtilityItem + { + FunctionName = EDIT_IMAGE_FN, + TemplateName = $"{EDIT_IMAGE_FN}.fn" + } + ] + } + }; + + utilities.AddRange(items); + } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs new file mode 100644 index 000000000..e2ed8cf44 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs @@ -0,0 +1,27 @@ +using BotSharp.Plugin.ImageHandler.Converters; +using BotSharp.Plugin.ImageHandler.Hooks; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Plugin.ImageHandler; + +public class FileHandlerPlugin : IBotSharpPlugin +{ + public string Id => "bac8bbf3-da91-4c92-98d8-db14d68e75ae"; + public string Name => "Image Handler"; + public string Description => "AI handles images."; + public string IconUrl => "https://lirp.cdn-website.com/6f8d6d8a/dms3rep/multi/opt/API_Icon-640w.png"; + public string[] AgentIds => []; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(provider => + { + var settingService = provider.GetRequiredService(); + return settingService.Bind("ImageHandler"); + }); + + services.AddScoped(); + services.AddScoped(); + } + +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs new file mode 100644 index 000000000..59eb16be6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextIn.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ImageHandler.LlmContexts; + +public class LlmContextIn +{ + [JsonPropertyName("user_request")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserRequest { get; set; } + + [JsonPropertyName("image_description")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImageDescription { get; set; } + + [JsonPropertyName("image_urls")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable? ImageUrls { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 000000000..822b49364 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.ImageHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("selected_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Selected { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs new file mode 100644 index 000000000..b8f147b90 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.ImageHandler.Settings; + +public class ImageHandlerSettings +{ + public ImageReadSettings? Reading { get; set; } + public ImageGenerationSettings? Generation { get; set; } + public ImageEditSettings? Edit { get; set; } +} + +public class ImageReadSettings : LlmBase +{ + public string? ImageDetailLevel { get; set; } +} + +public class ImageGenerationSettings : LlmBase +{ + +} + +public class ImageEditSettings : LlmBase +{ + public SettingBase? ImageConverter { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs new file mode 100644 index 000000000..88290b6fd --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Using.cs @@ -0,0 +1,34 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using System.Linq; +global using System.Text.Json; +global using System.Net.Mime; +global using System.Threading.Tasks; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using BotSharp.Abstraction.Agents; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Plugins; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Enums; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Converters; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.MLTasks; +global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Abstraction.Settings; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Options; +global using BotSharp.Core.Infrastructures; +global using BotSharp.Plugin.ImageHandler.Enums; +global using BotSharp.Plugin.ImageHandler.Settings; +global using BotSharp.Plugin.ImageHandler.LlmContexts; +global using BotSharp.Plugin.ImageHandler.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json similarity index 91% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json index 84a32e566..5392c6f11 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json @@ -1,5 +1,5 @@ { - "name": "util-file-edit_image", + "name": "util-image-edit_image", "description": "If the user requests you editting or changing an image or a picture, you can call this function to edit an image.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json similarity index 90% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json index ba96aef0d..014eb436b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json @@ -1,5 +1,5 @@ { - "name": "util-file-generate_image", + "name": "util-image-generate_image", "description": "If the user requests you providing or generating image or picture, you can call this function to generate image.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json similarity index 96% rename from src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json index 57a6d90da..7e1efae52 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json @@ -1,5 +1,5 @@ { - "name": "util-file-read_image", + "name": "util-image-read_image", "description": "If the user's request is related to describing or analyzing images, you can call this function to analyze images.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid new file mode 100644 index 000000000..7ddaeb606 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid @@ -0,0 +1 @@ +Please call util-image-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid new file mode 100644 index 000000000..788317c0b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid @@ -0,0 +1,2 @@ +** Please call util-image-generate_image if user wants you to provide or generate an image or picture. +** Please do not call util-image-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid new file mode 100644 index 000000000..1115ef155 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid @@ -0,0 +1,2 @@ +Please call function util-image-read_image if user wants to describe an image or images. +You can also call function util-image-read_image to access the image or images that user uploaded. \ No newline at end of file diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index e5b90ddde..5a7c6eb7b 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -37,6 +37,7 @@ + diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 1304313c9..f61fd92d2 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -621,6 +621,7 @@ "BotSharp.Plugin.MetaGLM", "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", + "BotSharp.Plugin.ImageHandler", "BotSharp.Plugin.EmailHandler", "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.ChartHandler", From 8c0be6109402b335cf204e476b3a63ccbcb388dd Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 9 Oct 2025 16:54:14 -0500 Subject: [PATCH 02/11] migrate image handle settings --- .../BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs | 2 +- src/WebStarter/appsettings.json | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs index 98ddc7e2c..8820ff55b 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs @@ -106,7 +106,7 @@ private List AssembleFiles(string conversationId, IEnumerable x?.Trim()) .Where(x => !string.IsNullOrWhiteSpace(x)) - .Select(x => new BotSharpFile { FileUrl = x }).ToList(); + .Select(x => new BotSharpFile { FileUrl = x }).ToList(); lastDialog.Files.AddRange(addnFiles); } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index f61fd92d2..e5794bb04 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -462,6 +462,8 @@ } }, + "Image" + "AudioHandler": { "Audio": { "Reading": { From 282773ba5cef3c07071748ad0bd87f333c36d80c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 9 Oct 2025 17:32:06 -0500 Subject: [PATCH 03/11] fix typo --- .../BotSharp.Plugin.FileHandler.csproj | 4 -- .../FileHandlerPlugin.cs | 5 +-- .../Hooks/ImageHandlerUtilityHook.cs | 22 +++++------ .../ImageHandlerPlugin.cs | 5 +-- src/WebStarter/appsettings.json | 37 +++++++++---------- 5 files changed, 33 insertions(+), 40 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj index 888b18960..410616b6b 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.FileHandler/BotSharp.Plugin.FileHandler.csproj @@ -28,10 +28,6 @@ - - - - diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs index a8ecac6dc..51fa33f1a 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/FileHandlerPlugin.cs @@ -7,8 +7,8 @@ public class FileHandlerPlugin : IBotSharpPlugin { public string Id => "65be5aee-48df-4ff8-a50a-05c8bcd2a793"; public string Name => "File Handler"; - public string Description => "AI reads files, such as image, pdf, excel"; - public string IconUrl => "https://lirp.cdn-website.com/6f8d6d8a/dms3rep/multi/opt/API_Icon-640w.png"; + public string Description => "AI handles files."; + public string IconUrl => "https://cdn-icons-png.flaticon.com/512/2567/2567656.png"; public string[] AgentIds => []; public void RegisterDI(IServiceCollection services, IConfiguration config) @@ -21,5 +21,4 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); } - } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs index 825fc0272..1a0f7b385 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs @@ -2,9 +2,9 @@ namespace BotSharp.Plugin.ImageHandler.Hooks; public class ImageHandlerUtilityHook : IAgentUtilityHook { - private const string READ_IMAGE_FN = "util-file-read_image"; - private const string GENERATE_IMAGE_FN = "util-file-generate_image"; - private const string EDIT_IMAGE_FN = "util-file-edit_image"; + private const string READ_IMAGE_FN = "util-image-read_image"; + private const string GENERATE_IMAGE_FN = "util-image-generate_image"; + private const string EDIT_IMAGE_FN = "util-image-edit_image"; public void AddUtilities(List utilities) { @@ -13,25 +13,25 @@ public void AddUtilities(List utilities) new AgentUtility { Category = "image", - Name = UtilityName.ImageGenerator, + Name = UtilityName.ImageReader, Items = [ new UtilityItem { - FunctionName = GENERATE_IMAGE_FN, - TemplateName = $"{GENERATE_IMAGE_FN}.fn" - } + FunctionName = READ_IMAGE_FN, + TemplateName = $"{READ_IMAGE_FN}.fn" + } ] }, new AgentUtility { Category = "image", - Name = UtilityName.ImageReader, + Name = UtilityName.ImageGenerator, Items = [ new UtilityItem { - FunctionName = READ_IMAGE_FN, - TemplateName = $"{READ_IMAGE_FN}.fn" - } + FunctionName = GENERATE_IMAGE_FN, + TemplateName = $"{GENERATE_IMAGE_FN}.fn" + } ] }, new AgentUtility diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs index e2ed8cf44..1a6268345 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/ImageHandlerPlugin.cs @@ -4,12 +4,12 @@ namespace BotSharp.Plugin.ImageHandler; -public class FileHandlerPlugin : IBotSharpPlugin +public class ImageHandlerPlugin : IBotSharpPlugin { public string Id => "bac8bbf3-da91-4c92-98d8-db14d68e75ae"; public string Name => "Image Handler"; public string Description => "AI handles images."; - public string IconUrl => "https://lirp.cdn-website.com/6f8d6d8a/dms3rep/multi/opt/API_Icon-640w.png"; + public string IconUrl => "https://cdn-icons-png.flaticon.com/512/8002/8002135.png"; public string[] AgentIds => []; public void RegisterDI(IServiceCollection services, IConfiguration config) @@ -23,5 +23,4 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); } - } \ No newline at end of file diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index e5794bb04..b067b0528 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -431,24 +431,6 @@ }, "FileHandler": { - "Image": { - "Reading": { - "LlmProvider": "openai", - "LlmModel": "gpt-5-mini", - "ImageDetailLevel": "auto" - }, - "Generation": { - "LlmProvider": "openai", - "LlmModel": "gpt-image-1" - }, - "Edit": { - "LlmProvider": "openai", - "LlmModel": "gpt-image-1", - "ImageConverter": { - "Provider": "file-handler" - } - } - }, "Pdf": { "Reading": { "LlmProvider": "google-ai", @@ -462,7 +444,24 @@ } }, - "Image" + "ImageHandler": { + "Reading": { + "LlmProvider": "openai", + "LlmModel": "gpt-5-mini", + "ImageDetailLevel": "auto" + }, + "Generation": { + "LlmProvider": "openai", + "LlmModel": "gpt-image-1" + }, + "Edit": { + "LlmProvider": "openai", + "LlmModel": "gpt-image-1", + "ImageConverter": { + "Provider": "image-handler" + } + } + }, "AudioHandler": { "Audio": { From c1a9de4a048ae5ff3dbac4fbc143df75d1070c00 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 9 Oct 2025 17:43:29 -0500 Subject: [PATCH 04/11] revert --- .../BotSharp.Plugin.ImageHandler.csproj | 12 ++++++------ .../Functions/EditImageFn.cs | 2 +- .../Functions/GenerateImageFn.cs | 2 +- .../Functions/ReadImageFn.cs | 2 +- .../Hooks/ImageHandlerUtilityHook.cs | 12 ++++++------ ...age-edit_image.json => util-file-edit_image.json} | 2 +- ...rate_image.json => util-file-generate_image.json} | 2 +- ...age-read_image.json => util-file-read_image.json} | 2 +- .../templates/util-file-edit_image.fn.liquid | 1 + .../templates/util-file-generate_image.fn.liquid | 2 ++ .../templates/util-file-read_image.fn.liquid | 2 ++ .../templates/util-image-edit_image.fn.liquid | 1 - .../templates/util-image-generate_image.fn.liquid | 2 -- .../templates/util-image-read_image.fn.liquid | 2 -- 14 files changed, 23 insertions(+), 23 deletions(-) rename src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-image-edit_image.json => util-file-edit_image.json} (91%) rename src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-image-generate_image.json => util-file-generate_image.json} (90%) rename src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/{util-image-read_image.json => util-file-read_image.json} (96%) create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid create mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid delete mode 100644 src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj index ee344a0fb..fc2f33bdf 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/BotSharp.Plugin.ImageHandler.csproj @@ -11,22 +11,22 @@ - + PreserveNewest - + PreserveNewest - + PreserveNewest - + PreserveNewest - + PreserveNewest - + PreserveNewest diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs index e493f1307..b59eb2c02 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs @@ -4,7 +4,7 @@ namespace BotSharp.Plugin.ImageHandler.Functions; public class EditImageFn : IFunctionCallback { - public string Name => "util-image-edit_image"; + public string Name => "util-file-edit_image"; public string Indication => "Editing image"; private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs index cd733260b..8ce7e8f93 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs @@ -2,7 +2,7 @@ namespace BotSharp.Plugin.ImageHandler.Functions; public class GenerateImageFn : IFunctionCallback { - public string Name => "util-image-generate_image"; + public string Name => "util-file-generate_image"; public string Indication => "Generating image"; private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs index 8820ff55b..236e9f60b 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs @@ -4,7 +4,7 @@ namespace BotSharp.Plugin.ImageHandler.Functions; public class ReadImageFn : IFunctionCallback { - public string Name => "util-image-read_image"; + public string Name => "util-file-read_image"; public string Indication => "Reading images"; private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs index 1a0f7b385..36853aa84 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Hooks/ImageHandlerUtilityHook.cs @@ -2,9 +2,9 @@ namespace BotSharp.Plugin.ImageHandler.Hooks; public class ImageHandlerUtilityHook : IAgentUtilityHook { - private const string READ_IMAGE_FN = "util-image-read_image"; - private const string GENERATE_IMAGE_FN = "util-image-generate_image"; - private const string EDIT_IMAGE_FN = "util-image-edit_image"; + private const string READ_IMAGE_FN = "util-file-read_image"; + private const string GENERATE_IMAGE_FN = "util-file-generate_image"; + private const string EDIT_IMAGE_FN = "util-file-edit_image"; public void AddUtilities(List utilities) { @@ -12,7 +12,7 @@ public void AddUtilities(List utilities) { new AgentUtility { - Category = "image", + Category = "file", Name = UtilityName.ImageReader, Items = [ new UtilityItem @@ -24,7 +24,7 @@ public void AddUtilities(List utilities) }, new AgentUtility { - Category = "image", + Category = "file", Name = UtilityName.ImageGenerator, Items = [ new UtilityItem @@ -36,7 +36,7 @@ public void AddUtilities(List utilities) }, new AgentUtility { - Category = "image", + Category = "file", Name = UtilityName.ImageEditor, Items = [ new UtilityItem diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json similarity index 91% rename from src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json index 5392c6f11..84a32e566 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-edit_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-edit_image.json @@ -1,5 +1,5 @@ { - "name": "util-image-edit_image", + "name": "util-file-edit_image", "description": "If the user requests you editting or changing an image or a picture, you can call this function to edit an image.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json similarity index 90% rename from src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json index 014eb436b..ba96aef0d 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-generate_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-generate_image.json @@ -1,5 +1,5 @@ { - "name": "util-image-generate_image", + "name": "util-file-generate_image", "description": "If the user requests you providing or generating image or picture, you can call this function to generate image.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json similarity index 96% rename from src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json rename to src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json index 7e1efae52..57a6d90da 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-image-read_image.json +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-file-read_image.json @@ -1,5 +1,5 @@ { - "name": "util-image-read_image", + "name": "util-file-read_image", "description": "If the user's request is related to describing or analyzing images, you can call this function to analyze images.", "parameters": { "type": "object", diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid new file mode 100644 index 000000000..54b722c91 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-edit_image.fn.liquid @@ -0,0 +1 @@ +Please call util-file-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid new file mode 100644 index 000000000..fcbae06a8 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-generate_image.fn.liquid @@ -0,0 +1,2 @@ +** Please call util-file-generate_image if user wants you to provide or generate an image or picture. +** Please do not call util-file-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid new file mode 100644 index 000000000..c245d21dc --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-file-read_image.fn.liquid @@ -0,0 +1,2 @@ +Please call function util-file-read_image if user wants to describe an image or images. +You can also call function util-file-read_image to access the image or images that user uploaded. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid deleted file mode 100644 index 7ddaeb606..000000000 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-edit_image.fn.liquid +++ /dev/null @@ -1 +0,0 @@ -Please call util-image-edit_image if user wants to edit, change or modify an image in the conversation. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid deleted file mode 100644 index 788317c0b..000000000 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-generate_image.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -** Please call util-image-generate_image if user wants you to provide or generate an image or picture. -** Please do not call util-image-generate_image, if user does not generate image explicitly or wants to change or edit the existing image. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid b/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid deleted file mode 100644 index 1115ef155..000000000 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/util-image-read_image.fn.liquid +++ /dev/null @@ -1,2 +0,0 @@ -Please call function util-image-read_image if user wants to describe an image or images. -You can also call function util-image-read_image to access the image or images that user uploaded. \ No newline at end of file From dc2e3028eca7306a89390be1e88dd59a302f3c93 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 10 Oct 2025 11:39:45 -0500 Subject: [PATCH 05/11] refine llm config --- .../Agents/Models/AgentLlmConfig.cs | 45 ++++++++++++++ .../Models/LlmConfigBase.cs | 18 +++--- .../FileInstructService.SelectFile.cs | 4 +- .../Functions/ReadAudioFn.cs | 30 +++++----- .../Settings/AudioHandlerSettings.cs | 2 +- .../Functions/PlotChartFn.cs | 27 ++++----- .../Functions/HandleEmailSenderFn.cs | 4 +- .../Functions/ReadPdfFn.cs | 34 +++-------- .../Settings/FileHandlerSettings.cs | 2 +- .../Functions/EditImageFn.cs | 58 +++++-------------- .../Functions/GenerateImageFn.cs | 52 ++++------------- .../Functions/ReadImageFn.cs | 26 ++------- .../Settings/ImageHandlerSettings.cs | 6 +- .../Functions/PyProgrammerFn.cs | 22 +++---- src/WebStarter/appsettings.json | 32 +++++----- 15 files changed, 154 insertions(+), 208 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs index 75e3aa93b..7c7c5325a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs @@ -41,4 +41,49 @@ public class AgentLlmConfig [JsonPropertyName("reasoning_effort_level")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? ReasoningEffortLevel { get; set; } + + /// + /// Image generation config + /// + [JsonPropertyName("image_generation")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmImageGenerationConfig? ImageGeneration { get; set; } + + /// + /// Image edit config + /// + [JsonPropertyName("image_edit")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmImageEditConfig? ImageEdit { get; set; } + + /// + /// Audio transcription config + /// + [JsonPropertyName("audio_transcription")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmAudioTranscriptionConfig? AudioTranscription { get; set; } + + /// + /// Realtime config + /// + [JsonPropertyName("realtime")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public LlmRealtimeConfig? Realtime { get; set; } +} + + +public class LlmImageGenerationConfig : LlmProviderModel +{ +} + +public class LlmImageEditConfig : LlmProviderModel +{ } + +public class LlmRealtimeConfig : LlmProviderModel +{ +} + +public class LlmAudioTranscriptionConfig : LlmProviderModel +{ +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs index 1d88942c4..899f6e7ab 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Models; -public class LlmConfigBase : LlmBase +public class LlmConfigBase : LlmProviderModel { /// /// Llm maximum output tokens @@ -13,15 +13,13 @@ public class LlmConfigBase : LlmBase public string? ReasoningEffortLevel { get; set; } } -public class LlmBase +public class LlmProviderModel { - /// - /// Llm provider - /// - public string? LlmProvider { get; set; } + [JsonPropertyName("provider")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Provider { get; set; } - /// - /// Llm model - /// - public string? LlmModel { get; set; } + [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Model { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 502645765..a4cf2d88b 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -153,8 +153,8 @@ private async Task> SelectFiles(IEnumerable Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); + var agentService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); var routingCtx = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + var wholeDialogs = routingCtx.GetDialogs(); if (wholeDialogs.IsNullOrEmpty()) { @@ -44,7 +47,7 @@ public async Task Execute(RoleDialogModel message) } var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetAudioTranscription(dialogs); + var response = await GetAudioTranscription(agent, dialogs); message.Content = response; dialogs.ForEach(x => x.Files = null); return true; @@ -54,7 +57,7 @@ private List AssembleFiles(string convId, List { if (dialogs.IsNullOrEmpty()) { - return new List(); + return []; } var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); @@ -85,13 +88,13 @@ private List AssembleFiles(string convId, List return dialogs; } - private async Task GetAudioTranscription(List dialogs) + private async Task GetAudioTranscription(Agent agent, List dialogs) { - var audioCompletion = PrepareModel(); + var audioCompletion = PrepareModel(agent); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).LastOrDefault(); var transcripts = new List(); - if (dialog != null) + if (dialog?.Files != null) { foreach (var file in dialog.Files) { @@ -129,9 +132,9 @@ private async Task GetAudioTranscription(List dialogs) return string.Join("\r\n\r\n", transcripts); } - private IAudioTranscription PrepareModel() + private IAudioTranscription PrepareModel(Agent agent) { - var (provider, model) = GetLlmProviderModel(); + var (provider, model) = GetLlmProviderModel(agent); return CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model); } @@ -142,21 +145,18 @@ private bool VerifyAudioFileType(string fileName) || !string.IsNullOrEmpty(FileUtility.GetFileContentType(fileName)); } - private (string, string) GetLlmProviderModel() + private (string, string) GetLlmProviderModel(Agent agent) { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("audio_read_llm_provider"); - var model = state.GetState("audio_read_llm_provider"); + var provider = agent?.LlmConfig?.AudioTranscription?.Provider; + var model = agent?.LlmConfig?.AudioTranscription?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = _settings?.Audio?.Reading?.LlmProvider; - model = _settings?.Audio?.Reading?.LlmModel; + provider = _settings?.Audio?.Reading?.Provider; + model = _settings?.Audio?.Reading?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index b6353b057..8c4c1af02 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -13,7 +13,7 @@ public class AudioSettings public AudioReadSettings? Reading { get; set; } } -public class AudioReadSettings : LlmBase +public class AudioReadSettings : LlmProviderModel { } #endregion \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs index 1cac2c81d..187e22ae9 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -12,7 +12,6 @@ public class PlotChartFn : IFunctionCallback public string Name => "util-chart-plot_chart"; public string Indication => "Plotting chart"; - public PlotChartFn( IServiceProvider services, ILogger logger, @@ -35,8 +34,8 @@ public async Task Execute(RoleDialogModel message) var inst = GetChartPlotInstruction(message.CurrentAgentId); var innerAgent = new Agent { - Id = agent.Id, - Name = agent.Name, + Id = agent?.Id ?? BuiltInAgentId.AIProgrammer, + Name = agent?.Name ?? "AI Programmer", Instruction = inst, LlmConfig = GetLlmConfig(), TemplateDict = new Dictionary @@ -132,16 +131,16 @@ private string GetChartPlotInstruction(string agentId) private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-5"; + var provider = _settings.ChartPlot?.Provider; + var model = _settings.ChartPlot?.Model; - var state = _services.GetRequiredService(); - provider = state.GetState("chart_plot_llm_provider") - .IfNullOrEmptyAs(_settings.ChartPlot?.LlmProvider) - .IfNullOrEmptyAs(provider); - model = state.GetState("chart_plot_llm_model") - .IfNullOrEmptyAs(_settings.ChartPlot?.LlmModel) - .IfNullOrEmptyAs(model); + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5"; return (provider, model); } @@ -151,10 +150,6 @@ private AgentLlmConfig GetLlmConfig() var maxOutputTokens = _settings?.ChartPlot?.MaxOutputTokens ?? 8192; var reasoningEffortLevel = _settings?.ChartPlot?.ReasoningEffortLevel ?? "minimal"; - var state = _services.GetRequiredService(); - maxOutputTokens = int.TryParse(state.GetState("chart_plot_max_output_tokens"), out var tokens) ? tokens : maxOutputTokens; - reasoningEffortLevel = state.GetState("chart_plot_reasoning_effort_level").IfNullOrEmptyAs(reasoningEffortLevel); - return new AgentLlmConfig { MaxOutputTokens = maxOutputTokens, diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index 8352a8482..e38233c3a 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -76,8 +76,8 @@ private async Task> GetConversationFiles() IsIncludeBotFiles = true, IsAttachFiles = true, MessageLimit = convSettings?.FileSelect?.MessageLimit, - LlmProvider = convSettings?.FileSelect?.LlmProvider, - LlmModel = convSettings?.FileSelect?.LlmModel, + Provider = convSettings?.FileSelect?.Provider, + Model = convSettings?.FileSelect?.Model, MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index a2c2f4e5c..c0f9236ee 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -29,20 +29,15 @@ public ReadPdfFn( public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agentService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); var routingCtx = _services.GetRequiredService(); - var agentService = _services.GetRequiredService(); - - Agent? fromAgent = null; - if (!string.IsNullOrEmpty(message.CurrentAgentId)) - { - fromAgent = await agentService.GetAgent(message.CurrentAgentId); - } - + + var fromAgent = await agentService.GetAgent(message.CurrentAgentId); var agent = new Agent { - Id = fromAgent?.Id ?? BuiltInAgentId.UtilityAssistant, - Name = fromAgent?.Name ?? "Utility Assistant", + Id = fromAgent?.Id ?? BuiltInAgentId.FileAssistant, + Name = fromAgent?.Name ?? "File Assistant", Instruction = fromAgent?.Instruction ?? args?.UserRequest ?? "Please describe the pdf file(s).", LlmConfig = fromAgent?.LlmConfig ?? new() }; @@ -140,27 +135,16 @@ private async Task GetChatCompletion(Agent agent, List private (string, string) GetLlmProviderModel() { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("pdf_read_llm_provider"); - var model = state.GetState("pdf_read_llm_model"); - - if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) - { - return (provider, model); - } - - provider = _settings?.Pdf?.Reading?.LlmProvider; - model = _settings?.Pdf?.Reading?.LlmModel; + var provider = _settings?.Pdf?.Reading?.Provider; + var model = _settings?.Pdf?.Reading?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = "openai"; - model = "gpt-5-mini"; + provider = "google-ai"; + model = "gemini-2.0-flash"; return (provider, model); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index 698896596..efbe8d114 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -13,7 +13,7 @@ public class PdfSettings public PdfReadSettings? Reading { get; set; } } -public class PdfReadSettings : LlmBase +public class PdfReadSettings : LlmProviderModel { public bool ConvertToImage { get; set; } public string? ImageDetailLevel { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs index d45c7237a..317727710 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/EditImageFn.cs @@ -11,7 +11,6 @@ public class EditImageFn : IFunctionCallback private readonly ILogger _logger; private readonly ImageHandlerSettings _settings; - private Agent _agent; private string _conversationId; private string _messageId; @@ -29,11 +28,15 @@ public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); var descrpition = args?.UserRequest ?? string.Empty; + + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + await Init(message); SetImageOptions(); var image = await SelectImage(descrpition); - var response = await GetImageEditGeneration(message, descrpition, image); + var response = await GetImageEdit(agent, message, descrpition, image); message.Content = response; message.StopCompletion = true; return true; @@ -41,10 +44,8 @@ public async Task Execute(RoleDialogModel message) private async Task Init(RoleDialogModel message) { - var agentService = _services.GetRequiredService(); var convService = _services.GetRequiredService(); - _agent = await agentService.GetAgent(message.CurrentAgentId); _conversationId = convService.ConversationId; _messageId = message.MessageId; } @@ -68,15 +69,15 @@ private void SetImageOptions() IsAttachFiles = true, ContentTypes = [MediaTypeNames.Image.Png, MediaTypeNames.Image.Jpeg], MessageLimit = convSettings?.FileSelect?.MessageLimit, - LlmProvider = convSettings?.FileSelect?.LlmProvider, - LlmModel = convSettings?.FileSelect?.LlmModel, + Provider = convSettings?.FileSelect?.Provider, + Model = convSettings?.FileSelect?.Model, MaxOutputTokens = convSettings?.FileSelect?.MaxOutputTokens, ReasoningEffortLevel = convSettings?.FileSelect?.ReasoningEffortLevel }); return selecteds?.FirstOrDefault(); } - private async Task GetImageEditGeneration(RoleDialogModel message, string description, MessageFileModel? image) + private async Task GetImageEdit(Agent agent, RoleDialogModel message, string description, MessageFileModel? image) { if (image == null) { @@ -85,15 +86,10 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin try { - var (provider, model) = GetLlmProviderModel(); + var (provider, model) = GetLlmProviderModel(agent); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, - Name = _agent?.Name ?? "File Assistant" - }; var fileStorage = _services.GetRequiredService(); var fileBinary = fileStorage.GetFileBytes(image.FileStorageUrl); @@ -112,7 +108,7 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin return response.Content; } - return await GetImageEditResponse(description, defaultContent: null); + return await GetImageEditResponse(agent, description); } catch (Exception ex) { @@ -122,45 +118,23 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin } } - private async Task GetImageEditResponse(string description, string? defaultContent) + private async Task GetImageEditResponse(Agent agent, string description) { - if (defaultContent != null) - { - return defaultContent; - } - - var llmConfig = _agent.LlmConfig; - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, - Name = _agent?.Name ?? "File Assistant", - LlmConfig = new AgentLlmConfig - { - Provider = llmConfig?.Provider ?? "openai", - Model = llmConfig?.Model ?? "gpt-5-mini", - MaxOutputTokens = llmConfig?.MaxOutputTokens, - ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel - } - }; - return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); } - private (string, string) GetLlmProviderModel() + private (string, string) GetLlmProviderModel(Agent agent) { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("image_edit_llm_provider"); - var model = state.GetState("image_edit_llm_provider"); + var provider = agent?.LlmConfig?.ImageEdit?.Provider; + var model = agent?.LlmConfig?.ImageEdit?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = _settings?.Edit?.LlmProvider; - model = _settings?.Edit?.LlmModel; + provider = _settings?.Edit?.Provider; + model = _settings?.Edit?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs index b73d5869d..ccb3e43cc 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/GenerateImageFn.cs @@ -9,7 +9,6 @@ public class GenerateImageFn : IFunctionCallback private readonly ILogger _logger; private readonly ImageHandlerSettings _settings; - private Agent _agent; private string _conversationId; private string _messageId; @@ -30,9 +29,9 @@ public async Task Execute(RoleDialogModel message) SetImageOptions(); var agentService = _services.GetRequiredService(); - _agent = await agentService.GetAgent(message.CurrentAgentId); + var currentAgent = await agentService.GetAgent(message.CurrentAgentId); - var response = await GetImageGeneration(message, args?.ImageDescription); + var response = await GetImageGeneration(currentAgent, message, args?.ImageDescription); message.Content = response; message.StopCompletion = true; return true; @@ -53,18 +52,11 @@ private void SetImageOptions() state.SetState("image_response_format", "bytes"); } - private async Task GetImageGeneration(RoleDialogModel message, string? description) + private async Task GetImageGeneration(Agent agent, RoleDialogModel message, string? description) { try { - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, - Name = _agent?.Name ?? "File Assistant", - Instruction = description - }; - - var (provider, model) = GetLlmProviderModel(); + var (provider, model) = GetLlmProviderModel(agent); var completion = CompletionProvider.GetImageCompletion(_services, provider: provider, model: model); var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; var dialog = RoleDialogModel.From(message, AgentRole.User, text); @@ -76,7 +68,7 @@ private async Task GetImageGeneration(RoleDialogModel message, string? d return result.Content; } - return await GetImageGenerationResponse(description, defaultContent: null); + return await GetImageGenerationResponse(agent, description); } catch (Exception ex) { @@ -86,45 +78,23 @@ private async Task GetImageGeneration(RoleDialogModel message, string? d } } - private async Task GetImageGenerationResponse(string description, string? defaultContent) + private async Task GetImageGenerationResponse(Agent agent, string description) { - if (defaultContent != null) - { - return defaultContent; - } - - var llmConfig = _agent.LlmConfig; - var agent = new Agent - { - Id = _agent?.Id ?? BuiltInAgentId.FileAssistant, - Name = _agent?.Name ?? "File Assistant", - LlmConfig = new AgentLlmConfig - { - Provider = llmConfig?.Provider ?? "openai", - Model = llmConfig?.Model ?? "gpt-5-mini", - MaxOutputTokens = llmConfig?.MaxOutputTokens, - ReasoningEffortLevel = llmConfig?.ReasoningEffortLevel - } - }; - return await AiResponseHelper.GetImageGenerationResponse(_services, agent, description); } - private (string, string) GetLlmProviderModel() + private (string, string) GetLlmProviderModel(Agent agent) { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("image_generate_llm_provider"); - var model = state.GetState("image_generate_llm_model"); + var provider = agent?.LlmConfig?.ImageGeneration?.Provider; + var model = agent?.LlmConfig?.ImageGeneration?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { return (provider, model); } - provider = _settings?.Generation?.LlmProvider; - model = _settings?.Generation?.LlmModel; + provider = _settings?.Generation?.Provider; + model = _settings?.Generation?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs index 236e9f60b..9a42f47d4 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Functions/ReadImageFn.cs @@ -30,16 +30,11 @@ public ReadImageFn( public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); + var agentService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); var routingCtx = _services.GetRequiredService(); - var agentService = _services.GetRequiredService(); - - Agent? fromAgent = null; - if (!string.IsNullOrEmpty(message.CurrentAgentId)) - { - fromAgent = await agentService.GetAgent(message.CurrentAgentId); - } - + + var fromAgent = await agentService.GetAgent(message.CurrentAgentId); var agent = new Agent { Id = fromAgent?.Id ?? BuiltInAgentId.FileAssistant, @@ -133,19 +128,8 @@ private async Task GetChatCompletion(Agent agent, List private (string, string) GetLlmProviderModel() { - var state = _services.GetRequiredService(); - var llmProviderService = _services.GetRequiredService(); - - var provider = state.GetState("image_read_llm_provider"); - var model = state.GetState("image_read_llm_model"); - - if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) - { - return (provider, model); - } - - provider = _settings?.Reading?.LlmProvider; - model = _settings?.Reading?.LlmModel; + var provider = _settings?.Reading?.Provider; + var model = _settings?.Reading?.Model; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs index b8f147b90..81a11c8d5 100644 --- a/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.ImageHandler/Settings/ImageHandlerSettings.cs @@ -9,17 +9,17 @@ public class ImageHandlerSettings public ImageEditSettings? Edit { get; set; } } -public class ImageReadSettings : LlmBase +public class ImageReadSettings : LlmProviderModel { public string? ImageDetailLevel { get; set; } } -public class ImageGenerationSettings : LlmBase +public class ImageGenerationSettings : LlmProviderModel { } -public class ImageEditSettings : LlmBase +public class ImageEditSettings : LlmProviderModel { public SettingBase? ImageConverter { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs index 0a69b7a8e..45a1e778a 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs @@ -186,16 +186,16 @@ private string GetPyCodeInterpreterInstruction(string agentId) private (string, string) GetLlmProviderModel() { - var provider = "openai"; - var model = "gpt-5"; + var provider = _settings.CodeGeneration?.Provider; + var model = _settings.CodeGeneration?.Model; - var state = _services.GetRequiredService(); - provider = state.GetState("py_intepreter_llm_provider") - .IfNullOrEmptyAs(_settings.CodeGeneration?.LlmProvider) - .IfNullOrEmptyAs(provider); - model = state.GetState("py_intepreter_llm_model") - .IfNullOrEmptyAs(_settings.CodeGeneration?.LlmModel) - .IfNullOrEmptyAs(model); + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-5"; return (provider, model); } @@ -205,10 +205,6 @@ private AgentLlmConfig GetLlmConfig() var maxOutputTokens = _settings?.CodeGeneration?.MaxOutputTokens ?? 8192; var reasoningEffortLevel = _settings?.CodeGeneration?.ReasoningEffortLevel ?? "minimal"; - var state = _services.GetRequiredService(); - maxOutputTokens = int.TryParse(state.GetState("py_intepreter_max_output_tokens"), out var tokens) ? tokens : maxOutputTokens; - reasoningEffortLevel = state.GetState("py_intepreter_reasoning_effort_level").IfNullOrEmptyAs(reasoningEffortLevel); - return new AgentLlmConfig { MaxOutputTokens = maxOutputTokens, diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b067b0528..276f853e7 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -301,8 +301,8 @@ "MinTimeSecondsBetweenMessages": 2 }, "FileSelect": { - "LlmProvider": "openai", - "LlmModel": "gpt-5-mini", + "Provider": "openai", + "Model": "gpt-5-mini", "MaxOutputTokens": 8192, "ReasoningEffortLevel": "low", "MessageLimit": 50 @@ -433,8 +433,8 @@ "FileHandler": { "Pdf": { "Reading": { - "LlmProvider": "google-ai", - "LlmModel": "gemini-2.0-flash", + "Provider": "google-ai", + "Model": "gemini-2.0-flash", "ConvertToImage": false, "ImageDetailLevel": "auto", "ImageConverter": { @@ -446,17 +446,17 @@ "ImageHandler": { "Reading": { - "LlmProvider": "openai", - "LlmModel": "gpt-5-mini", + "Provider": "openai", + "Model": "gpt-5-mini", "ImageDetailLevel": "auto" }, "Generation": { - "LlmProvider": "openai", - "LlmModel": "gpt-image-1" + "Provider": "openai", + "Model": "gpt-image-1" }, "Edit": { - "LlmProvider": "openai", - "LlmModel": "gpt-image-1", + "Provider": "openai", + "Model": "gpt-image-1", "ImageConverter": { "Provider": "image-handler" } @@ -466,8 +466,8 @@ "AudioHandler": { "Audio": { "Reading": { - "LlmProvider": "openai", - "LlmModel": "gpt-4o-mini-transcribe" + "Provider": "openai", + "Model": "gpt-4o-mini-transcribe" } } }, @@ -486,8 +486,8 @@ "ChartHandler": { "ChartPlot": { - "LlmProvider": "openai", - "LlmModel": "gpt-5", + "Provider": "openai", + "Model": "gpt-5", "MaxOutputTokens": 8192, "ReasoningEffortLevel": "minimal", "MessageLimit": 50 @@ -570,8 +570,8 @@ "InstallLocation": "C:/Users/xxx/AppData/Local/Programs/Python/Python313/python313.dll", "PythonVersion": "3.13.3", "CodeGeneration": { - "LlmProvider": "openai", - "LlmModel": "gpt-5", + "Provider": "openai", + "Model": "gpt-5", "MaxOutputTokens": 8192, "ReasoningEffortLevel": "minimal", "MessageLimit": 50 From 28e3518e41379c0f179a40f1a456697d63815fdc Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 10 Oct 2025 12:16:28 -0500 Subject: [PATCH 06/11] apply agent realtime config --- .../Services/RealtimeHub.cs | 41 ++++++++++++++++--- .../Realtime/RealTimeCompletionProvider.cs | 16 ++++---- .../Realtime/RealTimeCompletionProvider.cs | 2 +- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index b34f5fba4..b0a088ccc 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Models; @@ -10,7 +11,8 @@ namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub { private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly RealtimeModelSettings _settings; private RealtimeHubConnection _conn; public RealtimeHubConnection HubConn => _conn; @@ -18,10 +20,14 @@ public class RealtimeHub : IRealtimeHub private IRealTimeCompletion _completer; public IRealTimeCompletion Completer => _completer; - public RealtimeHub(IServiceProvider services, ILogger logger) + public RealtimeHub( + IServiceProvider services, + ILogger logger, + RealtimeModelSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task ConnectToModel( @@ -43,10 +49,10 @@ public async Task ConnectToModel( routing.Context.SetDialogs(dialogs); routing.Context.SetMessageId(_conn.ConversationId, Guid.Empty.ToString()); - var states = _services.GetRequiredService(); - var settings = _services.GetRequiredService(); + var (provider, model) = GetLlmProviderModel(agent); - _completer = _services.GetServices().First(x => x.Provider == settings.Provider); + _completer = _services.GetServices().First(x => x.Provider == provider); + _completer.SetModelName(model); _completer.SetOptions(options); await _completer.Connect( @@ -156,7 +162,7 @@ await HookEmitter.Emit(_services, async hook => await hook.OnRouti }, onInterruptionDetected: async () => { - if (settings.InterruptResponse) + if (_settings.InterruptResponse) { // Reset states _conn.ResetResponseState(); @@ -179,4 +185,27 @@ public RealtimeHubConnection SetHubConnection(string conversationId) return _conn; } + + private (string, string) GetLlmProviderModel(Agent agent) + { + var provider = agent?.LlmConfig?.Realtime?.Provider; + var model = agent?.LlmConfig?.Realtime?.Model; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = _settings.Provider; + model = _settings.Model; + + if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) + { + return (provider, model); + } + + provider = "openai"; + model = "gpt-realtime"; + return (provider, model); + } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 802088bdf..1bc5d74d4 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -18,11 +18,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private string _model = GoogleAIModels.Gemini2FlashLive001; private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; + private readonly GoogleAiSettings _settings; + private List renderedInstructions = []; private LlmRealtimeSession _session; - private readonly GoogleAiSettings _settings; + private RealtimeOptions? _realtimeOptions; private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; private readonly JsonSerializerOptions _jsonOptions = new() @@ -50,12 +52,12 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public GoogleRealTimeProvider( IServiceProvider services, - GoogleAiSettings settings, - ILogger logger) + ILogger logger, + GoogleAiSettings settings) { - _settings = settings; _services = services; _logger = logger; + _settings = settings; } public async Task Connect( @@ -84,7 +86,7 @@ public async Task Connect( var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); - _model = realtimeModelSettings.Model; + _model ??= realtimeModelSettings.Model; var modelSettings = settingsService.GetSetting(Provider, _model); Reset(); @@ -422,7 +424,7 @@ public void SetModelName(string model) public void SetOptions(RealtimeOptions? options) { - + _realtimeOptions = options; } #region Private methods diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index e6b614e67..269cafa20 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -67,7 +67,7 @@ public async Task Connect( var settingsService = _services.GetRequiredService(); var realtimeSettings = _services.GetRequiredService(); - _model = realtimeSettings.Model; + _model ??= realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); _session = new LlmRealtimeSession(_services, new ChatSessionOptions From 835a3c71efbaacd279019b9bd1a359597b0bdec3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 10 Oct 2025 14:16:35 -0500 Subject: [PATCH 07/11] sync mongo --- .../Agents/Models/AgentLlmConfig.cs | 4 +- .../Collections/AgentDocument.cs | 2 +- .../Models/AgentLlmConfigMongoElement.cs | 44 ---- .../Models/AgentLlmConfigMongoModel.cs | 193 ++++++++++++++++++ .../Models/LlmProviderModelMongoModel.cs | 38 ++++ .../Repository/MongoRepository.Agent.cs | 8 +- 6 files changed, 238 insertions(+), 51 deletions(-) delete mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoModel.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Models/LlmProviderModelMongoModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs index 7c7c5325a..6ed259a3f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs @@ -80,10 +80,10 @@ public class LlmImageEditConfig : LlmProviderModel { } -public class LlmRealtimeConfig : LlmProviderModel +public class LlmAudioTranscriptionConfig : LlmProviderModel { } -public class LlmAudioTranscriptionConfig : LlmProviderModel +public class LlmRealtimeConfig : LlmProviderModel { } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 161842650..7a2390267 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -26,7 +26,7 @@ public class AgentDocument : MongoBase public List Labels { get; set; } public List RoutingRules { get; set; } public List Rules { get; set; } - public AgentLlmConfigMongoElement? LlmConfig { get; set; } + public AgentLlmConfigMongoModel? LlmConfig { get; set; } public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs deleted file mode 100644 index 63bc3b20b..000000000 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoElement.cs +++ /dev/null @@ -1,44 +0,0 @@ -using BotSharp.Abstraction.Agents.Models; - -namespace BotSharp.Plugin.MongoStorage.Models; - -[BsonIgnoreExtraElements(Inherited = true)] -public class AgentLlmConfigMongoElement -{ - public string? Provider { get; set; } - public string? Model { get; set; } - public bool IsInherit { get; set; } - public int MaxRecursionDepth { get; set; } - public int? MaxOutputTokens { get; set; } - public string? ReasoningEffortLevel { get; set; } - - public static AgentLlmConfigMongoElement? ToMongoElement(AgentLlmConfig? config) - { - if (config == null) return null; - - return new AgentLlmConfigMongoElement - { - Provider = config.Provider, - Model = config.Model, - IsInherit = config.IsInherit, - MaxRecursionDepth = config.MaxRecursionDepth, - MaxOutputTokens = config.MaxOutputTokens, - ReasoningEffortLevel = config.ReasoningEffortLevel - }; - } - - public static AgentLlmConfig? ToDomainElement(AgentLlmConfigMongoElement? config) - { - if (config == null) return null; - - return new AgentLlmConfig - { - Provider = config.Provider, - Model = config.Model, - IsInherit = config.IsInherit, - MaxRecursionDepth = config.MaxRecursionDepth, - MaxOutputTokens = config.MaxOutputTokens, - ReasoningEffortLevel = config.ReasoningEffortLevel - }; - } -} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoModel.cs new file mode 100644 index 000000000..165a590a1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/AgentLlmConfigMongoModel.cs @@ -0,0 +1,193 @@ +using BotSharp.Abstraction.Agents.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +[BsonIgnoreExtraElements(Inherited = true)] +public class AgentLlmConfigMongoModel +{ + public string? Provider { get; set; } + public string? Model { get; set; } + public bool IsInherit { get; set; } + public int MaxRecursionDepth { get; set; } + public int? MaxOutputTokens { get; set; } + public string? ReasoningEffortLevel { get; set; } + + public LlmImageGenerationConfigMongoModel? ImageGeneration { get; set; } + public LlmImageEditConfigMongoModel? ImageEdit { get; set; } + public LlmAudioTranscriptionConfigMongoModel? AudioTranscription { get; set; } + public LlmRealtimeConfigMongoModel? Realtime { get; set; } + + + + public static AgentLlmConfigMongoModel? ToMongoElement(AgentLlmConfig? config) + { + if (config == null) + { + return null; + } + + return new AgentLlmConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, + MaxOutputTokens = config.MaxOutputTokens, + ReasoningEffortLevel = config.ReasoningEffortLevel, + ImageGeneration = LlmImageGenerationConfigMongoModel.ToMongoModel(config.ImageGeneration), + ImageEdit = LlmImageEditConfigMongoModel.ToMongoModel(config.ImageEdit), + AudioTranscription = LlmAudioTranscriptionConfigMongoModel.ToMongoModel(config.AudioTranscription), + Realtime = LlmRealtimeConfigMongoModel.ToMongoModel(config.Realtime) + }; + } + + public static AgentLlmConfig? ToDomainElement(AgentLlmConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new AgentLlmConfig + { + Provider = config.Provider, + Model = config.Model, + IsInherit = config.IsInherit, + MaxRecursionDepth = config.MaxRecursionDepth, + MaxOutputTokens = config.MaxOutputTokens, + ReasoningEffortLevel = config.ReasoningEffortLevel, + ImageGeneration = LlmImageGenerationConfigMongoModel.ToDomainModel(config.ImageGeneration), + ImageEdit = LlmImageEditConfigMongoModel.ToDomainModel(config.ImageEdit), + AudioTranscription = LlmAudioTranscriptionConfigMongoModel.ToDomainModel(config.AudioTranscription), + Realtime = LlmRealtimeConfigMongoModel.ToDomainModel(config.Realtime) + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmImageGenerationConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmImageGenerationConfig? ToDomainModel(LlmImageGenerationConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmImageGenerationConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmImageGenerationConfigMongoModel? ToMongoModel(LlmImageGenerationConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmImageGenerationConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmImageEditConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmImageEditConfig? ToDomainModel(LlmImageEditConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmImageEditConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmImageEditConfigMongoModel? ToMongoModel(LlmImageEditConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmImageEditConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmAudioTranscriptionConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmAudioTranscriptionConfig? ToDomainModel(LlmAudioTranscriptionConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmAudioTranscriptionConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmAudioTranscriptionConfigMongoModel? ToMongoModel(LlmAudioTranscriptionConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmAudioTranscriptionConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmRealtimeConfigMongoModel : LlmProviderModelMongoModel +{ + public static LlmRealtimeConfig? ToDomainModel(LlmRealtimeConfigMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmRealtimeConfig + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmRealtimeConfigMongoModel? ToMongoModel(LlmRealtimeConfig? config) + { + if (config == null) + { + return null; + } + + return new LlmRealtimeConfigMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/LlmProviderModelMongoModel.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/LlmProviderModelMongoModel.cs new file mode 100644 index 000000000..ae5e6b202 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/LlmProviderModelMongoModel.cs @@ -0,0 +1,38 @@ +using BotSharp.Abstraction.Models; + +namespace BotSharp.Plugin.MongoStorage.Models; + +[BsonIgnoreExtraElements(Inherited = true)] +public class LlmProviderModelMongoModel +{ + public string? Provider { get; set; } + public string? Model { get; set; } + + public static LlmProviderModel? ToDomainModel(LlmProviderModelMongoModel? config) + { + if (config == null) + { + return null; + } + + return new LlmProviderModel + { + Provider = config.Provider, + Model = config.Model, + }; + } + + public static LlmProviderModelMongoModel? ToMongoModel(LlmProviderModel? config) + { + if (config == null) + { + return null; + } + + return new LlmProviderModelMongoModel + { + Provider = config.Provider, + Model = config.Model, + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 76cadd324..044a52b13 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -333,7 +333,7 @@ private void UpdateAgentRules(string agentId, List rules) private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) { - var llmConfig = AgentLlmConfigMongoElement.ToMongoElement(config); + var llmConfig = AgentLlmConfigMongoModel.ToMongoElement(config); var filter = Builders.Filter.Eq(x => x.Id, agentId); var update = Builders.Update .Set(x => x.LlmConfig, llmConfig) @@ -377,7 +377,7 @@ private void UpdateAgentAllFields(Agent agent) .Set(x => x.McpTools, agent.McpTools.Select(u => AgentMcpToolMongoElement.ToMongoElement(u)).ToList()) .Set(x => x.KnowledgeBases, agent.KnowledgeBases.Select(u => AgentKnowledgeBaseMongoElement.ToMongoElement(u)).ToList()) .Set(x => x.Rules, agent.Rules.Select(e => AgentRuleMongoElement.ToMongoElement(e)).ToList()) - .Set(x => x.LlmConfig, AgentLlmConfigMongoElement.ToMongoElement(agent.LlmConfig)) + .Set(x => x.LlmConfig, AgentLlmConfigMongoModel.ToMongoElement(agent.LlmConfig)) .Set(x => x.IsPublic, agent.IsPublic) .Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -550,7 +550,7 @@ public void BulkInsertAgents(List agents) MaxMessageCount = x.MaxMessageCount, Profiles = x.Profiles ?? [], Labels = x.Labels ?? [], - LlmConfig = AgentLlmConfigMongoElement.ToMongoElement(x.LlmConfig), + LlmConfig = AgentLlmConfigMongoModel.ToMongoElement(x.LlmConfig), ChannelInstructions = x.ChannelInstructions?.Select(i => ChannelInstructionMongoElement.ToMongoElement(i))?.ToList() ?? [], Templates = x.Templates?.Select(t => AgentTemplateMongoElement.ToMongoElement(t))?.ToList() ?? [], Functions = x.Functions?.Select(f => FunctionDefMongoElement.ToMongoElement(f))?.ToList() ?? [], @@ -651,7 +651,7 @@ private Agent TransformAgentDocument(AgentDocument? agentDoc) Profiles = agentDoc.Profiles ?? [], Labels = agentDoc.Labels ?? [], MaxMessageCount = agentDoc.MaxMessageCount, - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig), + LlmConfig = AgentLlmConfigMongoModel.ToDomainElement(agentDoc.LlmConfig), ChannelInstructions = agentDoc.ChannelInstructions?.Select(i => ChannelInstructionMongoElement.ToDomainElement(i))?.ToList() ?? [], Templates = agentDoc.Templates?.Select(t => AgentTemplateMongoElement.ToDomainElement(t))?.ToList() ?? [], Functions = agentDoc.Functions?.Select(f => FunctionDefMongoElement.ToDomainElement(f)).ToList() ?? [], From 3e71f24d1ac64aedc99dfe424b5fb165507f24e0 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 10 Oct 2025 15:19:30 -0500 Subject: [PATCH 08/11] refine update agent code scripts --- .../Agents/IAgentService.cs | 10 ++++ .../Options/AgentCodeScriptUpdateOptions.cs | 8 ++++ .../Repositories/IBotSharpRepository.cs | 3 +- .../Models/AgentCodeScriptDbUpdateOptions.cs | 6 +++ .../Services/AgentService.CodeScripts.cs | 46 +++++++++++++++++++ .../Services/InstructService.Execute.cs | 5 +- .../FileRepository.AgentCodeScript.cs | 8 +++- .../MongoRepository.AgentCodeScript.cs | 5 +- 8 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Repositories/Models/AgentCodeScriptDbUpdateOptions.cs create mode 100644 src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CodeScripts.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index a2f2a17f2..e7b054d79 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Agents.Options; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; @@ -65,4 +66,13 @@ public interface IAgentService Task> GetUserAgents(string userId); PluginDef GetPlugin(string agentId); + + Task> GetAgentCodeScripts(string agentId, AgentCodeScriptFilter? filter = null) + => Task.FromResult(new List()); + + Task GetAgentCodeScript(string agentId, string scriptName, string scriptType = AgentCodeScriptType.Src) + => Task.FromResult(string.Empty); + + Task UpdateAgentCodeScripts(string agentId, List codeScripts, AgentCodeScriptUpdateOptions? options = null) + => Task.FromResult(false); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs new file mode 100644 index 000000000..4b5e9e957 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Options/AgentCodeScriptUpdateOptions.cs @@ -0,0 +1,8 @@ +using BotSharp.Abstraction.Repositories.Models; + +namespace BotSharp.Abstraction.Agents.Options; + +public class AgentCodeScriptUpdateOptions : AgentCodeScriptDbUpdateOptions +{ + public bool DeleteIfNotIncluded { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index f2ebe6a8f..63ca3da3e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; using BotSharp.Abstraction.Roles.Models; using BotSharp.Abstraction.Shared; using BotSharp.Abstraction.Statistics.Enums; @@ -111,7 +112,7 @@ List GetAgentCodeScripts(string agentId, AgentCodeScriptFilter? => throw new NotImplementedException(); string? GetAgentCodeScript(string agentId, string scriptName, string scriptType = AgentCodeScriptType.Src) => throw new NotImplementedException(); - bool UpdateAgentCodeScripts(string agentId, List scripts) + bool UpdateAgentCodeScripts(string agentId, List scripts, AgentCodeScriptDbUpdateOptions? options = null) => throw new NotImplementedException(); bool BulkInsertAgentCodeScripts(string agentId, List scripts) => throw new NotImplementedException(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/AgentCodeScriptDbUpdateOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/AgentCodeScriptDbUpdateOptions.cs new file mode 100644 index 000000000..9217445b9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Models/AgentCodeScriptDbUpdateOptions.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Repositories.Models; + +public class AgentCodeScriptDbUpdateOptions +{ + public bool IsUpsert { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CodeScripts.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CodeScripts.cs new file mode 100644 index 000000000..5c7c8bc00 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CodeScripts.cs @@ -0,0 +1,46 @@ +using BotSharp.Abstraction.Agents.Options; + +namespace BotSharp.Core.Agents.Services; + +public partial class AgentService +{ + public async Task> GetAgentCodeScripts(string agentId, AgentCodeScriptFilter? filter = null) + { + var db = _services.GetRequiredService(); + var scripts = db.GetAgentCodeScripts(agentId, filter); + return await Task.FromResult(scripts); + } + + public async Task GetAgentCodeScript(string agentId, string scriptName, string scriptType = AgentCodeScriptType.Src) + { + var db = _services.GetRequiredService(); + var script = db.GetAgentCodeScript(agentId, scriptName, scriptType); + return await Task.FromResult(script); + } + + public async Task UpdateAgentCodeScripts(string agentId, List codeScripts, AgentCodeScriptUpdateOptions? options = null) + { + if (string.IsNullOrWhiteSpace(agentId) || codeScripts.IsNullOrEmpty()) + { + return false; + } + + var db = _services.GetRequiredService(); + + var toDeleteScripts = new List(); + if (options?.DeleteIfNotIncluded == true) + { + var curDbScripts = await GetAgentCodeScripts(agentId); + var codePaths = codeScripts.Select(x => x.CodePath).ToList(); + toDeleteScripts = curDbScripts.Where(x => !codePaths.Contains(x.CodePath)).ToList(); + } + + var updateResult = db.UpdateAgentCodeScripts(agentId, codeScripts, options); + if (!toDeleteScripts.IsNullOrEmpty()) + { + db.DeleteAgentCodeScripts(agentId, toDeleteScripts); + } + + return updateResult; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index daa558033..be162e6d2 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -150,9 +150,8 @@ await hook.OnResponseGenerated(new InstructResponseModel return response; } - + var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); - var db = _services.GetRequiredService(); var hooks = _services.GetHooks(agent.Id); var codeProvider = codeOptions?.CodeInterpretProvider ?? "botsharp-py-interpreter"; @@ -187,7 +186,7 @@ await hook.OnResponseGenerated(new InstructResponseModel } // Get code script - var codeScript = db.GetAgentCodeScript(agent.Id, scriptName, scriptType: AgentCodeScriptType.Src); + var codeScript = await agentService.GetAgentCodeScript(agent.Id, scriptName, scriptType: AgentCodeScriptType.Src); if (string.IsNullOrWhiteSpace(codeScript)) { #if DEBUG diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs index 764287ddf..c332d147c 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentCodeScript.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Repositories.Models; using System.IO; namespace BotSharp.Core.Repository; @@ -73,7 +74,7 @@ public List GetAgentCodeScripts(string agentId, AgentCodeScript return string.Empty; } - public bool UpdateAgentCodeScripts(string agentId, List scripts) + public bool UpdateAgentCodeScripts(string agentId, List scripts, AgentCodeScriptDbUpdateOptions? options = null) { if (string.IsNullOrWhiteSpace(agentId) || scripts.IsNullOrEmpty()) { @@ -95,7 +96,10 @@ public bool UpdateAgentCodeScripts(string agentId, List scripts } var file = Path.Combine(dir, script.Name); - File.WriteAllText(file, script.Content); + if (options?.IsUpsert == true || File.Exists(file)) + { + File.WriteAllText(file, script.Content); + } } return true; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs index 3d551b7d0..8e8a8154d 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentCodeScript.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Repositories.Models; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -55,7 +56,7 @@ public List GetAgentCodeScripts(string agentId, AgentCodeScript return found?.Content; } - public bool UpdateAgentCodeScripts(string agentId, List scripts) + public bool UpdateAgentCodeScripts(string agentId, List scripts, AgentCodeScriptDbUpdateOptions? options = null) { if (string.IsNullOrWhiteSpace(agentId) || scripts.IsNullOrEmpty()) { @@ -73,7 +74,7 @@ public bool UpdateAgentCodeScripts(string agentId, List scripts }), Builders.Update.Set(y => y.Content, x.Content) .Set(x => x.UpdatedTime, DateTime.UtcNow) - )) + ) { IsUpsert = options?.IsUpsert ?? false }) .ToList(); var result = _dc.AgentCodeScripts.BulkWrite(ops, new BulkWriteOptions { IsOrdered = false }); From 77a79d9957854bea8553c5d2cb7932ff187cf34a Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 12 Oct 2025 22:47:11 -0500 Subject: [PATCH 09/11] fix role content --- .../Conversations/Models/RoleDialogModel.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index da9caecf3..e99b187cf 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -150,10 +150,6 @@ public string RoleContent { text = !string.IsNullOrWhiteSpace(Payload) ? Payload : Content; } - else - { - text = !string.IsNullOrWhiteSpace(Content) ? Content : RichContent?.Message?.Text; - } return text; } From e92d97f8b438997e477255beefa0f8b2baba341e Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 12 Oct 2025 22:50:12 -0500 Subject: [PATCH 10/11] minor change --- .../Functions/PyProgrammerFn.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs index 45a1e778a..15b618b07 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs @@ -115,6 +115,7 @@ public async Task Execute(RoleDialogModel message) dynamic stringIO = io.StringIO(); sys.stdout = stringIO; sys.stderr = stringIO; + sys.argv = new PyList(); // Set global items using var globals = new PyDict(); From 96262c35d2a750a0bbb9d1b10e0debde1ce59133 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 13 Oct 2025 11:53:36 -0500 Subject: [PATCH 11/11] add CodeScript in ProgramCodeTemplateMessage --- .../Conversations/Models/RoleDialogModel.cs | 2 +- .../RichContent/Template/ProgramCodeTemplateMessage.cs | 3 +++ .../Providers/ChatCompletionProvider.cs | 6 +++--- .../Providers/Chat/ChatCompletionProvider.cs | 6 +++--- .../Functions/PlotChartFn.cs | 1 + .../Providers/Chat/ChatCompletionProvider.cs | 6 +++--- .../Providers/Chat/GeminiChatCompletionProvider.cs | 8 ++++---- .../Providers/Chat/PalmChatCompletionProvider.cs | 6 +++--- .../Providers/Realtime/RealTimeCompletionProvider.cs | 10 +++++----- .../Providers/ChatCompletionProvider.cs | 4 ++-- .../Providers/ChatCompletionProvider.cs | 2 +- .../Providers/ChatCompletionProvider.cs | 8 ++++---- .../MicrosoftExtensionsAIChatCompletionProvider.cs | 6 +++--- .../Providers/Chat/ChatCompletionProvider.cs | 6 +++--- .../Providers/Realtime/RealTimeCompletionProvider.cs | 6 +++--- .../Functions/PyProgrammerFn.cs | 3 ++- .../Providers/ChatCompletionProvider.cs | 4 ++-- 17 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index e99b187cf..90cc44889 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -141,7 +141,7 @@ public class RoleDialogModel : ITrackableMessage public bool IsFromAssistant => Role == AgentRole.Assistant || Role == AgentRole.Model; [JsonIgnore(Condition = JsonIgnoreCondition.Always)] - public string RoleContent + public string LlmContent { get { diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs index 60f6464ad..a47584b44 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/Models/RichContent/Template/ProgramCodeTemplateMessage.cs @@ -8,6 +8,9 @@ public class ProgramCodeTemplateMessage : IRichMessage, ITemplateMessage [JsonPropertyName("text")] public string Text { get; set; } = string.Empty; + [JsonPropertyName("code_script")] + public string? CodeScript { get; set; } + [JsonPropertyName("template_type")] public virtual string TemplateType { get; set; } = TemplateTypeEnum.ProgramCode; diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 0d24bc41f..fcb639101 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -140,11 +140,11 @@ public Task GetChatCompletionsStreamingAsync(Agent agent, List< { if (message.Role == AgentRole.User) { - messages.Add(new Message(RoleType.User, message.RoleContent)); + messages.Add(new Message(RoleType.User, message.LlmContent)); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new Message(RoleType.Assistant, message.RoleContent)); + messages.Add(new Message(RoleType.Assistant, message.LlmContent)); } else if (message.Role == AgentRole.Function) { @@ -170,7 +170,7 @@ public Task GetChatCompletionsStreamingAsync(Agent agent, List< new ToolResultContent() { ToolUseId = message.ToolCallId, - Content = [new TextContent() { Text = message.RoleContent }] + Content = [new TextContent() { Text = message.LlmContent }] } } }); diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index b7efb11ff..dc9a0fbc5 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -432,11 +432,11 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; @@ -448,7 +448,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; diff --git a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs index 187e22ae9..71efda50d 100644 --- a/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs +++ b/src/Plugins/BotSharp.Plugin.ChartHandler/Functions/PlotChartFn.cs @@ -83,6 +83,7 @@ public async Task Execute(RoleDialogModel message) Message = new ProgramCodeTemplateMessage { Text = ret?.JsCode ?? string.Empty, + CodeScript = ret?.JsCode, Language = "javascript" } }; diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index eb143ebb3..6349b1ed0 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -399,11 +399,11 @@ public void SetModelName(string model) ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; @@ -415,7 +415,7 @@ public void SetModelName(string model) } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 1d2b4ef48..60e0ef025 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -263,17 +263,17 @@ public void SetModelName(string model) Name = message.FunctionName, Response = new JsonObject() { - ["result"] = message.RoleContent ?? string.Empty + ["result"] = message.LlmContent ?? string.Empty } } } ], AgentRole.Function)); - convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.RoleContent}"); + convPrompts.Add($"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.LlmContent}"); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var contentParts = new List { new() { Text = text } }; if (allowMultiModal && !message.Files.IsNullOrEmpty()) @@ -285,7 +285,7 @@ public void SetModelName(string model) } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var contentParts = new List { new() { Text = text } }; if (allowMultiModal && !message.Files.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index 8dfaa695b..bec65e177 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -108,7 +108,7 @@ public async Task GetChatCompletions(Agent agent, List new PalmChatMessage(c.RoleContent, c.Role == AgentRole.User ? "user" : "AI")) + var messages = conversations.Select(c => new PalmChatMessage(c.LlmContent, c.Role == AgentRole.User ? "user" : "AI")) .ToList(); if (!functions.IsNullOrEmpty()) @@ -124,8 +124,8 @@ public async Task GetChatCompletions(Agent agent, List {dialog.RoleContent}\r\n" : - $"{dialog.Role}: {dialog.RoleContent}\r\n"; + $"{dialog.Role}: {dialog.FunctionName} => {dialog.LlmContent}\r\n" : + $"{dialog.Role}: {dialog.LlmContent}\r\n"; } prompt += "\r\n\r\n" + router.Templates.FirstOrDefault(x => x.Name == "response_with_function").Content; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 1bc5d74d4..180d18c3f 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -567,25 +567,25 @@ await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) Name = message.FunctionName ?? string.Empty, Response = new JsonObject() { - ["result"] = message.RoleContent ?? string.Empty + ["result"] = message.LlmContent ?? string.Empty } } } ], AgentRole.Function)); convPrompts.Add( - $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.RoleContent}"); + $"{AgentRole.Assistant}: Call function {message.FunctionName}({message.FunctionArgs}) => {message.LlmContent}"); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; contents.Add(new Content(text, AgentRole.User)); convPrompts.Add($"{AgentRole.User}: {text}"); } else if (message.Role == AgentRole.Assistant) { - contents.Add(new Content(message.RoleContent, AgentRole.Model)); - convPrompts.Add($"{AgentRole.Assistant}: {message.RoleContent}"); + contents.Add(new Content(message.LlmContent, AgentRole.Model)); + convPrompts.Add($"{AgentRole.Assistant}: {message.LlmContent}"); } } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index ea6f75fdf..a24633939 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -42,7 +42,7 @@ public async Task GetChatCompletions(Agent agent, List $"{x.Role}: {x.RoleContent}")).Trim(); + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.LlmContent}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; var llama = _services.GetRequiredService(); @@ -118,7 +118,7 @@ public async Task GetChatCompletionsAsync(Agent agent, Func onMessageReceived, Func onFunctionExecuting) { - var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.RoleContent}")).Trim(); + var content = string.Join("\r\n", conversations.Select(x => $"{x.Role}: {x.LlmContent}")).Trim(); content += $"\r\n{AgentRole.Assistant}: "; var state = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs index a7a4adddf..a748ed2a7 100644 --- a/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LangChain/Providers/ChatCompletionProvider.cs @@ -41,7 +41,7 @@ public async Task GetChatCompletions(Agent agent, List new Message(c.RoleContent, c.Role == AgentRole.User ? MessageRole.Human : MessageRole.Ai)).ToList(); + .Select(c => new Message(c.LlmContent, c.Role == AgentRole.User ? MessageRole.Human : MessageRole.Ai)).ToList(); var response = await model.GenerateAsync(new ChatRequest { Messages = messages }, _settings); diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index 2505eed1e..38c3b398d 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -111,8 +111,8 @@ private string PrepareOptions(Agent agent, List conversations, foreach (var message in samples) { messages.Add(message.Role == AgentRole.User ? - new MessageItem("user", message.RoleContent) : - new MessageItem("assistant", message.RoleContent)); + new MessageItem("user", message.LlmContent) : + new MessageItem("assistant", message.LlmContent)); } foreach (var function in functions) @@ -129,13 +129,13 @@ private string PrepareOptions(Agent agent, List conversations, } else if (message.Role == "user") { - var userMessage = new MessageItem("user",message.RoleContent); + var userMessage = new MessageItem("user",message.LlmContent); messages.Add(userMessage); } else if (message.Role == "assistant") { - messages.Add(new MessageItem("assistant", message.RoleContent)); + messages.Add(new MessageItem("assistant", message.LlmContent)); } } diff --git a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs index 43f5134a0..888fe341d 100644 --- a/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MicrosoftExtensionsAI/MicrosoftExtensionsAIChatCompletionProvider.cs @@ -109,16 +109,16 @@ public async Task GetChatCompletions(Agent agent, List>(x.FunctionArgs ?? "{}")), - new FunctionResultContent(x.FunctionName, x.RoleContent) + new FunctionResultContent(x.FunctionName, x.LlmContent) ])); } else if (x.Role == AgentRole.System || x.Role == AgentRole.Assistant) { - messages.Add(new(x.Role == AgentRole.System ? ChatRole.System : ChatRole.Assistant, x.RoleContent)); + messages.Add(new(x.Role == AgentRole.System ? ChatRole.System : ChatRole.Assistant, x.LlmContent)); } else if (x.Role == AgentRole.User) { - List contents = [new TextContent(x.RoleContent)]; + List contents = [new TextContent(x.LlmContent)]; if (allowMultiModal) { foreach (var file in x.Files) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 3812a5916..7ab149e26 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -402,11 +402,11 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; @@ -418,7 +418,7 @@ public async Task GetChatCompletionsStreamingAsync(Agent agent, } else if (message.Role == AgentRole.Assistant) { - var text = message.RoleContent; + var text = message.LlmContent; var textPart = ChatMessageContentPart.CreateTextPart(text); var contentParts = new List { textPart }; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 269cafa20..379941b40 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -621,15 +621,15 @@ private async Task OnUserAudioTranscriptionCompleted(RealtimeHu ChatToolCall.CreateFunctionToolCall(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.RoleContent)); + messages.Add(new ToolChatMessage(message.ToolCallId.IfNullOrEmptyAs(message.FunctionName), message.LlmContent)); } else if (message.Role == AgentRole.User) { - messages.Add(new UserChatMessage(message.RoleContent)); + messages.Add(new UserChatMessage(message.LlmContent)); } else if (message.Role == AgentRole.Assistant) { - messages.Add(new AssistantChatMessage(message.RoleContent)); + messages.Add(new AssistantChatMessage(message.LlmContent)); } } diff --git a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs index 15b618b07..8dc9e7d50 100644 --- a/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs +++ b/src/Plugins/BotSharp.Plugin.PythonInterpreter/Functions/PyProgrammerFn.cs @@ -75,7 +75,8 @@ public async Task Execute(RoleDialogModel message) Recipient = new Recipient { Id = convService.ConversationId }, Message = new ProgramCodeTemplateMessage { - Text = ret.PythonCode ?? string.Empty, + Text = result, + CodeScript = ret.PythonCode, Language = "python" } }; diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 93b90d13f..5f0405c17 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -269,13 +269,13 @@ public void SetModelName(string model) } else if (message.Role == "user") { - var userMessage = ChatMessage.FromUser(message.RoleContent); + var userMessage = ChatMessage.FromUser(message.LlmContent); messages.Add(userMessage); } else if (message.Role == "assistant") { - messages.Add(ChatMessage.FromAssistant(message.RoleContent)); + messages.Add(ChatMessage.FromAssistant(message.LlmContent)); } }