diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 2247c212a..16e19a6f5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -17,6 +17,11 @@ public class LlmModelSetting /// public string Version { get; set; } = "1106-Preview"; + /// + /// Api version + /// + public string? ApiVersion { get; set; } + /// /// Deployment same functional model in a group. /// It can be used to deploy same model in different regions. diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs new file mode 100644 index 000000000..2cc560126 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/JsonExtensions.cs @@ -0,0 +1,41 @@ +using Newtonsoft.Json.Linq; +using Newtonsoft.Json; + +namespace BotSharp.Abstraction.Utilities; + +public static class JsonExtensions +{ + public static string FormatJson(this string? json, Formatting format = Formatting.Indented) + { + if (string.IsNullOrWhiteSpace(json)) + { + return "{}"; + } + + try + { + var parsedJson = JObject.Parse(json); + foreach (var item in parsedJson) + { + try + { + var key = item.Key; + var value = parsedJson[key].ToString(); + var parsedValue = JObject.Parse(value); + parsedJson[key] = parsedValue; + } + catch { continue; } + } + + var jsonSettings = new JsonSerializerSettings + { + Formatting = format + }; + return JsonConvert.SerializeObject(parsedJson, jsonSettings); + } + catch + { + return json; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs index ff817b257..4953e114a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs @@ -132,11 +132,11 @@ private async Task GetTextCompletion(string apiUrl, stri private string BuildApiUrl(LlmModelSetting modelSetting) { - var url = string.Empty; + var apiVersion = !string.IsNullOrWhiteSpace(modelSetting.ApiVersion) ? modelSetting.ApiVersion : "2023-09-15-preview"; var endpoint = modelSetting.Endpoint.EndsWith("/") ? modelSetting.Endpoint.Substring(0, modelSetting.Endpoint.Length - 1) : modelSetting.Endpoint; - url = $"{endpoint}/openai/deployments/{_model}/completions?api-version=2023-09-15-preview"; + var url = $"{endpoint}/openai/deployments/{_model}/completions?api-version={apiVersion}"; return url; } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs index b9e2a84be..bf2292ea3 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Messaging.Enums; -using BotSharp.Abstraction.Options; using Microsoft.AspNetCore.SignalR; namespace BotSharp.Plugin.ChatHub.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index ec3eb8a95..99bb19372 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -1,11 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Loggers; -using BotSharp.Abstraction.Loggers.Enums; -using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Options; -using BotSharp.Abstraction.Repositories; -using BotSharp.Abstraction.Routing; using Microsoft.AspNetCore.SignalR; using System.Text.Encodings.Web; using System.Text.Unicode; @@ -126,7 +118,7 @@ public override async Task OnFunctionExecuting(RoleDialogModel message) var agent = await _agentService.LoadAgent(message.CurrentAgentId); message.FunctionArgs = message.FunctionArgs ?? "{}"; - var args = FormatJson(message.FunctionArgs); + var args = message.FunctionArgs.FormatJson(); var log = $"{message.FunctionName} executing\r\n```json\r\n{args}\r\n```"; var input = new ContentLogInputModel(conversationId, message) @@ -567,40 +559,5 @@ private JsonSerializerOptions InitLocalJsonOptions(BotSharpOptions options) return localOptions; } - - private string FormatJson(string? json) - { - var defaultJson = "{}"; - if (string.IsNullOrWhiteSpace(json)) - { - return defaultJson; - } - - try - { - var parsedJson = JObject.Parse(json); - foreach (var item in parsedJson) - { - try - { - var key = item.Key; - var value = parsedJson[key].ToString(); - var parsedValue = JObject.Parse(value); - parsedJson[key] = parsedValue; - } - catch { continue; } - } - - var jsonSettings = new JsonSerializerSettings - { - Formatting = Newtonsoft.Json.Formatting.Indented - }; - return JsonConvert.SerializeObject(parsedJson, jsonSettings) ?? defaultJson; - } - catch - { - return defaultJson; - } - } #endregion } diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs index 22c02fdf1..c8afd3dda 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/WelcomeHook.cs @@ -1,8 +1,4 @@ -using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.Messaging; -using BotSharp.Abstraction.Templating; using Microsoft.AspNetCore.SignalR; -using BotSharp.Abstraction.Options; namespace BotSharp.Plugin.ChatHub.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs index 0e730db9f..7cc89c9db 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Using.cs @@ -19,4 +19,16 @@ global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Conversations.Models; global using BotSharp.OpenAPI.ViewModels.Conversations; -global using BotSharp.OpenAPI.ViewModels.Users; \ No newline at end of file +global using BotSharp.OpenAPI.ViewModels.Users; +global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Loggers; +global using BotSharp.Abstraction.Loggers.Enums; +global using BotSharp.Abstraction.Loggers.Models; +global using BotSharp.Abstraction.Options; +global using BotSharp.Abstraction.Repositories; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Messaging; +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Messaging.Models.RichContent; +global using BotSharp.Abstraction.Templating; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs index 525eae8bf..84be6e168 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs @@ -78,8 +78,9 @@ private async Task> GetConversationFiles() var conversationId = convService.ConversationId; var dialogs = convService.GetDialogHistory(fromBreakpoint: false); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); - return await SelectFiles(files, dialogs); + var userFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User); + var botFiles = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot); + return await SelectFiles(userFiles.Concat(botFiles), dialogs); } private async Task> SelectFiles(IEnumerable files, List dialogs) @@ -94,7 +95,7 @@ private async Task> SelectFiles(IEnumerable { - return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}"; + return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, author: {x.FileSource}, content_type: {x.ContentType}"; }).ToList(); var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "select_attachment_prompt"); prompt = render.Render(prompt, new Dictionary @@ -114,7 +115,8 @@ private async Task> SelectFiles(IEnumerable>(content) ?? new List(); + var selecteds = JsonSerializer.Deserialize(content); + var fids = selecteds?.Selecteds ?? new List(); return files.Where((x, idx) => fids.Contains(idx + 1)).ToList(); } catch (Exception ex) @@ -146,6 +148,6 @@ private async Task SendEmailBySMTP(MimeMessage mailMessage) await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls); await smtpClient.AuthenticateAsync(_emailSettings.EmailAddress, _emailSettings.Password); var response = await smtpClient.SendAsync(mailMessage); - return response; + return response ?? "Email sent"; } } diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 000000000..5371a1813 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.EmailHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("selected_ids")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IEnumerable Selecteds { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid index 0ad3e377f..9091f6bf3 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid @@ -1,2 +1,2 @@ -Please call handle_email_request if user wants to send out an email. -** Please take a look at the conversation, and decide whether user wants to send email with attachments or not. \ No newline at end of file +Please call handle_email_request if user wants to send email. +** Please take a look at the conversation and decide whether user wants to send email with files/attachments/images or not. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid index 0b1b87c3e..f96e286ae 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_attachment_prompt.liquid @@ -1,18 +1,42 @@ Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user. -Your response must be a list of file ids. -** Please only output the list. Do not prepend or append anything. -For example: -Suppose there are three files: +** Ensure the output is only in JSON format without any additional text. +** If no files are selected, you must output an empty list []. +** You may need to look at the file_name as a reference to find the correct file id. -id: 1, file_name: example_file.png, content_type: image/png -id: 2, file_name: example_file.jpeg, content_type: image/jpeg -id: 3, file_name: example_file.pdf, content_type: application/pdf +Here is the JSON format to use: +{ + "selected_ids": a list of id selected from the [FILES] section +} -If user wants the first file and the third file, the ouput should be [1, 3]. -If user wants the all the images, the output should be [1, 2]. -If user wants the pdf file, the output should be [3]. -If user does not want any files, the ouput should be []; +Suppose there are 4 files: + +id: 1, file_name: example_file.jpg, author: user, content_type: image/jpeg +id: 2, file_name: example_file.pdf, author: user, content_type: application/pdf +id: 3, file_name: example_file.png, author: bot, content_type: image/png +id: 4, file_name: example_file.png, author: bot, content_type: image/png + +===== +Example 1: +USER: I want to send the first file and the third file. +OUTPUT: { "selected_ids": [1, 3] } + +Example 2: +USER: Send all the images. +OUTPUT: { "selected_ids": [1, 2, 4] } + +Example 3: +USER: Send all the images I uploaded. +OUTPUT: { "selected_ids": [1] } + +Example 4: +USER: Send the image and the pdf file. +OUTPUT: { "selected_ids": [1, 2] } + +Example 5: +USER: Send the images generated by bot +OUTPUT: { "selected_ids": [3, 4] } +===== [FILES] {% for file in file_list -%} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 37adf4fbf..63938c5e3 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -28,7 +28,7 @@ public async Task Execute(RoleDialogModel message) Init(message); SetImageOptions(); - var image = await SelectConversationImage(); + var image = await SelectConversationImage(descrpition); var response = await GetImageEditGeneration(message, descrpition, image); message.Content = response; return true; @@ -48,17 +48,17 @@ private void SetImageOptions() state.SetState("image_count", "1"); } - private async Task SelectConversationImage() + private async Task SelectConversationImage(string? description) { var convService = _services.GetRequiredService(); var fileService = _services.GetRequiredService(); - var dialogs = convService.GetDialogHistory(fromBreakpoint: false); + var dialogs = convService.GetDialogHistory(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var images = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); - return await SelectImage(images, dialogs); + var userImages = fileService.GetMessageFiles(_conversationId, messageIds, FileSourceType.User, imageOnly: true); + return await SelectImage(userImages, dialogs.LastOrDefault(), description); } - private async Task SelectImage(IEnumerable images, List dialogs) + private async Task SelectImage(IEnumerable images, RoleDialogModel message, string? description) { if (images.IsNullOrEmpty()) return null; @@ -91,10 +91,15 @@ private void SetImageOptions() var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai"); var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4"); var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name); - var response = await completion.GetChatCompletions(agent, dialogs); + + var text = !string.IsNullOrWhiteSpace(description) ? description : message.Content; + var dialog = RoleDialogModel.From(message, AgentRole.User, text); + + var response = await completion.GetChatCompletions(agent, new List { dialog }); var content = response?.Content ?? string.Empty; - var fid = JsonSerializer.Deserialize(content); - return images.Where((x, idx) => idx == fid - 1).FirstOrDefault(); + var selected = JsonSerializer.Deserialize(content); + var fid = selected?.Selected ?? -1; + return fid > 0 ? images.Where((x, idx) => idx == fid - 1).FirstOrDefault() : null; } catch (Exception ex) { @@ -126,7 +131,7 @@ private async Task GetImageEditGeneration(RoleDialogModel message, strin stream.Close(); SaveGeneratedImage(result?.GeneratedImages?.FirstOrDefault()); - return !string.IsNullOrWhiteSpace(result?.Content) ? result.Content : "Image edit is completed."; + return $"Image \"{image.FileName}.{image.FileType}\" is successfylly editted."; } catch (Exception ex) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs new file mode 100644 index 000000000..ee7d40dcb --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.FileHandler/LlmContexts/LlmContextOut.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.FileHandler.LlmContexts; + +public class LlmContextOut +{ + [JsonPropertyName("selected_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Selected { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid index 2af9bf431..4d0505634 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid +++ b/src/Plugins/BotSharp.Plugin.FileHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/select_edit_image_prompt.liquid @@ -1,30 +1,38 @@ Please take a look at the images in the [IMAGES] section from the conversation and select ONLY one image based on the conversation with user. Your response must be an interger number. -** Please ONLY output the interger number. Do not prepend or append anything else. -** If you think user requests multiple images. Please ONLY select the first image and output its id. + +** Ensure the output is only in JSON format without any additional text. +** You may need to look at the image_name as a reference to find the correct image id. + +Here is the JSON format to use: +{ + "selected_id": the id selected from the [IMAGES] section +} + Suppose there are three images: id: 1, image_name: example_image_a.png id: 2, image_name: example_image_b.png id: 3, image_name: example_image_c.png +id: 4, image_name: example_image_d.png ===== Example 1: USER: I want to add a dog in the first file. -OUTPUT: 1 +OUTPUT: { "selected_id": 1 } Example 2: USER: Add a coffee cup in the second image I uploaded. -OUTPUT: 2 +OUTPUT: { "selected_id": 2 } Example 3: USER: Please remove the left tree in the third and the first images. -OUTPUT: 3 +OUTPUT: { "selected_id": 3 } Example 4: -USER: Add a boat in the images. -OUTPUT: 1 +USER: Circle the head of the dog in example_image_b.png. +OUTPUT: { "selected_id": 4 } =====