diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 79ba3c64a..33678eccb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -110,6 +110,7 @@ public class RoleDialogModel : ITrackableMessage /// /// Files to be used in conversation /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public List? Files { get; set; } /// @@ -133,6 +134,12 @@ public class RoleDialogModel : ITrackableMessage public bool IsStreaming { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool IsFromUser => Role == AgentRole.User; + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool IsFromAssistant => Role == AgentRole.Assistant || Role == AgentRole.Model; + public RoleDialogModel() { } diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs index 4e032b667..cd299f698 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs @@ -5,13 +5,19 @@ public class TokenStatsModel public string Provider { get; set; } public string Model { get; set; } public string Prompt { get; set; } + + #region Input public int TextInputTokens { get; set; } public int CachedTextInputTokens { get; set; } public int AudioInputTokens { get; set; } public int CachedAudioInputTokens { get; set; } + #endregion + + #region Output public int TextOutputTokens { get; set; } public int AudioOutputTokens { get; set; } - public AgentLlmConfig LlmConfig { get; set; } + #endregion + public int TotalInputTokens => TextInputTokens + CachedTextInputTokens + AudioInputTokens + CachedAudioInputTokens; public int TotalOutputTokens => TextOutputTokens + AudioOutputTokens; diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSourceType.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSource.cs similarity index 78% rename from src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSourceType.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSource.cs index 04c92df6a..cde8c79f6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSourceType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Enums/FileSource.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Enums; -public static class FileSourceType +public static class FileSource { public const string User = "user"; public const string Bot = "bot"; diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index a1a7730f7..1d66c80a7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,12 +5,5 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public SettingBase? Pdf2TextConverter { get; set; } - public SettingBase? Pdf2ImageConverter { get; set; } public SettingBase? ImageConverter { get; set; } -} - -public class SettingBase -{ - public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index ef40e11c3..716b86e2d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -19,24 +19,22 @@ public interface IFileStorageService #region Conversation /// - /// Get the message file screenshots for specific content types, e.g., pdf + /// Get the message file screenshots for pdf /// /// /// /// - Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds); + Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. /// /// /// - /// - /// + /// /// - IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); + IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, MessageFileOptions? options = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); - IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); bool SaveMessageFiles(string conversationId, string messageId, string source, List files); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 1f56d8dea..32f431912 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -6,7 +6,7 @@ public class MessageFileModel : FileInformation public string MessageId { get; set; } [JsonPropertyName("file_source")] - public string FileSource { get; set; } = FileSourceType.User; + public string FileSource { get; set; } = Enums.FileSource.User; [JsonPropertyName("file_index")] public string FileIndex { get; set; } = string.Empty; diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs new file mode 100644 index 000000000..a647ad611 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileOptions.cs @@ -0,0 +1,20 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class MessageFileOptions +{ + /// + /// File sources: user, bot + /// + public IEnumerable? Sources { get; set; } + + /// + /// File content types + /// + public IEnumerable? ContentTypes { get; set; } +} + + +public class MessageFileScreenshotOptions : MessageFileOptions +{ + public string ImageConvertProvider { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index b589e5a6d..b9c01612f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -20,9 +20,4 @@ public class KnowledgeTextEmbeddingSetting : SettingBase { public string Model { get; set; } public int Dimension { get; set; } -} - -public class SettingBase -{ - public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs index 59edd20dc..1d88942c4 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/LlmConfigBase.cs @@ -1,24 +1,27 @@ namespace BotSharp.Abstraction.Models; -public class LlmConfigBase +public class LlmConfigBase : LlmBase { /// - /// Llm provider + /// Llm maximum output tokens /// - public string? LlmProvider { get; set; } + public int? MaxOutputTokens { get; set; } /// - /// Llm model + /// Llm reasoning effort level /// - public string? LlmModel { get; set; } + public string? ReasoningEffortLevel { get; set; } +} +public class LlmBase +{ /// - /// Llm maximum output tokens + /// Llm provider /// - public int? MaxOutputTokens { get; set; } + public string? LlmProvider { get; set; } /// - /// Llm reasoning effort level + /// Llm model /// - public string? ReasoningEffortLevel { get; set; } -} + public string? LlmModel { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs b/src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs new file mode 100644 index 000000000..7def8d3fc --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Settings/SettingBase.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Abstraction.Settings; + +public class SettingBase +{ + public string Provider { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Using.cs b/src/Infrastructure/BotSharp.Abstraction/Using.cs index 38a6dca00..d20775375 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Using.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Using.cs @@ -20,4 +20,5 @@ global using BotSharp.Abstraction.Files.Enums; global using BotSharp.Abstraction.Knowledges.Models; global using BotSharp.Abstraction.Crontab.Models; -global using BotSharp.Abstraction.MCP.Models; \ No newline at end of file +global using BotSharp.Abstraction.MCP.Models; +global using BotSharp.Abstraction.Settings; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs index 4c94b619f..a2de1940f 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs @@ -27,13 +27,13 @@ public async Task> SelectMessageFiles(string conve dialogs = dialogs.TakeLast(options.MessageLimit.Value).ToList(); } - var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var files = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, options.ContentTypes); - if (options.IsIncludeBotFiles) + var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); + var files = _fileStorage.GetMessageFiles(conversationId, messageIds, options: new() { - var botFiles = _fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, options.ContentTypes); - files = MergeMessageFiles(messageIds, files, botFiles); - } + Sources = options.IsIncludeBotFiles ?[FileSource.User, FileSource.Bot] : [FileSource.User], + ContentTypes = options.ContentTypes + }); + files = MergeMessageFiles(messageIds, files); if (files.IsNullOrEmpty()) { @@ -43,19 +43,31 @@ public async Task> SelectMessageFiles(string conve return await SelectFiles(files, dialogs, options); } - private IEnumerable MergeMessageFiles(IEnumerable messageIds, IEnumerable userFiles, IEnumerable botFiles) + private IEnumerable MergeMessageFiles(IEnumerable messageIds, IEnumerable files) { - var files = new List(); + var mergedFiles = new List(); - if (messageIds.IsNullOrEmpty()) return files; + if (messageIds.IsNullOrEmpty()) + { + return mergedFiles; + } + + var userFiles = files.Where(x => x.FileSource.IsEqualTo(FileSource.User)); + var botFiles = files.Where(x => x.FileSource.IsEqualTo(FileSource.Bot)); foreach (var messageId in messageIds) { var users = userFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); var bots = botFiles.Where(x => x.MessageId == messageId).OrderBy(x => x.FileIndex, new MessageFileIndexComparer()).ToList(); - if (!users.IsNullOrEmpty()) files.AddRange(users); - if (!bots.IsNullOrEmpty()) files.AddRange(bots); + if (!users.IsNullOrEmpty()) + { + mergedFiles.AddRange(users); + } + if (!bots.IsNullOrEmpty()) + { + mergedFiles.AddRange(bots); + } } return files; @@ -74,25 +86,24 @@ private async Task> SelectFiles(IEnumerable(); var db = _services.GetRequiredService(); - try - { - // Handle dialogs and files - var innerDialogs = (dialogs ?? []).ToList(); - var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s)."; - innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList(); - - if (options.IsAttachFiles) - { - AssembleMessageFiles(innerDialogs, files, options); - } + // Handle dialogs and files + var innerDialogs = (dialogs ?? []).ToList(); + var text = !string.IsNullOrWhiteSpace(options.Description) ? options.Description : "Please follow the instruction and select file(s)."; + innerDialogs = innerDialogs.Concat([new RoleDialogModel(AgentRole.User, text)]).ToList(); + if (options.IsAttachFiles) + { + AssembleMessageFiles(innerDialogs, files, options); + } + try + { // Handle instruction var promptMessages = innerDialogs.Select(x => { var text = $"[Role] '{x.Role}': {x.RichContent?.Message?.Text ?? x.Payload ?? x.Content}"; var fileDescs = x.Files?.Select((f, fidx) => $"- message_id: '{x.MessageId}', file_index: '{f.FileIndex}', " + - $"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSourceType.User : FileSourceType.Bot)}'"); + $"content_type: '{f.ContentType}', author: '{(x.Role == AgentRole.User ? FileSource.User : FileSource.Bot)}'"); var desc = string.Empty; if (!fileDescs.IsNullOrEmpty()) @@ -164,6 +175,10 @@ private async Task> SelectFiles(IEnumerable x.Files = null); + } } private void AssembleMessageFiles(IEnumerable dialogs, IEnumerable files, SelectFileOptions options) @@ -184,10 +199,10 @@ private void AssembleMessageFiles(IEnumerable dialogs, IEnumera continue; } - var userMsg = group.FirstOrDefault(x => x.Role == AgentRole.User); + var userMsg = group.FirstOrDefault(x => x.IsFromUser); if (userMsg != null) { - var userFiles = found.Where(x => x.FileSource == FileSourceType.User); + var userFiles = found.Where(x => x.FileSource == FileSource.User); userMsg.Files = userFiles.Select(x => new BotSharpFile { ContentType = x.ContentType, @@ -199,10 +214,10 @@ private void AssembleMessageFiles(IEnumerable dialogs, IEnumera }).ToList(); } - var botMsg = group.LastOrDefault(x => x.Role == AgentRole.Assistant); + var botMsg = group.LastOrDefault(x => x.IsFromAssistant); if (botMsg != null) { - var botFiles = found.Where(x => x.FileSource == FileSourceType.Bot); + var botFiles = found.Where(x => x.FileSource == FileSource.Bot); botMsg.Files = botFiles.Select(x => new BotSharpFile { ContentType = x.ContentType, diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 1216dc3fe..6050edcff 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,16 +6,17 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) + || messageIds.IsNullOrEmpty() + || options.Sources.IsNullOrEmpty()) { return files; } - var source = FileSourceType.User; - var pathPrefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); + var baseUrl = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER); foreach (var messageId in messageIds) { @@ -24,35 +25,37 @@ public async Task> GetMessageFileScreenshotsAsync( continue; } - var dir = Path.Combine(pathPrefix, messageId, FileSourceType.User); - if (!ExistDirectory(dir)) - { - continue; - } - - foreach (var subDir in Directory.GetDirectories(dir)) + foreach (var source in options.Sources) { - var file = Directory.GetFiles(subDir).FirstOrDefault(); - if (file == null) + var dir = Path.Combine(baseUrl, messageId, source); + if (!ExistDirectory(dir)) { continue; } - var screenshots = await GetScreenshots(file, subDir, messageId, source); - if (screenshots.IsNullOrEmpty()) + foreach (var subDir in Directory.GetDirectories(dir)) { - continue; - } + var file = Directory.GetFiles(subDir).FirstOrDefault(); + if (file == null) + { + continue; + } - files.AddRange(screenshots); + var screenshots = await GetScreenshotsAsync(file, subDir, messageId, source, options); + if (screenshots.IsNullOrEmpty()) + { + continue; + } + + files.AddRange(screenshots); + } } } return files; } - public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, IEnumerable? contentTypes = null) + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, MessageFileOptions? options = null) { var files = new List(); if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) @@ -67,39 +70,56 @@ public IEnumerable GetMessageFiles(string conversationId, IEnu continue; } - var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId, source); - if (!ExistDirectory(dir)) + var baseDir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId); + if (!ExistDirectory(baseDir)) + { + continue; + } + + var sources = options?.Sources != null + ? options.Sources + : Directory.GetDirectories(baseDir).Select(x => x.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries).Last()); + if (sources.IsNullOrEmpty()) { continue; } - foreach (var subDir in Directory.GetDirectories(dir)) + foreach (var source in sources) { - var index = subDir.Split(Path.DirectorySeparatorChar).Last(); + var dir = Path.Combine(baseDir, source); + if (!ExistDirectory(dir)) + { + continue; + } - foreach (var file in Directory.GetFiles(subDir)) + foreach (var subDir in Directory.GetDirectories(dir)) { - var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) - { - continue; - } + var fileIndex = subDir.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries).Last(); - var fileName = Path.GetFileNameWithoutExtension(file); - var fileExtension = Path.GetExtension(file).Substring(1); - var model = new MessageFileModel() + foreach (var file in Directory.GetFiles(subDir)) { - MessageId = messageId, - FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}", - FileDownloadUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}/download", - FileStorageUrl = file, - FileName = fileName, - FileExtension = fileExtension, - ContentType = contentType, - FileSource = source, - FileIndex = index - }; - files.Add(model); + var contentType = FileUtility.GetFileContentType(file); + if (options?.ContentTypes != null && !options.ContentTypes.Contains(contentType)) + { + continue; + } + + var fileName = Path.GetFileNameWithoutExtension(file); + var fileExtension = Path.GetExtension(file).Substring(1); + var model = new MessageFileModel + { + MessageId = messageId, + FileUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{fileIndex}/{fileName}", + FileDownloadUrl = $"/conversation/{conversationId}/message/{messageId}/{source}/file/{fileIndex}/{fileName}/download", + FileStorageUrl = file, + FileName = fileName, + FileExtension = fileExtension, + ContentType = contentType, + FileSource = source, + FileIndex = fileIndex + }; + files.Add(model); + } } } } @@ -126,38 +146,6 @@ public string GetMessageFile(string conversationId, string messageId, string sou return found; } - public IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds) - { - var foundMsgs = new List(); - if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) - { - return foundMsgs; - } - - foreach (var messageId in messageIds) - { - if (string.IsNullOrWhiteSpace(messageId)) - { - continue; - } - - var prefix = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, FILE_FOLDER, messageId); - var userDir = Path.Combine(prefix, FileSourceType.User); - if (ExistDirectory(userDir)) - { - foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.User }); - } - - var botDir = Path.Combine(prefix, FileSourceType.Bot); - if (ExistDirectory(botDir)) - { - foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.Bot }); - } - } - - return foundMsgs; - } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (string.IsNullOrWhiteSpace(conversationId) @@ -299,40 +287,9 @@ private string GetConversationFileDirectory(string? conversationId, string? mess return dir; } - private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) - { - if (dialogs.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - if (offset.HasValue && offset < 1) - { - offset = 1; - } - - var messageIds = new List(); - if (offset.HasValue) - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - } - - return messageIds; - } - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) + private async Task> ConvertPdfToImagesAsync(string pdfLoc, string imageLoc, MessageFileScreenshotOptions options) { - var converters = _services.GetServices(); - if (converters.IsNullOrEmpty()) - { - return Enumerable.Empty(); - } - - var converter = GetPdf2ImageConverter(); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options.ImageConvertProvider); if (converter == null) { return Enumerable.Empty(); @@ -341,14 +298,7 @@ private async Task> ConvertPdfToImages(string pdfLoc, string return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IImageConverter? GetPdf2ImageConverter() - { - var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); - return converter; - } - - private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + private async Task> GetScreenshotsAsync(string file, string parentDir, string messageId, string source, MessageFileScreenshotOptions options) { var files = new List(); @@ -378,7 +328,7 @@ private async Task> GetScreenshots(string file, st } else if (contentType == MediaTypeNames.Application.Pdf) { - var images = await ConvertPdfToImages(file, screenshotDir); + var images = await ConvertPdfToImagesAsync(file, screenshotDir, options); foreach (var image in images) { var fileName = Path.GetFileNameWithoutExtension(image); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 5da7a784f..76425dfeb 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -96,7 +96,7 @@ public async Task> GetDialogs([FromRoute] string var fileStorage = _services.GetRequiredService(); var messageIds = history.Select(x => x.MessageId).Distinct().ToList(); - var fileMessages = fileStorage.GetMessagesWithFile(conversationId, messageIds); + var files = fileStorage.GetMessageFiles(conversationId, messageIds, options: new() { Sources = [FileSource.User, FileSource.Bot] }); var dialogs = new List(); foreach (var message in history) @@ -115,7 +115,7 @@ public async Task> GetDialogs([FromRoute] string Data = message.Data, Sender = UserDto.FromUser(user), Payload = message.Payload, - HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.User) + HasMessageFiles = files.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSource.User) }); } else if (message.Role == AgentRole.Assistant) @@ -136,7 +136,7 @@ public async Task> GetDialogs([FromRoute] string Role = message.Role, }, RichContent = message.SecondaryRichContent ?? message.RichContent, - HasMessageFiles = fileMessages.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSourceType.Bot) + HasMessageFiles = files.Any(x => x.MessageId.IsEqualTo(message.MessageId) && x.FileSource == FileSource.Bot) }); } } @@ -490,7 +490,7 @@ public async Task UploadConversationMessageFiles([FromRoute] string agen var conv = await convService.GetConversationRecordOrCreateNew(agentId); var fileStorage = _services.GetRequiredService(); var messageId = Guid.NewGuid().ToString(); - var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSourceType.User, input.Files); + var isSaved = fileStorage.SaveMessageFiles(conv.Id, messageId, FileSource.User, input.Files); return isSaved ? messageId : string.Empty; } @@ -498,8 +498,8 @@ public async Task UploadConversationMessageFiles([FromRoute] string agen public IEnumerable GetConversationMessageFiles([FromRoute] string conversationId, [FromRoute] string messageId, [FromRoute] string source) { var fileStorage = _services.GetRequiredService(); - var files = fileStorage.GetMessageFiles(conversationId, new List { messageId }, source); - return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? new List(); + var files = fileStorage.GetMessageFiles(conversationId, [messageId], options: new() { Sources = [source] }); + return files?.Select(x => MessageFileViewModel.Transform(x))?.ToList() ?? []; } [HttpGet("/conversation/{conversationId}/message/{messageId}/{source}/file/{index}/{fileName}")] diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index a114c2fae..e27b40544 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -13,7 +13,7 @@ public class HandleAudioRequestFn : IFunctionCallback private readonly ILogger _logger; private readonly BotSharpOptions _options; - private readonly IEnumerable _audioContentType = new List + private readonly IEnumerable _audioContentTypes = new List { AudioType.mp3.ToFileType(), AudioType.wav.ToFileType(), @@ -52,7 +52,11 @@ private List AssembleFiles(string convId, List } var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); + var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, options: new() + { + Sources = [FileSource.User], + ContentTypes = _audioContentTypes + }); audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index ec1530e89..670261ac7 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -150,7 +150,6 @@ private async Task GetImageEditResponse(string description, string? defa { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_edit_llm_provider"); var model = state.GetState("image_edit_llm_provider"); @@ -160,8 +159,8 @@ private async Task GetImageEditResponse(string description, string? defa return (provider, model); } - provider = fileSettings?.Image?.Edit?.LlmProvider; - model = fileSettings?.Image?.Edit?.LlmModel; + provider = _settings?.Image?.Edit?.LlmProvider; + model = _settings?.Image?.Edit?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -191,13 +190,13 @@ private IEnumerable SaveGeneratedImage(ImageGeneration? image) }; var fileStorage = _services.GetRequiredService(); - fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSource.Bot, files); return files.Select(x => x.FileName); } private async Task ConvertImageToPngWithRgba(BinaryData binaryFile) { - var provider = _settings?.ImageConverter?.Provider; + var provider = _settings?.Image?.Edit?.ImageConverter?.Provider; var converter = _services.GetServices().FirstOrDefault(x => x.Provider == provider); if (converter == null) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4a98d056a..33fa8e81f 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; - namespace BotSharp.Plugin.FileHandler.Functions; public class GenerateImageFn : IFunctionCallback @@ -9,6 +7,7 @@ public class GenerateImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; private Agent _agent; private string _conversationId; @@ -16,10 +15,12 @@ public class GenerateImageFn : IFunctionCallback public GenerateImageFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -113,7 +114,6 @@ private async Task GetImageGenerationResponse(string description, string { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_generate_llm_provider"); var model = state.GetState("image_generate_llm_model"); @@ -123,8 +123,8 @@ private async Task GetImageGenerationResponse(string description, string return (provider, model); } - provider = fileSettings?.Image?.Generation?.LlmProvider; - model = fileSettings?.Image?.Generation?.LlmModel; + provider = _settings?.Image?.Generation?.LlmProvider; + model = _settings?.Image?.Generation?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -151,7 +151,7 @@ private IEnumerable SaveGeneratedImages(List? images) }).ToList(); var fileStorage = _services.GetRequiredService(); - fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSourceType.Bot, files); + fileStorage.SaveMessageFiles(_conversationId, _messageId, FileSource.Bot, files); return files.Select(x => x.FileName); } } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs index 98d1f52f7..05848c994 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadImageFn.cs @@ -9,13 +9,22 @@ public class ReadImageFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; + + private readonly IEnumerable _imageContentTypes = new List + { + MediaTypeNames.Image.Png, + MediaTypeNames.Image.Jpeg + }; public ReadImageFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -47,6 +56,7 @@ public async Task Execute(RoleDialogModel message) var dialogs = AssembleFiles(conv.ConversationId, args?.ImageUrls, wholeDialogs); var response = await GetChatCompletion(agent, dialogs); + dialogs.ForEach(x => x.Files = null); message.Content = response; return true; } @@ -58,24 +68,30 @@ private List AssembleFiles(string conversationId, IEnumerable(); } - var contentTypes = new List - { - MediaTypeNames.Image.Png, - MediaTypeNames.Image.Jpeg - }; - var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var userImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.User, contentTypes); - var botImages = fileStorage.GetMessageFiles(conversationId, messageIds, FileSourceType.Bot, contentTypes); - var images = userImages.Concat(botImages); + var images = fileStorage.GetMessageFiles(conversationId, messageIds, options: new() + { + Sources = [FileSource.User, FileSource.Bot], + ContentTypes = _imageContentTypes + }); foreach (var dialog in dialogs) { var found = images.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; - dialog.Files = found.Select(x => new BotSharpFile + var targets = found; + if (dialog.Role == AgentRole.User) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.User)).ToList(); + } + else if (dialog.Role == AgentRole.Assistant || dialog.Role == AgentRole.Model) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.Bot)).ToList(); + } + + dialog.Files = targets.Select(x => new BotSharpFile { ContentType = x.ContentType, FileUrl = x.FileUrl, @@ -119,7 +135,6 @@ private async Task GetChatCompletion(Agent agent, List { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("image_read_llm_provider"); var model = state.GetState("image_read_llm_model"); @@ -129,8 +144,8 @@ private async Task GetChatCompletion(Agent agent, List return (provider, model); } - provider = fileSettings?.Image?.Reading?.LlmProvider; - model = fileSettings?.Image?.Reading?.LlmModel; + provider = _settings?.Image?.Reading?.LlmProvider; + model = _settings?.Image?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index d6f8d74b3..a2c2f4e5c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -9,6 +9,7 @@ public class ReadPdfFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly FileHandlerSettings _settings; private readonly IEnumerable _pdfContentTypes = new List { @@ -17,10 +18,12 @@ public class ReadPdfFn : IFunctionCallback public ReadPdfFn( IServiceProvider services, - ILogger logger) + ILogger logger, + FileHandlerSettings settings) { _services = services; _logger = logger; + _settings = settings; } public async Task Execute(RoleDialogModel message) @@ -52,6 +55,7 @@ public async Task Execute(RoleDialogModel message) var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); var response = await GetChatCompletion(agent, dialogs); + dialogs.ForEach(x => x.Files = null); message.Content = response; return true; } @@ -65,16 +69,47 @@ private async Task> AssembleFiles(string conversationId, L var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds); - if (screenshots.IsNullOrEmpty()) return dialogs; + IEnumerable files; + if (_settings.Pdf?.Reading?.ConvertToImage == true) + { + files = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds, options: new() + { + Sources = [FileSource.User], + ImageConvertProvider = _settings.Pdf?.Reading?.ImageConverter?.Provider + }); + } + else + { + files = fileStorage.GetMessageFiles(conversationId, messageIds, options: new() + { + Sources = [FileSource.User], + ContentTypes = _pdfContentTypes + }); + } + + + if (files.IsNullOrEmpty()) + { + return dialogs; + } foreach (var dialog in dialogs) { - var found = screenshots.Where(x => x.MessageId == dialog.MessageId).ToList(); + var found = files.Where(x => x.MessageId == dialog.MessageId).ToList(); if (found.IsNullOrEmpty()) continue; - dialog.Files = found.Select(x => new BotSharpFile + var targets = found; + if (dialog.IsFromUser) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.User)).ToList(); + } + else if (dialog.IsFromAssistant) + { + targets = found.Where(x => x.FileSource.IsEqualTo(FileSource.Bot)).ToList(); + } + + dialog.Files = targets.Select(x => new BotSharpFile { ContentType = x.ContentType, FileUrl = x.FileUrl, @@ -107,7 +142,6 @@ private async Task GetChatCompletion(Agent agent, List { var state = _services.GetRequiredService(); var llmProviderService = _services.GetRequiredService(); - var fileSettings = _services.GetRequiredService(); var provider = state.GetState("pdf_read_llm_provider"); var model = state.GetState("pdf_read_llm_model"); @@ -117,8 +151,8 @@ private async Task GetChatCompletion(Agent agent, List return (provider, model); } - provider = fileSettings?.Image?.Reading?.LlmProvider; - model = fileSettings?.Image?.Reading?.LlmModel; + provider = _settings?.Pdf?.Reading?.LlmProvider; + model = _settings?.Pdf?.Reading?.LlmModel; if (!string.IsNullOrEmpty(provider) && !string.IsNullOrEmpty(model)) { @@ -133,6 +167,11 @@ private async Task GetChatCompletion(Agent agent, List private void SetImageDetailLevel() { + if (_settings.Pdf?.Reading?.ConvertToImage != true) + { + return; + } + var state = _services.GetRequiredService(); var fileSettings = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs index cf1b8590d..9eb4c9ac0 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Settings/FileHandlerSettings.cs @@ -1,10 +1,11 @@ +using BotSharp.Abstraction.Models; + namespace BotSharp.Plugin.FileHandler.Settings; public class FileHandlerSettings { public ImageSettings? Image { get; set; } public PdfSettings? Pdf { get; set; } - public SettingBase? ImageConverter { get; set; } } #region Image @@ -16,22 +17,22 @@ public class ImageSettings public ImageVariationSettings? Variation { get; set; } } -public class ImageReadSettings : FileLlmSettingBase +public class ImageReadSettings : LlmBase { public string? ImageDetailLevel { get; set; } } -public class ImageGenerationSettings : FileLlmSettingBase +public class ImageGenerationSettings : LlmBase { } -public class ImageEditSettings : FileLlmSettingBase +public class ImageEditSettings : LlmBase { - + public SettingBase? ImageConverter { get; set; } } -public class ImageVariationSettings : FileLlmSettingBase +public class ImageVariationSettings : LlmBase { } @@ -43,15 +44,10 @@ public class PdfSettings public PdfReadSettings? Reading { get; set; } } -public class PdfReadSettings : FileLlmSettingBase +public class PdfReadSettings : LlmBase { public bool ConvertToImage { get; set; } public string? ImageDetailLevel { get; set; } + public SettingBase? ImageConverter { get; set; } } -#endregion - -public class FileLlmSettingBase -{ - public string? LlmProvider { get; set; } - public string? LlmModel { get; set; } -} \ No newline at end of file +#endregion \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7ac788cf6..722f50d9c 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -1,4 +1,5 @@ #pragma warning disable OPENAI001 +using BotSharp.Abstraction.Hooks; using OpenAI.Images; namespace BotSharp.Plugin.OpenAI.Providers.Image; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 799a0f545..0c774dd86 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.Files; using BotSharp.Abstraction.Files.Converters; -using BotSharp.Abstraction.Files.Enums; using BotSharp.Abstraction.Files.Utilities; using System.Net.Mime; @@ -8,74 +6,103 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options) { var files = new List(); - if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) + if (string.IsNullOrEmpty(conversationId) + || messageIds.IsNullOrEmpty() + || options.Sources.IsNullOrEmpty()) { return files; } - var source = FileSourceType.User; - var pathPrefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; + var baseDir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}"; foreach (var messageId in messageIds) { - var dir = $"{pathPrefix}/{messageId}/{source}"; - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + if (string.IsNullOrWhiteSpace(messageId)) { - var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); - if (file == null) continue; + continue; + } - var screenshots = await GetScreenshots(file, subDir, messageId, source); - if (screenshots.IsNullOrEmpty()) continue; + foreach (var source in options.Sources) + { + var dir = $"{baseDir}/{messageId}/{source}"; + foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + { + var file = _cosClient.BucketClient.GetDirFiles(subDir).FirstOrDefault(); + if (file == null) continue; - files.AddRange(screenshots); + var screenshots = await GetScreenshotsAsync(file, subDir, messageId, source, options); + if (screenshots.IsNullOrEmpty()) continue; + + files.AddRange(screenshots); + } } } return files; } - public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, - string source, IEnumerable? contentTypes = null) + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, MessageFileOptions? options = null) { var files = new List(); - if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return files; + if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) + { + return files; + } foreach (var messageId in messageIds) { - var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}/{source}"; - if (!ExistDirectory(dir)) + var baseDir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}"; + if (!ExistDirectory(baseDir)) { continue; } - foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + var sources = options?.Sources != null + ? options.Sources + : _cosClient.BucketClient.GetDirectories(baseDir).Select(x => x.Split("/", StringSplitOptions.RemoveEmptyEntries).Last()); + if (sources.IsNullOrEmpty()) { - foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) + continue; + } + + foreach (var source in sources) + { + var dir = Path.Combine(baseDir, source); + if (!ExistDirectory(dir)) { - var contentType = FileUtility.GetFileContentType(file); - if (!contentTypes.IsNullOrEmpty() && !contentTypes.Contains(contentType)) - { - continue; - } + continue; + } - var fileName = Path.GetFileNameWithoutExtension(file); - var fileExtension = Path.GetExtension(file).Substring(1); + foreach (var subDir in _cosClient.BucketClient.GetDirectories(dir)) + { var fileIndex = subDir.Split("/", StringSplitOptions.RemoveEmptyEntries).LastOrDefault() ?? string.Empty; - var model = new MessageFileModel() + + foreach (var file in _cosClient.BucketClient.GetDirFiles(subDir)) { - MessageId = messageId, - FileUrl = BuilFileUrl(file), - FileDownloadUrl = BuilFileUrl(file), - FileStorageUrl = file, - FileName = fileName, - FileExtension = fileExtension, - ContentType = contentType, - FileSource = source, - FileIndex = fileIndex - }; - files.Add(model); + var contentType = FileUtility.GetFileContentType(file); + if (options?.ContentTypes != null && !options.ContentTypes.Contains(contentType)) + { + continue; + } + + var fileName = Path.GetFileNameWithoutExtension(file); + var fileExtension = Path.GetExtension(file).Substring(1); + var model = new MessageFileModel() + { + MessageId = messageId, + FileUrl = BuilFileUrl(file), + FileDownloadUrl = BuilFileUrl(file), + FileStorageUrl = file, + FileName = fileName, + FileExtension = fileExtension, + ContentType = contentType, + FileSource = source, + FileIndex = fileIndex + }; + files.Add(model); + } } } } @@ -87,37 +114,13 @@ public IEnumerable GetMessageFiles(string conversationId, IEnu public string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName) { - var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}/"; + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{source}/{index}"; var fileList = _cosClient.BucketClient.GetDirFiles(dir); var found = fileList.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f).IsEqualTo(fileName)); return found; } - public IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds) - { - var foundMsgs = new List(); - if (string.IsNullOrWhiteSpace(conversationId) || messageIds.IsNullOrEmpty()) return foundMsgs; - - foreach (var messageId in messageIds) - { - var prefix = $"{CONVERSATION_FOLDER}/{conversationId}/{FILE_FOLDER}/{messageId}"; - var userDir = $"{prefix}/{FileSourceType.User}/"; - if (ExistDirectory(userDir)) - { - foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.User }); - } - - var botDir = $"{prefix}/{FileSourceType.Bot}"; - if (ExistDirectory(botDir)) - { - foundMsgs.Add(new MessageFileModel { MessageId = messageId, FileSource = FileSourceType.Bot }); - } - } - - return foundMsgs; - } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; @@ -218,35 +221,9 @@ private string GetConversationFileDirectory(string? conversationId, string? mess return dir; } - private IEnumerable GetMessageIds(IEnumerable dialogs, int? offset = null) + private async Task> ConvertPdfToImagesAsync(string pdfLoc, string imageLoc, MessageFileScreenshotOptions options) { - if (dialogs.IsNullOrEmpty()) return Enumerable.Empty(); - - if (offset.HasValue && offset < 1) - { - offset = 1; - } - - var messageIds = new List(); - if (offset.HasValue) - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().TakeLast(offset.Value).ToList(); - } - else - { - messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - } - - return messageIds; - } - - - private async Task> ConvertPdfToImages(string pdfLoc, string imageLoc) - { - var converters = _services.GetServices(); - if (converters.IsNullOrEmpty()) return Enumerable.Empty(); - - var converter = GetPdf2ImageConverter(); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == options.ImageConvertProvider); if (converter == null) { return Enumerable.Empty(); @@ -254,19 +231,12 @@ private async Task> ConvertPdfToImages(string pdfLoc, string return await converter.ConvertPdfToImages(pdfLoc, imageLoc); } - private IImageConverter? GetPdf2ImageConverter() - { - var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); - return converter; - } - private string BuilFileUrl(string file) { return $"https://{_fullBuketName}.cos.{_settings.Region}.myqcloud.com/{file}"; } - private async Task> GetScreenshots(string file, string parentDir, string messageId, string source) + private async Task> GetScreenshotsAsync(string file, string parentDir, string messageId, string source, MessageFileScreenshotOptions options) { var files = new List(); @@ -297,7 +267,7 @@ private async Task> GetScreenshots(string file, st } else if (contentType == MediaTypeNames.Application.Pdf) { - var images = await ConvertPdfToImages(file, screenshotDir); + var images = await ConvertPdfToImagesAsync(file, screenshotDir, options); foreach (var image in images) { var fileName = Path.GetFileNameWithoutExtension(image); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index e43f6eb0b..0ce8ce318 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -439,12 +439,6 @@ "FileCore": { "Storage": "LocalFileStorage", - "Pdf2TextConverter": { - "Provider": "" - }, - "Pdf2ImageConverter": { - "Provider": "" - }, "ImageConverter": { "Provider": "" } @@ -463,7 +457,10 @@ }, "Edit": { "LlmProvider": "openai", - "LlmModel": "gpt-image-1" + "LlmModel": "gpt-image-1", + "ImageConverter": { + "Provider": "file-handler" + } }, "Variation": { "LlmProvider": "", @@ -472,14 +469,14 @@ }, "Pdf": { "Reading": { - "LlmProvider": "openai", - "LlmModel": "gpt-5-mini", - "ConvertToImage": true, - "ImageDetailLevel": "auto" + "LlmProvider": "google-ai", + "LlmModel": "gemini-2.0-flash", + "ConvertToImage": false, + "ImageDetailLevel": "auto", + "ImageConverter": { + "Provider": null + } } - }, - "ImageConverter": { - "Provider": "file-handler" } }, diff --git a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs index bb1d0c444..0e1be86cb 100644 --- a/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs +++ b/tests/BotSharp.LLM.Tests/Core/NullFileStorageService.cs @@ -54,7 +54,7 @@ public string BuildDirectory(params string[] segments) return string.Join("/", segments); } - public Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) + public Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds, MessageFileScreenshotOptions options) { return Task.FromResult>(new List { @@ -63,8 +63,7 @@ public Task> GetMessageFileScreenshotsAsync(string }); } - public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, - IEnumerable? contentTypes = null) + public IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, MessageFileOptions? options = null) { return new List { @@ -78,15 +77,6 @@ public string GetMessageFile(string conversationId, string messageId, string sou return $"FakePath/{fileName}"; } - public IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds) - { - return new List - { - new MessageFileModel { FileName = "MessageFile1.jpg" }, - new MessageFileModel { FileName = "MessageFile2.png" } - }; - } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { return true;