Merge pull request #364 from iceljc/features/refine-breakpoint

add breakpoint
SciSharp · Mar 26, 2024 · b0eb240 · b0eb240
2 parents 71eeb01 + fc74da3
commit b0eb240
Show file tree

Hide file tree

Showing 12 changed files with 206 additions and 46 deletions.
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs
@@ -28,12 +28,6 @@ public class Conversation
 
     public DateTime UpdatedTime { get; set; } = DateTime.UtcNow;
     public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
-
-    /// <summary>
-    /// The default value will be same as CreatedTime
-    /// It used to insert a breakpoint in the conversation to hide the previous dialogs.
-    /// </summary>
-    public DateTime Breakpoint { get; set; } = DateTime.UtcNow.AddMilliseconds(-100);
 }
 
 public class DialogElement

diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationBreakpoint.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/ConversationBreakpoint.cs
@@ -0,0 +1,13 @@
+namespace BotSharp.Abstraction.Conversations.Models;
+
+public class ConversationBreakpoint
+{
+    [JsonPropertyName("message_id")]
+    public string? MessageId { get; set; }
+
+    [JsonPropertyName("breakpoint")]
+    public DateTime Breakpoint { get; set; }
+
+    [JsonPropertyName("created_time")]
+    public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
@@ -58,7 +58,8 @@ public interface IBotSharpRepository
     Conversation GetConversation(string conversationId);
     PagedItems<Conversation> GetConversations(ConversationFilter filter);
     void UpdateConversationTitle(string conversationId, string title);
-    void UpdateConversationBreakpoint(string conversationId, DateTime breakpoint);
+    void UpdateConversationBreakpoint(string conversationId, string messageId, DateTime breakpoint);
+    DateTime GetConversationBreakpoint(string conversationId);
     List<Conversation> GetLastConversations();
     List<string> GetIdleConversations(int batchSize, int messageLimit, int bufferHours);
     bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false);

diff --git a/...frastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs b/...frastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs
@@ -5,7 +5,9 @@ public partial class ConversationService : IConversationService
     public async Task UpdateBreakpoint(bool resetStates = false)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        db.UpdateConversationBreakpoint(_conversationId, DateTime.UtcNow);
+        var routingCtx = _services.GetRequiredService<IRoutingContext>();
+        var messageId = routingCtx.MessageId;
+        db.UpdateConversationBreakpoint(_conversationId, messageId, DateTime.UtcNow);
 
         // Reset states
         if (resetStates)

diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs
@@ -111,8 +111,8 @@ public List<RoleDialogModel> GetDialogHistory(int lastCount = 50, bool fromBreak
         if (fromBreakpoint)
         {
             var db = _services.GetRequiredService<IBotSharpRepository>();
-            var conversation = db.GetConversation(_conversationId);
-            dialogs = dialogs.Where(x => x.CreatedAt >= conversation.Breakpoint).ToList();
+            var breakpoint = db.GetConversationBreakpoint(_conversationId);
+            dialogs = dialogs.Where(x => x.CreatedAt >= breakpoint).ToList();
         }
 
         return dialogs

diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs
@@ -189,9 +189,12 @@ public void AppendConversationDialogs(string conversationId, List<DialogElement>
     public void UpdateConversationTitle(string conversationId, string title)
         => new NotImplementedException();
 
-    public void UpdateConversationBreakpoint(string conversationId, DateTime breakpoint)
+    public void UpdateConversationBreakpoint(string conversationId, string messageId, DateTime breakpoint)
         => new NotImplementedException();
-
+
+    public DateTime GetConversationBreakpoint(string conversationId)
+        => throw new NotImplementedException();
+
     public void UpdateConversationStates(string conversationId, List<StateKeyValue> states)
         => new NotImplementedException();
 

diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
@@ -10,6 +10,10 @@ public partial class FileRepository
     {
         public void CreateNewConversation(Conversation conversation)
         {
+            var utcNow = DateTime.UtcNow;
+            conversation.CreatedTime = utcNow;
+            conversation.UpdatedTime = utcNow;
+
             var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversation.Id);
             if (!Directory.Exists(dir))
             {
@@ -42,6 +46,20 @@ public void CreateNewConversation(Conversation conversation)
                 }).ToList();
                 File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options));
             }
+
+            var breakpointFile = Path.Combine(dir, BREAKPOINT_FILE);
+            if (!File.Exists(breakpointFile))
+            {
+                var initialBreakpoints = new List<ConversationBreakpoint>
+                {
+                    new ConversationBreakpoint()
+                    {
+                        Breakpoint = utcNow.AddMilliseconds(-100),
+                        CreatedTime = DateTime.UtcNow
+                    }
+                };
+                File.WriteAllText(breakpointFile, JsonSerializer.Serialize(initialBreakpoints, _options));
+            }
         }
 
         public bool DeleteConversations(IEnumerable<string> conversationIds)
@@ -142,21 +160,61 @@ public void UpdateConversationTitle(string conversationId, string title)
             }
         }
 
-        public void UpdateConversationBreakpoint(string conversationId, DateTime breakpoint)
+        public void UpdateConversationBreakpoint(string conversationId, string messageId, DateTime breakpoint)
         {
             var convDir = FindConversationDirectory(conversationId);
             if (!string.IsNullOrEmpty(convDir))
             {
-                var convFile = Path.Combine(convDir, CONVERSATION_FILE);
-                var content = File.ReadAllText(convFile);
-                var record = JsonSerializer.Deserialize<Conversation>(content, _options);
-                if (record != null)
+                var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE);
+
+                if (!File.Exists(breakpointFile))
                 {
-                    record.UpdatedTime = DateTime.UtcNow;
-                    record.Breakpoint = breakpoint;
-                    File.WriteAllText(convFile, JsonSerializer.Serialize(record, _options));
+                    File.Create(breakpointFile);
                 }
+
+                var content = File.ReadAllText(breakpointFile);
+                var records = JsonSerializer.Deserialize<List<ConversationBreakpoint>>(content, _options);
+                var newBreakpoint = new List<ConversationBreakpoint>()
+                {
+                    new ConversationBreakpoint
+                    {
+                        MessageId = messageId,
+                        Breakpoint = breakpoint,
+                        CreatedTime = DateTime.UtcNow
+                    }
+                };
+
+                if (records != null && !records.IsNullOrEmpty())
+                {
+                    records = records.Concat(newBreakpoint).ToList();
+                }
+                else
+                {
+                    records = newBreakpoint;
+                }
+
+                File.WriteAllText(breakpointFile, JsonSerializer.Serialize(records, _options));
+            }
+        }
+
+        public DateTime GetConversationBreakpoint(string conversationId)
+        {
+            var convDir = FindConversationDirectory(conversationId);
+            if (string.IsNullOrEmpty(convDir))
+            {
+                return default;
             }
+
+            var breakpointFile = Path.Combine(convDir, BREAKPOINT_FILE);
+            if (!File.Exists(breakpointFile))
+            {
+                File.Create(breakpointFile);
+            }
+
+            var content = File.ReadAllText(breakpointFile);
+            var records = JsonSerializer.Deserialize<List<ConversationBreakpoint>>(content, _options);
+
+            return records?.LastOrDefault()?.Breakpoint ?? default;
         }
 
         public ConversationState GetConversationStates(string conversationId)
@@ -425,6 +483,11 @@ public bool TruncateConversation(string conversationId, string messageId, bool c
             var states = CollectConversationStates(stateDir);
             isSaved = HandleTruncatedStates(stateDir, states, refTime);
 
+            // Handle truncated breakpoints
+            var breakpointDir = Path.Combine(convDir, BREAKPOINT_FILE);
+            var breakpoints = CollectConversationBreakpoints(breakpointDir);
+            isSaved = HandleTruncatedBreakpoints(breakpointDir, breakpoints, messageId);
+
             // Remove logs
             if (cleanLog)
             {
@@ -485,7 +548,7 @@ private List<string> ParseDialogElements(List<DialogElement> dialogs)
             foreach (var element in dialogs)
             {
                 var meta = element.MetaData;
-                var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.fff tt", CultureInfo.InvariantCulture);
+                var createTime = meta.CreateTime.ToString("MM/dd/yyyy hh:mm:ss.ffffff tt", CultureInfo.InvariantCulture);
                 var metaStr = $"{createTime}|{meta.Role}|{meta.AgentId}|{meta.MessageId}|{meta.SenderId}|{meta.FunctionName}|{element.RichContent}";
                 dialogTexts.Add(metaStr);
                 var content = $"  - {element.Content}";
@@ -507,6 +570,18 @@ private List<StateKeyValue> CollectConversationStates(string stateFile)
             return states ?? new List<StateKeyValue>();
         }
 
+        private List<ConversationBreakpoint> CollectConversationBreakpoints(string breakpointFile)
+        {
+            var breakpoints = new List<ConversationBreakpoint>();
+            if (!File.Exists(breakpointFile)) return breakpoints;
+
+            var content = File.ReadAllText(breakpointFile);
+            if (string.IsNullOrEmpty(content)) return breakpoints;
+
+            breakpoints = JsonSerializer.Deserialize<List<ConversationBreakpoint>>(content, _options);
+            return breakpoints ?? new List<ConversationBreakpoint>();
+        }
+
         private bool HandleTruncatedDialogs(string convDir, string dialogDir, List<DialogElement> dialogs, int foundIdx)
         {
             var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList();
@@ -538,6 +613,16 @@ private bool HandleTruncatedStates(string stateDir, List<StateKeyValue> states,
             return isSaved;
         }
 
+        private bool HandleTruncatedBreakpoints(string breakpointDir, List<ConversationBreakpoint> breakpoints, string refMessageId)
+        {
+            var targetIdx = breakpoints.FindIndex(x => x.MessageId == refMessageId);
+            var truncatedBreakpoints = breakpoints?.Where((x, idx) => idx < targetIdx)?
+                                                   .ToList() ?? new List<ConversationBreakpoint>();
+
+            var isSaved = SaveTruncatedBreakpoints(breakpointDir, truncatedBreakpoints);
+            return isSaved;
+        }
+
         private bool HandleTruncatedLogs(string convDir, DateTime refTime)
         {
             var contentLogDir = Path.Combine(convDir, "content_log");
@@ -595,6 +680,16 @@ private bool SaveTruncatedStates(string stateDir, List<StateKeyValue> states)
             File.WriteAllText(stateDir, stateStr);
             return true;
         }
+
+        private bool SaveTruncatedBreakpoints(string breakpointDir, List<ConversationBreakpoint> breakpoints)
+        {
+            if (string.IsNullOrEmpty(breakpointDir) || breakpoints == null) return false;
+            if (!File.Exists(breakpointDir)) File.Create(breakpointDir);
+
+            var breakpointStr = JsonSerializer.Serialize(breakpoints, _options);
+            File.WriteAllText(breakpointDir, breakpointStr);
+            return true;
+        }
         #endregion
     }
 }
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs
@@ -29,6 +29,7 @@ public partial class FileRepository : IBotSharpRepository
     private const string STATS_FILE = "stats.json";
     private const string DIALOG_FILE = "dialogs.txt";
     private const string STATE_FILE = "state.json";
+    private const string BREAKPOINT_FILE = "breakpoint.json";
     private const string EXECUTION_LOG_FILE = "execution.log";
     private const string PLUGIN_CONFIG_FILE = "config.json";
     private const string AGENT_TASK_PREFIX = "#metadata";

diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs
@@ -11,5 +11,4 @@ public class ConversationDocument : MongoBase
     public int DialogCount { get; set; }
     public DateTime CreatedTime { get; set; }
     public DateTime UpdatedTime { get; set; }
-    public DateTime Breakpoint { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationStateDocument.cs
@@ -6,4 +6,5 @@ public class ConversationStateDocument : MongoBase
 {
     public string ConversationId { get; set; }
     public List<StateMongoElement> States { get; set; }
+    public List<BreakpointMongoElement> Breakpoints { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/BreakpointMongoElement.cs
@@ -0,0 +1,8 @@
+namespace BotSharp.Plugin.MongoStorage.Models;
+
+public class BreakpointMongoElement
+{
+    public string? MessageId { get; set; }
+    public DateTime Breakpoint { get; set; }
+    public DateTime CreatedTime { get; set; }
+}