Fix split_text chunking bug

autogpt/processing/text.py

@@ -1,6 +1,5 @@
 """Text processing functions"""
-from typing import Dict, Generator, Optional
-
+from typing import Generator, Optional, Dict, List
 from selenium.webdriver.remote.webdriver import WebDriver
 
 from autogpt.config import Config
@@ -12,23 +11,25 @@
 
 
 def split_text(text: str, max_length: int = 8192) -> Generator[str, None, None]:
-    """Split text into chunks of a maximum length
-
-    Args:
-        text (str): The text to split
-        max_length (int, optional): The maximum length of each chunk. Defaults to 8192.
-
-    Yields:
-        str: The next chunk of text
-
-    Raises:
-        ValueError: If the text is longer than the maximum length
-    """
     paragraphs = text.split("\n")
     current_length = 0
     current_chunk = []
 
+    def split_long_paragraph(paragraph: str, max_length: int) -> List[str]:
+        return [
+            paragraph[i : i + max_length] for i in range(0, len(paragraph), max_length)
+        ]
+
     for paragraph in paragraphs:
+        if len(paragraph) > max_length:
+            long_paragraph_chunks = split_long_paragraph(paragraph, max_length)
+            for long_chunk in long_paragraph_chunks[:-1]:
+                if current_chunk:
+                    yield "\n".join(current_chunk)
+                    current_chunk = []
+                yield long_chunk
+            paragraph = long_paragraph_chunks[-1]
+
         if current_length + len(paragraph) + 1 <= max_length:
             current_chunk.append(paragraph)
             current_length += len(paragraph) + 1