Update tokenizer.ts as per the codereviwer ai comment

coderabbitai · Nov 30, 2023 · 4cf4e4d · 4cf4e4d
1 parent 44a12fb
commit 4cf4e4d
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/tokenizer.ts b/src/tokenizer.ts
@@ -23,7 +23,9 @@ export function splitPrompt(
   let remainingPrompt = prompt
   while (remainingPrompt.length > 0) {
      if (remainingPrompt.length > maxTokens && !remainingPrompt.includes(' ')) {
-     throw new Error(`Word length exceeds maxTokens: ${maxTokens}`);
+     const piece = remainingPrompt.substring(0, maxTokens).trim();
+       promptPieces.push(piece);
+       remainingPrompt = remainingPrompt.substring(maxTokens).trim();
   }
     const lastSpaceIndex = remainingPrompt.lastIndexOf(' ', maxTokens)
     if (lastSpaceIndex >= 0) {