Improved sliding window - trim old context #12
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,14 +5,17 @@ import com.aallam.openai.api.chat.ChatMessage | |
| import com.aallam.openai.api.chat.ChatRole | ||
| import com.aallam.openai.api.chat.FunctionCall | ||
|
|
||
| /** | ||
| * Wrapper for the final `ChatMessage` class so that we | ||
| * can count tokens and split the grounding from the | ||
| * user query for embedding-supported (RAG) requests | ||
| */ | ||
| class CustomChatMessage @OptIn(BetaOpenAI::class) constructor( | ||
| public val role: ChatRole, | ||
| public val grounding: String? = null, | ||
| public val userContent: String? = null, | ||
| public val name: String? = null, | ||
| public val functionCall: FunctionCall? = null | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| ) { | ||
|
|
||
| public fun summary(): String? { | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
@@ -26,8 +29,26 @@ class CustomChatMessage @OptIn(BetaOpenAI::class) constructor( | |
| } | ||
| } | ||
|
|
||
| /** | ||
| * Count the number of tokens in the user query PLUS the additional | ||
| * embedding data for grounding OR the functions when userContent is empty | ||
| * | ||
| * `userContent` can never be larger than `tokensAllowed` - if this happens in | ||
| * message history, the message will be dropped from the window. In theory | ||
| * it could happen if a user entered maxTokens worth of text in their | ||
| * chat query, but in practice that seems unlikely (and should probably | ||
| * be a validation error on the UI) | ||
| */ | ||
| public fun getTokenCount(includeGrounding: Boolean = true, tokensAllowed: Int = -1) : Int { | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| var messageContent = userContent ?: "" | ||
| if (includeGrounding) { | ||
| messageContent = if (tokensAllowed < 0) { | ||
| grounding + userContent ?: "" | ||
| } else { // only include as much of the grounding as will fit | ||
| Tokenizer.trimToTokenLimit(grounding, tokensAllowed) + userContent ?: "" | ||
| } | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| if (userContent.isNullOrEmpty()) { | ||
| messageContent = "" + functionCall?.name + functionCall?.arguments | ||
| } | ||
|
|
@@ -36,9 +57,30 @@ class CustomChatMessage @OptIn(BetaOpenAI::class) constructor( | |
| return messageTokens | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| */ | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| public fun canFitInTokenLimit(includeGrounding: Boolean = true, tokensAllowed: Int = -1): Boolean { | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if (tokensAllowed < 0) return true | ||
| if (getTokenCount(includeGrounding, tokensAllowed) <= tokensAllowed) return true | ||
| return false | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| /** | ||
| * Create `ChatMessage` instance to add to completion request | ||
| */ | ||
| @OptIn(BetaOpenAI::class) | ||
| public fun getChatMessage (includeGrounding: Boolean = true, tokensAllowed: Int = -1) : ChatMessage { | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| var content = userContent | ||
| if (includeGrounding) { | ||
| content += if (tokensAllowed < 0) { | ||
| grounding | ||
| } else { | ||
| // only include as much of the grounding as will fit | ||
| // TODO: preserve leading and trailing grounding instructions | ||
| Tokenizer.trimToTokenLimit(grounding, tokensAllowed) | ||
| } | ||
| } | ||
| return ChatMessage(role = role, content = content, name = name, functionCall = functionCall) | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,9 +11,12 @@ class SlidingWindow { | |
| /** | ||
| * Takes the conversation history and trims older ChatMessage | ||
| * objects (except for System messasge) from the start | ||
| * | ||
| * Only includes the most recent embedding, omits the additional | ||
| * grounding information from older messages | ||
| */ | ||
| @OptIn(BetaOpenAI::class) | ||
| fun chatHistoryToWindow (conversation: MutableList<CustomChatMessage>): MutableList<ChatMessage> { | ||
| Log.v("LLM-SW", "-- chatHistoryToWindow() max tokens ${Constants.OPENAI_MAX_TOKENS}") | ||
| // set parameters for sliding window | ||
| val tokenLimit = Constants.OPENAI_MAX_TOKENS | ||
|
|
@@ -24,6 +27,7 @@ class SlidingWindow { | |
| Log.v("LLM-SW", "-- tokens reserved for response $expectedResponseSizeTokens and functions $reservedForFunctionsTokens") | ||
| var tokensUsed = 0 | ||
| var systemMessage: ChatMessage? = null | ||
| var includeGrounding = true | ||
|
|
||
| /** maximum tokens for chat, after hardcoded functions and allowing for a given response size */ | ||
| val tokenMax = tokenLimit - expectedResponseSizeTokens - reservedForFunctionsTokens | ||
|
|
@@ -33,23 +37,30 @@ class SlidingWindow { | |
|
|
||
| // check for system message | ||
| if (conversation[0].role == ChatRole.System) { | ||
| systemMessage = conversation[0].getChatMessage() | ||
| var systemMessageTokenCount = Tokenizer.countTokensIn(systemMessage.content) | ||
| tokensUsed += systemMessageTokenCount | ||
| Log.v("LLM-SW", "-- tokens used by system message: $tokensUsed") | ||
| } | ||
|
|
||
| // loop through other messages | ||
| for (m in conversation.reversed()) { | ||
| if (m.role != ChatRole.System) { | ||
|
|
||
| Log.v("LLM-SW", "-- message (${m.role.role}) ${m.summary()}") | ||
| Log.v("LLM-SW", " contains tokens: ${m.getTokenCount(includeGrounding)}") | ||
| val tokensRemaining = tokenMax - tokensUsed | ||
| if (m.canFitInTokenLimit(includeGrounding, tokensRemaining)) { | ||
| messagesInWindow.add(m.getChatMessage(includeGrounding, tokensRemaining)) | ||
| tokensUsed += m.getTokenCount(includeGrounding, tokensRemaining) | ||
|
|
||
| if (m.role == ChatRole.User) { | ||
| Log.v("LLM-SW", " added (grounding:$includeGrounding). Still available: ${tokenMax - tokensUsed}") | ||
| // stop subsequent user messages from including grounding | ||
| includeGrounding = false | ||
| } else { | ||
| Log.v("LLM-SW", " added. Still available: ${tokenMax - tokensUsed}") | ||
| } | ||
| } else { | ||
| Log.v("LLM-SW", " NOT ADDED. Still available: ${tokenMax - tokensUsed} (inc response quota ${expectedResponseSizeTokens}) ") | ||
| break // could optionally keep adding subsequent, smaller messages to context up until token limit | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,5 +22,21 @@ class Tokenizer { | |
|
|
||
| return tokens | ||
| } | ||
|
|
||
| /** | ||
| * Trim the input text to be under the number of tokens specified | ||
| * | ||
| * @return substring of the text that's no longer than the | ||
| * specified number of tokens | ||
| */ | ||
| fun trimToTokenLimit (text: String?, tokenLimit: Int): String? { | ||
| // TODO: limit by tokens instead of the rough character approximation | ||
| val charLimit = tokenLimit * 4 | ||
| return if (text?.length!! <= charLimit) { | ||
conceptdev marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| text | ||
| } else { | ||
| text?.substring(0, charLimit) | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,7 +19,7 @@ class SessionsByTimeFunction { | |
| return "sessionsByTime" | ||
| } | ||
| fun description(): String { | ||
| return "ONLY WHEN a date and time is specified, return the sessions that start on that date, during that time." | ||
| } | ||
| fun params(): Parameters { | ||
| val params = Parameters.buildJsonObject { | ||
|
|
@@ -31,7 +31,7 @@ class SessionsByTimeFunction { | |
| } | ||
| putJsonObject("earliestTime") { | ||
| put("type", "string") | ||
| put("description", "The earliest time that the conference sessions might start, eg. 09:00.") | ||
| } | ||
| putJsonObject("latestTime") { | ||
| put("type", "string") | ||
|
|
@@ -40,6 +40,7 @@ class SessionsByTimeFunction { | |
| } | ||
| putJsonArray("required") { | ||
| add("date") | ||
| add("earliestTime") | ||
| } | ||
| } | ||
| return params | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is this still true with the custom chat message class? or can we remove the null check now