-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added support for streaming responses.
- Loading branch information
1 parent
8767527
commit 4b11a60
Showing
4 changed files
with
109 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,4 +6,3 @@ struct ChatCompletionResponse: Codable { | |
} | ||
let choices: [Choice] | ||
} | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
// Created by B.T. Franklin on 5/5/23 | ||
|
||
import Foundation | ||
|
||
struct ChatStreamedResponseChunk: Codable { | ||
struct Choice: Codable { | ||
struct Delta: Codable { | ||
let role: ChatMessage.Role? | ||
let content: String? | ||
} | ||
let delta: Delta | ||
|
||
enum FinishReason: String, Codable { | ||
case stop | ||
case length | ||
case contentFilter | ||
} | ||
let finishReason: FinishReason? | ||
} | ||
let choices: [Choice] | ||
} | ||
|
||
extension ChatStreamedResponseChunk { | ||
static private var CHUNK_DECODER: JSONDecoder = { | ||
let decoder = JSONDecoder() | ||
decoder.keyDecodingStrategy = .convertFromSnakeCase | ||
return decoder | ||
}() | ||
|
||
static func decode(from string: String) -> ChatStreamedResponseChunk? { | ||
guard string.hasPrefix("data: "), | ||
let data = string.dropFirst(6).data(using: .utf8) else { | ||
return nil | ||
} | ||
return try? CHUNK_DECODER.decode(ChatStreamedResponseChunk.self, from: data) | ||
} | ||
} |
71 changes: 71 additions & 0 deletions
71
Sources/CleverBird/chat/ChatThread+completeWithStreaming.swift
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
// Created by B.T. Franklin on 5/5/23 | ||
|
||
import Foundation | ||
|
||
extension ChatThread { | ||
public func completeWithStreaming() async throws -> AsyncThrowingStream<String, Swift.Error> { | ||
|
||
let requestBody = ChatCompletionRequestParameters( | ||
model: self.model, | ||
temperature: self.temperature, | ||
topP: self.topP, | ||
stream: true, | ||
stop: self.stop, | ||
presencePenalty: self.presencePenalty, | ||
frequencyPenalty: self.frequencyPenalty, | ||
user: self.user, | ||
messages: self.messages | ||
) | ||
|
||
// Define the callback closure that appends the message to the chat thread | ||
let onResponseStreamCompleted: (ChatMessage) -> Void = { message in | ||
_ = self.addMessage(message) | ||
} | ||
|
||
let asyncByteStream = try await self.connection.createAsyncByteStream(for: requestBody) | ||
return AsyncThrowingStream { continuation in | ||
Task { | ||
|
||
var responseMessageRole: ChatMessage.Role? | ||
var responseMessageContent: String? | ||
|
||
do { | ||
for try await line in asyncByteStream.lines { | ||
print(line) | ||
guard let responseChunk = ChatStreamedResponseChunk.decode(from: line) else { | ||
break | ||
} | ||
|
||
if let deltaRole = responseChunk.choices.first?.delta.role { | ||
responseMessageRole = deltaRole | ||
continue | ||
} | ||
|
||
guard let delta = responseChunk.choices.first?.delta else { | ||
continue | ||
} | ||
|
||
guard let deltaContent = delta.content else { | ||
continue | ||
} | ||
|
||
if let currentMessageContent = responseMessageContent { | ||
responseMessageContent = currentMessageContent + deltaContent | ||
} else { | ||
responseMessageContent = deltaContent | ||
} | ||
continuation.yield(deltaContent) | ||
} | ||
} catch { | ||
throw CleverBirdError.responseParsingFailed | ||
} | ||
|
||
if let responseMessageRole, let responseMessageContent { | ||
onResponseStreamCompleted(ChatMessage(role: responseMessageRole, content: responseMessageContent)) | ||
} | ||
continuation.finish() | ||
} | ||
} | ||
} | ||
|
||
} |