Skip to content

Commit d28caaf

Browse files
hellodanyloCatherineSueslin1237
authored
[router] Support complex assistant and tool messages in /chat/completions (#12860)
Co-authored-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: Simo Lin <linsimo.mark@gmail.com>
1 parent ad8d24c commit d28caaf

File tree

13 files changed

+127
-104
lines changed

13 files changed

+127
-104
lines changed

‎sgl-router/benches/request_processing.rs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use serde_json::{from_str, to_string, to_value, to_vec};
55
use sglang_router_rs::{
66
core::{BasicWorker, BasicWorkerBuilder, Worker, WorkerType},
77
protocols::{
8-
chat::{ChatCompletionRequest, ChatMessage, UserMessageContent},
8+
chat::{ChatCompletionRequest, ChatMessage, MessageContent},
99
common::StringOrArray,
1010
completion::CompletionRequest,
1111
generate::GenerateRequest,
@@ -148,11 +148,11 @@ fn create_sample_chat_completion_request() -> ChatCompletionRequest {
148148
model: "gpt-3.5-turbo".to_string(),
149149
messages: vec![
150150
ChatMessage::System {
151-
content: "You are a helpful assistant".to_string(),
151+
content: MessageContent::Text("You are a helpful assistant".to_string()),
152152
name: None,
153153
},
154154
ChatMessage::User {
155-
content: UserMessageContent::Text(
155+
content: MessageContent::Text(
156156
"Explain quantum computing in simple terms".to_string(),
157157
),
158158
name: None,
@@ -188,18 +188,20 @@ fn create_sample_completion_request() -> CompletionRequest {
188188
#[allow(deprecated)]
189189
fn create_large_chat_completion_request() -> ChatCompletionRequest {
190190
let mut messages = vec![ChatMessage::System {
191-
content: "You are a helpful assistant with extensive knowledge.".to_string(),
191+
content: MessageContent::Text(
192+
"You are a helpful assistant with extensive knowledge.".to_string(),
193+
),
192194
name: None,
193195
}];
194196

195197
// Add many user/assistant pairs to simulate a long conversation
196198
for i in 0..50 {
197199
messages.push(ChatMessage::User {
198-
content: UserMessageContent::Text(format!("Question {}: What do you think about topic number {} which involves complex reasoning about multiple interconnected systems and their relationships?", i, i)),
200+
content: MessageContent::Text(format!("Question {}: What do you think about topic number {} which involves complex reasoning about multiple interconnected systems and their relationships?", i, i)),
199201
name: None,
200202
});
201203
messages.push(ChatMessage::Assistant {
202-
content: Some(format!("Answer {}: This is a detailed response about topic {} that covers multiple aspects and provides comprehensive analysis of the interconnected systems you mentioned.", i, i)),
204+
content: Some(MessageContent::Text(format!("Answer {}: This is a detailed response about topic {} that covers multiple aspects and provides comprehensive analysis of the interconnected systems you mentioned.", i, i))),
203205
name: None,
204206
tool_calls: None,
205207
reasoning_content: None,

‎sgl-router/src/protocols/chat.rs‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,20 @@ use crate::protocols::{
2222
pub enum ChatMessage {
2323
#[serde(rename = "system")]
2424
System {
25-
content: String,
25+
content: MessageContent,
2626
#[serde(skip_serializing_if = "Option::is_none")]
2727
name: Option<String>,
2828
},
2929
#[serde(rename = "user")]
3030
User {
31-
content: UserMessageContent,
31+
content: MessageContent,
3232
#[serde(skip_serializing_if = "Option::is_none")]
3333
name: Option<String>,
3434
},
3535
#[serde(rename = "assistant")]
3636
Assistant {
3737
#[serde(skip_serializing_if = "Option::is_none")]
38-
content: Option<String>,
38+
content: Option<MessageContent>,
3939
#[serde(skip_serializing_if = "Option::is_none")]
4040
name: Option<String>,
4141
#[serde(skip_serializing_if = "Option::is_none")]
@@ -46,20 +46,38 @@ pub enum ChatMessage {
4646
},
4747
#[serde(rename = "tool")]
4848
Tool {
49-
content: String,
49+
content: MessageContent,
5050
tool_call_id: String,
5151
},
5252
#[serde(rename = "function")]
5353
Function { content: String, name: String },
5454
}
5555

56-
#[derive(Debug, Clone, Deserialize, Serialize)]
56+
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
5757
#[serde(untagged)]
58-
pub enum UserMessageContent {
58+
pub enum MessageContent {
5959
Text(String),
6060
Parts(Vec<ContentPart>),
6161
}
6262

63+
impl MessageContent {
64+
pub fn to_simple_string(&self) -> String {
65+
match self {
66+
MessageContent::Text(text) => text.clone(),
67+
MessageContent::Parts(parts) => {
68+
let texts: Vec<String> = parts
69+
.iter()
70+
.filter_map(|part| match part {
71+
ContentPart::Text { text } => Some(text.clone()),
72+
_ => None,
73+
})
74+
.collect();
75+
texts.join(" ")
76+
}
77+
}
78+
}
79+
}
80+
6381
// ============================================================================
6482
// Chat Completion Request
6583
// ============================================================================
@@ -320,12 +338,12 @@ fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::Validati
320338
for msg in messages.iter() {
321339
if let ChatMessage::User { content, .. } = msg {
322340
match content {
323-
UserMessageContent::Text(text) if text.is_empty() => {
341+
MessageContent::Text(text) if text.is_empty() => {
324342
return Err(validator::ValidationError::new(
325343
"message content cannot be empty",
326344
));
327345
}
328-
UserMessageContent::Parts(parts) if parts.is_empty() => {
346+
MessageContent::Parts(parts) if parts.is_empty() => {
329347
return Err(validator::ValidationError::new(
330348
"message content parts cannot be empty",
331349
));
@@ -589,35 +607,26 @@ impl GenerationRequest for ChatCompletionRequest {
589607
self.messages
590608
.iter()
591609
.filter_map(|msg| match msg {
592-
ChatMessage::System { content, .. } => Some(content.clone()),
593-
ChatMessage::User { content, .. } => match content {
594-
UserMessageContent::Text(text) => Some(text.clone()),
595-
UserMessageContent::Parts(parts) => {
596-
let texts: Vec<String> = parts
597-
.iter()
598-
.filter_map(|part| match part {
599-
ContentPart::Text { text } => Some(text.clone()),
600-
_ => None,
601-
})
602-
.collect();
603-
Some(texts.join(" "))
604-
}
605-
},
610+
ChatMessage::System { content, .. } => Some(content.to_simple_string()),
611+
ChatMessage::User { content, .. } => Some(content.to_simple_string()),
606612
ChatMessage::Assistant {
607613
content,
608614
reasoning_content,
609615
..
610616
} => {
611617
// Combine content and reasoning content for routing decisions
612-
let main_content = content.clone().unwrap_or_default();
618+
let main_content = content
619+
.as_ref()
620+
.map(|c| c.to_simple_string())
621+
.unwrap_or_default();
613622
let reasoning = reasoning_content.clone().unwrap_or_default();
614623
if main_content.is_empty() && reasoning.is_empty() {
615624
None
616625
} else {
617626
Some(format!("{} {}", main_content, reasoning).trim().to_string())
618627
}
619628
}
620-
ChatMessage::Tool { content, .. } => Some(content.clone()),
629+
ChatMessage::Tool { content, .. } => Some(content.to_simple_string()),
621630
ChatMessage::Function { content, .. } => Some(content.clone()),
622631
})
623632
.collect::<Vec<String>>()

‎sgl-router/src/protocols/common.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ impl StringOrArray {
7777
// Content Parts (for multimodal messages)
7878
// ============================================================================
7979

80-
#[derive(Debug, Clone, Deserialize, Serialize)]
80+
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
8181
#[serde(tag = "type")]
8282
pub enum ContentPart {
8383
#[serde(rename = "text")]
@@ -86,7 +86,7 @@ pub enum ContentPart {
8686
ImageUrl { image_url: ImageUrl },
8787
}
8888

89-
#[derive(Debug, Clone, Deserialize, Serialize)]
89+
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
9090
pub struct ImageUrl {
9191
pub url: String,
9292
#[serde(skip_serializing_if = "Option::is_none")]

‎sgl-router/src/routers/grpc/harmony/builder.rs‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use tracing::debug;
1616

1717
use super::types::HarmonyBuildOutput;
1818
use crate::protocols::{
19-
chat::{ChatCompletionRequest, ChatMessage, UserMessageContent},
19+
chat::{ChatCompletionRequest, ChatMessage, MessageContent},
2020
common::{ContentPart, Tool},
2121
responses::{
2222
ReasoningEffort as ResponsesReasoningEffort, ResponseContentPart, ResponseInput,
@@ -704,7 +704,7 @@ impl HarmonyBuilder {
704704
},
705705
recipient: None,
706706
content: vec![Content::Text(TextContent {
707-
text: content.clone(),
707+
text: content.to_simple_string(),
708708
})],
709709
channel: None,
710710
content_type: None,
@@ -715,8 +715,8 @@ impl HarmonyBuilder {
715715
ChatMessage::User { content, name } => {
716716
// Extract text from user content
717717
let text = match content {
718-
UserMessageContent::Text(text) => text.clone(),
719-
UserMessageContent::Parts(parts) => {
718+
MessageContent::Text(text) => text.clone(),
719+
MessageContent::Parts(parts) => {
720720
// For multimodal content, extract text parts
721721
parts
722722
.iter()
@@ -772,7 +772,11 @@ impl HarmonyBuilder {
772772
} else {
773773
// Regular assistant message with content
774774
// Combine content with reasoning if present
775-
let mut text = content.clone().unwrap_or_default();
775+
let mut text = content
776+
.as_ref()
777+
.map(|c| c.to_simple_string())
778+
.unwrap_or_default();
779+
776780
if let Some(reasoning) = reasoning_content {
777781
if !text.is_empty() {
778782
text.push('\n');
@@ -813,7 +817,7 @@ impl HarmonyBuilder {
813817
},
814818
recipient: Some("assistant".to_string()),
815819
content: vec![Content::Text(TextContent {
816-
text: content.clone(),
820+
text: content.to_simple_string(),
817821
})],
818822
channel: None,
819823
content_type: None,

‎sgl-router/src/routers/grpc/regular/responses/conversions.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
1010
use crate::{
1111
protocols::{
12-
chat::{ChatCompletionRequest, ChatCompletionResponse, ChatMessage, UserMessageContent},
12+
chat::{ChatCompletionRequest, ChatCompletionResponse, ChatMessage, MessageContent},
1313
common::{
1414
FunctionCallResponse, JsonSchemaFormat, ResponseFormat, StreamOptions, ToolCall,
1515
UsageInfo,
@@ -38,7 +38,7 @@ pub fn responses_to_chat(req: &ResponsesRequest) -> Result<ChatCompletionRequest
3838
// 1. Add system message if instructions provided
3939
if let Some(instructions) = &req.instructions {
4040
messages.push(ChatMessage::System {
41-
content: instructions.clone(),
41+
content: MessageContent::Text(instructions.clone()),
4242
name: None,
4343
});
4444
}
@@ -48,7 +48,7 @@ pub fn responses_to_chat(req: &ResponsesRequest) -> Result<ChatCompletionRequest
4848
ResponseInput::Text(text) => {
4949
// Simple text input → user message
5050
messages.push(ChatMessage::User {
51-
content: UserMessageContent::Text(text.clone()),
51+
content: MessageContent::Text(text.clone()),
5252
name: None,
5353
});
5454
}
@@ -111,7 +111,7 @@ pub fn responses_to_chat(req: &ResponsesRequest) -> Result<ChatCompletionRequest
111111
// Add tool result message if output exists
112112
if let Some(output_text) = output {
113113
messages.push(ChatMessage::Tool {
114-
content: output_text.clone(),
114+
content: MessageContent::Text(output_text.clone()),
115115
tool_call_id: id.clone(),
116116
});
117117
}
@@ -140,7 +140,7 @@ pub fn responses_to_chat(req: &ResponsesRequest) -> Result<ChatCompletionRequest
140140
// Note: The function name is looked up from prev_outputs in Harmony path
141141
// For Chat path, we just use the call_id
142142
messages.push(ChatMessage::Tool {
143-
content: output.clone(),
143+
content: MessageContent::Text(output.clone()),
144144
tool_call_id: call_id.clone(),
145145
});
146146
}
@@ -213,23 +213,23 @@ fn extract_text_from_content(content: &[ResponseContentPart]) -> String {
213213
fn role_to_chat_message(role: &str, text: String) -> ChatMessage {
214214
match role {
215215
"user" => ChatMessage::User {
216-
content: UserMessageContent::Text(text),
216+
content: MessageContent::Text(text),
217217
name: None,
218218
},
219219
"assistant" => ChatMessage::Assistant {
220-
content: Some(text),
220+
content: Some(MessageContent::Text(text)),
221221
name: None,
222222
tool_calls: None,
223223
reasoning_content: None,
224224
},
225225
"system" => ChatMessage::System {
226-
content: text,
226+
content: MessageContent::Text(text),
227227
name: None,
228228
},
229229
_ => {
230230
// Unknown role, treat as user message
231231
ChatMessage::User {
232-
content: UserMessageContent::Text(text),
232+
content: MessageContent::Text(text),
233233
name: None,
234234
}
235235
}

‎sgl-router/src/routers/grpc/utils.rs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -948,7 +948,7 @@ mod tests {
948948
use super::*;
949949
use crate::{
950950
protocols::{
951-
chat::{ChatMessage, UserMessageContent},
951+
chat::{ChatMessage, MessageContent},
952952
common::{ContentPart, ImageUrl},
953953
},
954954
tokenizer::chat_template::ChatTemplateContentFormat,
@@ -957,7 +957,7 @@ mod tests {
957957
#[test]
958958
fn test_transform_messages_string_format() {
959959
let messages = vec![ChatMessage::User {
960-
content: UserMessageContent::Parts(vec![
960+
content: MessageContent::Parts(vec![
961961
ContentPart::Text {
962962
text: "Hello".to_string(),
963963
},
@@ -990,7 +990,7 @@ mod tests {
990990
#[test]
991991
fn test_transform_messages_openai_format() {
992992
let messages = vec![ChatMessage::User {
993-
content: UserMessageContent::Parts(vec![
993+
content: MessageContent::Parts(vec![
994994
ContentPart::Text {
995995
text: "Describe this image:".to_string(),
996996
},
@@ -1024,7 +1024,7 @@ mod tests {
10241024
#[test]
10251025
fn test_transform_messages_simple_string_content() {
10261026
let messages = vec![ChatMessage::User {
1027-
content: UserMessageContent::Text("Simple text message".to_string()),
1027+
content: MessageContent::Text("Simple text message".to_string()),
10281028
name: None,
10291029
}];
10301030

@@ -1044,11 +1044,11 @@ mod tests {
10441044
fn test_transform_messages_multiple_messages() {
10451045
let messages = vec![
10461046
ChatMessage::System {
1047-
content: "System prompt".to_string(),
1047+
content: MessageContent::Text("System prompt".to_string()),
10481048
name: None,
10491049
},
10501050
ChatMessage::User {
1051-
content: UserMessageContent::Parts(vec![
1051+
content: MessageContent::Parts(vec![
10521052
ContentPart::Text {
10531053
text: "User message".to_string(),
10541054
},
@@ -1079,7 +1079,7 @@ mod tests {
10791079
#[test]
10801080
fn test_transform_messages_empty_text_parts() {
10811081
let messages = vec![ChatMessage::User {
1082-
content: UserMessageContent::Parts(vec![ContentPart::ImageUrl {
1082+
content: MessageContent::Parts(vec![ContentPart::ImageUrl {
10831083
image_url: ImageUrl {
10841084
url: "https://example.com/image.jpg".to_string(),
10851085
detail: None,
@@ -1101,11 +1101,11 @@ mod tests {
11011101
fn test_transform_messages_mixed_content_types() {
11021102
let messages = vec![
11031103
ChatMessage::User {
1104-
content: UserMessageContent::Text("Plain text".to_string()),
1104+
content: MessageContent::Text("Plain text".to_string()),
11051105
name: None,
11061106
},
11071107
ChatMessage::User {
1108-
content: UserMessageContent::Parts(vec![
1108+
content: MessageContent::Parts(vec![
11091109
ContentPart::Text {
11101110
text: "With image".to_string(),
11111111
},

0 commit comments

Comments
 (0)