Bump SGLang on Modal to fix flaky e2e tests (#4009)

* Bump SGLang on Modal to fix flaky e2e tests

SGLang now emits many duplicate tool calls, instead of not emitting
any tool calls at all. I've adjusted our tests to allow this for sglang.

When running locally, the tests almost always pass on the first try
(with the occasional retry), instead of the ~9 retries we were seeing
on our daily cron job

* Fix clippy

* Change warmup url

Author: Aaron1011

.github/workflows/merge-queue.yml

@@ -83,7 +83,7 @@ jobs:
       - name: Warm up Modal instances
         run: |
           curl -H "Modal-Key: $MODAL_KEY" -H "Modal-Secret: $MODAL_SECRET" https://tensorzero--vllm-inference-vllm-inference.modal.run/docs > vllm_modal_logs.txt &
-          curl -H "Modal-Key: $MODAL_KEY" -H "Modal-Secret: $MODAL_SECRET" https://tensorzero--sglang-inference-sglang-inference.modal.run/ > sglang_modal_logs.txt &
+          curl -H "Modal-Key: $MODAL_KEY" -H "Modal-Secret: $MODAL_SECRET" https://tensorzero--sglang-0-4-10-inference-sglang-inference.modal.run/ > sglang_modal_logs.txt &
           # TODO: Re-enable once we can switch to a T4 GPU
           # curl -H "Modal-Key: $MODAL_KEY" -H "Modal-Secret: $MODAL_SECRET" https://tensorzero--vllm-gpt-oss-20b-serve.modal.run/ > vllm_gpt_oss_modal_logs.txt &
 

ci/buildkite/modal-warmup.sh

@@ -23,7 +23,7 @@ curl -H "Modal-Key: $MODAL_KEY" -H "Modal-Secret: $MODAL_SECRET" \
 
 echo "Warming up SGLang Modal instance..."
 curl -H "Modal-Key: $MODAL_KEY" -H "Modal-Secret: $MODAL_SECRET" \
-     https://tensorzero--sglang-inference-sglang-inference.modal.run/ \
+     https://tensorzero--sglang-0-4-10-inference-sglang-inference.modal.run/ \
      > sglang_modal_logs.txt &
 
 echo "Waiting for warmup requests to complete..."

tensorzero-core/fixtures/deployment/sgl-modal/sgl_inference.py

@@ -7,7 +7,9 @@
 
 sgl_image = (
     modal.Image.from_registry(f"nvidia/cuda:{tag}", add_python="3.12")
-    .pip_install("sglang[all]>=0.4.6.post1", "huggingface_hub[hf_transfer]==0.30.2")
+    # This is the last version of sglang that supports SM75 GPUs (e.g. the Nvidia T4)
+    .pip_install("sglang[all]==0.4.10.post2", "huggingface_hub[hf_transfer]==0.34")
+    .apt_install("numactl")
     .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
 )
 
@@ -20,7 +22,7 @@
 sglang_cache_vol = modal.Volume.from_name("sglang-cache", create_if_missing=True)
 
 N_GPU = 1
-app = modal.App(name="sglang-inference")
+app = modal.App(name="sglang-0.4.10-inference")
 
 
 @app.function(
@@ -47,6 +49,8 @@ def sglang_inference():
         "python",
         "-m",
         "sglang.launch_server",
+        # This prevents the container from OOMing on startup
+        "--disable-cuda-graph",
         "--model-path",
         MODEL_NAME,
         "--tool-call-parser",

tensorzero-core/tests/e2e/providers/common.rs

@@ -7010,11 +7010,14 @@ pub async fn check_tool_use_tool_choice_specific_inference_response(
         .collect();
 
     // Assert at most one tool call (a model could decide to call no tools if to reads the `self_destruct` description).
-    assert!(
-        tool_call_blocks.len() <= 1,
-        "Expected at most one tool call, found {}",
-        tool_call_blocks.len()
-    );
+    // Sglang likes to emit lots of tool calls
+    if provider.model_provider_name != "sglang" {
+        assert!(
+            tool_call_blocks.len() <= 1,
+            "Expected at most one tool call, found {}",
+            tool_call_blocks.len()
+        );
+    }
 
     let tool_call_block = tool_call_blocks.first();
     match tool_call_block {
@@ -7143,10 +7146,19 @@ pub async fn test_tool_use_tool_choice_specific_streaming_inference_request_with
                     let block_tool_id = block.get("id").unwrap().as_str().unwrap();
                     match &tool_id {
                         None => tool_id = Some(block_tool_id.to_string()),
-                        Some(tool_id) => assert_eq!(
-                            tool_id, block_tool_id,
-                            "Provider returned multiple tool calls"
-                        ),
+                        Some(tool_id) => {
+                            if provider.model_provider_name == "sglang" {
+                                // Sglang likes to emit lots of duplicate tool calls
+                                if tool_id != block_tool_id {
+                                    continue;
+                                }
+                            } else {
+                                assert_eq!(
+                                    tool_id, block_tool_id,
+                                    "Provider returned multiple tool calls"
+                                );
+                            }
+                        }
                     }
 
                     let chunk_arguments = block.get("raw_arguments").unwrap().as_str().unwrap();
@@ -7417,11 +7429,14 @@ pub async fn test_tool_use_tool_choice_specific_streaming_inference_request_with
         .collect();
 
     // Assert at most one tool call (a model could decide to call no tools if to reads the `self_destruct` description).
-    assert!(
-        tool_call_blocks.len() <= 1,
-        "Expected at most one tool call, found {}",
-        tool_call_blocks.len()
-    );
+    // Sglang likes to emit lots of tool calls
+    if provider.model_provider_name != "sglang" {
+        assert!(
+            tool_call_blocks.len() <= 1,
+            "Expected at most one tool call, found {}",
+            tool_call_blocks.len()
+        );
+    }
 
     let tool_call_block = tool_call_blocks.first();
     match tool_call_block {
@@ -7683,16 +7698,25 @@ pub async fn check_tool_use_tool_choice_allowed_tools_inference_response(
         .filter(|block| matches!(block, StoredContentBlock::ToolCall(_)))
         .collect();
 
-    // Assert exactly one tool call
-    assert_eq!(tool_call_blocks.len(), 1, "Expected exactly one tool call");
+    if provider.model_provider_name == "sglang" {
+        // Sglang likes to emit lots of duplicate tool calls
+        assert!(
+            !tool_call_blocks.is_empty(),
+            "Expected at least one tool call"
+        );
+    } else {
+        // Assert exactly one tool call
+        assert_eq!(tool_call_blocks.len(), 1, "Expected exactly one tool call");
+    }
 
-    let tool_call_block = tool_call_blocks[0];
-    match tool_call_block {
-        StoredContentBlock::ToolCall(tool_call) => {
-            assert_eq!(tool_call.name, "get_humidity");
-            serde_json::from_str::<Value>(&tool_call.arguments.to_lowercase()).unwrap();
+    for tool_call_block in tool_call_blocks {
+        match tool_call_block {
+            StoredContentBlock::ToolCall(tool_call) => {
+                assert_eq!(tool_call.name, "get_humidity");
+                serde_json::from_str::<Value>(&tool_call.arguments.to_lowercase()).unwrap();
+            }
+            _ => panic!("Unreachable"),
         }
-        _ => panic!("Unreachable"),
     }
 }
 
@@ -7789,10 +7813,13 @@ pub async fn test_tool_use_allowed_tools_streaming_inference_request_with_provid
                     if let Some(block_raw_name) = block.get("raw_name") {
                         match tool_name {
                             Some(_) => {
-                                assert!(
-                                    block_raw_name.as_str().unwrap().is_empty(),
-                                    "Raw name already seen, got {block:#?}"
-                                );
+                                // Sglang likes to emit lots of duplicate tool calls
+                                if provider.model_provider_name != "sglang" {
+                                    assert!(
+                                        block_raw_name.as_str().unwrap().is_empty(),
+                                        "Raw name already seen, got {block:#?}"
+                                    );
+                                }
                             }
                             None => {
                                 tool_name = Some(block_raw_name.as_str().unwrap().to_string());
@@ -7803,7 +7830,16 @@ pub async fn test_tool_use_allowed_tools_streaming_inference_request_with_provid
                     let block_tool_id = block.get("id").unwrap().as_str().unwrap();
                     match &tool_id {
                         None => tool_id = Some(block_tool_id.to_string()),
-                        Some(tool_id) => assert_eq!(tool_id, block_tool_id),
+                        Some(tool_id) => {
+                            if provider.model_provider_name == "sglang" {
+                                // Sglang likes to emit lots of duplicate tool calls
+                                if tool_id != block_tool_id {
+                                    continue;
+                                }
+                            } else {
+                                assert_eq!(tool_id, block_tool_id);
+                            }
+                        }
                     }
 
                     let chunk_arguments = block.get("raw_arguments").unwrap().as_str().unwrap();
@@ -8035,16 +8071,25 @@ pub async fn test_tool_use_allowed_tools_streaming_inference_request_with_provid
         .filter(|block| matches!(block, StoredContentBlock::ToolCall(_)))
         .collect();
 
-    // Assert exactly one tool call
-    assert_eq!(tool_call_blocks.len(), 1, "Expected exactly one tool call");
+    // Sglang likes to emit lots of tool calls
+    if provider.model_provider_name == "sglang" {
+        assert!(
+            !tool_call_blocks.is_empty(),
+            "Expected at least one tool call"
+        );
+    } else {
+        // Assert exactly one tool call
+        assert_eq!(tool_call_blocks.len(), 1, "Expected exactly one tool call");
+    }
 
-    let tool_call_block = tool_call_blocks[0];
-    match tool_call_block {
-        StoredContentBlock::ToolCall(tool_call) => {
-            assert_eq!(tool_call.name, "get_humidity");
-            serde_json::from_str::<Value>(&tool_call.arguments.to_lowercase()).unwrap();
+    for tool_call_block in tool_call_blocks {
+        match tool_call_block {
+            StoredContentBlock::ToolCall(tool_call) => {
+                assert_eq!(tool_call.name, "get_humidity");
+                serde_json::from_str::<Value>(&tool_call.arguments.to_lowercase()).unwrap();
+            }
+            _ => panic!("Unreachable"),
         }
-        _ => panic!("Unreachable"),
     }
 }
 

tensorzero-core/tests/e2e/providers/sglang.rs

@@ -64,7 +64,8 @@ async fn get_providers() -> E2ETestProviders {
             ),
             (
                 "api_base".to_string(),
-                "https://tensorzero--sglang-inference-sglang-inference.modal.run/v1/".to_string(),
+                "https://tensorzero--sglang-0-4-10-inference-sglang-inference.modal.run/v1/"
+                    .to_string(),
             ),
         ]),
         use_modal_headers: true,

tensorzero-core/tests/e2e/tensorzero.toml

@@ -403,7 +403,7 @@ routing = ["sglang"]
 [models."Qwen/Qwen2.5-1.5B-Instruct".providers.sglang]
 type = "sglang"
 model_name = "Qwen/Qwen2.5-1.5B-Instruct"
-api_base = "https://tensorzero--sglang-inference-sglang-inference.modal.run/v1/"
+api_base = "https://tensorzero--sglang-0-4-10-inference-sglang-inference.modal.run/v1/"
 
 [models."qwen2.5-0.5b-instruct-vllm"]
 routing = ["vllm"]