feat(config): deployment-wide default for upstream request/stream timeouts

config.example.yaml

@@ -160,6 +160,21 @@ ratelimit:
 # The values below are the defaults; uncomment to override. Every
 # duration accepts 0 to switch that knob off.
 upstream:
+  # Deployment-wide default for `Model.timeout`: the end-to-end deadline
+  # for non-streaming calls, and the fallback streaming budget (below),
+  # for every model that does not set its own. A backstop against an
+  # upstream that accepted the connection and then goes silent forever —
+  # not a responsiveness target (set per-model `timeout` for that), so it
+  # is deliberately generous: deep-reasoning calls can run past 10
+  # minutes. A model opts out with `timeout: 0`; setting 0 here restores
+  # the old "no deadline unless configured" behaviour.
+  # timeout_ms: 6000000
+
+  # Deployment-wide default for `Model.stream_timeout`: the maximum gap
+  # between streaming chunks. 0 falls back to `timeout_ms`, mirroring how
+  # an unset `Model.stream_timeout` falls back to `Model.timeout`.
+  # stream_timeout_ms: 0
+
   # Max time for DNS + TCP + TLS before an attempt fails. Without it a
   # black-holed upstream is bounded only by the model's own timeout.
   # connect_timeout_ms: 5000

config.managed.yaml

@@ -107,6 +107,8 @@ cache:
 # the gateway expires them — the symptom is intermittent transport
 # errors against an otherwise healthy upstream.
 upstream:
+  # timeout_ms: 6000000
+  # stream_timeout_ms: 0
   # connect_timeout_ms: 5000
   # tcp_keepalive_secs: 60
   # tcp_keepalive_interval_secs: 30

crates/aisix-core/src/config.rs

@@ -784,6 +784,24 @@ pub enum RateLimitBackend {
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields, default)]
 pub struct UpstreamConfig {
+    /// Deployment-wide default for `Model.timeout`: the end-to-end deadline
+    /// in milliseconds for non-streaming upstream calls (and the fallback
+    /// budget for streaming ones, below). Applies to every model that sets
+    /// neither its own `timeout` nor a group-level one. `0` restores the
+    /// pre-default behaviour: no deadline at all.
+    ///
+    /// The default matches the LiteLLM proxy's `request_timeout` (6000 s).
+    /// It is a backstop against an upstream that accepted the connection
+    /// and then goes silent forever — not a responsiveness target, which
+    /// is what per-model `timeout` is for. Deliberately generous so it can
+    /// never cut down a legitimate long request (deep-reasoning calls run
+    /// past 10 minutes).
+    pub timeout_ms: u64,
+    /// Deployment-wide default for `Model.stream_timeout`: the maximum gap
+    /// in milliseconds between upstream streaming chunks. `0` (the
+    /// default) falls back to `timeout_ms`, mirroring how an unset
+    /// `Model.stream_timeout` falls back to `Model.timeout`.
+    pub stream_timeout_ms: u64,
     /// Max time for DNS + TCP + TLS before an attempt fails. Without it a
     /// black-holed upstream is bounded only by the model's overall timeout.
     pub connect_timeout_ms: u64,
@@ -818,6 +836,8 @@ pub struct UpstreamConfig {
 impl Default for UpstreamConfig {
     fn default() -> Self {
         Self {
+            timeout_ms: DEFAULT_UPSTREAM_TIMEOUT_MS,
+            stream_timeout_ms: 0,
             connect_timeout_ms: 5_000,
             tcp_keepalive_secs: 60,
             tcp_keepalive_interval_secs: 30,
@@ -834,6 +854,10 @@ impl Default for UpstreamConfig {
 /// `router_settings.num_retries` nor `litellm_settings.num_retries` is set.
 pub const DEFAULT_UPSTREAM_RETRIES: u32 = 2;
 
+/// Deployment-wide request-timeout default: 6000 s, matching the LiteLLM
+/// proxy's `request_timeout`. See [`UpstreamConfig::timeout_ms`].
+pub const DEFAULT_UPSTREAM_TIMEOUT_MS: u64 = 6_000_000;
+
 /// Connection-layer settings for the inbound side — the client (or the
 /// gateway in front of this one) talking to the proxy and admin listeners.
 ///
@@ -1372,6 +1396,8 @@ admin:
 "#,
         );
         let cfg = Config::load_from_path(Some(f.path())).unwrap();
+        assert_eq!(cfg.upstream.timeout_ms, 6_000_000);
+        assert_eq!(cfg.upstream.stream_timeout_ms, 0);
         assert_eq!(cfg.upstream.connect_timeout_ms, 5_000);
         assert_eq!(cfg.upstream.tcp_keepalive_secs, 60);
         assert_eq!(cfg.upstream.tcp_keepalive_interval_secs, 30);
@@ -1395,13 +1421,17 @@ admin:
   addr: "127.0.0.1:3001"
   admin_keys: ["k1"]
 upstream:
+  timeout_ms: 0
+  stream_timeout_ms: 30000
   connect_timeout_ms: 2000
   pool_idle_timeout_secs: 10
   tcp_keepalive_secs: 0
   pool_max_idle_per_host: 16
 "#,
         );
         let cfg = Config::load_from_path(Some(f.path())).unwrap();
+        assert_eq!(cfg.upstream.timeout_ms, 0);
+        assert_eq!(cfg.upstream.stream_timeout_ms, 30_000);
         assert_eq!(cfg.upstream.connect_timeout_ms, 2_000);
         assert_eq!(cfg.upstream.pool_idle_timeout_secs, 10);
         assert_eq!(cfg.upstream.tcp_keepalive_secs, 0);

crates/aisix-core/src/models/ensemble.rs

@@ -103,8 +103,7 @@ impl EnsembleConfig {
     }
 
     /// Per-call upstream deadline applied to each panel member and the
-    /// judge call. Folds the `0`/absent sentinel into `None` like
-    /// [`Model::request_timeout`](super::Model::request_timeout) so callers
+    /// judge call. Folds the `0`/absent sentinel into `None` so callers
     /// can apply it unconditionally.
     pub fn timeout(&self) -> Option<std::time::Duration> {
         self.timeout_ms

crates/aisix-core/src/models/model.rs

@@ -228,11 +228,11 @@ pub struct Model {
     #[schemars(length(min = 1))]
     pub provider_key_id: Option<String>,
 
-    /// End-to-end timeout in milliseconds for non-streaming upstream calls. `0` or absent disables the non-streaming timeout.
+    /// End-to-end timeout in milliseconds for non-streaming upstream calls. Absent falls back to the group's `timeout`, then to the deployment-wide `upstream.timeout_ms` default. `0` disables the non-streaming timeout for this model.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub timeout: Option<u64>,
 
-    /// Maximum gap in milliseconds between upstream streaming chunks. `0` or absent falls back to `timeout`.
+    /// Maximum gap in milliseconds between upstream streaming chunks. `0` or absent falls back to the group's `stream_timeout`, then to the model's (or group's) `timeout`, then to the deployment-wide `upstream.stream_timeout_ms` / `timeout_ms` defaults.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub stream_timeout: Option<u64>,
 
@@ -325,37 +325,27 @@ impl Model {
         self.model_name.as_deref()
     }
 
-    /// Non-streaming request deadline derived from `timeout`. Folds the
-    /// `0`/absent "no timeout" sentinel into `None` so callers can apply
-    /// it unconditionally with `if let Some(d) = ...`.
-    pub fn request_timeout(&self) -> Option<std::time::Duration> {
+    /// This resource's own non-streaming deadline, as one level of the
+    /// model → group → `upstream.timeout_ms` resolution performed by the
+    /// proxy's `effective_timeouts`. Tri-state: `None` defers to the next
+    /// level, `Some(None)` is an explicit `0` ("no deadline, stop
+    /// resolving"), `Some(Some(d))` is a configured deadline.
+    pub fn request_timeout_level(&self) -> Option<Option<std::time::Duration>> {
         self.timeout
-            .filter(|&ms| ms > 0)
-            .map(std::time::Duration::from_millis)
+            .map(|ms| (ms > 0).then(|| std::time::Duration::from_millis(ms)))
     }
 
-    /// Streaming per-chunk read deadline derived from `stream_timeout`.
-    /// Same `0`/absent → `None` folding as [`Model::request_timeout`].
+    /// This resource's own streaming per-chunk deadline, as one level of
+    /// the model → group → `upstream.stream_timeout_ms` → resolved
+    /// `timeout` chain. Unlike [`Model::request_timeout_level`], `0` and
+    /// absent both defer — `stream_timeout` has always used `0` as "fall
+    /// back", not "disable".
     pub fn stream_read_timeout(&self) -> Option<std::time::Duration> {
         self.stream_timeout
             .filter(|&ms| ms > 0)
             .map(std::time::Duration::from_millis)
     }
 
-    /// Effective deadline for a streaming request: a positive
-    /// `stream_timeout`, otherwise the non-streaming `timeout`. Applied to the
-    /// connect phase, the per-chunk read timeout, and the first-chunk
-    /// failover gate. Because `stream_read_timeout()` folds `0` to `None`,
-    /// `stream_timeout: 0` is treated the same as absent — it falls back to
-    /// `timeout` rather than disabling the streaming timeout. `None` (both
-    /// unset or `0`) = no streaming timeout. Note: a model that sets only a
-    /// small `timeout` therefore also gets that value as its streaming
-    /// budget.
-    pub fn stream_timeout_effective(&self) -> Option<std::time::Duration> {
-        self.stream_read_timeout()
-            .or_else(|| self.request_timeout())
-    }
-
     /// Whether a client at `source_ip` may access this model (#557).
     ///
     /// Returns `true` when no `allowed_cidrs` restriction is configured (the
@@ -500,59 +490,30 @@ mod tests {
         .unwrap();
         assert_eq!(m.stream_timeout, Some(2_500));
         assert_eq!(
-            m.request_timeout(),
-            Some(std::time::Duration::from_millis(30_000))
+            m.request_timeout_level(),
+            Some(Some(std::time::Duration::from_millis(30_000)))
         );
         assert_eq!(
             m.stream_read_timeout(),
             Some(std::time::Duration::from_millis(2_500))
         );
 
-        // Absent → None.
+        // Absent → defer to the next resolution level.
         let none: Model = serde_json::from_str(
             r#"{"display_name":"x","provider":"openai","model_name":"g","provider_key_id":"pk-1"}"#,
         )
         .unwrap();
-        assert_eq!(none.request_timeout(), None);
+        assert_eq!(none.request_timeout_level(), None);
         assert_eq!(none.stream_read_timeout(), None);
 
-        // Explicit 0 is the "no timeout" sentinel → None.
+        // Explicit `timeout: 0` resolves to "no deadline" and stops the
+        // chain; explicit `stream_timeout: 0` defers like absent.
         let zero: Model = serde_json::from_str(
             r#"{"display_name":"x","provider":"openai","model_name":"g","provider_key_id":"pk-1","timeout":0,"stream_timeout":0}"#,
         )
         .unwrap();
-        assert_eq!(zero.request_timeout(), None);
+        assert_eq!(zero.request_timeout_level(), Some(None));
         assert_eq!(zero.stream_read_timeout(), None);
-
-        // stream_timeout_effective cascade: prefer stream_timeout when set.
-        assert_eq!(
-            m.stream_timeout_effective(),
-            Some(std::time::Duration::from_millis(2_500))
-        );
-        // Falls back to `timeout` when stream_timeout is absent.
-        let timeout_only: Model = serde_json::from_str(
-            r#"{"display_name":"x","provider":"openai","model_name":"g","provider_key_id":"pk-1","timeout":5000}"#,
-        )
-        .unwrap();
-        assert_eq!(
-            timeout_only.stream_timeout_effective(),
-            Some(std::time::Duration::from_millis(5_000))
-        );
-        // None when neither is set, and when both are the 0 sentinel.
-        assert_eq!(none.stream_timeout_effective(), None);
-        assert_eq!(zero.stream_timeout_effective(), None);
-
-        // Explicit `stream_timeout: 0` folds to absent → falls back to
-        // `timeout`, not "disable streaming".
-        let stream_zero_timeout_set: Model = serde_json::from_str(
-            r#"{"display_name":"x","provider":"openai","model_name":"g","provider_key_id":"pk-1","timeout":5000,"stream_timeout":0}"#,
-        )
-        .unwrap();
-        assert_eq!(stream_zero_timeout_set.stream_read_timeout(), None);
-        assert_eq!(
-            stream_zero_timeout_set.stream_timeout_effective(),
-            Some(std::time::Duration::from_millis(5_000))
-        );
     }
 
     #[test]

crates/aisix-core/src/models/semantic.rs

@@ -167,8 +167,7 @@ impl Semantic {
     }
 
     /// Per-call embedding deadline. Folds the `0`/absent sentinel into
-    /// `None` like [`Model::request_timeout`](super::Model::request_timeout)
-    /// so callers can apply it unconditionally.
+    /// `None` so callers can apply it unconditionally.
     pub fn embedding_timeout(&self) -> Option<std::time::Duration> {
         self.embedding_timeout_ms
             .filter(|&ms| ms > 0)

crates/aisix-proxy/src/audio.rs

@@ -654,7 +654,9 @@ async fn multipart_dispatch(
             // the per-model E2E request timeout like the other direct-upstream
             // paths (count_tokens/rerank/responses) so a slow/blackholed audio
             // provider fails over and the model's timeout cooldown can engage.
-            if let Some(d) = model.request_timeout() {
+            if let Some(d) =
+                crate::routing::effective_timeouts(model, None, state.default_timeouts).request
+            {
                 req = req.timeout(d);
             }
             async move {
@@ -1039,7 +1041,9 @@ async fn speech_dispatch(
                 .json(&body);
             // #554/#911: speech synthesis is non-streaming; apply the per-model
             // E2E request timeout (same as count_tokens/rerank/responses).
-            if let Some(d) = model.request_timeout() {
+            if let Some(d) =
+                crate::routing::effective_timeouts(model, None, state.default_timeouts).request
+            {
                 req = req.timeout(d);
             }
             async move {

crates/aisix-proxy/src/chat.rs

@@ -1277,14 +1277,20 @@ async fn dispatch(
                 Arc::new(pk_entry.value.clone()),
                 Some(client),
             );
-            if let Some(d) = model.stream_timeout_effective() {
+            // Effective streaming budget, resolved target → group →
+            // `upstream.stream_timeout_ms`/`timeout_ms`. Used for the
+            // connect deadline (above) AND the per-chunk read timeout +
+            // first-chunk peek below, so the budget is applied
+            // consistently.
+            let timeouts = crate::routing::effective_timeouts(
+                model,
+                Some(&virtual_entry.value),
+                state.default_timeouts,
+            );
+            let stream_budget = timeouts.stream;
+            if let Some(d) = stream_budget {
                 ctx = ctx.with_deadline(d);
             }
-            // Effective streaming budget: `stream_timeout`, falling back to
-            // `timeout`. Used for the connect deadline (above) AND the
-            // per-chunk read timeout + first-chunk peek below, so the budget
-            // is applied consistently.
-            let stream_budget = model.stream_timeout_effective();
 
             // How many times to re-hit the SAME target (with backoff) on a
             // retryable failure before failing over to the next one.
@@ -1351,19 +1357,20 @@ async fn dispatch(
                     }
                 };
                 let attempt_started = Instant::now();
-                // Connect, then — only when a streaming budget is configured —
-                // peek the first chunk so a slow or erroring first token fails
-                // over before the 200 is committed. Without a budget there is
-                // nothing to gate on, so the stream is committed directly (a
-                // first-chunk error then surfaces in-band, exactly like the
-                // pre-#554 behavior). The read-timeout wrapper is a no-op when
-                // the budget is None.
+                // Connect, then — only when a streaming budget is configured
+                // ON THE RESOURCES — peek the first chunk so a slow or
+                // erroring first token fails over before the 200 is
+                // committed. The deployment-default budget does not peek
+                // (see `TimeoutBudget::stream_configured`): it stays a read
+                // timeout, so the 200 commits directly and the SSE
+                // heartbeats cover the wait for the first token. The
+                // read-timeout wrapper is a no-op when the budget is None.
                 let attempt_stream: Result<aisix_gateway::ChatChunkStream, BridgeError> =
                     match bridge.chat_stream(req, &ctx).await {
                         Err(e) => Err(e),
                         Ok(up) => {
                             let up = crate::stream_timeout::with_read_timeout(up, stream_budget);
-                            if stream_budget.is_some() {
+                            if timeouts.stream_configured {
                                 let mut up = up;
                                 match up.next().await {
                                     // Re-prepend the peeked chunk so the SSE pump
@@ -2175,7 +2182,13 @@ async fn dispatch(
             Arc::new(pk_entry.value.clone()),
             Some(client),
         );
-        if let Some(d) = model.request_timeout() {
+        if let Some(d) = crate::routing::effective_timeouts(
+            model,
+            Some(&virtual_entry.value),
+            state.default_timeouts,
+        )
+        .request
+        {
             ctx = ctx.with_deadline(d);
         }
         // Per-target retry budget — see the streaming loop above.
@@ -2870,7 +2883,9 @@ async fn dispatch_ensemble(
             Arc::new(judge_pk.value.clone()),
             Some(client),
         );
-        if let Some(deadline) = judge_model.request_timeout() {
+        if let Some(deadline) =
+            crate::routing::effective_timeouts(judge_model, None, state.default_timeouts).request
+        {
             judge_ctx = judge_ctx.with_deadline(deadline);
         }
 

crates/aisix-proxy/src/completions.rs

@@ -323,7 +323,8 @@ async fn dispatch(
         Arc::new(pk_entry.value.clone()),
         Some(client_ctx),
     );
-    if let Some(d) = model.request_timeout() {
+    if let Some(d) = crate::routing::effective_timeouts(model, None, state.default_timeouts).request
+    {
         ctx = ctx.with_deadline(d);
     }
 

crates/aisix-proxy/src/count_tokens.rs

@@ -267,6 +267,11 @@ async fn dispatch(
                 body,
                 &target.model,
                 &target.id,
+                crate::routing::effective_timeouts(
+                    &target.model,
+                    Some(&model_entry.value),
+                    state.default_timeouts,
+                ),
                 request_id,
                 client,
             )
@@ -319,6 +324,9 @@ async fn count_tokens_to_target(
     body: &Value,
     model: &aisix_core::Model,
     model_id: &str,
+    // Deadlines resolved by the caller across target → group → deployment
+    // default (`routing::effective_timeouts`); this fn only applies them.
+    timeouts: crate::routing::TimeoutBudget,
     request_id: &str,
     client: &ClientContext,
 ) -> Result<Response, ProxyError> {
@@ -402,7 +410,7 @@ async fn count_tokens_to_target(
     let client = crate::http_client::client();
     let mut req = client.post(&url).headers(headers).json(&body);
     // #554: count_tokens is non-streaming; apply the E2E request timeout.
-    if let Some(d) = model.request_timeout() {
+    if let Some(d) = timeouts.request {
         req = req.timeout(d);
     }
     let send_started = Instant::now();