fix(secrets): stop vault refresh task leaks and handle token expiry

Author: jjleng

voice/engine/src/server.rs

@@ -185,6 +185,29 @@ impl RegisteredSession {
     }
 }
 
+impl Drop for RegisteredSession {
+    fn drop(&mut self) {
+        // Abort the background secret refresh task when the session is dropped.
+        //
+        // This covers all early-exit paths where `run_session_with_transport` is
+        // never reached (and thus never calls `h.abort()`):
+        //   - TTL cleanup task evicts unconnected sessions from the DashMap
+        //   - WebRTC ICE connection timeout / failure
+        //   - Telephony session_id mismatch (start-message timeout → random UUID)
+        //
+        // Note: `take_refresh_handle` removes the handle from the inner Option,
+        // so for the normal path — where the caller already took the handle and
+        // called `h.abort()` inside `run_session_with_transport` — this no-ops.
+        if let Some(arc) = &self.secret_refresh_handle {
+            if let Ok(mut guard) = arc.lock() {
+                if let Some(h) = guard.take() {
+                    h.abort();
+                }
+            }
+        }
+    }
+}
+
 impl std::fmt::Debug for RegisteredSession {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("RegisteredSession")
@@ -545,7 +568,7 @@ async fn handle_registered_socket(socket: WebSocket, session_id: String, state:
         run_voice_session(
             session_id,
             socket,
-            registered.config,
+            registered.config.clone(),
             &registered.providers,
             secrets,
             refresh_handle,
@@ -1100,7 +1123,7 @@ async fn rtc_offer_registered(
     rtc_create_session(
         session_id,
         body,
-        registered.config,
+        registered.config.clone(),
         &registered.providers,
         secrets,
         refresh_handle,
@@ -1572,8 +1595,8 @@ async fn handle_telephony_session(
             let from_number = reg.from_number.clone();
             (
                 effective_session_id.clone(),
-                reg.config,
-                reg.providers,
+                reg.config.clone(),
+                reg.providers.clone(),
                 secrets,
                 refresh_handle,
                 reg_tracer,

voice/server/src/secrets.rs

@@ -2,18 +2,27 @@
 //!
 //! Owns the lifecycle of per-session secrets:
 //!   1. Initial resolution at session registration time
-//!   2. Background refresh task that re-queries the vault every 90 seconds
+//!   2. Background refresh task that re-queries the vault every 90 seconds,
+//!      automatically renewing the scoped vault token if it expires.
 //!
 //! voice-engine receives a pre-populated `SharedSecretMap` and never touches
 //! the vault directly. This keeps voice-engine free of infrastructure concerns.
 
 use std::sync::Arc;
+use std::time::Duration;
 
 use agent_kit::agent_backends::{SecretMap, SharedSecretMap};
 use tracing::warn;
 
+use crate::cred::VaultHandle;
 use crate::vault_client;
 
+/// Scoped vault token TTL issued to each session.
+///
+/// If a session runs longer than 1 hour, the token is automatically renewed
+/// by the background refresh task.
+pub const VAULT_TOKEN_TTL: Duration = Duration::from_secs(3600);
+
 /// Cached vault address from `VAULT_ADDR` environment variable.
 ///
 /// Returns `None` if `VAULT_ADDR` is not set. The value is read once and
@@ -67,14 +76,16 @@ const SECRET_REFRESH_INTERVAL_SECS: u64 = 90;
 /// and updates the shared secret map.
 ///
 /// Returns a `JoinHandle` — the caller should abort it when the session ends.
-/// If `vault_token` is `None`, no task is spawned (returns `None`).
+/// If `vault` or `vault_token` is `None`, no task is spawned (returns `None`).
 pub fn spawn_secret_refresh_task(
     agent_id: String,
+    vault: Option<Arc<VaultHandle>>,
     vault_token: Option<String>,
     secrets: SharedSecretMap,
 ) -> Option<tokio::task::JoinHandle<()>> {
     let addr = vault_addr()?.to_string();
-    let token = vault_token?;
+    let vault = vault?;
+    let mut token = vault_token?;
 
     let Ok(agent_uuid) = uuid::Uuid::parse_str(&agent_id) else {
         return None;
@@ -112,6 +123,16 @@ pub fn spawn_secret_refresh_task(
                         }
                     }
                 }
+                Err(vault_client::VaultResolveError::TokenExpired) => {
+                    // The scoped token expired — mint a fresh one and retry
+                    // immediately on the next tick rather than stopping the task.
+                    // This keeps secrets refreshing for arbitrarily long sessions.
+                    tracing::info!(
+                        agent_id = %agent_id,
+                        "Vault scoped token expired — renewing for session"
+                    );
+                    token = vault.create_scoped_token(agent_uuid, VAULT_TOKEN_TTL);
+                }
                 Err(e) => {
                     tracing::warn!(
                         agent_id = %agent_id,

voice/server/src/telephony.rs

@@ -12,7 +12,7 @@
 //! are NOT duplicated here.
 
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Instant;
 
 use crate::cred::{EncryptionEngine, VaultHandle};
 use crate::observability;
@@ -65,7 +65,7 @@ impl AppState {
     pub fn vault_token_for(&self, agent_uuid: Uuid) -> Option<String> {
         self.vault
             .as_ref()
-            .map(|v| v.create_scoped_token(agent_uuid, Duration::from_secs(3600)))
+            .map(|v| v.create_scoped_token(agent_uuid, crate::secrets::VAULT_TOKEN_TTL))
     }
 }
 
@@ -498,6 +498,7 @@ pub async fn register_session(
         crate::secrets::resolve_vault_secrets(&agent_id_for_secrets, vault_token.as_deref()).await;
     let refresh_handle = crate::secrets::spawn_secret_refresh_task(
         agent_id_for_secrets,
+        state.vault.clone(),
         vault_token_for_refresh,
         secrets.clone(),
     );

voice/server/src/vault_client.rs

@@ -53,6 +53,14 @@ pub async fn resolve_secrets(
                 );
                 return Ok(SecretMap::new());
             }
+            Err(vaultrs::error::ClientError::APIError { code: 403, .. }) => {
+                // Scoped token has expired.
+                warn!(
+                    agent_id = %agent_id,
+                    "Vault scoped token expired (403)"
+                );
+                return Err(VaultResolveError::TokenExpired);
+            }
             Err(e) => {
                 warn!(
                     agent_id = %agent_id,
@@ -85,4 +93,7 @@ pub enum VaultResolveError {
     ClientConfig(String),
     #[error("failed to read secrets from vault: {0}")]
     ReadFailed(String),
+    /// The scoped vault token has expired (HTTP 403).
+    #[error("vault scoped token expired")]
+    TokenExpired,
 }