From 383998430fac1c75bc98e287921c70cdb765166d Mon Sep 17 00:00:00 2001 From: iamninihuang Date: Fri, 1 May 2026 03:15:36 +0800 Subject: [PATCH 1/2] feat(gateway): support inbound media and file attachments --- src/gateway.rs | 95 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/main.rs | 1 + 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/src/gateway.rs b/src/gateway.rs index 5b82b270..e921f8c5 100644 --- a/src/gateway.rs +++ b/src/gateway.rs @@ -1,4 +1,7 @@ use crate::adapter::{AdapterRouter, ChannelRef, ChatAdapter, MessageRef, SenderContext}; +use crate::acp::ContentBlock; +use crate::config::SttConfig; +use crate::media; use anyhow::Result; use async_trait::async_trait; use futures_util::{SinkExt, StreamExt}; @@ -44,12 +47,23 @@ struct GwSender { is_bot: bool, } +#[derive(Clone, Debug, Deserialize)] +pub struct GwAttachment { + pub url: String, + #[serde(rename = "contentType")] + pub content_type: Option, + pub filename: Option, + pub size: Option, +} + #[derive(Clone, Debug, Deserialize)] struct GwContent { #[allow(dead_code)] #[serde(rename = "type")] content_type: String, - text: String, + text: Option, + #[serde(default)] + attachments: Vec, } #[derive(Serialize)] @@ -268,6 +282,7 @@ pub struct GatewayParams { pub allowed_channels: Vec, pub allow_all_users: bool, pub allowed_users: Vec, + pub stt_config: SttConfig, } pub async fn run_gateway_adapter( @@ -284,6 +299,7 @@ pub async fn run_gateway_adapter( let allowed_channels = params.allowed_channels; let allow_all_users = params.allow_all_users; let allowed_users = params.allowed_users; + let stt_config = params.stt_config; let connect_url = match ¶ms.token { Some(token) => { @@ -414,9 +430,82 @@ pub async fn run_gateway_adapter( let adapter = adapter.clone(); let router = router.clone(); - let prompt = event.content.text.clone(); + let prompt = event.content.text.clone().unwrap_or_default(); + let attachments = event.content.attachments.clone(); + let stt_config = stt_config.clone(); tasks.spawn(async move { + let mut extra_blocks = Vec::new(); + let mut text_file_bytes: u64 = 0; + let mut text_file_count: u32 = 0; + const TEXT_TOTAL_CAP: u64 = 1024 * 1024; // 1 MB total for all text file attachments + const TEXT_FILE_COUNT_CAP: u32 = 5; + + for attachment in attachments { + let mime = attachment.content_type.as_deref().unwrap_or(""); + let filename = attachment.filename.as_deref().unwrap_or("attachment"); + let size = attachment.size.unwrap_or(0); + + if media::is_audio_mime(mime) { + if stt_config.enabled { + let mime_clean = mime.split(';').next().unwrap_or(mime).trim(); + if let Some(transcript) = media::download_and_transcribe( + &attachment.url, + filename, + mime_clean, + size, + &stt_config, + None, + ).await { + tracing::debug!(filename = %filename, chars = transcript.len(), "voice transcript injected"); + extra_blocks.insert(0, ContentBlock::Text { + text: format!("[Voice message transcript]: {transcript}"), + }); + } + } else { + tracing::warn!(filename = %filename, "skipping audio attachment (STT disabled)"); + } + } else if media::is_text_file(filename, attachment.content_type.as_deref()) { + if text_file_count >= TEXT_FILE_COUNT_CAP { + tracing::warn!(filename = %filename, count = text_file_count, "text file count cap reached, skipping"); + continue; + } + if text_file_bytes + size > TEXT_TOTAL_CAP { + tracing::warn!(filename = %filename, total = text_file_bytes, "text attachments total exceeds 1MB cap, skipping remaining"); + continue; + } + if let Some((block, actual_bytes)) = media::download_and_read_text_file( + &attachment.url, + filename, + size, + None, + ).await { + text_file_bytes += actual_bytes; + text_file_count += 1; + tracing::debug!(filename = %filename, "adding text file attachment"); + extra_blocks.push(block); + } + } else if let Some(block) = media::download_and_encode_image( + &attachment.url, + attachment.content_type.as_deref(), + filename, + size, + None, + ).await { + tracing::debug!(url = %attachment.url, filename = %filename, "adding image attachment"); + extra_blocks.push(block); + } else { + tracing::debug!(url = %attachment.url, filename = %filename, mime = %mime, "ignoring unsupported attachment type"); + extra_blocks.push(ContentBlock::Text { + text: format!("[User uploaded a file: {filename} ({mime})]"), + }); + } + } + + if prompt.is_empty() && extra_blocks.is_empty() { + return; + } + // If supergroup with no thread_id, create a forum topic let thread_channel = if event.channel.channel_type == "supergroup" && channel.thread_id.is_none() @@ -439,7 +528,7 @@ pub async fn run_gateway_adapter( &thread_channel, &sender_json, &prompt, - vec![], + extra_blocks, &trigger_msg, false, ) diff --git a/src/main.rs b/src/main.rs index 706fbfee..9c3cb930 100644 --- a/src/main.rs +++ b/src/main.rs @@ -226,6 +226,7 @@ async fn main() -> anyhow::Result<()> { allowed_channels: gw_cfg.allowed_channels, allow_all_users: config::resolve_allow_all(gw_cfg.allow_all_users, &gw_cfg.allowed_users), allowed_users: gw_cfg.allowed_users, + stt_config: cfg.stt.clone(), }; Some(tokio::spawn(async move { if let Err(e) = gateway::run_gateway_adapter(params, router, shutdown_rx).await { From 8ced19b000352e03dca36aa8ec18ab35df944d74 Mon Sep 17 00:00:00 2001 From: iamninihuang Date: Fri, 1 May 2026 03:19:13 +0800 Subject: [PATCH 2/2] style(gateway): polish attachment schema and visibility --- src/gateway.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/gateway.rs b/src/gateway.rs index e921f8c5..a6b95086 100644 --- a/src/gateway.rs +++ b/src/gateway.rs @@ -48,12 +48,11 @@ struct GwSender { } #[derive(Clone, Debug, Deserialize)] -pub struct GwAttachment { - pub url: String, - #[serde(rename = "contentType")] - pub content_type: Option, - pub filename: Option, - pub size: Option, +struct GwAttachment { + url: String, + content_type: Option, + filename: Option, + size: Option, } #[derive(Clone, Debug, Deserialize)] @@ -299,7 +298,6 @@ pub async fn run_gateway_adapter( let allowed_channels = params.allowed_channels; let allow_all_users = params.allow_all_users; let allowed_users = params.allowed_users; - let stt_config = params.stt_config; let connect_url = match ¶ms.token { Some(token) => { @@ -432,7 +430,7 @@ pub async fn run_gateway_adapter( let router = router.clone(); let prompt = event.content.text.clone().unwrap_or_default(); let attachments = event.content.attachments.clone(); - let stt_config = stt_config.clone(); + let stt_config = params.stt_config.clone(); tasks.spawn(async move { let mut extra_blocks = Vec::new();