diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index ad30f5eac7..027840a1d4 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -11,7 +11,7 @@ import pytest from deltachat_rpc_client import Contact, EventType, Message, events -from deltachat_rpc_client.const import DownloadState, MessageState +from deltachat_rpc_client.const import MessageState from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS from deltachat_rpc_client.rpc import JsonRpcError diff --git a/src/context.rs b/src/context.rs index 93e5d0ce07..70172822bf 100644 --- a/src/context.rs +++ b/src/context.rs @@ -591,7 +591,7 @@ impl Context { convert_folder_meaning(self, folder_meaning).await? { connection - .fetch_move_delete(self, &mut session, &watch_folder, folder_meaning) + .fetch_move_delete(self, &mut session, true, &watch_folder, folder_meaning) .await?; } } @@ -605,6 +605,12 @@ impl Context { warn!(self, "Failed to update quota: {err:#}."); } } + + // OPTIONAL TODO: if time left start downloading messages + // while (msg = download_when_normal_starts) { + // if not time_left {break;} + // connection.download_message(msg) } + // } } info!( diff --git a/src/download.rs b/src/download.rs index 1cff3a3c66..1069c95da7 100644 --- a/src/download.rs +++ b/src/download.rs @@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize}; use crate::context::Context; use crate::imap::session::Session; -use crate::log::info; -use crate::message::{Message, MsgId}; +use crate::log::{info, warn}; +use crate::message::{self, Message, MsgId, rfc724_mid_exists}; use crate::{EventType, chatlist_events}; /// If a message is downloaded only partially @@ -26,8 +26,7 @@ pub(crate) const PRE_MESSAGE_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000; /// Max message size to be fetched in the background. /// This limit defines what messages are fully fetched in the background. /// This is for all messages that don't have the full message header. -#[allow(unused)] -pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000; +pub(crate) const MAX_FETCH_MSG_SIZE: u32 = 1_000_000; /// Max size for pre messages. A warning is emitted when this is exceeded. /// Should be well below `MAX_FETCH_MSG_SIZE` @@ -78,11 +77,17 @@ impl MsgId { } DownloadState::InProgress => return Err(anyhow!("Download already in progress.")), DownloadState::Available | DownloadState::Failure => { + if msg.rfc724_mid().is_empty() { + return Err(anyhow!("Download not possible, message has no rfc724_mid")); + } self.update_download_state(context, DownloadState::InProgress) .await?; context .sql - .execute("INSERT INTO download (msg_id) VALUES (?)", (self,)) + .execute( + "INSERT INTO download (msg_id) VALUES (?)", + (msg.rfc724_mid(),), + ) .await?; context.scheduler.interrupt_inbox().await; } @@ -131,25 +136,14 @@ impl Message { /// Most messages are downloaded automatically on fetch instead. pub(crate) async fn download_msg( context: &Context, - msg_id: MsgId, + rfc724_mid: String, session: &mut Session, ) -> Result<()> { - let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else { - // If partially downloaded message was already deleted - // we do not know its Message-ID anymore - // so cannot download it. - // - // Probably the message expired due to `delete_device_after` - // setting or was otherwise removed from the device, - // so we don't want it to reappear anyway. - return Ok(()); - }; - let row = context .sql .query_row_optional( "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''", - (&msg.rfc724_mid,), + (&rfc724_mid,), |row| { let server_uid: u32 = row.get(0)?; let server_folder: String = row.get(1)?; @@ -164,7 +158,7 @@ pub(crate) async fn download_msg( }; session - .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone()) + .fetch_single_msg(context, &server_folder, server_uid, rfc724_mid) .await?; Ok(()) } @@ -206,6 +200,142 @@ impl Session { } } +async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> { + if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? { + // Update download state to failure + // so it can be retried. + // + // On success update_download_state() is not needed + // as receive_imf() already + // set the state and emitted the event. + msg_id + .update_download_state(context, DownloadState::Failure) + .await?; + } + Ok(()) +} + +async fn available_full_msgs_contains_rfc724_mid( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(context + .sql + .query_get_value::( + "SELECT rfc724_mid FROM available_full_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await? + .is_some()) +} + +async fn remove_from_available_full_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute( + "DELETE FROM available_full_msgs WHERE rfc724_mid=?", + (&rfc724_mid,), + ) + .await?; + Ok(()) +} + +async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> { + context + .sql + .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,)) + .await?; + Ok(()) +} + +// this is a dedicated method because it is used in multiple places. +pub(crate) async fn premessage_is_downloaded_for( + context: &Context, + rfc724_mid: &str, +) -> Result { + Ok(message::rfc724_mid_exists(context, rfc724_mid) + .await? + .is_some()) +} + +pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM download", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + + for rfc724_mid in &rfc724_mids { + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err + ); + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // This is probably a classical email that vanished before we could download it + warn!( + context, + "{rfc724_mid} is probably a classical email that vanished before we could download it" + ); + remove_from_download_table(context, rfc724_mid).await?; + } else if available_full_msgs_contains_rfc724_mid(context, rfc724_mid).await? { + // set the message to DownloadState::Failure - probably it was deleted on the server in the meantime + set_msg_state_to_failed(context, rfc724_mid).await?; + remove_from_download_table(context, rfc724_mid).await?; + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } else { + // leave the message in DownloadState::InProgress; + // it will be downloaded once it arrives. + } + } + } + + Ok(()) +} + +/// Download known full messages without pre_message +/// in order to guard against lost pre-messages: +// TODO better fn name +pub(crate) async fn download_known_full_messages_without_pre_message( + context: &Context, + session: &mut Session, +) -> Result<()> { + let rfc724_mids = context + .sql + .query_map_vec("SELECT rfc724_mid FROM available_full_msgs", (), |row| { + let rfc724_mid: String = row.get(0)?; + Ok(rfc724_mid) + }) + .await?; + for rfc724_mid in &rfc724_mids { + if !premessage_is_downloaded_for(context, rfc724_mid).await? { + // Download the full-message unconditionally, + // because the pre-message got lost. + // The message may be in the wrong order, + // but at least we have it at all. + let res = download_msg(context, rfc724_mid.clone(), session).await; + if res.is_ok() { + remove_from_available_full_msgs_table(context, rfc724_mid).await?; + } + if let Err(err) = res { + warn!( + context, + "download_known_full_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.", + err + ); + } + } + } + Ok(()) +} + #[cfg(test)] mod tests { use mailparse::MailHeaderMap; diff --git a/src/headerdef.rs b/src/headerdef.rs index eca9bc2fb7..40a4a51899 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -102,10 +102,15 @@ pub enum HeaderDef { /// used to encrypt and decrypt messages. /// This secret is sent to a new member in the member-addition message. ChatBroadcastSecret, + /// This message announces a bigger message with attachment that is refereced by rfc724_mid. #[strum(serialize = "Chat-Full-Message-ID")] // correct casing ChatFullMessageId, + /// Announce full message attachment size inside of a pre-message. + #[strum(serialize = "Chat-Full-Message-Size")] // correct casing + ChatFullMessageSize, + /// This message has a pre-message /// and thus this message can be skipped while fetching messages. /// This is a cleartext / unproteced header. diff --git a/src/imap.rs b/src/imap.rs index 1258d944a0..519267211e 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -23,7 +23,6 @@ use num_traits::FromPrimitive; use ratelimit::Ratelimit; use url::Url; -use crate::calls::{create_fallback_ice_servers, create_ice_servers_from_metadata}; use crate::chat::{self, ChatId, ChatIdBlocked, add_device_msg}; use crate::chatlist_events; use crate::config::Config; @@ -48,6 +47,10 @@ use crate::tools::{self, create_id, duration_to_str, time}; use crate::transport::{ ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params, }; +use crate::{ + calls::{create_fallback_ice_servers, create_ice_servers_from_metadata}, + download::MAX_FETCH_MSG_SIZE, +}; pub(crate) mod capabilities; mod client; @@ -503,6 +506,7 @@ impl Imap { &mut self, context: &Context, session: &mut Session, + is_background_fetch: bool, watch_folder: &str, folder_meaning: FolderMeaning, ) -> Result<()> { @@ -512,7 +516,13 @@ impl Imap { } let msgs_fetched = self - .fetch_new_messages(context, session, watch_folder, folder_meaning) + .fetch_new_messages( + context, + session, + is_background_fetch, + watch_folder, + folder_meaning, + ) .await .context("fetch_new_messages")?; if msgs_fetched && context.get_config_delete_device_after().await?.is_some() { @@ -538,6 +548,7 @@ impl Imap { &mut self, context: &Context, session: &mut Session, + is_background_fetch: bool, folder: &str, folder_meaning: FolderMeaning, ) -> Result { @@ -565,7 +576,13 @@ impl Imap { let mut read_cnt = 0; loop { let (n, fetch_more) = self - .fetch_new_msg_batch(context, session, folder, folder_meaning) + .fetch_new_msg_batch( + context, + session, + is_background_fetch, + folder, + folder_meaning, + ) .await?; read_cnt += n; if !fetch_more { @@ -579,6 +596,7 @@ impl Imap { &mut self, context: &Context, session: &mut Session, + is_background_fetch: bool, folder: &str, folder_meaning: FolderMeaning, ) -> Result<(usize, bool)> { @@ -597,10 +615,22 @@ impl Imap { let read_cnt = msgs.len(); let mut uids_fetch = Vec::::with_capacity(msgs.len() + 1); + let mut available_full_msgs = Vec::::with_capacity(msgs.len()); + let mut download_when_normal_starts = Vec::::with_capacity(msgs.len()); let mut uid_message_ids = BTreeMap::new(); let mut largest_uid_skipped = None; let delete_target = context.get_delete_msgs_target().await?; + let download_limit = { + let download_limit: Option = + context.get_config_parsed(Config::DownloadLimit).await?; + if download_limit == Some(0) { + None + } else { + download_limit + } + }; + // Store the info about IMAP messages in the database. for (uid, ref fetch_response) in msgs { let headers = match get_fetch_headers(fetch_response) { @@ -612,6 +642,9 @@ impl Imap { }; let message_id = prefetch_get_message_id(&headers); + let size = fetch_response + .size + .context("imap fetch response does not contain size")?; // Determine the target folder where the message should be moved to. // @@ -679,8 +712,45 @@ impl Imap { ) .await.context("prefetch_should_download")? { - uids_fetch.push(uid); - uid_message_ids.insert(uid, message_id); + let fetch_now: bool = if headers + .get_header_value(HeaderDef::ChatIsFullMessage) + .is_some() + { + // This is a full-message + available_full_msgs.push(message_id.clone()); + + // whether it fits download size limit + if download_limit.is_none_or(|download_limit| size < download_limit) { + if is_background_fetch { + download_when_normal_starts.push(message_id.clone()); + false + } else { + true + } + } else { + false + } + } else { + // This is not a full message + if is_background_fetch { + if size < MAX_FETCH_MSG_SIZE { + // may be a pre-message or a pure-text message, fetch now + true + } else { + // This is e.g. a classical email + // Queue for full download, in order to prevent missing messages + download_when_normal_starts.push(message_id.clone()); + false + } + } else { + true + } + }; + + if fetch_now { + uids_fetch.push(uid); + uid_message_ids.insert(uid, message_id); + } } else { largest_uid_skipped = Some(uid); } @@ -752,6 +822,25 @@ impl Imap { chat::mark_old_messages_as_noticed(context, received_msgs).await?; + // TODO: is there correct place for this? + if fetch_res.is_ok() { + for rfc724_mid in available_full_msgs { + context + .sql + .insert("INSERT INTO available_full_msgs VALUES (?)", (rfc724_mid,)) + .await?; + } + for rfc724_mid in download_when_normal_starts { + context + .sql + .insert( + "INSERT INTO download (rfc724_mid) VALUES (?)", + (rfc724_mid,), + ) + .await?; + } + } + // Now fail if fetching failed, so we will // establish a new session if this one is broken. fetch_res?; diff --git a/src/imap/scan_folders.rs b/src/imap/scan_folders.rs index d3208c8158..72f142e06f 100644 --- a/src/imap/scan_folders.rs +++ b/src/imap/scan_folders.rs @@ -76,7 +76,7 @@ impl Imap { && folder_meaning != FolderMeaning::Trash && folder_meaning != FolderMeaning::Unknown { - self.fetch_move_delete(context, session, folder.name(), folder_meaning) + self.fetch_move_delete(context, session, false, folder.name(), folder_meaning) .await .context("Can't fetch new msgs in scanned folder") .log_err(context) diff --git a/src/imap/session.rs b/src/imap/session.rs index a633974d4b..f426c25a26 100644 --- a/src/imap/session.rs +++ b/src/imap/session.rs @@ -17,6 +17,7 @@ use crate::tools; /// - Chat-Version to check if a message is a chat message /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message, /// not necessarily sent by Delta Chat. +/// - Chat-Is-Full-Message to skip it in background fetch or when it is too large const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\ MESSAGE-ID \ DATE \ @@ -24,6 +25,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE FROM \ IN-REPLY-TO REFERENCES \ CHAT-VERSION \ + CHAT-IS-FULL-MESSAGE \ AUTO-SUBMITTED \ AUTOCRYPT-SETUP-MESSAGE\ )])"; diff --git a/src/message.rs b/src/message.rs index b42b162fcb..3b04e5fcaa 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1671,9 +1671,18 @@ pub async fn delete_msgs_ex( let update_db = |trans: &mut rusqlite::Transaction| { trans.execute( "UPDATE imap SET target=? WHERE rfc724_mid=?", - (target, msg.rfc724_mid), + (target, &msg.rfc724_mid), )?; trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?; + trans.execute( + "DELETE FROM download WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; + // TODO: is the following nessesary? + trans.execute( + "DELETE FROM available_full_msgs WHERE rfc724_mid=?", + (&msg.rfc724_mid,), + )?; Ok(()) }; if let Err(e) = context.sql.transaction(update_db).await { diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 355b6e8d49..ee4dbe19a3 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1878,6 +1878,15 @@ impl MimeFactory { // add attachment part if msg.viewtype.has_file() { if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode { + let attachment_size = msg + .get_filebytes(context) + .await? + .context("attachment exists, but get_filebytes returned nothing")? + .to_string(); + headers.push(( + HeaderDef::ChatFullMessageSize.get_headername(), + mail_builder::headers::raw::Raw::new(attachment_size).into(), + )); // TODO: generate thumbnail and attach it instead (if it makes sense) } else { let file_part = build_body_file(context, &msg).await?; diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 3698db2932..143ad43190 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -147,6 +147,23 @@ pub(crate) struct MimeMessage { /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized /// clocks, but not too much. pub(crate) timestamp_sent: i64, + + pub(crate) pre_message: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum PreMessageMode { + /// This is full messages + /// it replaces it's pre-message attachment if it exists already, + /// and if the pre-message does not exist it is treated as normal message + FullMessage, + /// This is a pre-message, + /// it adds a message preview for a full message + /// and it is ignored if the full message was downloaded already + PreMessage { + full_msg_rfc724_mid: String, + attachment_size: u64, + }, } #[derive(Debug, PartialEq)] @@ -239,6 +256,9 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup"; impl MimeMessage { /// Parse a mime message. + /// + /// This method has some side-effects, + /// such as saving blobs and saving found public keys to the database. pub(crate) async fn from_bytes(context: &Context, body: &[u8]) -> Result { let mail = mailparse::parse_mail(body)?; @@ -346,6 +366,29 @@ impl MimeMessage { let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into()); + let pre_message = if let Some(full_msg_rfc724_mid) = + mail.headers.get_header_value(HeaderDef::ChatFullMessageId) + { + let attachment_size: u64 = mail + .headers + .get_header_value(HeaderDef::ChatFullMessageSize) + .unwrap_or_default() + .parse() + .unwrap_or_default(); + Some(PreMessageMode::PreMessage { + full_msg_rfc724_mid, + attachment_size, + }) + } else if mail + .headers + .get_header_value(HeaderDef::ChatIsFullMessage) + .is_some() + { + Some(PreMessageMode::FullMessage) + } else { + None + }; + let mail_raw; // Memory location for a possible decrypted message. let decrypted_msg; // Decrypted signed OpenPGP message. let secrets: Vec = context @@ -609,6 +652,7 @@ impl MimeMessage { is_bot: None, timestamp_rcvd, timestamp_sent, + pre_message, }; match mail { diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 460736d4fe..1708219a06 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -20,7 +20,7 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX, use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc_inner; -use crate::download::DownloadState; +use crate::download::{DownloadState, premessage_is_downloaded_for}; use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed}; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; @@ -29,7 +29,6 @@ use crate::key::{DcKey, Fingerprint}; use crate::key::{self_fingerprint, self_fingerprint_opt}; use crate::log::LogExt; use crate::log::{info, warn}; -use crate::logged_debug_assert; use crate::message::{ self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists, }; @@ -45,6 +44,7 @@ use crate::stock_str; use crate::sync::Sync::*; use crate::tools::{self, buf_compress, remove_subject_prefix, validate_broadcast_secret}; use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location}; +use crate::{logged_debug_assert, mimeparser}; /// This is the struct that is returned after receiving one email (aka MIME message). /// @@ -1087,6 +1087,38 @@ async fn decide_chat_assignment( { info!(context, "Chat edit/delete/iroh/sync message (TRASH)."); true + } else if let Some(pre_message) = &mime_parser.pre_message { + use crate::mimeparser::PreMessageMode::*; + match pre_message { + FullMessage => { + // if pre message exist, then trash after replacing, otherwise treat as normal message + let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?; + info!( + context, + "Message is a Full Message ({}).", + if pre_message_exists { + "pre-message exists already, so trash after replacing attachment" + } else { + "no pre-message -> Keep" + } + ); + pre_message_exists + } + PreMessage { + full_msg_rfc724_mid, + .. + } => { + // if full message already exists, then trash/ignore + let full_msg_exists = + premessage_is_downloaded_for(context, full_msg_rfc724_mid).await?; + info!( + context, + "Message is a Pre-Message (full_msg_exists:{full_msg_exists})." + ); + full_msg_exists + // TODO find out if trashing affects multi device usage? + } + } } else if mime_parser.is_system_message == SystemMessage::CallAccepted || mime_parser.is_system_message == SystemMessage::CallEnded { @@ -1900,6 +1932,7 @@ async fn add_parts( } handle_edit_delete(context, mime_parser, from_id).await?; + handle_full_message(context, mime_parser, from_id).await?; if mime_parser.is_system_message == SystemMessage::CallAccepted || mime_parser.is_system_message == SystemMessage::CallEnded @@ -2033,7 +2066,11 @@ RETURNING id sort_timestamp, if trash { 0 } else { mime_parser.timestamp_sent }, if trash { 0 } else { mime_parser.timestamp_rcvd }, - if trash { Viewtype::Unknown } else { typ }, + if trash { + Viewtype::Unknown + } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message { + Viewtype::Text + } else { typ }, if trash { MessageState::Undefined } else { state }, if trash { MessengerMessage::No } else { is_dc_message }, if trash || hidden { "" } else { msg }, @@ -2045,7 +2082,16 @@ RETURNING id param.to_string() }, !trash && hidden, - if trash { 0 } else { part.bytes as isize }, + if trash { + 0 + } else if let Some(mimeparser::PreMessageMode::PreMessage { + attachment_size, + .. + }) = mime_parser.pre_message { + attachment_size as isize + } else { + part.bytes as isize + }, if save_mime_modified && !(trash || hidden) { mime_headers.clone() } else { @@ -2253,6 +2299,76 @@ async fn handle_edit_delete( Ok(()) } +async fn handle_full_message( + context: &Context, + mime_parser: &MimeMessage, + from_id: ContactId, +) -> Result<()> { + if let Some(mimeparser::PreMessageMode::FullMessage) = &mime_parser.pre_message { + // if pre message exist, replace attachment + // only replacing attachment ensures that doesn't overwrite the text if it was edited before. + let rfc724_mid = mime_parser + .get_rfc724_mid() + .context("expected full message to have a message id")?; + + let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else { + warn!( + context, + "Download Full-Message: Database entry does not exist." + ); + return Ok(()); + }; + let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else { + // else: message is processed like a normal message + warn!( + context, + "Download Full-Message: pre message was not downloaded, yet so treat as normal message" + ); + return Ok(()); + }; + + if original_msg.from_id != from_id { + warn!(context, "Download Full-Message: Bad sender."); + return Ok(()); + } + if let Some(part) = mime_parser.parts.first() { + if !part.typ.has_file() { + warn!( + context, + "Download Full-Message: First mime part's message-viewtype has no file" + ); + return Ok(()); + } + + let edit_msg_showpadlock = part + .param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or_default(); + if edit_msg_showpadlock || !original_msg.get_showpadlock() { + context + .sql + .execute( + "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?", + ( + part.param.to_string(), + part.typ, + part.bytes as isize, + part.error.as_deref().unwrap_or_default(), + DownloadState::Done as u32, + original_msg.id, + ), + ) + .await?; + context.emit_msgs_changed(original_msg.chat_id, original_msg.id); + } else { + warn!(context, "Download Full-Message: Not encrypted."); + } + } + } + + Ok(()) +} + async fn tweak_sort_timestamp( context: &Context, mime_parser: &mut MimeMessage, diff --git a/src/scheduler.rs b/src/scheduler.rs index 2a3537daf4..0a064c415c 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -16,13 +16,12 @@ pub(crate) use self::connectivity::ConnectivityStore; use crate::config::{self, Config}; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; -use crate::download::{DownloadState, download_msg}; +use crate::download::{download_known_full_messages_without_pre_message, download_msgs}; use crate::ephemeral::{self, delete_expired_imap_messages}; use crate::events::EventType; use crate::imap::{FolderMeaning, Imap, session::Session}; use crate::location; use crate::log::{LogExt, error, info, warn}; -use crate::message::MsgId; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; use crate::stats::maybe_send_stats; @@ -345,38 +344,6 @@ pub(crate) struct Scheduler { recently_seen_loop: RecentlySeenLoop, } -async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> { - let msg_ids = context - .sql - .query_map_vec("SELECT msg_id FROM download", (), |row| { - let msg_id: MsgId = row.get(0)?; - Ok(msg_id) - }) - .await?; - - for msg_id in msg_ids { - if let Err(err) = download_msg(context, msg_id, session).await { - warn!(context, "Failed to download message {msg_id}: {:#}.", err); - - // Update download state to failure - // so it can be retried. - // - // On success update_download_state() is not needed - // as receive_imf() already - // set the state and emitted the event. - msg_id - .update_download_state(context, DownloadState::Failure) - .await?; - } - context - .sql - .execute("DELETE FROM download WHERE msg_id=?", (msg_id,)) - .await?; - } - - Ok(()) -} - async fn inbox_loop( ctx: Context, started: oneshot::Sender<()>, @@ -596,7 +563,7 @@ async fn fetch_idle( { // Fetch the watched folder. connection - .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning) + .fetch_move_delete(ctx, &mut session, true, &watch_folder, folder_meaning) .await .context("fetch_move_delete")?; @@ -607,6 +574,11 @@ async fn fetch_idle( delete_expired_imap_messages(ctx) .await .context("delete_expired_imap_messages")?; + + //------- + // TODO: verify that this is the correct position for this call + // in order to guard against lost pre-messages: + download_known_full_messages_without_pre_message(ctx, &mut session).await?; } else if folder_config == Config::ConfiguredInboxFolder { session.last_full_folder_scan.lock().await.take(); } @@ -635,7 +607,7 @@ async fn fetch_idle( // no new messages. We want to select the watched folder anyway before going IDLE // there, so this does not take additional protocol round-trip. connection - .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning) + .fetch_move_delete(ctx, &mut session, true, &watch_folder, folder_meaning) .await .context("fetch_move_delete after scan_folders")?; } diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index f2e0bbb291..8ae8661145 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1339,6 +1339,26 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); .await?; } + inc_and_check(&mut migration_version, 139)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE download_new ( + rfc724_mid TEXT NOT NULL + ); + INSERT INTO download_new (rfc724_mid) + SELECT m.rfc724_mid FROM download d + JOIN msgs m ON d.msg_id = m.id + WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != ''; + DROP TABLE download; + ALTER TABLE download_new RENAME TO download; + CREATE TABLE available_full_msgs ( + rfc724_mid TEXT NOT NULL + );", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await?