From 033801208edad3da7963dfe919250d1bd7dc1e31 Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Thu, 31 May 2018 16:52:54 +0300 Subject: [PATCH] imapc: Don't use RFC822.SIZE values for message body size calculation If the RFC822.SIZE doesn't match the exact stream size, the body size calculation will become wrong. The only downside to this patch is that now the body size will need to be calculated by parsing the stream, but there shouldn't be any need to do that unless the body was already FETCHed, so it shouldn't cause any extra IMAP traffic. --- src/lib-storage/index/imapc/imapc-mail-fetch.c | 7 ++++++- src/lib-storage/index/imapc/imapc-mail.c | 10 +++++++++- src/lib-storage/index/index-mail.c | 14 ++++++++++++-- src/lib-storage/index/index-mail.h | 3 +++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c index 496026ad97..f6b3d1c29f 100644 --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c @@ -589,6 +589,8 @@ void imapc_mail_init_stream(struct imapc_mail *mail) smaller than the fetched message header. In this case change the size as well, otherwise reading via istream-mail will fail. */ if (mail->body_fetched || imail->data.physical_size < size) { + if (mail->body_fetched) + imail->data.inexact_total_sizes = FALSE; imail->data.physical_size = size; /* we'll assume that the remote server is working properly and sending CRLF linefeeds */ @@ -839,8 +841,11 @@ void imapc_mail_fetch_update(struct imapc_mail *mail, } else if (strcasecmp(key, "RFC822.SIZE") == 0) { if (imap_arg_get_atom(&args[i+1], &value) && str_to_uoff(value, &size) == 0 && - IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) { mail->imail.data.physical_size = size; + mail->imail.data.virtual_size = size; + mail->imail.data.inexact_total_sizes = TRUE; + } match = TRUE; } else if (strcasecmp(key, "X-GM-MSGID") == 0 || strcasecmp(key, "X-GUID") == 0) { diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c index 002d709b7d..4c730dd97a 100644 --- a/src/lib-storage/index/imapc/imapc-mail.c +++ b/src/lib-storage/index/imapc/imapc-mail.c @@ -169,7 +169,9 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) && data->stream == NULL) { - /* trust RFC822.SIZE to be correct */ + /* Trust RFC822.SIZE to be correct enough to present to the + IMAP client. However, it can be wrong in some implementation + so try not to trust it too much. */ if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0) return -1; if (data->physical_size == (uoff_t)-1) { @@ -406,8 +408,14 @@ static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving) { struct imapc_mail *imail = IMAPC_MAIL(_mail); struct index_mail *mail = &imail->imail; + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; index_mail_set_seq(_mail, seq, saving); + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) { + /* RFC822.SIZE may be read from vsize record or cache. It may + not be exactly correct. */ + mail->data.inexact_total_sizes = TRUE; + } /* searching code handles prefetching internally, elsewhere we want to do it immediately */ diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c index 0a6e9e05a8..52d74d4c19 100644 --- a/src/lib-storage/index/index-mail.c +++ b/src/lib-storage/index/index-mail.c @@ -466,9 +466,14 @@ static void index_mail_try_set_body_size(struct index_mail *mail) { struct index_mail_data *data = &mail->data; - if (data->hdr_size_set && + if (data->hdr_size_set && !data->inexact_total_sizes && data->physical_size != (uoff_t)-1 && data->virtual_size != (uoff_t)-1) { + /* We know the total size of this mail and we know the + header size, so we can calculate also the body size. + However, don't do this if there's a possibility that + physical_size or virtual_size don't actually match the + mail stream's size (e.g. buggy imapc servers). */ data->body_size.physical_size = data->physical_size - data->hdr_size.physical_size; data->body_size.virtual_size = data->virtual_size - @@ -1230,7 +1235,7 @@ int index_mail_init_stream(struct index_mail *mail, struct mail *_mail = &mail->mail.mail; struct index_mail_data *data = &mail->data; struct istream *input; - bool has_nuls; + bool has_nuls, body_size_from_stream = FALSE; int ret; if (mail->mail.get_stream_reason != NULL && @@ -1301,6 +1306,7 @@ int index_mail_init_stream(struct index_mail *mail, } data->body_size_set = TRUE; } + body_size_from_stream = TRUE; } *body_size = data->body_size; @@ -1311,6 +1317,10 @@ int index_mail_init_stream(struct index_mail *mail, data->body_size.virtual_size; data->physical_size = data->hdr_size.physical_size + data->body_size.physical_size; + if (body_size_from_stream) { + /* the sizes were just calculated */ + data->inexact_total_sizes = FALSE; + } } ret = index_mail_stream_check_failure(mail); diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h index 2cd80c6aee..8053704df2 100644 --- a/src/lib-storage/index/index-mail.h +++ b/src/lib-storage/index/index-mail.h @@ -129,6 +129,9 @@ struct index_mail_data { bool destroy_callback_set:1; bool prefetch_sent:1; bool header_parser_initialized:1; + /* virtual_size and physical_size may not match the stream size. + Try to avoid trusting them too much. */ + bool inexact_total_sizes:1; }; struct index_mail {