imapc: Don't use RFC822.SIZE values for message body size calculation

If the RFC822.SIZE doesn't match the exact stream size, the body size
calculation will become wrong. The only downside to this patch is that
now the body size will need to be calculated by parsing the stream, but
there shouldn't be any need to do that unless the body was already
FETCHed, so it shouldn't cause any extra IMAP traffic.

src/lib-storage/index/imapc/imapc-mail-fetch.c

@@ -589,6 +589,8 @@ void imapc_mail_init_stream(struct imapc_mail *mail)
 	   smaller than the fetched message header. In this case change the
 	   size as well, otherwise reading via istream-mail will fail. */
 	if (mail->body_fetched || imail->data.physical_size < size) {
+		if (mail->body_fetched)
+			imail->data.inexact_total_sizes = FALSE;
 		imail->data.physical_size = size;
 		/* we'll assume that the remote server is working properly and
 		   sending CRLF linefeeds */
@@ -839,8 +841,11 @@ void imapc_mail_fetch_update(struct imapc_mail *mail,
 		} else if (strcasecmp(key, "RFC822.SIZE") == 0) {
 			if (imap_arg_get_atom(&args[i+1], &value) &&
 			    str_to_uoff(value, &size) == 0 &&
-			    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE))
+			    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
 				mail->imail.data.physical_size = size;
+				mail->imail.data.virtual_size = size;
+				mail->imail.data.inexact_total_sizes = TRUE;
+			}
 			match = TRUE;
 		} else if (strcasecmp(key, "X-GM-MSGID") == 0 ||
 			   strcasecmp(key, "X-GUID") == 0) {

src/lib-storage/index/imapc/imapc-mail.c

@@ -169,7 +169,9 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
 
 	if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) &&
 	    data->stream == NULL) {
-		/* trust RFC822.SIZE to be correct */
+		/* Trust RFC822.SIZE to be correct enough to present to the
+		   IMAP client. However, it can be wrong in some implementation
+		   so try not to trust it too much. */
 		if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0)
 			return -1;
 		if (data->physical_size == (uoff_t)-1) {
@@ -406,8 +408,14 @@ static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving)
 {
 	struct imapc_mail *imail = IMAPC_MAIL(_mail);
 	struct index_mail *mail = &imail->imail;
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
 
 	index_mail_set_seq(_mail, seq, saving);
+	if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) {
+		/* RFC822.SIZE may be read from vsize record or cache. It may
+		   not be exactly correct. */
+		mail->data.inexact_total_sizes = TRUE;
+	}
 
 	/* searching code handles prefetching internally,
 	   elsewhere we want to do it immediately */

src/lib-storage/index/index-mail.c

@@ -466,9 +466,14 @@ static void index_mail_try_set_body_size(struct index_mail *mail)
 {
 	struct index_mail_data *data = &mail->data;
 
-	if (data->hdr_size_set &&
+	if (data->hdr_size_set && !data->inexact_total_sizes &&
 	    data->physical_size != (uoff_t)-1 &&
 	    data->virtual_size != (uoff_t)-1) {
+		/* We know the total size of this mail and we know the
+		   header size, so we can calculate also the body size.
+		   However, don't do this if there's a possibility that
+		   physical_size or virtual_size don't actually match the
+		   mail stream's size (e.g. buggy imapc servers). */
 		data->body_size.physical_size = data->physical_size -
 			data->hdr_size.physical_size;
 		data->body_size.virtual_size = data->virtual_size -
@@ -1230,7 +1235,7 @@ int index_mail_init_stream(struct index_mail *mail,
 	struct mail *_mail = &mail->mail.mail;
 	struct index_mail_data *data = &mail->data;
 	struct istream *input;
-	bool has_nuls;
+	bool has_nuls, body_size_from_stream = FALSE;
 	int ret;
 
 	if (mail->mail.get_stream_reason != NULL &&
@@ -1301,6 +1306,7 @@ int index_mail_init_stream(struct index_mail *mail,
 				}
 				data->body_size_set = TRUE;
 			}
+			body_size_from_stream = TRUE;
 		}
 
 		*body_size = data->body_size;
@@ -1311,6 +1317,10 @@ int index_mail_init_stream(struct index_mail *mail,
 			data->body_size.virtual_size;
 		data->physical_size = data->hdr_size.physical_size +
 			data->body_size.physical_size;
+		if (body_size_from_stream) {
+			/* the sizes were just calculated */
+			data->inexact_total_sizes = FALSE;
+		}
 	}
 	ret = index_mail_stream_check_failure(mail);
 

src/lib-storage/index/index-mail.h

@@ -129,6 +129,9 @@ struct index_mail_data {
 	bool destroy_callback_set:1;
 	bool prefetch_sent:1;
 	bool header_parser_initialized:1;
+	/* virtual_size and physical_size may not match the stream size.
+	   Try to avoid trusting them too much. */
+	bool inexact_total_sizes:1;
 };
 
 struct index_mail {