imapc: Added imapc_features=fetch-fix-broken-mails

This allows ignoring a missing FETCH reply. It shouldn't be used normally,
but if there's a mail that the IMAP server simply won't serve, this can be
used to finish dsync successfully.

src/lib-storage/index/imapc/imapc-mail.c

@@ -59,28 +59,35 @@ static bool imapc_mail_is_expunged(struct mail *_mail)
 	return !imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq);
 }
 
-static void imapc_mail_failed(struct mail *mail, const char *field)
+static int imapc_mail_failed(struct mail *mail, const char *field)
 {
 	struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box;
+	bool fix_broken_mail = FALSE;
 
 	if (mail->expunged || imapc_mail_is_expunged(mail)) {
 		mail_set_expunged(mail);
 	} else if (!imapc_client_mailbox_is_opened(mbox->client_box)) {
 		/* we've already logged a disconnection error */
 		mail_storage_set_internal_error(mail->box->storage);
 	} else {
-		/* NOTE: earlier we didn't treat this as a failure, because
-		   old Exchange versions fail to return any data for messages
-		   in Calendars mailbox. But it's a bad idea to always assume
-		   that a missing field is intentional, because there's
-		   potential for data loss. Ideally we could detect whether
-		   this is an Exchange issue or not, but I don't have access
-		   to such an old Exchange anymore. So at least for now until
-		   someone complains, the Exchange workaround is disabled. */
+		/* By default we'll assume that this is a critical failure,
+		   because we don't want to lose any data. We can be here
+		   either because it's a temporary failure on the server or
+		   it's a permanent failure. Unfortunately we can't know
+		   which case it is, so permanent failures need to be worked
+		   around by setting imapc_features=fetch-fix-broken-mails.
+
+		   One reason for permanent failures was that earlier Exchange
+		   versions failed to return any data for messages in Calendars
+		   mailbox. This seems to be fixed in newer versions.
+		   */
+		fix_broken_mail = IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS);
 		mail_storage_set_critical(mail->box->storage,
-			"imapc: Remote server didn't send %s for UID %u in %s",
-			field, mail->uid, mail->box->vname);
+			"imapc: Remote server didn't send %s for UID %u in %s%s",
+			field, mail->uid, mail->box->vname,
+			fix_broken_mail ? " - treating it as empty" : "");
 	}
+	return fix_broken_mail ? 0 : -1;
 }
 
 static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
@@ -95,8 +102,11 @@ static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
 		if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE, NULL) < 0)
 			return -1;
 		if (data->received_date == (time_t)-1) {
-			imapc_mail_failed(_mail, "INTERNALDATE");
-			return -1;
+			if (imapc_mail_failed(_mail, "INTERNALDATE") < 0)
+				return -1;
+			/* assume that the server never returns INTERNALDATE
+			   for this mail (see BODY[] failure handling) */
+			data->received_date = 0;
 		}
 	}
 	*date_r = data->received_date;
@@ -138,8 +148,11 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r)
 		if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE, NULL) < 0)
 			return -1;
 		if (data->physical_size == (uoff_t)-1) {
-			imapc_mail_failed(_mail, "RFC822.SIZE");
-			return -1;
+			if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0)
+				return -1;
+			/* assume that the server never returns RFC822.SIZE
+			   for this mail (see BODY[] failure handling) */
+			data->physical_size = 0;
 		}
 		*size_r = data->physical_size;
 		return 0;
@@ -271,8 +284,14 @@ imapc_mail_get_stream(struct mail *_mail, bool get_body,
 			return -1;
 
 		if (data->stream == NULL) {
-			imapc_mail_failed(_mail, "BODY[]");
-			return -1;
+			if (imapc_mail_failed(_mail, "BODY[]"))
+				return -1;
+			i_assert(data->stream == NULL);
+
+			/* return the broken email as empty */
+			mail->body_fetched = TRUE;
+			data->stream = i_stream_create_from_data(NULL, 0);
+			imapc_mail_init_stream(mail);
 		}
 	}
 
@@ -471,7 +490,7 @@ static int imapc_mail_get_guid(struct mail *_mail, const char **value_r)
 		if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID, NULL) < 0)
 			return -1;
 		if (imail->data.guid == NULL) {
-			imapc_mail_failed(_mail, mbox->guid_fetch_field_name);
+			(void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name);
 			return -1;
 		}
 	} else {

src/lib-storage/index/imapc/imapc-settings.c

@@ -92,6 +92,7 @@ static const struct imapc_feature_list imapc_feature_list[] = {
 	{ "no-examine", IMAPC_FEATURE_NO_EXAMINE },
 	{ "proxyauth", IMAPC_FEATURE_PROXYAUTH },
 	{ "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS },
+	{ "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS },
 	{ NULL, 0 }
 };
 

src/lib-storage/index/imapc/imapc-settings.h

@@ -13,7 +13,8 @@ enum imapc_features {
 	IMAPC_FEATURE_ZIMBRA_WORKAROUNDS	= 0x20,
 	IMAPC_FEATURE_NO_EXAMINE		= 0x40,
 	IMAPC_FEATURE_PROXYAUTH			= 0x80,
-	IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS	= 0x100
+	IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS	= 0x100,
+	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200
 };
 /* </settings checks> */