imapc: Added support for imapc_features=modseq

If the remote server supports CONDSTORE or QRESYNC extensions we'll use the
remote's MODSEQ and HIGHESTMODSEQ counts.

There are some situations where the HIGHESTMODSEQ isn't updated exactly
correctly on an open mailbox, so this feature shouldn't be fully relied on.
It was primarily implemented for dsync+imapc support - both for preserving
modseqs and also for HIGHESTMODSEQ lookups.

src/lib-storage/index/imapc/imapc-mail.c

@@ -96,6 +96,26 @@ static int imapc_mail_failed(struct mail *mail, const char *field)
 	return fix_broken_mail ? 0 : -1;
 }
 
+static uint64_t imapc_mail_get_modseq(struct mail *_mail)
+{
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+	struct imapc_msgmap *msgmap;
+	const uint64_t *modseqs;
+	unsigned int count;
+	uint32_t rseq;
+
+	if (!imapc_storage_has_modseqs(mbox->storage))
+		return index_mail_get_modseq(_mail);
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	if (imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) {
+		modseqs = array_get(&mbox->rseq_modseqs, &count);
+		if (rseq <= count)
+			return modseqs[rseq-1];
+	}
+	return 1; /* unknown modseq */
+}
+
 static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
@@ -561,7 +581,7 @@ struct mail_vfuncs imapc_mail_vfuncs = {
 	index_mail_get_flags,
 	index_mail_get_keywords,
 	index_mail_get_keyword_indexes,
-	index_mail_get_modseq,
+	imapc_mail_get_modseq,
 	index_mail_get_pvt_modseq,
 	index_mail_get_parts,
 	index_mail_get_date,

src/lib-storage/index/imapc/imapc-mailbox.c

@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "ioloop.h"
+#include "mail-index-modseq.h"
 #include "imap-arg.h"
 #include "imap-seqset.h"
 #include "imap-util.h"
@@ -282,11 +283,12 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 	uint32_t lseq, rseq = reply->num;
 	struct imapc_fetch_request *const *fetch_requestp;
 	struct imapc_mail *const *mailp;
-	const struct imap_arg *list, *flags_list;
+	const struct imap_arg *list, *flags_list, *modseq_list;
 	const char *atom, *guid = NULL;
 	const struct mail_index_record *rec = NULL;
 	enum mail_flags flags;
 	uint32_t fetch_uid, uid;
+	uint64_t modseq = 0;
 	unsigned int i, j;
 	ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
 	bool seen_flags = FALSE, have_labels = FALSE;
@@ -319,6 +321,15 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 					array_append(&keywords, &atom, 1);
 				}
 			}
+		} else if (strcasecmp(atom, "MODSEQ") == 0 &&
+			   imapc_storage_has_modseqs(mbox->storage)) {
+			/* (modseq-number) */
+			if (!imap_arg_get_list(&list[i+1], &modseq_list))
+				return;
+			if (!imap_arg_get_atom(&modseq_list[0], &atom) ||
+			    str_to_uint64(atom, &modseq) < 0 ||
+			    modseq_list[1].type != IMAP_ARG_EOL)
+				return;
 		} else if (strcasecmp(atom, "X-GM-MSGID") == 0 &&
 			   !mbox->initial_sync_done) {
 			if (imap_arg_get_atom(&list[i+1], &atom))
@@ -414,6 +425,11 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 		}
 		mail_index_keywords_unref(&kw);
 	}
+	if (modseq != 0) {
+		if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < modseq)
+			mail_index_update_modseq(mbox->delayed_sync_trans, lseq, modseq);
+		array_idx_set(&mbox->rseq_modseqs, rseq-1, &modseq);
+	}
 	if (guid != NULL) {
 		struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
 		const enum index_cache_field guid_cache_idx =
@@ -454,6 +470,7 @@ static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
 	}
 	uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
 	imapc_msgmap_expunge(msgmap, rseq);
+	array_delete(&mbox->rseq_modseqs, rseq-1, 1);
 
 	imapc_mailbox_init_delayed_trans(mbox);
 	if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq))
@@ -578,6 +595,19 @@ imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply,
 	mbox->sync_uid_next = uid_next;
 }
 
+static void
+imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply,
+			      struct imapc_mailbox *mbox)
+{
+	uint64_t highestmodseq;
+
+	if (mbox == NULL ||
+	    str_to_uint64(reply->resp_text_value, &highestmodseq) < 0)
+		return;
+
+	mbox->sync_highestmodseq = highestmodseq;
+}
+
 static void
 imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply,
 			       struct imapc_mailbox *mbox)
@@ -646,6 +676,8 @@ void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox)
 					 imapc_resp_text_uidvalidity);
 	imapc_mailbox_register_resp_text(mbox, "UIDNEXT",
 					 imapc_resp_text_uidnext);
+	imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ",
+					 imapc_resp_text_highestmodseq);
 	imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS",
 					 imapc_resp_text_permanentflags);
 }

src/lib-storage/index/imapc/imapc-settings.c

@@ -93,6 +93,7 @@ static const struct imapc_feature_list imapc_feature_list[] = {
 	{ "proxyauth", IMAPC_FEATURE_PROXYAUTH },
 	{ "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS },
 	{ "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS },
+	{ "modseq", IMAPC_FEATURE_MODSEQ },
 	{ NULL, 0 }
 };
 

src/lib-storage/index/imapc/imapc-settings.h

@@ -14,7 +14,8 @@ enum imapc_features {
 	IMAPC_FEATURE_NO_EXAMINE		= 0x40,
 	IMAPC_FEATURE_PROXYAUTH			= 0x80,
 	IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS	= 0x100,
-	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200
+	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200,
+	IMAPC_FEATURE_MODSEQ			= 0x400
 };
 /* </settings checks> */
 

src/lib-storage/index/imapc/imapc-storage.c

@@ -55,6 +55,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
 				  struct imapc_storage_client *client);
 static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
 				     struct imapc_storage_client *client);
+static int imapc_mailbox_run_status(struct mailbox *box,
+				    enum mailbox_status_items items,
+				    struct mailbox_status *status_r);
 
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r)
 {
@@ -72,6 +75,16 @@ bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r)
 	return FALSE;
 }
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage)
+{
+	enum imapc_capability capa =
+		imapc_client_get_capabilities(storage->client->client);
+
+	return (capa & (IMAPC_CAPABILITY_CONDSTORE |
+			IMAPC_CAPABILITY_QRESYNC)) != 0 &&
+		IMAPC_HAS_FEATURE(storage, IMAPC_FEATURE_MODSEQ);
+}
+
 static struct mail_storage *imapc_storage_alloc(void)
 {
 	struct imapc_storage *storage;
@@ -603,6 +616,13 @@ static int imapc_mailbox_open(struct mailbox *box)
 		return -1;
 	}
 
+	if (imapc_storage_has_modseqs(mbox->storage)) {
+		if (!array_is_created(&mbox->rseq_modseqs))
+			i_array_init(&mbox->rseq_modseqs, 32);
+		else
+			array_clear(&mbox->rseq_modseqs);
+	}
+
 	if (imapc_mailbox_select(mbox) < 0) {
 		mailbox_close(box);
 		return -1;
@@ -635,6 +655,8 @@ static void imapc_mailbox_close(struct mailbox *box)
 		if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0)
 			mailbox_set_index_error(&mbox->box);
 	}
+	if (array_is_created(&mbox->rseq_modseqs))
+		array_free(&mbox->rseq_modseqs);
 	if (mbox->sync_view != NULL)
 		mail_index_view_close(&mbox->sync_view);
 	if (mbox->to_idle_delay != NULL)
@@ -727,6 +749,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
 			status->uidvalidity = num;
 		else if (strcasecmp(key, "UNSEEN") == 0)
 			status->unseen = num;
+		else if (strcasecmp(key, "HIGHESTMODSEQ") == 0 &&
+			 imapc_storage_has_modseqs(storage))
+			status->highest_modseq = num;
 	}
 }
 
@@ -765,15 +790,32 @@ static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
 	}
 }
 
-static void imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
-					      enum mailbox_status_items items,
-					      struct mailbox_status *status_r)
+static int imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
+					     enum mailbox_status_items items,
+					     struct mailbox_status *status_r)
 {
+	int ret = 0;
+
 	index_storage_get_open_status(&mbox->box, items, status_r);
 	if ((items & STATUS_PERMANENT_FLAGS) != 0)
 		status_r->permanent_flags = mbox->permanent_flags;
 	if ((items & STATUS_FIRST_RECENT_UID) != 0)
 		status_r->first_recent_uid = mbox->highest_nonrecent_uid + 1;
+	if ((items & STATUS_HIGHESTMODSEQ) != 0) {
+		/* FIXME: this doesn't work perfectly. we're now just returning
+		   the HIGHESTMODSEQ from the current index, which may or may
+		   not be correct. with QRESYNC enabled we could be returning
+		   sync_highestmodseq, but that would require implementing
+		   VANISHED replies. and without QRESYNC we'd have to issue
+		   STATUS (HIGHESTMODSEQ), which isn't efficient since we get
+		   here constantly (after every IMAP command). */
+	}
+	if (imapc_storage_has_modseqs(mbox->storage)) {
+		/* even if local indexes are only in memory, we still
+		   have modseqs on the IMAP server itself. */
+		status_r->nonpermanent_modseqs = FALSE;
+	}
+	return ret;
 }
 
 static int imapc_mailbox_delete(struct mailbox *box)
@@ -802,6 +844,9 @@ static int imapc_mailbox_run_status(struct mailbox *box,
 		str_append(str, " UIDVALIDITY");
 	if ((items & STATUS_UNSEEN) != 0)
 		str_append(str, " UNSEEN");
+	if ((items & STATUS_HIGHESTMODSEQ) != 0 &&
+	    imapc_storage_has_modseqs(mbox->storage))
+		str_append(str, " HIGHESTMODSEQ");
 
 	if (str_len(str) == 0) {
 		/* nothing requested */
@@ -833,14 +878,17 @@ static int imapc_mailbox_get_status(struct mailbox *box,
 		status_r->have_guids = TRUE;
 
 	if (box->opened) {
-		imapc_mailbox_get_selected_status(mbox, items, status_r);
+		if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0) {
+			/* can't do anything about this */
+		}
 	} else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS |
 			     STATUS_PERMANENT_FLAGS |
 			     STATUS_FIRST_RECENT_UID)) != 0) {
 		/* getting these requires opening the mailbox */
 		if (mailbox_open(box) < 0)
 			return -1;
-		imapc_mailbox_get_selected_status(mbox, items, status_r);
+		if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0)
+			return -1;
 	} else {
 		if (imapc_mailbox_run_status(box, items, status_r) < 0)
 			return -1;

src/lib-storage/index/imapc/imapc-storage.h

@@ -108,9 +108,11 @@ struct imapc_mailbox {
 	enum mail_flags permanent_flags;
 	uint32_t highest_nonrecent_uid;
 
+	ARRAY(uint64_t) rseq_modseqs;
 	ARRAY_TYPE(uint32_t) delayed_expunged_uids;
 	uint32_t sync_uid_validity;
 	uint32_t sync_uid_next;
+	uint64_t sync_highestmodseq;
 	uint32_t sync_fetch_first_uid;
 	uint32_t sync_next_lseq;
 	uint32_t sync_next_rseq;
@@ -165,6 +167,7 @@ void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox);
 void imapc_mail_cache_free(struct imapc_mail_cache *cache);
 int imapc_mailbox_select(struct imapc_mailbox *mbox);
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage);
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r);
 void imapc_copy_error_from_reply(struct imapc_storage *storage,
 				 enum mail_error default_error,

src/lib-storage/index/imapc/imapc-sync.c

@@ -5,6 +5,7 @@
 #include "str.h"
 #include "imap-util.h"
 #include "mail-cache.h"
+#include "mail-index-modseq.h"
 #include "index-sync-private.h"
 #include "imapc-client.h"
 #include "imapc-msgmap.h"
@@ -245,6 +246,13 @@ static void imapc_sync_uid_next(struct imapc_sync_context *ctx)
 	}
 }
 
+static void imapc_sync_highestmodseq(struct imapc_sync_context *ctx)
+{
+	if (imapc_storage_has_modseqs(ctx->mbox->storage) &&
+	    mail_index_modseq_get_highest(ctx->sync_view) < ctx->mbox->sync_highestmodseq)
+		mail_index_update_highest_modseq(ctx->trans, ctx->mbox->sync_highestmodseq);
+}
+
 static void
 imapc_initial_sync_check(struct imapc_sync_context *ctx, bool nooped)
 {
@@ -311,6 +319,10 @@ imapc_sync_send_commands(struct imapc_sync_context *ctx, uint32_t first_uid)
 	string_t *cmd = t_str_new(64);
 
 	str_printfa(cmd, "UID FETCH %u:* (FLAGS", first_uid);
+	if (imapc_storage_has_modseqs(ctx->mbox->storage)) {
+		str_append(cmd, " MODSEQ");
+		mail_index_modseq_enable(ctx->mbox->box.index);
+	}
 	if (IMAPC_BOX_HAS_FEATURE(ctx->mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
 		enum mailbox_info_flags flags;
 
@@ -393,8 +405,9 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
 		imapc_mailbox_run(mbox);
 	array_free(&ctx->expunged_uids);
 
-	/* add uidnext after all appends */
+	/* add uidnext & highestmodseq after all appends */
 	imapc_sync_uid_next(ctx);
+	imapc_sync_highestmodseq(ctx);
 
 	if (!ctx->failed)
 		imapc_sync_expunge_eom(ctx);