diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
index 8651483615..edfcf7dcc1 100644
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -368,6 +368,15 @@ imapc_mail_get_wanted_fetch_fields(struct imapc_mail *mail)
 	return fields;
 }
 
+void imapc_mail_try_init_stream_from_cache(struct imapc_mail *mail)
+{
+	struct mail *_mail = &mail->imail.mail.mail;
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+
+	if (mbox->prev_mail_cache.uid == _mail->uid)
+		imapc_mail_cache_get(mail, &mbox->prev_mail_cache);
+}
+
 bool imapc_mail_prefetch(struct mail *_mail)
 {
 	struct imapc_mail *mail = (struct imapc_mail *)_mail;
@@ -376,10 +385,13 @@ bool imapc_mail_prefetch(struct mail *_mail)
 	enum mail_fetch_field fields;
 	const char *const *headers = NULL;
 
-	if (mbox->prev_mail_cache.uid == _mail->uid)
-		imapc_mail_cache_get(mail, &mbox->prev_mail_cache);
 	/* try to get as much from cache as possible */
 	imapc_mail_update_access_parts(&mail->imail);
+	/* If mail is already cached we can avoid re-FETCHing the mail.
+	   However, don't initialize the stream if we don't actually want to
+	   access the mail. */
+	if (mail->imail.data.access_part != 0)
+		imapc_mail_try_init_stream_from_cache(mail);
 
 	fields = imapc_mail_get_wanted_fetch_fields(mail);
 	if (data->wanted_headers != NULL && data->stream == NULL &&
diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 1bbb313719..de2ad35e5e 100644
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -302,6 +302,10 @@ imapc_mail_get_stream(struct mail *_mail, bool get_body,
 		data->hdr_size_set = FALSE;
 	}
 
+	/* See if we can get it from cache. If the wanted_fields/headers are
+	   set properly, this is usually already done by prefetching. */
+	imapc_mail_try_init_stream_from_cache(mail);
+
 	if (data->stream == NULL) {
 		if (!data->initialized) {
 			/* coming here from mail_set_seq() */
diff --git a/src/lib-storage/index/imapc/imapc-mail.h b/src/lib-storage/index/imapc/imapc-mail.h
index ddc06ffcbf..0a7611b40e 100644
--- a/src/lib-storage/index/imapc/imapc-mail.h
+++ b/src/lib-storage/index/imapc/imapc-mail.h
@@ -33,6 +33,7 @@ imapc_mail_alloc(struct mailbox_transaction_context *t,
 		 struct mailbox_header_lookup_ctx *wanted_headers);
 int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields,
 		     const char *const *headers);
+void imapc_mail_try_init_stream_from_cache(struct imapc_mail *mail);
 bool imapc_mail_prefetch(struct mail *mail);
 void imapc_mail_fetch_flush(struct imapc_mailbox *mbox);
 void imapc_mail_init_stream(struct imapc_mail *mail);