Skip to content

Commit

Permalink
pop3-migration: Try to assign UIDLs based on dovecot.index.cache
Browse files Browse the repository at this point in the history 
Add the UIDLs to (imapc) mailbox cache after they've been assigned.
Try to read them from the cache on the next sync to avoid reading
mails' headers.

This can be disabled with pop3_migration_skip_uidl_cache=yes, just in case
there's some kind of a problem with it.
  • Loading branch information
@sirainen
sirainen committed Aug 25, 2017
1 parent f7590c1 commit 5af0c9f
Showing 1 changed file with 126 additions and 6 deletions.
132 changes: 126 additions & 6 deletions src/plugins/pop3-migration/pop3-migration-plugin.c
View file
Expand Up @@ -57,6 +57,7 @@ struct pop3_migration_mail_storage {
unsigned int pop3_all_hdr_sha1_set:1;
unsigned int ignore_missing_uidls:1;
unsigned int skip_size_check:1;
unsigned int skip_uidl_cache:1;
};

struct pop3_migration_mailbox {
Expand Down Expand Up @@ -113,6 +114,18 @@ static int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1,
return 0;
}

static int pop3_uidl_map_uidl_cmp(const struct pop3_uidl_map *map1,
const struct pop3_uidl_map *map2)
{
return strcmp(map1->pop3_uidl, map2->pop3_uidl);
}

static int imap_msg_map_uidl_cmp(const struct imap_msg_map *map1,
const struct imap_msg_map *map2)
{
return null_strcmp(map1->pop3_uidl, map2->pop3_uidl);
}

static int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
const struct pop3_uidl_map *map2)
{
Expand Down Expand Up @@ -525,15 +538,19 @@ pop3_map_read_hdr_hashes(struct mail_storage *storage, struct mailbox *pop3_box,
static int imap_map_read(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
const unsigned int uidl_cache_idx =
ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
struct mailbox_status status;
struct mailbox_transaction_context *t;
struct mail_search_args *search_args;
struct mail_search_context *ctx;
struct mail *mail;
struct imap_msg_map *map;
uoff_t psize = (uoff_t)-1;
string_t *uidl;
int ret = 0;

mailbox_get_open_status(box, STATUS_MESSAGES, &status);
Expand All @@ -549,6 +566,7 @@ static int imap_map_read(struct mailbox *box)
MAIL_FETCH_PHYSICAL_SIZE, NULL);
mail_search_args_unref(&search_args);

uidl = t_str_new(64);
while (mailbox_search_next(ctx, &mail)) {
if (mstorage->skip_size_check)
;
Expand All @@ -560,9 +578,16 @@ static int imap_map_read(struct mailbox *box)
break;
}

if (!mstorage->skip_uidl_cache) {
str_truncate(uidl, 0);
(void)mail_cache_lookup_field(mail->transaction->cache_view,
uidl, mail->seq, uidl_cache_idx);
}

map = array_append_space(&mbox->imap_msg_map);
map->uid = mail->uid;
map->psize = psize;
map->pop3_uidl = p_strdup_empty(box->pool, str_c(uidl));
}

if (mailbox_search_deinit(&ctx) < 0) {
Expand All @@ -582,6 +607,45 @@ static int imap_map_read_hdr_hashes(struct mailbox *box)
mbox->first_unfound_idx+1);
}

static void pop3_uidl_assign_cached(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct pop3_uidl_map *pop3_map;
struct imap_msg_map *imap_map;
unsigned int imap_idx, pop3_idx, pop3_count, imap_count;
int ret;

if (mstorage->skip_uidl_cache)
return;

array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_uidl_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_uidl_cmp);

pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);

/* see if we can match the messages using sizes */
for (imap_idx = pop3_idx = 0; imap_idx < imap_count; imap_idx++) {
if (imap_map[imap_idx].pop3_uidl == NULL)
continue;

ret = 1;
for (; pop3_idx < pop3_count; pop3_idx++) {
ret = strcmp(imap_map[imap_idx].pop3_uidl,
pop3_map[pop3_idx].pop3_uidl);
if (ret >= 0)
break;
}
if (ret == 0) {
imap_map[imap_idx].pop3_seq =
pop3_map[pop3_idx].pop3_seq;
pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
}
}
}

static bool pop3_uidl_assign_by_size(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
Expand All @@ -590,30 +654,42 @@ static bool pop3_uidl_assign_by_size(struct mailbox *box)
struct pop3_uidl_map *pop3_map;
struct imap_msg_map *imap_map;
unsigned int i, pop3_count, imap_count, count;

if (mstorage->skip_size_check)
return FALSE;
unsigned int size_match = 0, uidl_match = 0;

pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
count = I_MIN(pop3_count, imap_count);

/* see if we can match the messages using sizes */
for (i = 0; i < count; i++) {
if (pop3_map[i].size != imap_map[i].psize)
if (imap_map[i].pop3_uidl != NULL) {
/* some of the UIDLs were already found cached. */
if (strcmp(pop3_map[i].pop3_uidl, imap_map[i].pop3_uidl) == 0) {
uidl_match++;
continue;
}
/* mismatch - can't trust the sizes */
break;
}

if (pop3_map[i].size != imap_map[i].psize ||
mstorage->skip_size_check)
break;
if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
/* two messages with same size, don't trust them */
break;
}

size_match++;
pop3_map[i].imap_uid = imap_map[i].uid;
imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
imap_map[i].pop3_seq = pop3_map[i].pop3_seq;
}
mbox->first_unfound_idx = i;
if (box->storage->user->mail_debug)
i_debug("pop3_migration: %u/%u mails matched by size", i, count);
if (box->storage->user->mail_debug) {
i_debug("pop3_migration: cached uidls=%u, size matches=%u, total=%u",
uidl_match, size_match, count);
}
return i == count;
}

Expand Down Expand Up @@ -708,9 +784,42 @@ pop3_uidl_assign_by_hdr_hash(struct mailbox *box, struct mailbox *pop3_box)
return 0;
}

static void imap_uidls_add_to_cache(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct mailbox_transaction_context *t;
struct mail *mail;
struct index_mail *imail;
struct imap_msg_map *imap_map;
unsigned int i, count;
unsigned int field_idx;

t = mailbox_transaction_begin(box, 0);
mail = mail_alloc(t, 0, NULL);
imail = (struct index_mail *)mail;
field_idx = imail->ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;

imap_map = array_get_modifiable(&mbox->imap_msg_map, &count);
for (i = 0; i < count; i++) {
if (imap_map[i].pop3_uidl == NULL)
continue;

if (!mail_set_uid(mail, imap_map[i].uid))
i_unreached();
if (mail_cache_field_can_add(t->cache_trans, mail->seq, field_idx)) {
index_mail_cache_add_idx(imail, field_idx,
imap_map[i].pop3_uidl, strlen(imap_map[i].pop3_uidl)+1);
}
}
mail_free(&mail);
(void)mailbox_transaction_commit(&t);
}

static int pop3_migration_uidl_sync(struct mailbox *box)
{
struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
struct pop3_migration_mail_storage *mstorage =
POP3_MIGRATION_CONTEXT(box->storage);
struct mailbox *pop3_box;

pop3_box = pop3_mailbox_alloc(box->storage);
Expand All @@ -723,6 +832,11 @@ static int pop3_migration_uidl_sync(struct mailbox *box)
return -1;
}

pop3_uidl_assign_cached(box);

array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);

if (!pop3_uidl_assign_by_size(box)) {
/* everything wasn't assigned, figure out the rest with
header hashes */
Expand All @@ -732,6 +846,9 @@ static int pop3_migration_uidl_sync(struct mailbox *box)
}
}

if (!mstorage->skip_uidl_cache)
imap_uidls_add_to_cache(box);

mbox->uidl_synced = TRUE;
mailbox_free(&pop3_box);
return 0;
Expand Down Expand Up @@ -899,6 +1016,9 @@ static void pop3_migration_mail_storage_created(struct mail_storage *storage)
mstorage->skip_size_check =
mail_user_plugin_getenv(storage->user,
"pop3_migration_skip_size_check") != NULL;
mstorage->skip_uidl_cache =
mail_user_plugin_getenv(storage->user,
"pop3_migration_skip_uidl_cache") != NULL;

MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
}
Expand Down

0 comments on commit 5af0c9f

Please sign in to comment.