Skip to content

Commit

Permalink
imapc: Try to merge STOREs together as much as possible when syncing
Browse files Browse the repository at this point in the history
  • Loading branch information
@sirainen
sirainen authored and GitLab committed May 12, 2017
1 parent 0b45d81 commit 635f1b1
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 25 deletions.
138 changes: 113 additions & 25 deletions src/lib-storage/index/imapc/imapc-sync.c
View file
Expand Up @@ -2,7 +2,9 @@

#include "lib.h"
#include "ioloop.h"
#include "hash.h"
#include "str.h"
#include "sort.h"
#include "imap-util.h"
#include "mail-cache.h"
#include "mail-index-modseq.h"
Expand Down Expand Up @@ -75,10 +77,103 @@ imapc_sync_cmd(struct imapc_sync_context *ctx, const char *cmd_str)
return imapc_sync_cmd_full(ctx, cmd_str, FALSE);
}

static struct imapc_command *
imapc_sync_store_cmd(struct imapc_sync_context *ctx, const char *cmd_str)
static unsigned int imapc_sync_store_hash(const struct imapc_sync_store *store)
{
return str_hash(store->flags) ^ store->modify_type;
}

static int imapc_sync_store_cmp(const struct imapc_sync_store *store1,
const struct imapc_sync_store *store2)
{
if (store1->modify_type != store2->modify_type)
return 1;
return strcmp(store1->flags, store2->flags);
}

static const char *imapc_sync_flags_sort(const char *flags)
{
if (strchr(flags, ' ') == NULL)
return flags;

const char **str = t_strsplit(flags, " ");
i_qsort(str, str_array_length(str), sizeof(const char *),
i_strcasecmp_p);
return t_strarray_join(str, " ");
}

static void
imapc_sync_store_flush(struct imapc_sync_context *ctx)
{
struct imapc_sync_store *store;
const char *sorted_flags;

if (ctx->prev_uid1 == 0)
return;

sorted_flags = imapc_sync_flags_sort(str_c(ctx->prev_flags));
struct imapc_sync_store store_lookup = {
.modify_type = ctx->prev_modify_type,
.flags = sorted_flags,
};
store = hash_table_lookup(ctx->stores, &store_lookup);
if (store == NULL) {
store = p_new(ctx->pool, struct imapc_sync_store, 1);
store->modify_type = ctx->prev_modify_type;
store->flags = p_strdup(ctx->pool, sorted_flags);
p_array_init(&store->uids, ctx->pool, 4);
hash_table_insert(ctx->stores, store, store);
}
seq_range_array_add_range(&store->uids, ctx->prev_uid1, ctx->prev_uid2);
}

static void
imapc_sync_store(struct imapc_sync_context *ctx,
enum modify_type modify_type, uint32_t uid1, uint32_t uid2,
const char *flags)
{
if (ctx->prev_flags == NULL) {
ctx->prev_flags = str_new(ctx->pool, 128);
hash_table_create(&ctx->stores, ctx->pool, 0,
imapc_sync_store_hash, imapc_sync_store_cmp);
}

if (ctx->prev_uid1 != uid1 || ctx->prev_uid2 != uid2 ||
ctx->prev_modify_type != modify_type) {
imapc_sync_store_flush(ctx);
ctx->prev_uid1 = uid1;
ctx->prev_uid2 = uid2;
ctx->prev_modify_type = modify_type;
str_truncate(ctx->prev_flags, 0);
}
if (str_len(ctx->prev_flags) > 0)
str_append_c(ctx->prev_flags, ' ');
str_append(ctx->prev_flags, flags);
}

static void
imapc_sync_finish_store(struct imapc_sync_context *ctx)
{
return imapc_sync_cmd_full(ctx, cmd_str, TRUE);
struct hash_iterate_context *iter;
struct imapc_sync_store *store;
string_t *cmd = t_str_new(128);

imapc_sync_store_flush(ctx);

if (!hash_table_is_created(ctx->stores))
return;

iter = hash_table_iterate_init(ctx->stores);
while (hash_table_iterate(iter, ctx->stores, &store, &store)) {
str_truncate(cmd, 0);
str_append(cmd, "UID STORE ");
imap_write_seq_range(cmd, &store->uids);
str_printfa(cmd, " %cFLAGS (%s)",
store->modify_type == MODIFY_ADD ? '+' : '-',
store->flags);
imapc_sync_cmd_full(ctx, str_c(cmd), TRUE);
}
hash_table_iterate_deinit(&iter);
hash_table_destroy(&ctx->stores);
}

static void
Expand All @@ -87,7 +182,6 @@ imapc_sync_add_missing_deleted_flags(struct imapc_sync_context *ctx,
{
const struct mail_index_record *rec;
uint32_t seq, uid1, uid2;
const char *cmd;

/* if any of them has a missing \Deleted flag,
just add it to all of them. */
Expand All @@ -100,9 +194,8 @@ imapc_sync_add_missing_deleted_flags(struct imapc_sync_context *ctx,
if (seq <= seq2) {
mail_index_lookup_uid(ctx->sync_view, seq1, &uid1);
mail_index_lookup_uid(ctx->sync_view, seq2, &uid2);
cmd = t_strdup_printf("UID STORE %u:%u +FLAGS \\Deleted",
uid1, uid2);
imapc_sync_store_cmd(ctx, cmd);

imapc_sync_store(ctx, MODIFY_ADD, uid1, uid2, "\\Deleted");
}
}

Expand All @@ -115,50 +208,42 @@ static void imapc_sync_index_flags(struct imapc_sync_context *ctx,

if (sync_rec->add_flags != 0) {
i_assert((sync_rec->add_flags & MAIL_RECENT) == 0);
str_printfa(str, "UID STORE %u:%u +FLAGS (",
sync_rec->uid1, sync_rec->uid2);

imap_write_flags(str, sync_rec->add_flags, NULL);
str_append_c(str, ')');
imapc_sync_store_cmd(ctx, str_c(str));
imapc_sync_store(ctx, MODIFY_ADD, sync_rec->uid1,
sync_rec->uid2, str_c(str));
}

if (sync_rec->remove_flags != 0) {
i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0);
str_truncate(str, 0);
str_printfa(str, "UID STORE %u:%u -FLAGS (",
sync_rec->uid1, sync_rec->uid2);
imap_write_flags(str, sync_rec->remove_flags, NULL);
str_append_c(str, ')');
imapc_sync_store_cmd(ctx, str_c(str));
imapc_sync_store(ctx, MODIFY_REMOVE, sync_rec->uid1,
sync_rec->uid2, str_c(str));
}
}

static void
imapc_sync_index_keyword(struct imapc_sync_context *ctx,
const struct mail_index_sync_rec *sync_rec)
{
string_t *str = t_str_new(128);
const char *const *kw_p;
char change_char;
enum modify_type modify_type;

switch (sync_rec->type) {
case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
change_char = '+';
modify_type = MODIFY_ADD;
break;
case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
change_char = '-';
modify_type = MODIFY_REMOVE;
break;
default:
i_unreached();
}

str_printfa(str, "UID STORE %u:%u %cFLAGS (",
sync_rec->uid1, sync_rec->uid2, change_char);

kw_p = array_idx(ctx->keywords, sync_rec->keyword_idx);
str_append(str, *kw_p);
str_append_c(str, ')');
imapc_sync_store_cmd(ctx, str_c(str));
imapc_sync_store(ctx, modify_type, sync_rec->uid1,
sync_rec->uid2, *kw_p);
}

static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx)
Expand Down Expand Up @@ -359,6 +444,7 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)

i_array_init(&ctx->expunged_uids, 64);
ctx->keywords = mail_index_get_keywords(mbox->box.index);
ctx->pool = pool_alloconly_create("imapc sync pool", 1024);

imapc_sync_uid_validity(ctx);
while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN {
Expand All @@ -381,6 +467,8 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
break;
}
} T_END;
imapc_sync_finish_store(ctx);
pool_unref(&ctx->pool);

if (!mbox->initial_sync_done) {
/* with initial syncing we're fetching all messages' flags and
Expand Down
14 changes: 14 additions & 0 deletions src/lib-storage/index/imapc/imapc-sync.h
View file
Expand Up @@ -4,6 +4,13 @@
struct mailbox;
struct mailbox_sync_status;

struct imapc_sync_store {
enum modify_type modify_type;
const char *flags;

ARRAY_TYPE(seq_range) uids;
};

struct imapc_sync_context {
struct imapc_mailbox *mbox;
struct mail_index_sync_ctx *index_sync_ctx;
Expand All @@ -14,6 +21,13 @@ struct imapc_sync_context {
ARRAY_TYPE(seq_range) expunged_uids;
unsigned int sync_command_count;

pool_t pool;
HASH_TABLE(struct imapc_sync_store *, struct imapc_sync_store *) stores;

uint32_t prev_uid1, prev_uid2;
enum modify_type prev_modify_type;
string_t *prev_flags;

bool failed:1;
};

Expand Down

0 comments on commit 635f1b1

Please sign in to comment.