From b44033e45e9f48f8a6e1ac5905234fec5de6d6cc Mon Sep 17 00:00:00 2001 From: Aki Tuomi Date: Thu, 20 Oct 2016 19:06:22 +0300 Subject: [PATCH] director: Keep per-tag directory --- src/director/director-connection.c | 6 +- src/director/director-request.c | 32 ++++++---- src/director/director.c | 95 ++++++++++++++++++++++-------- src/director/director.h | 4 +- src/director/doveadm-connection.c | 23 +++++--- src/director/mail-host.c | 44 +++++++++----- src/director/mail-host.h | 28 ++++++++- src/director/main.c | 12 +++- src/director/notify-connection.c | 13 ++-- 9 files changed, 184 insertions(+), 73 deletions(-) diff --git a/src/director/director-connection.c b/src/director/director-connection.c index ea1818b97e..f5aa44ab33 100644 --- a/src/director/director-connection.c +++ b/src/director/director-connection.c @@ -502,7 +502,7 @@ director_user_refresh(struct director_connection *conn, struct director *dir = conn->dir; struct user *user; bool ret = FALSE, unset_weak_user = FALSE; - struct user_directory *users = dir->users; + struct user_directory *users = host->tag->users; *forced_r = FALSE; @@ -1236,7 +1236,7 @@ static bool director_handshake_cmd_done(struct director_connection *conn) if (conn->users_unsorted && conn->user_iter == NULL) { /* we sent our user list before receiving remote's */ conn->users_unsorted = FALSE; - user_directory_sort(conn->dir->users); + mail_hosts_sort_users(conn->dir->mail_hosts); } str = t_str_new(128); @@ -1908,7 +1908,7 @@ static int director_connection_send_users(struct director_connection *conn) if (conn->users_unsorted && conn->handshake_received) { /* we received remote's list of users before sending ours */ conn->users_unsorted = FALSE; - user_directory_sort(conn->dir->users); + mail_hosts_sort_users(conn->dir->mail_hosts); } ret = o_stream_flush(conn->output); diff --git a/src/director/director-request.c b/src/director/director-request.c index bf1eb88e91..bdcb0e20da 100644 --- a/src/director/director-request.c +++ b/src/director/director-request.c @@ -96,8 +96,12 @@ static void director_request_timeout(struct director *dir) DIRECTOR_REQUEST_TIMEOUT_SECS > ioloop_time) break; - user = user_directory_lookup(request->dir->users, - request->username_hash); + const char *tag_name = request->username_tag == NULL ? "" : + request->username_tag; + struct mail_tag *tag = mail_tag_find(dir->mail_hosts, tag_name); + user = tag == NULL ? NULL : + user_directory_lookup(tag->users, request->username_hash); + errormsg = director_request_get_timeout_error(request, user, str); if (user != NULL && @@ -177,11 +181,9 @@ static void ring_log_delayed_warning(struct director *dir) } static bool -director_request_existing(struct director_request *request, struct user *user, - const char *tag) +director_request_existing(struct director_request *request, struct user *user) { struct director *dir = request->dir; - struct user_directory *users = dir->users; struct mail_host *host; if (USER_IS_BEING_KILLED(user)) { @@ -208,12 +210,13 @@ director_request_existing(struct director_request *request, struct user *user, request->username_hash); return FALSE; } - if (!user_directory_user_is_near_expiring(users, user)) + if (!user_directory_user_is_near_expiring(user->host->tag->users, user)) return TRUE; /* user is close to being expired. another director may have already expired it. */ - host = mail_host_get_by_hash(dir->mail_hosts, user->username_hash, tag); + host = mail_host_get_by_hash(dir->mail_hosts, user->username_hash, + user->host->tag->name); if (!dir->ring_synced) { /* try again later once ring is synced */ request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; @@ -272,10 +275,10 @@ director_request_existing(struct director_request *request, struct user *user, bool director_request_continue(struct director_request *request) { struct director *dir = request->dir; - struct user_directory *users = dir->users; struct mail_host *host; struct user *user; const char *tag; + struct mail_tag *mail_tag; if (!dir->ring_handshaked) { /* delay requests until ring handshaking is complete */ @@ -286,12 +289,16 @@ bool director_request_continue(struct director_request *request) return FALSE; } - user = user_directory_lookup(users, request->username_hash); tag = request->username_tag == NULL ? "" : request->username_tag; + mail_tag = mail_tag_find(dir->mail_hosts, tag); + user = mail_tag == NULL ? NULL : + user_directory_lookup(mail_tag->users, request->username_hash); + if (user != NULL) { - if (!director_request_existing(request, user, tag)) + i_assert(user->host->tag == mail_tag); + if (!director_request_existing(request, user)) return FALSE; - user_directory_refresh(users, user); + user_directory_refresh(mail_tag->users, user); dir_debug("request: %u refreshed timeout to %u", request->username_hash, user->timestamp); } else { @@ -312,7 +319,8 @@ bool director_request_continue(struct director_request *request) request->username_hash); return FALSE; } - user = user_directory_add(users, request->username_hash, + user = user_directory_add(host->tag->users, + request->username_hash, host, ioloop_time); dir_debug("request: %u added timeout to %u (hosts_hash=%u)", request->username_hash, user->timestamp, diff --git a/src/director/director.c b/src/director/director.c index 90280b7181..9eef0b4b62 100644 --- a/src/director/director.c +++ b/src/director/director.c @@ -626,7 +626,7 @@ void director_remove_host(struct director *dir, struct director_host *src, struct director_host *orig_src, struct mail_host *host) { - struct user_directory *users = dir->users; + struct user_directory *users = host->tag->users; if (src != NULL) { if (orig_src == NULL) { @@ -649,7 +649,7 @@ void director_flush_host(struct director *dir, struct director_host *src, struct director_host *orig_src, struct mail_host *host) { - struct user_directory *users = dir->users; + struct user_directory *users = host->tag->users; if (orig_src == NULL) { orig_src = dir->self_host; @@ -716,11 +716,13 @@ static void director_flush_user_continue(int result, struct director_kill_context *ctx) { struct director *dir = ctx->dir; - struct user *user = - user_directory_lookup(dir->users, ctx->username_hash); - ctx->callback_pending = FALSE; + struct user *user = user_directory_lookup(ctx->tag->users, + ctx->username_hash); + if (user != NULL) + director_user_kill_finish_delayed(dir, user,result == 1); + if (result == 0) { struct istream *is = iostream_temp_finish(&ctx->reply, (size_t)-1); char *data; @@ -921,7 +923,6 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state, const char *data, void *context) { struct director_kill_context *ctx = context; - struct user_directory *users = ctx->dir->users; struct user *user; /* this is an asynchronous notification about user being killed. @@ -944,7 +945,7 @@ static void director_kill_user_callback(enum ipc_client_cmd_state state, ctx->callback_pending = FALSE; - user = user_directory_lookup(users, ctx->username_hash); + user = user_directory_lookup(ctx->tag->users, ctx->username_hash); if (!DIRECTOR_KILL_CONTEXT_IS_VALID(user, ctx)) { /* user was already freed - ignore */ i_assert(ctx->to_move == NULL); @@ -998,6 +999,7 @@ director_kill_user(struct director *dir, struct director_host *src, user->kill_ctx = ctx = i_new(struct director_kill_context, 1); ctx->dir = dir; + ctx->tag = old_host->tag; ctx->username_hash = user->username_hash; ctx->kill_is_self_initiated = src->self; if (old_host != NULL) @@ -1027,7 +1029,7 @@ void director_move_user(struct director *dir, struct director_host *src, struct director_host *orig_src, unsigned int username_hash, struct mail_host *host) { - struct user_directory *users = dir->users; + struct user_directory *users = host->tag->users; struct user *user; /* 1. move this user's host, and set its "killing" flag to delay all of @@ -1167,12 +1169,13 @@ director_send_user_killed_everywhere(struct director *dir, username_hash)); } -void director_user_killed(struct director *dir, unsigned int username_hash) +static void +director_user_tag_killed(struct director *dir, struct mail_tag *tag, + unsigned int username_hash) { - struct user_directory *users = dir->users; struct user *user; - user = user_directory_lookup(users, username_hash); + user = user_directory_lookup(tag->users, username_hash); if (user == NULL || !USER_IS_BEING_KILLED(user)) return; @@ -1203,14 +1206,24 @@ void director_user_killed(struct director *dir, unsigned int username_hash) } } -void director_user_killed_everywhere(struct director *dir, - struct director_host *src, - struct director_host *orig_src, - unsigned int username_hash) +void director_user_killed(struct director *dir, unsigned int username_hash) +{ + struct mail_tag *const *tagp; + + array_foreach(mail_hosts_get_tags(dir->mail_hosts), tagp) + director_user_tag_killed(dir, *tagp, username_hash); +} + +static void +director_user_tag_killed_everywhere(struct director *dir, + struct mail_tag *tag, + struct director_host *src, + struct director_host *orig_src, + unsigned int username_hash) { struct user *user; - user = user_directory_lookup(dir->users, username_hash); + user = user_directory_lookup(tag->users, username_hash); if (user == NULL) { dir_debug("User %u no longer exists - ignoring USER-KILLED-EVERYWHERE", username_hash); @@ -1231,6 +1244,19 @@ void director_user_killed_everywhere(struct director *dir, director_send_user_killed_everywhere(dir, src, orig_src, username_hash); } +void director_user_killed_everywhere(struct director *dir, + struct director_host *src, + struct director_host *orig_src, + unsigned int username_hash) +{ + struct mail_tag *const *tagp; + + array_foreach(mail_hosts_get_tags(dir->mail_hosts), tagp) { + director_user_tag_killed_everywhere(dir, *tagp, src, orig_src, + username_hash); + } +} + static void director_state_callback_timeout(struct director *dir) { timeout_remove(&dir->to_callback); @@ -1299,9 +1325,9 @@ director_init(const struct director_settings *set, i_array_init(&dir->dir_hosts, 16); i_array_init(&dir->pending_requests, 16); i_array_init(&dir->connections, 8); - dir->users = user_directory_init(set->director_user_expire, - director_user_freed); - dir->mail_hosts = mail_hosts_init(set->director_consistent_hashing); + dir->mail_hosts = mail_hosts_init(set->director_user_expire, + set->director_consistent_hashing, + director_user_freed); dir->ipc_proxy = ipc_client_init(DIRECTOR_IPC_PROXY_PATH); dir->ring_min_version = DIRECTOR_VERSION_MINOR; @@ -1322,7 +1348,6 @@ void director_deinit(struct director **_dir) director_connection_deinit(&conn, "Shutting down"); } - user_directory_deinit(&dir->users); mail_hosts_deinit(&dir->mail_hosts); mail_hosts_deinit(&dir->orig_config_hosts); @@ -1365,28 +1390,48 @@ void dir_debug(const char *fmt, ...) } struct director_user_iter { + struct director *dir; + unsigned int tag_idx; struct user_directory_iter *user_iter; }; struct director_user_iter *director_iterate_users_init(struct director *dir) { struct director_user_iter *iter = i_new(struct director_user_iter, 1); - - iter->user_iter = user_directory_iter_init(dir->users); + iter->dir = dir; return iter; } struct user *director_iterate_users_next(struct director_user_iter *iter) { - return user_directory_iter_next(iter->user_iter); + const ARRAY_TYPE(mail_tag) *tags; + struct user *user; + + i_assert(iter != NULL); + + if (iter->user_iter == NULL) { + tags = mail_hosts_get_tags(iter->dir->mail_hosts); + if (iter->tag_idx >= array_count(tags)) + return NULL; + struct mail_tag *const *tagp = array_idx(tags, iter->tag_idx); + iter->user_iter = user_directory_iter_init((*tagp)->users); + } + user = user_directory_iter_next(iter->user_iter); + if (user == NULL) { + user_directory_iter_deinit(&iter->user_iter); + iter->tag_idx++; + return director_iterate_users_next(iter); + } else + return user; } void director_iterate_users_deinit(struct director_user_iter **_iter) { + i_assert(_iter != NULL && *_iter != NULL); struct director_user_iter *iter = *_iter; - *_iter = NULL; - user_directory_iter_deinit(&iter->user_iter); + if (iter->user_iter != NULL) + user_directory_iter_deinit(&iter->user_iter); i_free(iter); } diff --git a/src/director/director.h b/src/director/director.h index 590366b39b..ffb5c3187e 100644 --- a/src/director/director.h +++ b/src/director/director.h @@ -34,6 +34,7 @@ struct director; struct mail_host; struct user; +struct director_user_init; enum user_kill_state { /* User isn't being killed */ @@ -72,6 +73,7 @@ typedef void director_state_change_callback_t(struct director *dir); struct director_kill_context { struct director *dir; + struct mail_tag *tag; unsigned int username_hash; struct ip_addr old_host_ip; bool kill_is_self_initiated; @@ -115,8 +117,6 @@ struct director { /* original mail hosts configured in config file. this is used only for doveadm lookups */ struct mail_host_list *orig_config_hosts; - /* temporary user -> host associations */ - struct user_directory *users; /* Number of users currently being moved */ unsigned int users_moving_count; diff --git a/src/director/doveadm-connection.c b/src/director/doveadm-connection.c index 99395a9a69..5527dfba93 100644 --- a/src/director/doveadm-connection.c +++ b/src/director/doveadm-connection.c @@ -31,7 +31,7 @@ struct director_reset_cmd { struct director *dir; struct doveadm_connection *_conn; - struct user_directory_iter *iter; + struct director_user_iter *iter; unsigned int host_idx, hosts_count; unsigned int max_moving_users; }; @@ -82,7 +82,9 @@ static void doveadm_cmd_host_list_removed(struct doveadm_connection *conn) string_t *str = t_str_new(1024); int ret; - orig_hosts_list = mail_hosts_init(conn->dir->set->director_consistent_hashing); + orig_hosts_list = mail_hosts_init(conn->dir->set->director_user_expire, + conn->dir->set->director_consistent_hashing, + NULL); (void)mail_hosts_parse_and_add(orig_hosts_list, conn->dir->set->director_mail_servers); @@ -428,7 +430,7 @@ static void doveadm_reset_cmd_free(struct director_reset_cmd *cmd) DLLIST_REMOVE(&reset_cmds, cmd); if (cmd->iter != NULL) - user_directory_iter_deinit(&cmd->iter); + director_iterate_users_deinit(&cmd->iter); if (cmd->_conn != NULL) cmd->_conn->reset_cmd = NULL; i_free(cmd); @@ -439,7 +441,6 @@ director_host_reset_users(struct director_reset_cmd *cmd, struct mail_host *host) { struct director *dir = cmd->dir; - struct director_user_iter *iter; struct user *user; struct mail_host *new_host; @@ -451,7 +452,8 @@ director_host_reset_users(struct director_reset_cmd *cmd, if (cmd->iter == NULL) cmd->iter = director_iterate_users_init(dir); - while ((user = director_iterate_users_next(iter)) != NULL) { + + while ((user = director_iterate_users_next(cmd->iter)) != NULL) { if (user->host != host) continue; new_host = mail_host_get_by_hash(dir->mail_hosts, @@ -552,11 +554,11 @@ static int doveadm_cmd_user_lookup(struct doveadm_connection *conn, const char *const *args) { - struct user_directory *users = conn->dir->users; struct user *user; struct mail_host *host; const char *username, *tag; unsigned int username_hash; + struct mail_tag *mail_tag; string_t *str = t_str_new(256); if (args[0] == NULL) { @@ -575,7 +577,9 @@ doveadm_cmd_user_lookup(struct doveadm_connection *conn, } /* get user's current host */ - user = user_directory_lookup(users, username_hash); + mail_tag = mail_tag_find(conn->dir->mail_hosts, tag); + user = mail_tag == NULL ? NULL : + user_directory_lookup(mail_tag->users, username_hash); if (user == NULL) str_append(str, "\t0"); else { @@ -639,7 +643,7 @@ doveadm_cmd_user_list(struct doveadm_connection *conn, const char *const *args) static int doveadm_cmd_user_move(struct doveadm_connection *conn, const char *const *args) { - struct user_directory *users = conn->dir->users; + struct user_directory *users; unsigned int username_hash; struct user *user; struct mail_host *host; @@ -663,7 +667,8 @@ doveadm_cmd_user_move(struct doveadm_connection *conn, const char *const *args) return 1; } } - user = user_directory_lookup(users, username_hash); + + user = user_directory_lookup(host->tag->users, username_hash); if (user != NULL && USER_IS_BEING_KILLED(user)) { o_stream_nsend_str(conn->output, "TRYAGAIN\n"); return 1; diff --git a/src/director/mail-host.c b/src/director/mail-host.c index edff8a576c..e8da77e060 100644 --- a/src/director/mail-host.c +++ b/src/director/mail-host.c @@ -5,25 +5,17 @@ #include "bsearch-insert-pos.h" #include "crc32.h" #include "md5.h" +#include "user-directory.h" #include "mail-host.h" #define VHOST_MULTIPLIER 100 -struct mail_vhost { - unsigned int hash; - struct mail_host *host; -}; - -struct mail_tag { - /* "" = no tag */ - char *name; - ARRAY(struct mail_vhost) vhosts; -}; - struct mail_host_list { - ARRAY(struct mail_tag *) tags; + ARRAY_TYPE(mail_tag) tags; ARRAY_TYPE(mail_host) hosts; + user_free_hook_t *user_free_hook; unsigned int hosts_hash; + unsigned int user_expire_secs; bool consistent_hashing; bool vhosts_unsorted; bool have_vhosts; @@ -153,7 +145,7 @@ mail_hosts_sort(struct mail_host_list *list) } } -static struct mail_tag * +struct mail_tag * mail_tag_find(struct mail_host_list *list, const char *tag_name) { struct mail_tag *const *tagp; @@ -175,6 +167,8 @@ mail_tag_get(struct mail_host_list *list, const char *tag_name) tag = i_new(struct mail_tag, 1); tag->name = i_strdup(tag_name); i_array_init(&tag->vhosts, 16*VHOST_MULTIPLIER); + tag->users = user_directory_init(list->user_expire_secs, + list->user_free_hook); array_append(&list->tags, &tag, 1); } return tag; @@ -182,6 +176,7 @@ mail_tag_get(struct mail_host_list *list, const char *tag_name) static void mail_tag_free(struct mail_tag *tag) { + user_directory_deinit(&tag->users); array_free(&tag->vhosts); i_free(tag->name); i_free(tag); @@ -508,12 +503,22 @@ bool mail_hosts_have_tags(struct mail_host_list *list) return FALSE; } -struct mail_host_list *mail_hosts_init(bool consistent_hashing) +const ARRAY_TYPE(mail_tag) *mail_hosts_get_tags(struct mail_host_list *list) +{ + return &list->tags; +} + +struct mail_host_list * +mail_hosts_init(unsigned int user_expire_secs, bool consistent_hashing, + user_free_hook_t *user_free_hook) { struct mail_host_list *list; list = i_new(struct mail_host_list, 1); + list->user_expire_secs = user_expire_secs; list->consistent_hashing = consistent_hashing; + list->user_free_hook = user_free_hook; + i_array_init(&list->hosts, 16); i_array_init(&list->tags, 4); return list; @@ -551,7 +556,8 @@ struct mail_host_list *mail_hosts_dup(const struct mail_host_list *src) struct mail_host_list *dest; struct mail_host *const *hostp, *dest_host; - dest = mail_hosts_init(src->consistent_hashing); + dest = mail_hosts_init(src->user_expire_secs, src->consistent_hashing, + src->user_free_hook); array_foreach(&src->hosts, hostp) { dest_host = mail_host_dup(*hostp); array_append(&dest->hosts, &dest_host, 1); @@ -559,3 +565,11 @@ struct mail_host_list *mail_hosts_dup(const struct mail_host_list *src) mail_hosts_sort(dest); return dest; } + +void mail_hosts_sort_users(struct mail_host_list *list) +{ + struct mail_tag *const *tagp; + + array_foreach(&list->tags, tagp) + user_directory_sort((*tagp)->users); +} diff --git a/src/director/mail-host.h b/src/director/mail-host.h index c0e4412285..002c11d8da 100644 --- a/src/director/mail-host.h +++ b/src/director/mail-host.h @@ -6,6 +6,22 @@ struct mail_host_list; +struct mail_vhost { + unsigned int hash; + struct mail_host *host; +}; + +/* mail_tags aren't removed/freed before mail_hosts_deinit(), so it's safe + to add pointers to them. */ +struct mail_tag { + /* "" = no tag */ + char *name; + ARRAY(struct mail_vhost) vhosts; + /* temporary user -> host associations */ + struct user_directory *users; +}; +ARRAY_DEFINE_TYPE(mail_tag, struct mail_tag *); + struct mail_host { struct mail_host_list *list; @@ -52,9 +68,19 @@ bool mail_hosts_have_usable(struct mail_host_list *list); const ARRAY_TYPE(mail_host) *mail_hosts_get(struct mail_host_list *list); bool mail_hosts_have_tags(struct mail_host_list *list); -struct mail_host_list *mail_hosts_init(bool consistent_hashing); +const ARRAY_TYPE(mail_tag) *mail_hosts_get_tags(struct mail_host_list *list); +struct mail_tag * +mail_tag_find(struct mail_host_list *list, const char *tag_name); +struct user * +mail_hosts_find_user(struct mail_host_list *list, const char *tag_name, + unsigned int username_hash); + +struct mail_host_list * +mail_hosts_init(unsigned int user_expire_secs, bool consistent_hashing, + user_free_hook_t *user_free_hook); void mail_hosts_deinit(struct mail_host_list **list); struct mail_host_list *mail_hosts_dup(const struct mail_host_list *src); +void mail_hosts_sort_users(struct mail_host_list *list); #endif diff --git a/src/director/main.c b/src/director/main.c index c54434f9c0..5bc355078e 100644 --- a/src/director/main.c +++ b/src/director/main.c @@ -40,13 +40,23 @@ static struct notify_connection *notify_conn; static struct timeout *to_proctitle_refresh; static ARRAY(enum director_socket_type) listener_socket_types; +static unsigned int director_total_users_count(void) +{ + struct mail_tag *const *tagp; + unsigned int count = 0; + + array_foreach(mail_hosts_get_tags(director->mail_hosts), tagp) + count += user_directory_count((*tagp)->users); + return count; +} + static void director_refresh_proctitle_timeout(void *context ATTR_UNUSED) { static uint64_t prev_requests = 0, prev_input = 0, prev_output; string_t *str; str = t_str_new(64); - str_printfa(str, "[%u users", user_directory_count(director->users)); + str_printfa(str, "[%u users", director_total_users_count()); if (director->users_moving_count > 0) str_printfa(str, ", %u moving", director->users_moving_count); str_printfa(str, ", %lu req/s", diff --git a/src/director/notify-connection.c b/src/director/notify-connection.c index 493940c53a..ca2d685049 100644 --- a/src/director/notify-connection.c +++ b/src/director/notify-connection.c @@ -1,6 +1,7 @@ /* Copyright (c) 2010-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "ioloop.h" #include "istream.h" #include "master-service.h" @@ -17,13 +18,13 @@ struct notify_connection { struct director *dir; }; -static void notify_update_user(struct director *dir, const char *username, - unsigned int username_hash) +static void notify_update_user(struct director *dir, struct mail_tag *tag, + const char *username, unsigned int username_hash) { struct user *user; int diff; - user = user_directory_lookup(dir->users, username_hash); + user = user_directory_lookup(tag->users, username_hash); if (user == NULL) return; @@ -32,19 +33,21 @@ static void notify_update_user(struct director *dir, const char *username, i_warning("notify: User %s refreshed too late (%d secs)", username, diff); } - user_directory_refresh(dir->users, user); + user_directory_refresh(tag->users, user); director_update_user(dir, dir->self_host, user); } static void notify_connection_input(struct notify_connection *conn) { + struct mail_tag *const *tagp; const char *line; unsigned int hash; while ((line = i_stream_read_next_line(conn->input)) != NULL) { if (!director_get_username_hash(conn->dir, line, &hash)) continue; - notify_update_user(conn->dir, line, hash); + array_foreach(mail_hosts_get_tags(conn->dir->mail_hosts), tagp) + notify_update_user(conn->dir, *tagp, line, hash); } if (conn->input->eof) { i_error("notify: read() unexpectedly returned EOF");