doveadm-mail-index.c

/* Copyright (c) 2010-2016 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "str.h"
#include "strescape.h"
#include "net.h"
#include "write-full.h"
#include "mail-namespace.h"
#include "mail-storage.h"
#include "mail-search-build.h"
#include "mailbox-list-iter.h"
#include "doveadm-settings.h"
#include "doveadm-mail.h"

#include <stdio.h>

#define INDEXER_SOCKET_NAME "indexer"
#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"

struct index_cmd_context {
	struct doveadm_mail_cmd_context ctx;

	int queue_fd;
	unsigned int max_recent_msgs;
	unsigned int queue:1;
	unsigned int have_wildcards:1;
};

static int cmd_index_box_precache(struct mailbox *box)
{
	struct mailbox_status status;
	struct mailbox_transaction_context *trans;
	struct mail_search_args *search_args;
	struct mail_search_context *ctx;
	struct mail *mail;
	struct mailbox_metadata metadata;
	uint32_t seq;
	unsigned int counter = 0, max;
	int ret = 0;

	if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
				 &metadata) < 0) {
		i_error("Mailbox %s: Precache-fields lookup failed: %s",
			mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
	}
	if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
			       &status) < 0) {
		i_error("Mailbox %s: Status lookup failed: %s",
			mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
		return -1;
	}

	seq = status.last_cached_seq + 1;
	if (seq > status.messages) {
		if (doveadm_verbose) {
			i_info("%s: Cache is already up to date",
			       mailbox_get_vname(box));
		}
		return 0;
	}
	if (doveadm_verbose) {
		i_info("%s: Caching mails seq=%u..%u",
		       mailbox_get_vname(box), seq, status.messages);
	}

	trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC);
	search_args = mail_search_build_init();
	mail_search_build_add_seqset(search_args, seq, status.messages);
	ctx = mailbox_search_init(trans, search_args, NULL,
				  metadata.precache_fields, NULL);
	mail_search_args_unref(&search_args);

	max = status.messages - seq + 1;
	while (mailbox_search_next(ctx, &mail)) {
		mail_precache(mail);
		if (doveadm_verbose && ++counter % 100 == 0) {
			printf("\r%u/%u", counter, max);
			fflush(stdout);
		}
	}
	if (doveadm_verbose)
		printf("\r%u/%u\n", counter, max);
	if (mailbox_search_deinit(&ctx) < 0) {
		i_error("Mailbox %s: Mail search failed: %s",
			mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
		ret = -1;
	}
	if (mailbox_transaction_commit(&trans) < 0) {
		i_error("Mailbox %s: Transaction commit failed: %s",
			mailbox_get_vname(box), mailbox_get_last_error(box, NULL));
		ret = -1;
	}
	return ret;
}

static int
cmd_index_box(struct index_cmd_context *ctx, const struct mailbox_info *info)
{
	struct mailbox *box;
	struct mailbox_status status;
	int ret = 0;

	box = mailbox_alloc(info->ns->list, info->vname,
			    MAILBOX_FLAG_IGNORE_ACLS);
	if (ctx->max_recent_msgs != 0) {
		/* index only if there aren't too many recent messages.
		   don't bother syncing the mailbox, that alone can take a
		   while with large maildirs. */
		if (mailbox_open(box) < 0) {
			i_error("Opening mailbox %s failed: %s", info->vname,
				mail_storage_get_last_error(mailbox_get_storage(box), NULL));
			doveadm_mail_failed_mailbox(&ctx->ctx, box);
			mailbox_free(&box);
			return -1;
		} 

		mailbox_get_open_status(box, STATUS_RECENT, &status);
		if (status.recent > ctx->max_recent_msgs) {
			mailbox_free(&box);
			return 0;
		}
	}

	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
		i_error("Syncing mailbox %s failed: %s", info->vname,
			mail_storage_get_last_error(mailbox_get_storage(box), NULL));
		doveadm_mail_failed_mailbox(&ctx->ctx, box);
		ret = -1;
	} else {
		if (cmd_index_box_precache(box) < 0) {
			doveadm_mail_failed_mailbox(&ctx->ctx, box);
			ret = -1;
		}
	}
	mailbox_free(&box);
	return ret;
}

static void index_queue_connect(struct index_cmd_context *ctx)
{
	const char *path;

	path = t_strconcat(doveadm_settings->base_dir,
			   "/"INDEXER_SOCKET_NAME, NULL);
	ctx->queue_fd = net_connect_unix(path);
	if (ctx->queue_fd == -1)
		i_fatal("net_connect_unix(%s) failed: %m", path);
	if (write_full(ctx->queue_fd, INDEXER_HANDSHAKE,
		       strlen(INDEXER_HANDSHAKE)) < 0)
		i_fatal("write(indexer) failed: %m");
}

static void cmd_index_queue(struct index_cmd_context *ctx,
			    struct mail_user *user, const char *mailbox)
{
	if (ctx->queue_fd == -1)
		index_queue_connect(ctx);

	T_BEGIN {
		string_t *str = t_str_new(256);

		str_append(str, "APPEND\t0\t");
		str_append_tabescaped(str, user->username);
		str_append_c(str, '\t');
		str_append_tabescaped(str, mailbox);
		str_printfa(str, "\t%u\n", ctx->max_recent_msgs);
		if (write_full(ctx->queue_fd, str_data(str), str_len(str)) < 0)
			i_fatal("write(indexer) failed: %m");
	} T_END;
}

static int
cmd_index_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
{
	struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
	const enum mailbox_list_iter_flags iter_flags =
		MAILBOX_LIST_ITER_NO_AUTO_BOXES |
		MAILBOX_LIST_ITER_RETURN_NO_FLAGS |
		MAILBOX_LIST_ITER_STAR_WITHIN_NS;
	const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL;
	struct mailbox_list_iterate_context *iter;
	const struct mailbox_info *info;
	unsigned int i;
	int ret = 0;

	if (ctx->queue && !ctx->have_wildcards) {
		/* we can do this quickly without going through the mailboxes */
		for (i = 0; _ctx->args[i] != NULL; i++)
			cmd_index_queue(ctx, user, _ctx->args[i]);
		return 0;
	}

	iter = mailbox_list_iter_init_namespaces(user->namespaces, _ctx->args,
						 ns_mask, iter_flags);
	while ((info = mailbox_list_iter_next(iter)) != NULL) {
		if ((info->flags & (MAILBOX_NOSELECT |
				    MAILBOX_NONEXISTENT)) == 0) T_BEGIN {
			if (ctx->queue)
				cmd_index_queue(ctx, user, info->vname);
			else {
				if (cmd_index_box(ctx, info) < 0)
					ret = -1;
			}
		} T_END;
	}
	if (mailbox_list_iter_deinit(&iter) < 0) {
		i_error("Listing mailboxes failed: %s",
			mailbox_list_get_last_error(user->namespaces->list, NULL));
		doveadm_mail_failed_error(_ctx, MAIL_ERROR_TEMP);
		ret = -1;
	}
	return ret;
}

static void cmd_index_init(struct doveadm_mail_cmd_context *_ctx,
			   const char *const args[])
{
	struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
	unsigned int i;

	if (args[0] == NULL)
		doveadm_mail_help_name("index");
	for (i = 0; args[i] != NULL; i++) {
		if (strchr(args[i], '*') != NULL ||
		    strchr(args[i], '%') != NULL) {
			ctx->have_wildcards = TRUE;
			break;
		}
	}
}

static void cmd_index_deinit(struct doveadm_mail_cmd_context *_ctx)
{
	struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;

	if (ctx->queue_fd != -1) {
		net_disconnect(ctx->queue_fd);
		ctx->queue_fd = -1;
	}
}

static bool
cmd_index_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
{
	struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;

	switch (c) {
	case 'q':
		ctx->queue = TRUE;
		break;
	case 'n':
		if (str_to_uint(optarg, &ctx->max_recent_msgs) < 0) {
			i_fatal_status(EX_USAGE,
				"Invalid -n parameter number: %s", optarg);
		}
		break;
	default:
		return FALSE;
	}
	return TRUE;
}

static struct doveadm_mail_cmd_context *cmd_index_alloc(void)
{
	struct index_cmd_context *ctx;

	ctx = doveadm_mail_cmd_alloc(struct index_cmd_context);
	ctx->queue_fd = -1;
	ctx->ctx.getopt_args = "qn:";
	ctx->ctx.v.parse_arg = cmd_index_parse_arg;
	ctx->ctx.v.init = cmd_index_init;
	ctx->ctx.v.deinit = cmd_index_deinit;
	ctx->ctx.v.run = cmd_index_run;
	return &ctx->ctx;
}

struct doveadm_cmd_ver2 doveadm_cmd_index_ver2 = {
	.name = "index",
	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-q] [-n <max recent>] <mailbox mask>",
	.mail_cmd = cmd_index_alloc,
DOVEADM_CMD_PARAMS_START
DOVEADM_CMD_MAIL_COMMON
DOVEADM_CMD_PARAM('q',"queue",CMD_PARAM_BOOL,0)
DOVEADM_CMD_PARAM('n',"max-recent",CMD_PARAM_INT64,0)
DOVEADM_CMD_PARAM('\0',"mailbox-mask",CMD_PARAM_STR,CMD_PARAM_FLAG_POSITIONAL)
DOVEADM_CMD_PARAMS_END
};