dsync: Add support for features

Add empty_header_workaround as first feature

src/doveadm/doveadm-dsync.c

@@ -17,6 +17,7 @@
 #include "settings-parser.h"
 #include "imap-util.h"
 #include "master-service.h"
+#include "master-service-settings.h"
 #include "master-service-ssl-settings.h"
 #include "mail-storage.h"
 #include "mail-storage-service.h"
@@ -103,6 +104,7 @@ struct dsync_cmd_context {
 	bool local_location_from_arg:1;
 	bool replicator_notify:1;
 	bool exited:1;
+	bool empty_hdr_workaround:1;
 };
 
 static bool legacy_dsync = FALSE;
@@ -573,6 +575,7 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
 	set.lock_timeout_secs = ctx->lock_timeout;
 	set.state = ctx->state_input;
 	set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0];
+
 	if (array_count(&ctx->exclude_mailboxes) > 0) {
 		/* array is NULL-terminated in init() */
 		set.exclude_mailboxes = array_idx(&ctx->exclude_mailboxes, 0);
@@ -622,6 +625,8 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
 		brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC;
 	if (ctx->oneway)
 		brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE;
+	if (ctx->empty_hdr_workaround)
+		brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
 	if (doveadm_debug)
 		brain_flags |= DSYNC_BRAIN_FLAG_DEBUG;
 
@@ -1064,6 +1069,8 @@ static struct doveadm_mail_cmd_context *cmd_dsync_alloc(void)
 			     DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
 	p_array_init(&ctx->exclude_mailboxes, ctx->ctx.pool, 4);
 	p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4);
+        if ((doveadm_settings->parsed_features & DSYNC_FEATURE_EMPTY_HDR_WORKAROUND) != 0)
+                ctx->empty_hdr_workaround = TRUE;
 	return &ctx->ctx;
 }
 
@@ -1179,6 +1186,7 @@ static struct doveadm_mail_cmd_context *cmd_dsync_server_alloc(void)
 	ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg;
 	ctx->ctx.v.run = cmd_dsync_server_run;
 	ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED;
+
 	ctx->fd_in = STDIN_FILENO;
 	ctx->fd_out = STDOUT_FILENO;
 	return &ctx->ctx;

src/doveadm/doveadm-settings.c

@@ -71,6 +71,7 @@ static const struct setting_define doveadm_setting_defines[] = {
 	DEF(SET_STR, ssl_client_ca_file),
 	DEF(SET_STR, director_username_hash),
 	DEF(SET_STR, doveadm_api_key),
+	DEF(SET_STR, dsync_features),
 
 	{ SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL },
 
@@ -92,6 +93,7 @@ const struct doveadm_settings doveadm_default_settings = {
 	.doveadm_allowed_commands = "",
 	.dsync_alt_char = "_",
 	.dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U",
+	.dsync_features = "",
 	.ssl_client_ca_dir = "",
 	.ssl_client_ca_file = "",
 	.director_username_hash = "%Lu",
@@ -129,6 +131,43 @@ fix_base_path(struct doveadm_settings *set, pool_t pool, const char **str)
 }
 
 /* <settings checks> */
+struct dsync_feature_list {
+	const char *name;
+	enum dsync_features num;
+};
+
+static const struct dsync_feature_list dsync_feature_list[] = {
+	{ "empty_header_workaround", DSYNC_FEATURE_EMPTY_HDR_WORKAROUND },
+	{ NULL, 0 }
+};
+
+static int
+dsync_settings_parse_features(struct doveadm_settings *set,
+			      const char **error_r)
+{
+	enum dsync_features features = 0;
+	const struct dsync_feature_list *list;
+	const char *const *str;
+
+	str = t_strsplit_spaces(set->dsync_features, " ,");
+	for (; *str != NULL; str++) {
+		list = dsync_feature_list;
+		for (; list->name != NULL; list++) {
+			if (strcasecmp(*str, list->name) == 0) {
+				features |= list->num;
+				break;
+			}
+		}
+		if (list->name == NULL) {
+			*error_r = t_strdup_printf("dsync_features: "
+				"Unknown feature: %s", *str);
+			return -1;
+		}
+	}
+	set->parsed_features = features;
+	return 0;
+}
+
 static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED,
 				   const char **error_r)
 {
@@ -142,6 +181,8 @@ static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED,
 		*error_r = "dsync_alt_char must not be empty";
 		return FALSE;
 	}
+	if (dsync_settings_parse_features(set, error_r) != 0)
+		return FALSE;
 	return TRUE;
 }
 /* </settings checks> */

src/doveadm/doveadm-settings.h

@@ -3,6 +3,12 @@
 
 #include "net.h"
 
+/* <settings checks> */
+enum dsync_features {
+	DSYNC_FEATURE_EMPTY_HDR_WORKAROUND = 0x1,
+};
+/* </settings checks> */
+
 struct doveadm_settings {
 	const char *base_dir;
 	const char *libexec_dir;
@@ -22,7 +28,8 @@ struct doveadm_settings {
 	const char *ssl_client_ca_file;
 	const char *director_username_hash;
 	const char *doveadm_api_key;
-
+	const char *dsync_features;
+	enum dsync_features parsed_features;
 	ARRAY(const char *) plugin_envs;
 };
 

src/doveadm/doveadm.c

@@ -269,6 +269,8 @@ static void doveadm_read_settings(void)
 	set = master_service_settings_get_others(master_service)[0];
 	doveadm_settings = settings_dup(&doveadm_setting_parser_info, set,
 					pool_datastack_create());
+
+	doveadm_settings->parsed_features = set->parsed_features; /* copy this value by hand */
 }
 
 static struct doveadm_cmd *doveadm_cmdline_commands[] = {

src/doveadm/dsync/dsync-brain-mailbox.c

@@ -220,6 +220,8 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
 		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY;
 	if (brain->hdr_hash_v2)
 		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2;
+	if (brain->empty_hdr_workaround)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND;
 
 	brain->box_importer = brain->backup_send ? NULL :
 		dsync_mailbox_import_init(brain->box, brain->virtual_all_box,

src/doveadm/dsync/dsync-brain-private.h

@@ -114,6 +114,7 @@ struct dsync_brain {
 	bool no_notify:1;
 	bool hdr_hash_v2:1;
 	bool failed:1;
+	bool empty_hdr_workaround:1;
 };
 
 extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1];

src/doveadm/dsync/dsync-brain.c

@@ -143,6 +143,7 @@ dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags)
 	brain->no_mailbox_renames =
 		(flags & DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES) != 0;
 	brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0;
+	brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0;
 }
 
 static void

src/doveadm/dsync/dsync-brain.h

@@ -32,7 +32,9 @@ enum dsync_brain_flags {
 	   be removed once the renaming logic has no more bugs.. */
 	DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES	= 0x200,
 	/* Add MAILBOX_TRANSACTION_FLAG_NO_NOTIFY to transactions. */
-	DSYNC_BRAIN_FLAG_NO_NOTIFY		= 0x400
+	DSYNC_BRAIN_FLAG_NO_NOTIFY		= 0x400,
+	/* Workaround missing Date/Message-ID headers */
+	DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND	= 0x800,
 };
 
 enum dsync_brain_sync_type {

src/doveadm/dsync/dsync-ibc-stream.c

@@ -731,6 +731,8 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
 		dsync_serializer_encode_add(encoder, "purge_remote", "");
 	if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0)
 		dsync_serializer_encode_add(encoder, "no_notify", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0)
+		dsync_serializer_encode_add(encoder, "empty_hdr_workaround", "");
 
 	dsync_serializer_encode_finish(&encoder, str);
 	dsync_ibc_stream_send_string(ibc, str);
@@ -839,6 +841,8 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
 		set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE;
 	if (dsync_deserializer_decode_try(decoder, "no_notify", &value))
 		set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY;
+	if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
 	set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2;
 
 	*set_r = set;

src/doveadm/dsync/dsync-mail.h

@@ -88,6 +88,12 @@ dsync_mail_get_hash_headers(struct mailbox *box);
 
 int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
 			    const char **hdr_hash_r);
+static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash)
+{
+	/* md5(\n) */
+	return strcmp(hdr_hash, "68b329da9893e34099c7d8ad5cb9c940") == 0;
+}
+
 int dsync_mail_fill(struct mail *mail, bool minimal_fill,
 		    struct dsync_mail *dmail_r, const char **error_field_r);
 int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,

src/doveadm/dsync/dsync-mailbox-import.c

@@ -122,6 +122,7 @@ struct dsync_mailbox_importer {
 	bool mails_have_guids:1;
 	bool mails_use_guid128:1;
 	bool delete_mailbox:1;
+	bool empty_hdr_workaround:1;
 };
 
 static const char *dsync_mail_change_type_names[] = {
@@ -278,6 +279,8 @@ dsync_mailbox_import_init(struct mailbox *box,
 		(flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0;
 	importer->hdr_hash_version =
 		(flags & DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2) != 0 ? 2 : 1;
+	importer->empty_hdr_workaround =
+		(flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
 
 	mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
 				STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
@@ -782,7 +785,16 @@ static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
 		i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
 	}
 
-	diff = importer_mail_cmp(&m1, &m2);
+	if (importer->empty_hdr_workaround && !importer->mails_have_guids &&
+	    importer->cur_mail != NULL && save_change != NULL &&
+	    (dsync_mail_hdr_hash_is_empty(m1.guid) ||
+	     dsync_mail_hdr_hash_is_empty(m2.guid))) {
+		/* one of the headers is empty. assume it's broken and that
+		   the header matches what we have currently. */
+		diff = 0;
+	} else {
+		diff = importer_mail_cmp(&m1, &m2);
+	}
 	if (diff < 0) {
 		/* add a record for local mail */
 		i_assert(importer->cur_mail != NULL);

src/doveadm/dsync/dsync-mailbox-import.h

@@ -11,7 +11,8 @@ enum dsync_mailbox_import_flags {
 	DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS	= 0x10,
 	DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128	= 0x20,
 	DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY		= 0x40,
-	DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2		= 0x80
+	DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2		= 0x80,
+	DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND	= 0x100
 };
 
 struct mailbox;