doveadm-sync: Add end-date support

src/doveadm/doveadm-dsync.c

@@ -39,7 +39,7 @@
 #include <ctype.h>
 #include <sys/wait.h>
 
-#define DSYNC_COMMON_GETOPT_ARGS "+1a:dDEfg:I:l:m:n:NO:Pr:Rs:t:T:Ux:"
+#define DSYNC_COMMON_GETOPT_ARGS "+1a:dDEfg:I:l:m:n:NO:Pr:Rs:t:e:T:Ux:"
 #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30
 /* The broken_char is mainly set to get a proper error message when trying to
    convert a mailbox with a name that can't be used properly translated between
@@ -67,6 +67,7 @@ struct dsync_cmd_context {
 	ARRAY_TYPE(const_string) exclude_mailboxes;
 	ARRAY_TYPE(const_string) namespace_prefixes;
 	time_t sync_since_timestamp;
+	time_t sync_until_timestamp;
 	uoff_t sync_max_size;
 	unsigned int io_timeout_secs;
 
@@ -569,6 +570,13 @@ cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
 			"%s ", net_ip2addr(&_ctx->cur_client_ip));
 	}
 	set.sync_since_timestamp = ctx->sync_since_timestamp;
+	set.sync_until_timestamp = ctx->sync_until_timestamp;
+
+	if (set.sync_since_timestamp > 0 && set.sync_until_timestamp > 0 &&
+	    set.sync_since_timestamp > set.sync_until_timestamp) {
+		i_fatal("start date is later than end date");
+	}
+
 	set.sync_max_size = ctx->sync_max_size;
 	set.sync_box = ctx->mailbox;
 	set.sync_flag = ctx->sync_flags;
@@ -1040,6 +1048,10 @@ cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
 		if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0)
 			i_fatal("Invalid -t parameter: %s", optarg);
 		break;
+	case 'e':
+		if (mail_parse_human_timestamp(optarg, &ctx->sync_until_timestamp) < 0)
+			i_fatal("Invalid -e parameter: %s", optarg);
+		break;
 	case 'I':
 		if (settings_get_size(optarg, &ctx->sync_max_size, &error) < 0)
 			i_fatal("Invalid -I parameter '%s': %s", optarg, error);

src/doveadm/dsync/dsync-brain-mailbox.c

@@ -233,6 +233,7 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
 					  remote_dsync_box->highest_modseq,
 					  remote_dsync_box->highest_pvt_modseq,
 					  brain->sync_since_timestamp,
+					  brain->sync_until_timestamp,
 					  brain->sync_max_size,
 					  brain->sync_flag,
 					  import_flags);
@@ -321,7 +322,8 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
 	if (brain->no_mail_prefetch)
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
-	if (brain->sync_since_timestamp > 0)
+	if (brain->sync_since_timestamp > 0 ||
+	    brain->sync_until_timestamp > 0)
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
 	if (brain->sync_max_size > 0)
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES;

src/doveadm/dsync/dsync-brain-private.h

@@ -58,6 +58,7 @@ struct dsync_brain {
 	const char *const *exclude_mailboxes;
 	enum dsync_brain_sync_type sync_type;
 	time_t sync_since_timestamp;
+	time_t sync_until_timestamp;
 	uoff_t sync_max_size;
 	const char *sync_flag;
 	char alt_char;

src/doveadm/dsync/dsync-brain.c

@@ -192,6 +192,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
 	brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
 		set->mailbox_alt_char;
 	brain->sync_since_timestamp = set->sync_since_timestamp;
+	brain->sync_until_timestamp = set->sync_until_timestamp;
 	brain->sync_max_size = set->sync_max_size;
 	brain->sync_flag = p_strdup(brain->pool, set->sync_flag);
 	brain->sync_box = p_strdup(brain->pool, set->sync_box);
@@ -231,6 +232,7 @@ dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
 	ibc_set.virtual_all_box = set->virtual_all_box;
 	ibc_set.exclude_mailboxes = set->exclude_mailboxes;
 	ibc_set.sync_since_timestamp = set->sync_since_timestamp;
+	ibc_set.sync_until_timestamp = set->sync_until_timestamp;
 	ibc_set.sync_max_size = set->sync_max_size;
 	ibc_set.sync_flags = set->sync_flag;
 	memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
@@ -487,6 +489,7 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
 	brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
 		p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
 	brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
+	brain->sync_until_timestamp = ibc_set->sync_until_timestamp;
 	brain->sync_max_size = ibc_set->sync_max_size;
 	brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags);
 	memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,

src/doveadm/dsync/dsync-brain.h

@@ -69,6 +69,8 @@ struct dsync_brain_settings {
 	char mailbox_alt_char;
 	/* Sync only mails with received timestamp at least this high. */
 	time_t sync_since_timestamp;
+	/* Sync only mails with received timestamp less or equal than this */
+	time_t sync_until_timestamp;
 	/* Don't sync mails larger than this. */
 	uoff_t sync_max_size;
 	/* Sync only mails which contains / doesn't contain this flag.

src/doveadm/dsync/dsync-ibc-pipe.c

@@ -180,6 +180,7 @@ dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc,
 	memcpy(item->u.set.sync_box_guid, set->sync_box_guid,
 	       sizeof(item->u.set.sync_box_guid));
 	item->u.set.sync_since_timestamp = set->sync_since_timestamp;
+	item->u.set.sync_until_timestamp = set->sync_until_timestamp;
 	item->u.set.sync_max_size = set->sync_max_size;
 	item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags);
 }

src/doveadm/dsync/dsync-ibc-stream.c

@@ -77,7 +77,7 @@ static const struct {
 	  	"debug sync_visible_namespaces exclude_mailboxes  "
 	  	"send_mail_requests backup_send backup_recv lock_timeout "
 	  	"no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote "
-		"no_notify sync_since_timestamp sync_max_size sync_flags "
+		"no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp"
 	  	"virtual_all_box"
 	},
 	{ .name = "mailbox_state",
@@ -708,6 +708,10 @@ dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
 		dsync_serializer_encode_add(encoder, "sync_since_timestamp",
 			t_strdup_printf("%ld", (long)set->sync_since_timestamp));
 	}
+	if (set->sync_until_timestamp > 0) {
+		dsync_serializer_encode_add(encoder, "sync_until_timestamp",
+			t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+	}
 	if (set->sync_max_size > 0) {
 		dsync_serializer_encode_add(encoder, "sync_max_size",
 			t_strdup_printf("%llu", (unsigned long long)set->sync_max_size));
@@ -824,6 +828,14 @@ dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
 			return DSYNC_IBC_RECV_RET_TRYAGAIN;
 		}
 	}
+	if (dsync_deserializer_decode_try(decoder, "sync_until_timestamp", &value)) {
+		if (str_to_time(value, &set->sync_until_timestamp) < 0 ||
+		    set->sync_until_timestamp == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid sync_until_timestamp: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
 	if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) {
 		if (str_to_uoff(value, &set->sync_max_size) < 0 ||
 		    set->sync_max_size == 0) {

src/doveadm/dsync/dsync-ibc.h

@@ -58,6 +58,8 @@ struct dsync_ibc_settings {
 	const char *const *exclude_mailboxes;
 	/* Sync only mails with received timestamp at least this high. */
 	time_t sync_since_timestamp;
+	/* Sync only mails with received timestamp less or equal than this */
+	time_t sync_until_timestamp;
 	/* Don't sync mails larger than this. */
 	uoff_t sync_max_size;
 	/* Sync only mails with specified flags. */

src/doveadm/dsync/dsync-mail.h

@@ -79,7 +79,7 @@ struct dsync_mail_change {
 	/* +add, -remove, =final, &add_and_final. */
 	ARRAY_TYPE(const_string) keyword_changes;
 
-	/* Received timestamp for saves, if brain.sync_since_timestamp is set */
+	/* Received timestamp for saves, if brain.sync_since/until_timestamp is set */
 	time_t received_timestamp;
 	/* Mail's size for saves if brain.sync_max_size is set,
 	   (uoff_t)-1 otherwise. */

src/doveadm/dsync/dsync-mailbox-import.c

@@ -62,6 +62,7 @@ struct dsync_mailbox_importer {
 	uint32_t remote_first_recent_uid;
 	uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
 	time_t sync_since_timestamp;
+	time_t sync_until_timestamp;
 	uoff_t sync_max_size;
 	enum mailbox_transaction_flags transaction_flags;
 	unsigned int hdr_hash_version;
@@ -217,7 +218,9 @@ dsync_mailbox_import_init(struct mailbox *box,
 			  uint32_t remote_first_recent_uid,
 			  uint64_t remote_highest_modseq,
 			  uint64_t remote_highest_pvt_modseq,
-			  time_t sync_since_timestamp, uoff_t sync_max_size,
+			  time_t sync_since_timestamp,
+			  time_t sync_until_timestamp,
+			  uoff_t sync_max_size,
 			  const char *sync_flag,
 			  enum dsync_mailbox_import_flags flags)
 {
@@ -241,6 +244,7 @@ dsync_mailbox_import_init(struct mailbox *box,
 	importer->remote_highest_modseq = remote_highest_modseq;
 	importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
 	importer->sync_since_timestamp = sync_since_timestamp;
+	importer->sync_until_timestamp = sync_until_timestamp;
 	importer->sync_max_size = sync_max_size;
 	importer->stateful_import = importer->last_common_uid_found;
 	if (sync_flag != NULL) {
@@ -1350,6 +1354,14 @@ dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer,
 			return FALSE;
 		}
 	}
+	if (importer->sync_until_timestamp > 0) {
+		i_assert(change->received_timestamp > 0);
+		if (change->received_timestamp > importer->sync_until_timestamp) {
+			/* mail has too new timestamp - skip it */
+			*result_r = "Ignoring missing local mail with too new timestamp";
+			return FALSE;
+		}
+	}
 	if (importer->sync_max_size > 0) {
 		i_assert(change->virtual_size != (uoff_t)-1);
 		if (change->virtual_size < importer->sync_max_size) {
@@ -1644,7 +1656,9 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
 	int ret;
 
 	i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
-		 ((change->received_timestamp > 0 || importer->sync_since_timestamp == 0) &&
+		 ((change->received_timestamp > 0 ||
+		   (importer->sync_since_timestamp == 0 &&
+		    importer->sync_until_timestamp == 0)) &&
 		  (change->virtual_size != (uoff_t)-1 || importer->sync_max_size == 0)));
 
 	/* try to find the matching local mail */

src/doveadm/dsync/dsync-mailbox-import.h

@@ -32,7 +32,9 @@ dsync_mailbox_import_init(struct mailbox *box,
 			  uint32_t remote_first_recent_uid,
 			  uint64_t remote_highest_modseq,
 			  uint64_t remote_highest_pvt_modseq,
-			  time_t sync_since_timestamp, uoff_t sync_max_size,
+			  time_t sync_since_timestamp,
+			  time_t sync_until_timestamp,
+			  uoff_t sync_max_size,
 			  const char *sync_flag,
 			  enum dsync_mailbox_import_flags flags);
 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,