lib-index: Don't increase modseq for backend/dirty flag changes

These flags are used only for internal changes and they shouldn't be triggering any modseq changes. To avoid modseqs from unexpectedly shrinking, the new modseq counting behavior is enabled only for newly rotated transaction log files that have a new minor_version.
dovecot · Apr 3, 2017 · b6a369a · b6a369a
1 parent 0713b92
commit b6a369a
Show file tree

Hide file tree

Showing 10 changed files with 96 additions and 18 deletions.
diff --git a/src/doveadm/doveadm-dump-log.c b/src/doveadm/doveadm-dump-log.c
@@ -11,7 +11,8 @@
 
 static struct mail_transaction_ext_intro prev_intro;
 
-static void dump_hdr(struct istream *input, uint64_t *modseq_r)
+static void dump_hdr(struct istream *input, uint64_t *modseq_r,
+		     unsigned int *version_r)
 {
 	struct mail_transaction_log_header hdr;
 	const unsigned char *data;
@@ -42,6 +43,7 @@ static void dump_hdr(struct istream *input, uint64_t *modseq_r)
 	       (unsigned long long)hdr.initial_modseq);
 	printf("compat flags = %x\n", hdr.compat_flags);
 	*modseq_r = hdr.initial_modseq;
+	*version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr);
 }
 
 static const char *log_record_type(unsigned int type)
@@ -464,7 +466,8 @@ static void log_record_print(const struct mail_transaction_header *hdr,
 	}
 }
 
-static int dump_record(struct istream *input, uint64_t *modseq)
+static int dump_record(struct istream *input, uint64_t *modseq,
+		       unsigned int version)
 {
 	struct mail_transaction_header hdr;
 	unsigned int hdr_size;
@@ -505,7 +508,7 @@ static int dump_record(struct istream *input, uint64_t *modseq)
 	}
 
 	uint64_t prev_modseq = *modseq;
-	mail_transaction_update_modseq(&hdr, data, modseq);
+	mail_transaction_update_modseq(&hdr, data, modseq, version);
 	if (*modseq > prev_modseq)
 		printf(", modseq=%llu", (unsigned long long)*modseq);
 	printf("\n");
@@ -519,13 +522,14 @@ static void cmd_dump_log(int argc ATTR_UNUSED, char *argv[])
 {
 	struct istream *input;
 	uint64_t modseq;
+	unsigned int version;
 	int ret;
 
 	input = i_stream_create_file(argv[1], (size_t)-1);
-	dump_hdr(input, &modseq);
+	dump_hdr(input, &modseq, &version);
 	do {
 		T_BEGIN {
-			ret = dump_record(input, &modseq);
+			ret = dump_record(input, &modseq, version);
 		} T_END;
 	} while (ret > 0);
 	i_stream_unref(&input);

diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c
@@ -536,6 +536,32 @@ mail_index_transaction_keywords_count_modseq_incs(struct mail_index_transaction
 	return count;
 }
 
+static bool
+transaction_flag_updates_have_non_internal(struct mail_index_transaction *t)
+{
+	struct mail_transaction_log_file *file = t->view->index->log->head;
+	const uint8_t internal_flags =
+		MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+	const struct mail_index_flag_update *u;
+	const unsigned int hdr_version =
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr);
+
+	if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, HIDE_INTERNAL_MODSEQS)) {
+		/* this check can be a bit racy if the call isn't done while
+		   transaction log is locked. practically it won't matter
+		   now though. */
+		return array_count(&t->updates) > 0;
+	}
+
+	array_foreach(&t->updates, u) {
+		uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+		if ((changed_flags & ~internal_flags) != 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
 uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction *t)
 {
 	struct mail_transaction_log_file *file = t->view->index->log->head;
@@ -565,7 +591,8 @@ uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction
 		/* sorting may change the order of keyword_updates,  */
 		new_highest_modseq++;
 	}
-	if (array_is_created(&t->updates) && array_count(&t->updates) > 0)
+	if (array_is_created(&t->updates) &&
+	    transaction_flag_updates_have_non_internal(t) > 0)
 		new_highest_modseq++;
 	if (array_is_created(&t->keyword_updates)) {
 		new_highest_modseq +=

diff --git a/src/lib-index/mail-transaction-log-append.c b/src/lib-index/mail-transaction-log-append.c
@@ -31,7 +31,8 @@ void mail_transaction_log_append_add(struct mail_transaction_log_append_ctx *ctx
 	buffer_append(ctx->output, &hdr, sizeof(hdr));
 	buffer_append(ctx->output, data, size);
 
-	mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq);
+	mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq,
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&ctx->log->head->hdr));
 	ctx->transaction_count++;
 }
 

diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
@@ -1007,8 +1007,31 @@ log_file_track_mailbox_sync_offset_hdr(struct mail_transaction_log_file *file,
 	return 0;
 }
 
+static bool
+flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+			       unsigned int count, unsigned int version)
+{
+	const uint8_t internal_flags =
+		MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+
+	/* Hide internal flags from modseqs if the log file's version
+	   is new enough. This allows upgrading without the modseqs suddenly
+	   shrinking. */
+	if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS))
+		return TRUE;
+
+	for (unsigned int i = 0; i < count; i++) {
+		uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+		if ((changed_flags & ~internal_flags) != 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-				    const void *data, uint64_t *cur_modseq)
+				    const void *data, uint64_t *cur_modseq,
+				    unsigned int version)
 {
 	uint32_t trans_size;
 
@@ -1046,13 +1069,21 @@ void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
 			break;
 		}
 	case MAIL_TRANSACTION_APPEND:
-	case MAIL_TRANSACTION_FLAG_UPDATE:
 	case MAIL_TRANSACTION_KEYWORD_UPDATE:
 	case MAIL_TRANSACTION_KEYWORD_RESET:
 	case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
 		/* these changes increase modseq */
 		*cur_modseq += 1;
 		break;
+	case MAIL_TRANSACTION_FLAG_UPDATE: {
+		const struct mail_transaction_flag_update *rec = data;
+		unsigned int count;
+
+		count = (trans_size - sizeof(*hdr)) / sizeof(*rec);
+		if (flag_updates_have_non_internal(rec, count, version))
+			*cur_modseq += 1;
+		break;
+	}
 	case MAIL_TRANSACTION_MODSEQ_UPDATE: {
 		const struct mail_transaction_modseq_update *rec, *end;
 
@@ -1209,7 +1240,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
 	while (cur_offset < offset) {
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return- 1;
-		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 	}
 
 	/* @UNSAFE: cache the value */
@@ -1280,7 +1312,8 @@ int mail_transaction_log_file_get_modseq_next_offset(
 	while (cur_offset < file->sync_offset) {
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return -1;
-		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 		if (cur_modseq >= modseq)
 			break;
 	}
@@ -1312,8 +1345,8 @@ log_file_track_sync(struct mail_transaction_log_file *file,
 	const void *data = hdr + 1;
 	int ret;
 
-	mail_transaction_update_modseq(hdr, hdr + 1,
-				       &file->sync_highest_modseq);
+	mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq,
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 	if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
 		return 1;
 

diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h
@@ -148,7 +148,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file,
 				      const char *lock_reason);
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-				    const void *data, uint64_t *cur_modseq);
+				    const void *data, uint64_t *cur_modseq,
+				    unsigned int version);
 int mail_transaction_log_file_get_highest_modseq_at(
 		struct mail_transaction_log_file *file,
 		uoff_t offset, uint64_t *highest_modseq_r);

diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c
@@ -792,7 +792,8 @@ log_view_get_next(struct mail_transaction_log_view *view,
 		ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
 	} T_END;
 	if (ret > 0) {
-		mail_transaction_update_modseq(hdr, data, &view->prev_modseq);
+		mail_transaction_update_modseq(hdr, data, &view->prev_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 		*hdr_r = hdr;
 		*data_r = data;
 		view->cur_offset += full_size;

diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
@@ -218,6 +218,13 @@ bool mail_transaction_log_want_rotate(struct mail_transaction_log *log)
 {
 	struct mail_transaction_log_file *file = log->head;
 
+	if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
+	    (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+	     file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {
+		/* upgrade immediately to a new log file format */
+		return TRUE;
+	}
+
 	if (file->sync_offset > log->index->log_rotate_max_size) {
 		/* file is too large, definitely rotate */
 		return TRUE;

diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h
@@ -6,7 +6,7 @@
 #define MAIL_TRANSACTION_LOG_SUFFIX ".log"
 
 #define MAIL_TRANSACTION_LOG_MAJOR_VERSION 1
-#define MAIL_TRANSACTION_LOG_MINOR_VERSION 2
+#define MAIL_TRANSACTION_LOG_MINOR_VERSION 3
 #define MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE 24
 
 #define MAIL_TRANSACTION_LOG_VERSION_FULL(major, minor) \
@@ -18,6 +18,8 @@
 
 #define MAIL_TRANSACTION_LOG_VERSION_COMPAT_FLAGS \
 	MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2)
+#define MAIL_TRANSACTION_LOG_VERSION_HIDE_INTERNAL_MODSEQS \
+	MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3)
 
 struct mail_transaction_log_header {
 	uint8_t major_version;

diff --git a/src/lib-index/test-mail-transaction-log-append.c b/src/lib-index/test-mail-transaction-log-append.c
@@ -27,7 +27,8 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file ATT
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
 				    const void *data ATTR_UNUSED,
-				    uint64_t *cur_modseq)
+				    uint64_t *cur_modseq,
+				    unsigned int version ATTR_UNUSED)
 {
 	if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0)
 		*cur_modseq += 1;

diff --git a/src/lib-index/test-mail-transaction-log-view.c b/src/lib-index/test-mail-transaction-log-view.c
@@ -51,7 +51,8 @@ int mail_transaction_log_file_get_highest_modseq_at(
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr ATTR_UNUSED,
 				    const void *data ATTR_UNUSED,
-				    uint64_t *cur_modseq)
+				    uint64_t *cur_modseq,
+				    unsigned int version ATTR_UNUSED)
 {
 	*cur_modseq += 1;
 }