replication: prevent boot when rs uuid mismatches

When an instance was being bootstrapped, it didn't check if its
replication sources had the same replicaset UUID.

As a result, if they didn't match, it used to boot from any of
them almost randomly (using selection among non-read-only nodes,
and min uuid among these) and raise an error about the mismatching
ones later.

Obviously, there is something wrong, such replication config is
not valid and the node should not boot at all.

The patch tries to prevent such instance's bootstrap if it sees at
least 2 replicas with different replicaset UUID.

It does not solve the problem for all the cases because still one
of the replicas simply could be not available. Then the new node
would give up and boot from the other node successfully, and
notice replicaset UUID mismatch only when the connection to the
first node is restored.

But it should help for most of the boot attempts.

Closes #5613

@TarantoolBot document
Title: New field in IPROTO_BALLOT

`IPROTO_BALLOT(0x29)` is a response for `IPROTO_VOTE(0x44)`. It
used to contain 5 fields (is_ro, vclock, gc_vclock, etc). Now it
also contains `IPROTO_BALLOT_REPLICASET_UUID (0x06)`. It is a
UUID string showing replicaset UUID of the sender. It can be nil
UUID (when all digits are 0) when not known. It is optional. When
omitted, it is assumed to be not known.

changelogs/unreleased/gh-5613-cross-boot-uuid.md

@@ -0,0 +1,6 @@
+## bugfix/replication
+
+* Fixed an error when a replica at attempt to boot from instances of different
+  replicasets (with not the same replicaset UUID) used to boot from one of the
+  instances and raise an error about the others. Now it won't boot at all if
+  detects this situation (gh-5613).

src/box/box.cc

@@ -2872,6 +2872,7 @@ box_process_vote(struct ballot *ballot)
 	ballot->is_loading = is_ro;
 	vclock_copy(&ballot->vclock, &replicaset.vclock);
 	vclock_copy(&ballot->gc_vclock, &gc.vclock);
+	ballot->replicaset_uuid = REPLICASET_UUID;
 }
 
 /** Insert a new cluster into _schema */

src/box/errcode.h

@@ -277,6 +277,7 @@ struct errcode_record {
 	/*222 */_(ER_QUORUM_WAIT,		"Couldn't wait for quorum %d: %s") \
 	/*223 */_(ER_INTERFERING_PROMOTE,	"Instance with replica id %u was promoted first") \
 	/*224 */_(ER_RAFT_DISABLED,		"Elections were turned off while running box.ctl.promote()")\
+	/*225 */_(ER_REPLICASET_UUID_CONFLICT,	"Conflicting replicaset UUIDs %s and %s") \
 
 /*
  * !IMPORTANT! Please follow instructions at start of the file

src/box/iproto_constants.h

@@ -167,6 +167,7 @@ enum iproto_ballot_key {
 	IPROTO_BALLOT_GC_VCLOCK = 0x03,
 	IPROTO_BALLOT_IS_LOADING = 0x04,
 	IPROTO_BALLOT_IS_ANON = 0x05,
+	IPROTO_BALLOT_REPLICASET_UUID = 0x06,
 };
 
 #define bit(c) (1ULL<<IPROTO_##c)

src/box/replication.cc

@@ -702,6 +702,8 @@ replicaset_connect(struct applier **appliers, int count,
 	/* Memory for on_state triggers registered in appliers */
 	struct applier_on_connect triggers[VCLOCK_MAX];
 
+	struct tt_uuid replicaset_uuid = uuid_nil;
+
 	struct replicaset_connect_state state;
 	state.connected = state.failed = 0;
 	fiber_cond_create(&state.wakeup);
@@ -767,6 +769,18 @@ replicaset_connect(struct applier **appliers, int count,
 		struct applier *applier = appliers[i];
 		if (applier->state != APPLIER_CONNECTED)
 			applier_stop(applier);
+		struct tt_uuid *uuid = &applier->ballot.replicaset_uuid;
+		if (tt_uuid_is_nil(uuid))
+			continue;
+		if (tt_uuid_is_nil(&replicaset_uuid)) {
+			replicaset_uuid = *uuid;
+			continue;
+		}
+		if (tt_uuid_is_equal(&replicaset_uuid, uuid))
+			continue;
+		diag_set(ClientError, ER_REPLICASET_UUID_CONFLICT,
+			 tt_uuid_str(&replicaset_uuid), tt_uuid_str(uuid));
+		goto error;
 	}
 
 	/* Now all the appliers are connected, update the replica set. */

src/box/xrow.c

@@ -449,15 +449,17 @@ iproto_reply_vote(struct obuf *out, const struct ballot *ballot,
 		  uint64_t sync, uint32_t schema_version)
 {
 	size_t max_size = IPROTO_HEADER_LEN + mp_sizeof_map(1) +
-		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_map(5) +
+		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_map(6) +
 		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_bool(ballot->is_ro) +
 		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_bool(ballot->is_loading) +
 		mp_sizeof_uint(IPROTO_BALLOT_IS_ANON) +
 		mp_sizeof_bool(ballot->is_anon) +
 		mp_sizeof_uint(UINT32_MAX) +
 		mp_sizeof_vclock_ignore0(&ballot->vclock) +
 		mp_sizeof_uint(UINT32_MAX) +
-		mp_sizeof_vclock_ignore0(&ballot->gc_vclock);
+		mp_sizeof_vclock_ignore0(&ballot->gc_vclock) +
+		mp_sizeof_uint(UINT32_MAX) +
+		mp_sizeof_str(UUID_STR_LEN);
 
 	char *buf = obuf_reserve(out, max_size);
 	if (buf == NULL) {
@@ -469,7 +471,7 @@ iproto_reply_vote(struct obuf *out, const struct ballot *ballot,
 	char *data = buf + IPROTO_HEADER_LEN;
 	data = mp_encode_map(data, 1);
 	data = mp_encode_uint(data, IPROTO_BALLOT);
-	data = mp_encode_map(data, 5);
+	data = mp_encode_map(data, 6);
 	data = mp_encode_uint(data, IPROTO_BALLOT_IS_RO);
 	data = mp_encode_bool(data, ballot->is_ro);
 	data = mp_encode_uint(data, IPROTO_BALLOT_IS_LOADING);
@@ -480,6 +482,8 @@ iproto_reply_vote(struct obuf *out, const struct ballot *ballot,
 	data = mp_encode_vclock_ignore0(data, &ballot->vclock);
 	data = mp_encode_uint(data, IPROTO_BALLOT_GC_VCLOCK);
 	data = mp_encode_vclock_ignore0(data, &ballot->gc_vclock);
+	data = mp_encode_uint(data, IPROTO_BALLOT_REPLICASET_UUID);
+	data = xrow_encode_uuid(data, &ballot->replicaset_uuid);
 	size_t size = data - buf;
 	assert(size <= max_size);
 
@@ -1361,6 +1365,7 @@ xrow_decode_ballot(struct xrow_header *row, struct ballot *ballot)
 	ballot->is_loading = false;
 	ballot->is_anon = false;
 	vclock_create(&ballot->vclock);
+	ballot->replicaset_uuid = uuid_nil;
 
 	const char *start = NULL;
 	const char *end = NULL;
@@ -1424,6 +1429,11 @@ xrow_decode_ballot(struct xrow_header *row, struct ballot *ballot)
 						     &ballot->gc_vclock) != 0)
 				goto err;
 			break;
+		case IPROTO_BALLOT_REPLICASET_UUID:
+			if (xrow_decode_uuid(&data,
+					     &ballot->replicaset_uuid) != 0)
+				goto err;
+			break;
 		default:
 			mp_next(&data);
 		}

src/box/xrow.h

@@ -382,6 +382,8 @@ struct ballot {
 	struct vclock vclock;
 	/** Oldest vclock available on the instance. */
 	struct vclock gc_vclock;
+	/** Replicaset UUID of the sender. Nil when unknown. */
+	struct tt_uuid replicaset_uuid;
 };
 
 /**

test/box/error.result

@@ -443,6 +443,7 @@ t;
  |   222: box.error.QUORUM_WAIT
  |   223: box.error.INTERFERING_PROMOTE
  |   224: box.error.RAFT_DISABLED
+ |   225: box.error.REPLICASET_UUID_CONFLICT
  | ...
 
 test_run:cmd("setopt delimiter ''");

test/replication/gh-5613-cross-bootstrap.result

@@ -0,0 +1,54 @@
+-- test-run result file version 2
+test_run = require('test_run').new()
+ | ---
+ | ...
+
+test_run:cmd('create server master1 with script="replication/gh-5613-master1.lua"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('start server master1')
+ | ---
+ | - true
+ | ...
+test_run:cmd('create server master2 with script="replication/gh-5613-master2.lua"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('start server master2')
+ | ---
+ | - true
+ | ...
+
+test_run:cmd('create server replica with script="replication/gh-5613-replica.lua"')
+ | ---
+ | - true
+ | ...
+test_run:cmd('start server replica with crash_expected=True')
+ | ---
+ | - false
+ | ...
+opts = {filename = 'gh-5613-replica.log'}
+ | ---
+ | ...
+assert(test_run:grep_log(nil, 'ER_REPLICASET_UUID_CONFLICT', nil, opts) ~= nil)
+ | ---
+ | - true
+ | ...
+
+test_run:cmd('stop server master2')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server master2')
+ | ---
+ | - true
+ | ...
+test_run:cmd('stop server master1')
+ | ---
+ | - true
+ | ...
+test_run:cmd('delete server master1')
+ | ---
+ | - true
+ | ...

test/replication/gh-5613-cross-bootstrap.test.lua

@@ -0,0 +1,16 @@
+test_run = require('test_run').new()
+
+test_run:cmd('create server master1 with script="replication/gh-5613-master1.lua"')
+test_run:cmd('start server master1')
+test_run:cmd('create server master2 with script="replication/gh-5613-master2.lua"')
+test_run:cmd('start server master2')
+
+test_run:cmd('create server replica with script="replication/gh-5613-replica.lua"')
+test_run:cmd('start server replica with crash_expected=True')
+opts = {filename = 'gh-5613-replica.log'}
+assert(test_run:grep_log(nil, 'ER_REPLICASET_UUID_CONFLICT', nil, opts) ~= nil)
+
+test_run:cmd('stop server master2')
+test_run:cmd('delete server master2')
+test_run:cmd('stop server master1')
+test_run:cmd('delete server master1')

test/replication/gh-5613-master1.lua

@@ -0,0 +1,4 @@
+#!/usr/bin/env tarantool
+
+require('console').listen(os.getenv('ADMIN'))
+box.cfg({listen = 'unix/:./master1.sock'})

test/replication/gh-5613-master2.lua

@@ -0,0 +1,4 @@
+#!/usr/bin/env tarantool
+
+require('console').listen(os.getenv('ADMIN'))
+box.cfg({listen = 'unix/:./master2.sock'})

test/replication/gh-5613-replica.lua

@@ -0,0 +1,4 @@
+#!/usr/bin/env tarantool
+
+require('console').listen(os.getenv('ADMIN'))
+box.cfg({replication = {'unix/:./master1.sock', 'unix/:./master2.sock'}})

test/replication/suite.cfg

@@ -44,6 +44,7 @@
     "gh-5435-qsync-clear-synchro-queue-commit-all.test.lua": {},
     "gh-5536-wal-limit.test.lua": {},
     "gh-5566-final-join-synchro.test.lua": {},
+    "gh-5613-cross-bootstrap.test.lua": {},
     "gh-6032-promote-wal-write.test.lua": {},
     "gh-6057-qsync-confirm-async-no-wal.test.lua": {},
     "gh-6094-rs-uuid-mismatch.test.lua": {},