replication: introduce ballot.is_booted

The new field reports whether the instance has finished its
bootstrap/recovery, or IOW has finished box.cfg().

The new field will help in fixing #5613 so as not to try to join
to a replicaset via non-bootstrapped instances if there are
others.

The problem is that otherwise, if all nodes are booted but
are read-only, new instances bootstrap their own independent
replicaset. It would be better to just fail and terminate the
process than do such a bizarre action.

Part of #5613

(cherry picked from commit f8a150c)

src/box/box.cc

@@ -2835,6 +2835,7 @@ box_process_vote(struct ballot *ballot)
 	ballot->is_ro_cfg = cfg_geti("read_only") != 0;
 	ballot->is_anon = replication_anon;
 	ballot->is_ro = is_ro_summary;
+	ballot->is_booted = is_box_configured;
 	vclock_copy(&ballot->vclock, &replicaset.vclock);
 	vclock_copy(&ballot->gc_vclock, &gc.vclock);
 }

src/box/iproto_constants.h

@@ -167,6 +167,7 @@ enum iproto_ballot_key {
 	IPROTO_BALLOT_GC_VCLOCK = 0x03,
 	IPROTO_BALLOT_IS_RO = 0x04,
 	IPROTO_BALLOT_IS_ANON = 0x05,
+	IPROTO_BALLOT_IS_BOOTED = 0x06,
 };
 
 #define bit(c) (1ULL<<IPROTO_##c)

src/box/xrow.c

@@ -449,11 +449,13 @@ iproto_reply_vote(struct obuf *out, const struct ballot *ballot,
 		  uint64_t sync, uint32_t schema_version)
 {
 	size_t max_size = IPROTO_HEADER_LEN + mp_sizeof_map(1) +
-		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_map(5) +
+		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_map(6) +
 		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_bool(ballot->is_ro_cfg) +
 		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_bool(ballot->is_ro) +
 		mp_sizeof_uint(IPROTO_BALLOT_IS_ANON) +
 		mp_sizeof_bool(ballot->is_anon) +
+		mp_sizeof_uint(IPROTO_BALLOT_IS_BOOTED) +
+		mp_sizeof_bool(ballot->is_booted) +
 		mp_sizeof_uint(UINT32_MAX) +
 		mp_sizeof_vclock_ignore0(&ballot->vclock) +
 		mp_sizeof_uint(UINT32_MAX) +
@@ -469,13 +471,15 @@ iproto_reply_vote(struct obuf *out, const struct ballot *ballot,
 	char *data = buf + IPROTO_HEADER_LEN;
 	data = mp_encode_map(data, 1);
 	data = mp_encode_uint(data, IPROTO_BALLOT);
-	data = mp_encode_map(data, 5);
+	data = mp_encode_map(data, 6);
 	data = mp_encode_uint(data, IPROTO_BALLOT_IS_RO_CFG);
 	data = mp_encode_bool(data, ballot->is_ro_cfg);
 	data = mp_encode_uint(data, IPROTO_BALLOT_IS_RO);
 	data = mp_encode_bool(data, ballot->is_ro);
 	data = mp_encode_uint(data, IPROTO_BALLOT_IS_ANON);
 	data = mp_encode_bool(data, ballot->is_anon);
+	data = mp_encode_uint(data, IPROTO_BALLOT_IS_BOOTED);
+	data = mp_encode_bool(data, ballot->is_booted);
 	data = mp_encode_uint(data, IPROTO_BALLOT_VCLOCK);
 	data = mp_encode_vclock_ignore0(data, &ballot->vclock);
 	data = mp_encode_uint(data, IPROTO_BALLOT_GC_VCLOCK);
@@ -1360,6 +1364,7 @@ xrow_decode_ballot(struct xrow_header *row, struct ballot *ballot)
 	ballot->is_ro_cfg = false;
 	ballot->is_ro = false;
 	ballot->is_anon = false;
+	ballot->is_booted = true;
 	vclock_create(&ballot->vclock);
 
 	const char *start = NULL;
@@ -1424,6 +1429,11 @@ xrow_decode_ballot(struct xrow_header *row, struct ballot *ballot)
 						     &ballot->gc_vclock) != 0)
 				goto err;
 			break;
+		case IPROTO_BALLOT_IS_BOOTED:
+			if (mp_typeof(*data) != MP_BOOL)
+				goto err;
+			ballot->is_booted = mp_decode_bool(&data);
+			break;
 		default:
 			mp_next(&data);
 		}

src/box/xrow.h

@@ -379,6 +379,8 @@ struct ballot {
 	 * finished recovery/bootstrap; or anything else.
 	 */
 	bool is_ro;
+	/** Set if the instance has finished its bootstrap/recovery. */
+	bool is_booted;
 	/** Current instance vclock. */
 	struct vclock vclock;
 	/** Oldest vclock available on the instance. */