replication: ballot.is_loading -> is_ro

Is_loading in the ballot used to mean the following: "the instance
did not finish its box.cfg() or has read_only = true". Which is
quite a strange property.

For instance, it was 'true' even if the instance is not really
loading anymore but has read_only = true.

The patch renames it to 'is_ro' (which existed here before, but
also with a wrong meaning).

Its behaviour is slightly changed to report the RO state of the
instance. Not its read_only. This way it incorporates all the
possible RO conditions. Such as not finished bootstrap, having
read_only = true, being a Raft follower, and so on.

The patch is done in scope of #5613 where the ballot is going to
be extended and used a bit differently in the join-master search
algorithm.

Part of #5613

(cherry picked from commit 71d2a56)

src/box/box.cc

@@ -2834,14 +2834,7 @@ box_process_vote(struct ballot *ballot)
 {
 	ballot->is_ro_cfg = cfg_geti("read_only") != 0;
 	ballot->is_anon = replication_anon;
-	/*
-	 * is_ro is true on initial load and is set to box.cfg.read_only
-	 * after box_cfg() returns, during dynamic box.cfg parameters setting.
-	 * We would like to prefer already bootstrapped instances to the ones
-	 * still bootstrapping and the ones still bootstrapping, but writeable
-	 * to the ones that have box.cfg.read_only = true.
-	 */
-	ballot->is_loading = is_ro;
+	ballot->is_ro = is_ro_summary;
 	vclock_copy(&ballot->vclock, &replicaset.vclock);
 	vclock_copy(&ballot->gc_vclock, &gc.vclock);
 }

src/box/iproto_constants.h

@@ -165,7 +165,7 @@ enum iproto_ballot_key {
 	IPROTO_BALLOT_IS_RO_CFG = 0x01,
 	IPROTO_BALLOT_VCLOCK = 0x02,
 	IPROTO_BALLOT_GC_VCLOCK = 0x03,
-	IPROTO_BALLOT_IS_LOADING = 0x04,
+	IPROTO_BALLOT_IS_RO = 0x04,
 	IPROTO_BALLOT_IS_ANON = 0x05,
 };
 

src/box/replication.cc

@@ -987,7 +987,7 @@ replicaset_round(bool skip_ro)
 		 * Try to find a replica which has already left
 		 * orphan mode.
 		 */
-		if (ballot->is_loading && !leader_ballot->is_loading)
+		if (ballot->is_ro && !leader_ballot->is_ro)
 			continue;
 		/*
 		 * Choose the replica with the most advanced

src/box/xrow.c

@@ -451,7 +451,7 @@ iproto_reply_vote(struct obuf *out, const struct ballot *ballot,
 	size_t max_size = IPROTO_HEADER_LEN + mp_sizeof_map(1) +
 		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_map(5) +
 		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_bool(ballot->is_ro_cfg) +
-		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_bool(ballot->is_loading) +
+		mp_sizeof_uint(UINT32_MAX) + mp_sizeof_bool(ballot->is_ro) +
 		mp_sizeof_uint(IPROTO_BALLOT_IS_ANON) +
 		mp_sizeof_bool(ballot->is_anon) +
 		mp_sizeof_uint(UINT32_MAX) +
@@ -472,8 +472,8 @@ iproto_reply_vote(struct obuf *out, const struct ballot *ballot,
 	data = mp_encode_map(data, 5);
 	data = mp_encode_uint(data, IPROTO_BALLOT_IS_RO_CFG);
 	data = mp_encode_bool(data, ballot->is_ro_cfg);
-	data = mp_encode_uint(data, IPROTO_BALLOT_IS_LOADING);
-	data = mp_encode_bool(data, ballot->is_loading);
+	data = mp_encode_uint(data, IPROTO_BALLOT_IS_RO);
+	data = mp_encode_bool(data, ballot->is_ro);
 	data = mp_encode_uint(data, IPROTO_BALLOT_IS_ANON);
 	data = mp_encode_bool(data, ballot->is_anon);
 	data = mp_encode_uint(data, IPROTO_BALLOT_VCLOCK);
@@ -1358,7 +1358,7 @@ int
 xrow_decode_ballot(struct xrow_header *row, struct ballot *ballot)
 {
 	ballot->is_ro_cfg = false;
-	ballot->is_loading = false;
+	ballot->is_ro = false;
 	ballot->is_anon = false;
 	vclock_create(&ballot->vclock);
 
@@ -1404,10 +1404,10 @@ xrow_decode_ballot(struct xrow_header *row, struct ballot *ballot)
 				goto err;
 			ballot->is_ro_cfg = mp_decode_bool(&data);
 			break;
-		case IPROTO_BALLOT_IS_LOADING:
+		case IPROTO_BALLOT_IS_RO:
 			if (mp_typeof(*data) != MP_BOOL)
 				goto err;
-			ballot->is_loading = mp_decode_bool(&data);
+			ballot->is_ro = mp_decode_bool(&data);
 			break;
 		case IPROTO_BALLOT_IS_ANON:
 			if (mp_typeof(*data) != MP_BOOL)

src/box/xrow.h

@@ -374,10 +374,11 @@ struct ballot {
 	 */
 	bool is_anon;
 	/**
-	 * Set if the instance hasn't finished bootstrap or recovery, or
-	 * is syncing with other replicas in the replicaset.
+	 * Set if the instance is not writable due to any reason. Could be
+	 * config read_only=true; being orphan; being a Raft follower; not
+	 * finished recovery/bootstrap; or anything else.
 	 */
-	bool is_loading;
+	bool is_ro;
 	/** Current instance vclock. */
 	struct vclock vclock;
 	/** Oldest vclock available on the instance. */