Skip to content

Commit

Permalink
config: forbid attempt make an anon replica RW
Browse files Browse the repository at this point in the history 
This commit adds several checks that are specific for
`replication.failover` mode.

* `replication.failover: off`: an anonymous replica shouldn't be set to
  read-write mode using `database.mode` option.
* `replication.failover: manual`: an anonymous replica shouldn't be
  configured as a replicaset leader using `<replicaset>.leader` option.
* `replication.failover: election`: an anonymous replica can't be
  configured with `replication.election_mode` other than `off`.

This commit also adjusts default `replication.election_mode` to `off`
for an anonymous replica if it is part of a `replication.failover:
election` replicaset (the default for a non-anonymous instance is
`candidate`).

Part of tarantool#9432

NO_DOC=The documentation request is in the last commit of the series.
  • Loading branch information
@Totktonada
Totktonada committed Dec 6, 2023
1 parent 151edc0 commit f732a86
Show file tree
Hide file tree
Showing 4 changed files with 271 additions and 13 deletions.
6 changes: 0 additions & 6 deletions changelogs/unreleased/config-anonymous-replica.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,6 @@

There are caveats that are not resolved yet:

* An attempt to configure an anonymous replica in read-write mode (using
`database.mode` or `<replicaset>.leader`) should lead to an error on config
validation, before configuration applying.
* An attempt to configure an anonymous replica with
`replication.election_mode` != `off` should lead to an error on config
validation, before configuration applying.
* An anonymous replica can't be bootstrapped from a replicaset, where all the
instance are in read-only mode, however there are no technical problems
there (just too tight validation).
19 changes: 17 additions & 2 deletions src/box/lua/config/applier/box_cfg.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,9 @@ local function apply(config)
-- NB: configdata.lua verifies that there is at least one
-- non-anonymous instance. So, an anonymous replica is
-- read-only by default.
--
-- NB: configdata.lua also verifies that read-write mode
-- is not enabled for an anonymous replica in the config.
local mode = configdata:get('database.mode', {use_default = true})
if mode == 'ro' then
box_cfg.read_only = true
Expand All @@ -392,11 +395,23 @@ local function apply(config)
--
-- NB: If there is no configured leader, all the instances
-- of the given replicaset are configured as read-only.
--
-- NB: configdata.lua verifies that an anonymous replica
-- is not set as a leader.
box_cfg.read_only = not configdata:is_leader()
elseif failover == 'election' then
-- Enable leader election.
-- Enable leader election on non-anonymous instances.
if box_cfg.election_mode == nil then
box_cfg.election_mode = 'candidate'
box_cfg.election_mode = is_anon and 'off' or 'candidate'
end

-- An anonymous replica can be configured with
-- `election_mode: off`, but not other modes.
--
-- The validation is performed in configdata.lua for all
-- the peers of the given replicaset.
if is_anon then
assert(box_cfg.election_mode == 'off')
end

-- A particular instance may participate in the replicaset
Expand Down
123 changes: 118 additions & 5 deletions src/box/lua/config/configdata.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,12 +445,21 @@ local function validate_failover(found, peers, failover, leader)
end
end

-- Verify that the given replicaset contains at least one
-- non-anonymous replica.
-- Verify replication.anon = true prerequisites.
--
-- This check doesn't verify the whole cluster config, only the
-- First, it verifies that the given replicaset contains at least
-- one non-anonymous replica.
--
-- The key idea of the rest of the checks is that an anonymous
-- replica must be in the read-only mode.
--
-- Different failover modes control read-only/read-write mode in
-- different ways, so we need specific checks for each of them in
-- regard of an anonymous replica.
--
-- These checks don't verify the whole cluster config, only the
-- given replicaset.
local function validate_anon(found, peers)
local function validate_anon(found, peers, failover, leader)
-- failover: <any>
--
-- A replicaset can't consist of only anonymous replicas.
Expand All @@ -474,6 +483,105 @@ local function validate_anon(found, peers)
'forbidden, because it looks like there is no meaningful ' ..
'use case'):format(found.replicaset_name, found.group_name), 0)
end

-- failover: off
--
-- An anonymous replica shouldn't be set to RW.
if failover == 'off' then
for peer_name, peer in pairs(peers) do
local is_anon =
instance_config:get(peer.iconfig_def, 'replication.anon')
local mode =
instance_config:get(peer.iconfig_def, 'database.mode')
if is_anon and mode == 'rw' then
error(('database.mode = "rw" is set for instance %q of ' ..
'replicaset %q of group %q, but this option cannot be ' ..
'used together with replication.anon = true'):format(
peer_name, found.replicaset_name, found.group_name), 0)
end
end
end

-- failover: manual
--
-- An anonymous replica can't be a leader.
if failover == 'manual' and leader ~= nil then
assert(peers[leader] ~= nil)
local iconfig_def = peers[leader].iconfig_def
local is_anon = instance_config:get(iconfig_def, 'replication.anon')
if is_anon then
error(('replication.anon = true is set for instance %q of ' ..
'replicaset %q of group %q that is configured as a ' ..
'leader; a leader can not be an anonymous replica'):format(
leader, found.replicaset_name, found.group_name), 0)
end
end

-- failover: election
--
-- An anonymous replica can be in `election_mode: off`, but
-- not any other.
--
-- Let's look on illustrative examples below. The following
-- one works.
--
-- replicasets:
-- r-001:
-- replication:
-- failover: election
-- instances:
-- i-001: {} # candidate
-- i-002: {} # candidate
-- i-003: {} # candidate
-- i-004: # off --------+
-- replication: # +--> OK
-- anon: true # anonymous --+
--
-- All the non-anonymous instances have effective default
-- 'replication.election_mode: candidate', while anonymous
-- replicas default to 'off'.
--
-- However, the following example doesn't work.
--
-- replicasets:
-- r-001:
-- replication:
-- failover: election
-- election_mode: candidate # !!
-- instances:
-- i-001: {} # candidate
-- i-002: {} # candidate
-- i-003: {} # candidate
-- i-004: # candidate --+
-- replication: # +--> error
-- anon: true # anonymous --+
--
-- The default 'off' is not applied, because the explicit
-- 'candidate' value is set in the replicaset scope. It can be
-- fixed like so:
--
-- <...>
-- i-004:
-- replication:
-- anon: true
-- election_mode: off # !!
if failover == 'election' then
for peer_name, peer in pairs(peers) do
local is_anon = instance_config:get(peer.iconfig_def,
'replication.anon')
local election_mode = instance_config:get(peer.iconfig_def,
'replication.election_mode')
if is_anon and election_mode ~= nil and election_mode ~= 'off' then
error(('replication.election_mode = %q is set for instance ' ..
'%q of replicaset %q of group %q, but this option ' ..
'cannot be used together with replication.anon = true; ' ..
'consider setting replication.election_mode = "off" ' ..
'explicitly for this instance'):format(
election_mode, peer_name, found.replicaset_name,
found.group_name), 0)
end
end
end
end

local function new(iconfig, cconfig, instance_name)
Expand Down Expand Up @@ -580,7 +688,12 @@ local function new(iconfig, cconfig, instance_name)

-- Verify that there is at least one non-anonymous replica in
-- the given replicaset.
validate_anon(found, peers)
--
-- Verify that `replication.anon: true` (if any) doesn't
-- conflict with any other option (say, database.mode,
-- <replicaset>.leader or replication.election_mode).
validate_anon(found, peers, failover, leader)


-- Verify "replication.failover" = "supervised" strategy
-- prerequisites.
Expand Down
136 changes: 136 additions & 0 deletions test/config-luatest/anonymous_replica_test.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,3 +195,139 @@ g.test_all_anonymous = function(g)
is no meaningful use case
]]))
end

-- Verify that an anonymous replica can't be configured in
-- read-write mode.
--
-- The whole replicaset refuses to start if this misconfiguration
-- is found.
--
-- replication.failover: off
g.test_anonymous_replica_rw_mode = function(g)
local config = cbuilder.new()
:add_instance('instance-001', {
database = {
mode = 'rw',
},
})
:add_instance('instance-002', {})
:add_instance('instance-003', {})
:add_instance('instance-004', {
database = {
mode = 'rw',
},
replication = {
anon = true,
},
})
:config()

replicaset.startup_error(g, config, toline([[
database.mode = "rw" is set for instance "instance-004" of replicaset
"replicaset-001" of group "group-001", but this option cannot be used
together with replication.anon = true
]]))
end

-- Verify that an anonymous replica can't be assigned as a leader.
--
-- The whole replicaset refuses to start if this misconfiguration
-- is found.
--
-- replication.failover: manual
g.test_anonymous_replica_leader = function(g)
local config = cbuilder.new()
:set_replicaset_option('replication.failover', 'manual')
:set_replicaset_option('leader', 'instance-004')
:add_instance('instance-001', {})
:add_instance('instance-002', {})
:add_instance('instance-003', {})
:add_instance('instance-004', {
replication = {
anon = true,
},
})
:config()

replicaset.startup_error(g, config, toline([[
replication.anon = true is set for instance "instance-004" of replicaset
"replicaset-001" of group "group-001" that is configured as a leader; a
leader can not be an anonymous replica
]]))
end

-- Verify that an anonymous replica can't be configured with
-- replication.election_mode parameter other than null or "off".
--
-- The whole replicaset refuses to start if this misconfiguration
-- is found.
--
-- replication.failover: election
g.test_anonymous_replica_election_mode_other_than_off = function(g)
local error_t = toline([[
replication.election_mode = %q is set for instance "instance-004" of
replicaset "replicaset-001" of group "group-001", but this option cannot
be used together with replication.anon = true; consider setting
replication.election_mode = "off" explicitly for this instance
]])

for _, election_mode in ipairs({'candidate', 'voter', 'manual'}) do
local config = cbuilder.new()
:set_replicaset_option('replication.failover', 'election')
:add_instance('instance-001', {})
:add_instance('instance-002', {})
:add_instance('instance-003', {})
:add_instance('instance-004', {
replication = {
anon = true,
election_mode = election_mode,
},
})
:config()

replicaset.startup_error(g, config, error_t:format(election_mode))
end
end

-- Verify that the election mode defaults to 'off' for an
-- anonymous replica in a replicaset with election failover.
g.test_anonymous_replica_election_mode_off = function(g)
-- Three non-anonymous instances, two anonymous replicas.
--
-- The replicaset is in `failover: election` mode.
local config = cbuilder.new()
:set_replicaset_option('replication.failover', 'election')
:add_instance('instance-001', {})
:add_instance('instance-002', {})
:add_instance('instance-003', {})
:add_instance('instance-004', {
replication = {
anon = true,
},
})
:add_instance('instance-005', {
replication = {
anon = true,
},
})
:config()

local replicaset = replicaset.new(g, config)
replicaset:start()

local function verify_election_mode_candidate()
t.assert_equals(box.cfg.election_mode, 'candidate')
end

local function verify_election_mode_off()
t.assert_equals(box.cfg.election_mode, 'off')
end

-- Verify that non-anonymous instances have election mode
-- 'candidate', while anonymous replicas are 'off'.
replicaset['instance-001']:exec(verify_election_mode_candidate)
replicaset['instance-002']:exec(verify_election_mode_candidate)
replicaset['instance-003']:exec(verify_election_mode_candidate)
replicaset['instance-004']:exec(verify_election_mode_off)
replicaset['instance-005']:exec(verify_election_mode_off)
end

0 comments on commit f732a86

Please sign in to comment.