Skip to content

stable-25-3: Add overrides in state storage self heal #26103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 3, 2025
+114 −32
Merged

stable-25-3: Add overrides in state storage self heal #26103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion ydb/core/blobstorage/nodewarden/distconf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,10 @@ namespace NKikimr::NStorage {
bool GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss
, const NKikimrBlobStorage::TStorageConfig& baseConfig
, std::unordered_set<ui32>& usedNodes
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig = {});
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig = {}
, ui32 overrideReplicasInRingCount = 0
, ui32 overrideRingsCount = 0
);

////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Bridge ops
Expand Down
7 changes: 5 additions & 2 deletions ydb/core/blobstorage/nodewarden/distconf_generate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -583,7 +583,10 @@ namespace NKikimr::NStorage {

bool TDistributedConfigKeeper::GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss
, const NKikimrBlobStorage::TStorageConfig& baseConfig, std::unordered_set<ui32>& usedNodes
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig) {
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig
, ui32 overrideReplicasInRingCount
, ui32 overrideRingsCount
) {
std::map<TBridgePileId, THashMap<TString, std::vector<std::tuple<ui32, TNodeLocation>>>> nodes;
bool goodConfig = true;
for (const auto& node : baseConfig.GetAllNodes()) {
Expand All @@ -592,7 +595,7 @@ namespace NKikimr::NStorage {
nodes[pileId][location.GetDataCenterId()].emplace_back(node.GetNodeId(), location);
}
for (auto& [pileId, nodesByDataCenter] : nodes) {
TStateStoragePerPileGenerator generator(nodesByDataCenter, SelfHealNodesState, pileId, usedNodes, oldConfig);
TStateStoragePerPileGenerator generator(nodesByDataCenter, SelfHealNodesState, pileId, usedNodes, oldConfig, overrideReplicasInRingCount, overrideRingsCount);
generator.AddRingGroup(ss);
goodConfig &= generator.IsGoodConfig();
}
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/blobstorage/nodewarden/distconf_invoke.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,12 @@ namespace NKikimr::NStorage {
void ReassignStateStorageNode(const TQuery::TReassignStateStorageNode& cmd);
void ReconfigStateStorage(const NKikimrBlobStorage::TStateStorageConfig& cmd);
void SelfHealStateStorage(const TQuery::TSelfHealStateStorage& cmd);
void SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas);
void SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount);
void SelfHealNodesStateUpdate(const TQuery::TSelfHealNodesStateUpdate& cmd);
void GetStateStorageConfig(const TQuery::TGetStateStorageConfig& cmd);

void GetCurrentStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool getNodesState);
bool GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas);
bool GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount);
void AdjustRingGroupActorIdOffsetInRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Storage configuration YAML manipulation
Expand Down
21 changes: 11 additions & 10 deletions ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,19 @@ namespace NKikimr::NStorage {

using TInvokeRequestHandlerActor = TDistributedConfigKeeper::TInvokeRequestHandlerActor;

bool TInvokeRequestHandlerActor::GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas) {
bool TInvokeRequestHandlerActor::GetRecommendedStateStorageConfig(NKikimrBlobStorage::TStateStorageConfig* currentConfig, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount) {
const NKikimrBlobStorage::TStorageConfig& config = *Self->StorageConfig;
bool result = true;
std::unordered_set<ui32> usedNodes;
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig());
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig(), overrideReplicasInRingCount, overrideRingsCount);
if (pileupReplicas) {
usedNodes.clear();
}
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig());
result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig(), overrideReplicasInRingCount, overrideRingsCount);
if (pileupReplicas) {
usedNodes.clear();
}
result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig());
result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig(), overrideReplicasInRingCount, overrideRingsCount);
return result;
}

Expand Down Expand Up @@ -97,7 +97,7 @@ namespace NKikimr::NStorage {
auto* currentConfig = record->MutableStateStorageConfig();

if (cmd.GetRecommended()) {
GetRecommendedStateStorageConfig(currentConfig, cmd.GetPileupReplicas());
GetRecommendedStateStorageConfig(currentConfig, cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount());
AdjustRingGroupActorIdOffsetInRecommendedStateStorageConfig(currentConfig);
} else {
GetCurrentStateStorageConfig(currentConfig, cmd.GetNodesState());
Expand All @@ -112,19 +112,20 @@ namespace NKikimr::NStorage {
Self->SelfHealNodesState[node.GetNodeId()] = node.GetState();
}
if (cmd.GetEnableSelfHealStateStorage()) {
SelfHealStateStorage(cmd.GetWaitForConfigStep(), true, cmd.GetPileupReplicas());
SelfHealStateStorage(cmd.GetWaitForConfigStep(), true, cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount());
}
}

void TInvokeRequestHandlerActor::SelfHealStateStorage(const TQuery::TSelfHealStateStorage& cmd) {
SelfHealStateStorage(cmd.GetWaitForConfigStep(), cmd.GetForceHeal(), cmd.GetPileupReplicas());
SelfHealStateStorage(cmd.GetWaitForConfigStep(), cmd.GetForceHeal(), cmd.GetPileupReplicas(), cmd.GetOverrideReplicasInRingCount(), cmd.GetOverrideRingsCount());
}

void TInvokeRequestHandlerActor::SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas) {
void TInvokeRequestHandlerActor::SelfHealStateStorage(ui32 waitForConfigStep, bool forceHeal, bool pileupReplicas, ui32 overrideReplicasInRingCount, ui32 overrideRingsCount) {
RunCommonChecks();
STLOG(PRI_DEBUG, BS_NODE, NW105, "TInvokeRequestHandlerActor::SelfHealStateStorage", (waitForConfigStep, waitForConfigStep), (forceHeal, forceHeal), (pileupReplicas, pileupReplicas));
STLOG(PRI_DEBUG, BS_NODE, NW105, "TInvokeRequestHandlerActor::SelfHealStateStorage", (waitForConfigStep, waitForConfigStep),
(forceHeal, forceHeal), (pileupReplicas, pileupReplicas), (overrideReplicasInRingCount, overrideReplicasInRingCount), (overrideRingsCount, overrideRingsCount));
NKikimrBlobStorage::TStateStorageConfig targetConfig;
if (!GetRecommendedStateStorageConfig(&targetConfig, pileupReplicas) && !forceHeal) {
if (!GetRecommendedStateStorageConfig(&targetConfig, pileupReplicas, overrideReplicasInRingCount, overrideRingsCount) && !forceHeal) {
throw TExError() << "Recommended configuration has faulty nodes and can not be applyed";
}

Expand Down
30 changes: 19 additions & 11 deletions ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@ namespace NKikimr::NStorage {
, TBridgePileId pileId
, std::unordered_set<ui32>& usedNodes
, const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig
, ui32 overrideReplicasInRingCount
, ui32 overrideRingsCount
)
: PileId(pileId)
, SelfHealNodesState(selfHealNodesState)
, UsedNodes(usedNodes)
, OldConfig(oldConfig)
, OverrideReplicasInRingCount(overrideReplicasInRingCount)
, OverrideRingsCount(overrideRingsCount)
{
FillNodeGroups(nodes);
CalculateRingsParameters();
Expand Down Expand Up @@ -56,25 +60,29 @@ namespace NKikimr::NStorage {
void TStateStoragePerPileGenerator::CalculateRingsParameters() {
ui32 minNodesInGroup = NodeGroups[0].Nodes.size();
if (NodeGroups.size() == 1) {
if (minNodesInGroup < 5) {
RingsInGroupCount = minNodesInGroup;
NToSelect = minNodesInGroup < 3 ? 1 : 3;
} else {
RingsInGroupCount = OverrideRingsCount;
if (RingsInGroupCount == 0)
RingsInGroupCount = minNodesInGroup < 8 ? minNodesInGroup : 8;
NToSelect = 5;
if (RingsInGroupCount > minNodesInGroup) {
RingsInGroupCount = minNodesInGroup;
}
ReplicasInRingCount = 1 + minNodesInGroup / 1000;
NToSelect = RingsInGroupCount < 3 ? 1 : (RingsInGroupCount < 5 ? 3 : 5);
ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + minNodesInGroup / 1000);
} else {
RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3;
if (OverrideRingsCount == 3 || OverrideRingsCount == 9) {
RingsInGroupCount = OverrideRingsCount / 3;
} else {
RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3;
}
NToSelect = RingsInGroupCount < 3 ? 3 : 9;
ui32 nodesCnt = 0;
for (auto& n : NodeGroups) {
nodesCnt += n.Nodes.size();
}
ReplicasInRingCount = 1 + nodesCnt / 1000;
if (ReplicasInRingCount * RingsInGroupCount > minNodesInGroup) {
ReplicasInRingCount = 1;
}
ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + nodesCnt / 1000);
}
if (ReplicasInRingCount * RingsInGroupCount > minNodesInGroup) {
ReplicasInRingCount = 1;
}
}

Expand Down
7 changes: 6 additions & 1 deletion ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ namespace NKikimr::NStorage {
const std::unordered_map<ui32, ui32>& selfHealNodesState,
TBridgePileId pileId,
std::unordered_set<ui32>& usedNodes,
const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig);
const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig,
ui32 overrideReplicasInRingCount,
ui32 overrideRingsCount
);
bool IsGoodConfig() const;
void AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss);

Expand Down Expand Up @@ -41,5 +44,7 @@ namespace NKikimr::NStorage {
ui32 RingsInGroupCount = 1;
ui32 ReplicasInRingCount = 1;
ui32 NToSelect = 1;
ui32 OverrideReplicasInRingCount = 0;
ui32 OverrideRingsCount = 0;
};
}
36 changes: 32 additions & 4 deletions ydb/core/blobstorage/nodewarden/distconf_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,29 @@ namespace NBlobStorageNodeWardenTest{

Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {

NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}) {
NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0) {
NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true);
NKikimrConfig::TDomainsConfig::TStateStorage ss;
NKikimrBlobStorage::TStorageConfig config;
for (ui32 i : xrange(nodes)) {
auto *node = config.AddAllNodes();
node->SetNodeId(i + 1);
}
keeper.GenerateStateStorageConfig(&ss, config, usedNodes);
keeper.GenerateStateStorageConfig(&ss, config, usedNodes, {}, overrideReplicasInRingCount, overrideRingsCount);
return ss;
}

NKikimrConfig::TDomainsConfig::TStateStorage GenerateDCStateStorage(ui32 dcCnt, ui32 racksCnt, ui32 nodesInRack, std::unordered_map<ui32, ui32> nodesState = {}, std::unordered_set<ui32> usedNodes = {}, std::vector<ui32> oldConfig = {}, ui32 oldNToSelect = 9) {
NKikimrConfig::TDomainsConfig::TStateStorage GenerateDCStateStorage(
ui32 dcCnt
, ui32 racksCnt
, ui32 nodesInRack
, std::unordered_map<ui32, ui32> nodesState = {}
, std::unordered_set<ui32> usedNodes = {}
, std::vector<ui32> oldConfig = {}
, ui32 oldNToSelect = 9
, ui32 overrideReplicasInRingCount = 0
, ui32 overrideRingsCount = 0
) {
NKikimrBlobStorage::TStorageConfig config;
ui32 nodeId = 1;
NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true);
Expand Down Expand Up @@ -63,7 +73,7 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {
for (auto [nodeId, state] : nodesState) {
keeper.SelfHealNodesState[nodeId] = state;
}
keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS);
keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS, overrideReplicasInRingCount, overrideRingsCount);
return ss;
}

Expand Down Expand Up @@ -173,6 +183,24 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) {
// DC disconnected - use previous config for this DC
CheckStateStorage(GenerateDCStateStorage(3, 3, 3, { {10, 2}, {11, 4}, {13, 3}, {14, 4}, {15, 2} }, {}, {1, 5, 8, 10, 14, 17, 19, 22, 25}, 9), 9, {1, 4, 7, 10, 14, 17, 19, 22, 25});
}

Y_UNIT_TEST(GenerateConfigReplicasDensity) {
CheckStateStorage(GenerateSimpleStateStorage(100, {}, 0, 0), 5, {1, 2, 3, 4, 5, 6, 7, 8});
CheckStateStorage(GenerateSimpleStateStorage(100, {}, 1, 1), 1, {1});
CheckStateStorage2(GenerateSimpleStateStorage(100, {}, 3, 3),
"{ RingGroups { NToSelect: 3 Ring { Node: 1 Node: 2 Node: 3 }"
" Ring { Node: 4 Node: 5 Node: 6 } Ring { Node: 7 Node: 8 Node: 9 } } }");
CheckStateStorage(GenerateSimpleStateStorage(100, {}, 20, 10), 5, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10});
CheckStateStorage2(GenerateSimpleStateStorage(16, {}, 4, 2), "{ RingGroups { NToSelect: 1 Ring { Node: 1 Node: 2 Node: 3 Node: 4 } Ring { Node: 5 Node: 6 Node: 7 Node: 8 } } }");
CheckStateStorage(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, 0), 9, {1, 4, 7, 10, 13, 16, 19, 22, 25});
CheckStateStorage(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 1, 3), 3, {1, 10, 19});
CheckStateStorage2(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 2, 3), "{ RingGroups { NToSelect: 3 Ring { Node: 1 Node: 2 } Ring { Node: 10 Node: 11 } Ring { Node: 19 Node: 20 } } }");
CheckStateStorage2(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 3, 3), "{ RingGroups { NToSelect: 3 Ring { Node: 1 Node: 2 Node: 3 } Ring { Node: 10 Node: 11 Node: 12 } Ring { Node: 19 Node: 20 Node: 21 } } }");
CheckStateStorage2(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 2, 1),"{ RingGroups { NToSelect: 9 "
"Ring { Node: 1 Node: 2 } Ring { Node: 4 Node: 5 } Ring { Node: 7 Node: 8 } "
"Ring { Node: 10 Node: 11 } Ring { Node: 13 Node: 14 } Ring { Node: 16 Node: 17 } "
"Ring { Node: 19 Node: 20 } Ring { Node: 22 Node: 23 } Ring { Node: 25 Node: 26 } } }");
}
}
}
}
6 changes: 6 additions & 0 deletions ydb/core/cms/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ struct TCmsSentinelConfig {
TDuration WaitForConfigStep;
TDuration RelaxTime;
bool PileupReplicas;
ui32 OverrideReplicasInRingCount;
ui32 OverrideRingsCount;

void Serialize(NKikimrCms::TCmsConfig::TSentinelConfig::TStateStorageSelfHealConfig &config) const {
config.SetEnable(Enable);
Expand All @@ -30,6 +32,8 @@ struct TCmsSentinelConfig {
config.SetWaitForConfigStep(WaitForConfigStep.GetValue());
config.SetRelaxTime(RelaxTime.GetValue());
config.SetPileupReplicas(PileupReplicas);
config.SetOverrideReplicasInRingCount(OverrideReplicasInRingCount);
config.SetOverrideRingsCount(OverrideRingsCount);
}

void Deserialize(const NKikimrCms::TCmsConfig::TSentinelConfig::TStateStorageSelfHealConfig &config) {
Expand All @@ -40,6 +44,8 @@ struct TCmsSentinelConfig {
WaitForConfigStep = TDuration::MicroSeconds(config.GetWaitForConfigStep());
RelaxTime = TDuration::MicroSeconds(config.GetRelaxTime());
PileupReplicas = config.GetPileupReplicas();
OverrideReplicasInRingCount = config.GetOverrideReplicasInRingCount();
OverrideRingsCount = config.GetOverrideRingsCount();
}
};

Expand Down
2 changes: 2 additions & 0 deletions ydb/core/cms/sentinel.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1151,6 +1151,8 @@ class TSentinel: public TActorBootstrapped<TSentinel> {
updateRequest->SetWaitForConfigStep(Config.StateStorageSelfHealConfig.WaitForConfigStep.GetValue() / 1000000); // milliseconds -> seconds
updateRequest->SetEnableSelfHealStateStorage(Config.StateStorageSelfHealConfig.Enable);
updateRequest->SetPileupReplicas(Config.StateStorageSelfHealConfig.PileupReplicas);
updateRequest->SetOverrideReplicasInRingCount(Config.StateStorageSelfHealConfig.OverrideReplicasInRingCount);
updateRequest->SetOverrideRingsCount(Config.StateStorageSelfHealConfig.OverrideRingsCount);
for (auto& [nodeId, node] : SentinelState->Nodes) {
SentinelState->NeedSelfHealStateStorage |= node.Compute();
auto* nodeState = updateRequest->AddNodesState();
Expand Down
6 changes: 6 additions & 0 deletions ydb/core/protos/blobstorage_distributed_config.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,12 +225,16 @@ message TEvNodeConfigInvokeOnRoot {
optional bool Recommended = 1;
optional bool PileupReplicas = 2;
optional bool NodesState = 3;
optional uint32 OverrideReplicasInRingCount = 4;
optional uint32 OverrideRingsCount = 5;
}

message TSelfHealStateStorage {
optional uint32 WaitForConfigStep = 1;
optional bool ForceHeal = 2;
optional bool PileupReplicas = 3;
optional uint32 OverrideReplicasInRingCount = 4;
optional uint32 OverrideRingsCount = 5;
}

message TSelfHealNodesStateUpdate {
Expand All @@ -242,6 +246,8 @@ message TEvNodeConfigInvokeOnRoot {
optional uint32 WaitForConfigStep = 2;
repeated TNodeState NodesState = 3;
optional bool PileupReplicas = 4;
optional uint32 OverrideReplicasInRingCount = 5;
optional uint32 OverrideRingsCount = 6;
}

message TAdvanceGeneration
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/protos/cms.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,8 @@ message TCmsConfig {
optional uint32 WaitForConfigStep = 5 [default = 60000000];
optional uint32 RelaxTime = 6 [default = 600000000];
optional bool PileupReplicas = 7 [default = false];
optional uint32 OverrideReplicasInRingCount = 8 [default = 0];
optional uint32 OverrideRingsCount = 9 [default = 0];
}

message TStateLimit {
Expand Down
Loading
Loading