Skip to content

25-3-1: Avoid excessive cpu usage when registering large number of service counters #29773

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
snaury merged 3 commits into from
Dec 1, 2025
Merged

25-3-1: Avoid excessive cpu usage when registering large number of service counters #29773

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a82fb5
Delay monlib index SortPages until the next render
Nov 26, 2025
c7aec57
Allow FindSubgroup in dynamic counters to find unknown values
Nov 26, 2025
96c9a09
Use fixed database labels order and avoid quadratic lookups (#29667)
snaury Nov 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions library/cpp/monlib/dynamic_counters/counters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,18 @@ TIntrusivePtr<TDynamicCounters> TDynamicCounters::GetSubgroup(const TString& nam
return res;
}

TIntrusivePtr<TDynamicCounters> TDynamicCounters::FindSubgroup(const TString& name) const {
TReadGuard g(Lock);
const auto it = Counters.lower_bound({name, TString()});
if (it != Counters.end() && it->first.LabelName == name) {
const auto it2 = std::next(it);
if (it2 == Counters.end() || it2->first.LabelName != name) {
return AsDynamicCounters(it->second);
}
}
return nullptr;
}

TIntrusivePtr<TDynamicCounters> TDynamicCounters::FindSubgroup(const TString& name, const TString& value) const {
TReadGuard g(Lock);
const auto it = Counters.find({name, value});
Expand Down
1 change: 1 addition & 0 deletions library/cpp/monlib/dynamic_counters/counters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ namespace NMonitoring {
void RemoveSubgroupChain(const std::vector<std::pair<TString, TString>>& chain);

TIntrusivePtr<TDynamicCounters> GetSubgroup(const TString& name, const TString& value);
TIntrusivePtr<TDynamicCounters> FindSubgroup(const TString& name) const;
TIntrusivePtr<TDynamicCounters> FindSubgroup(const TString& name, const TString& value) const;
bool RemoveSubgroup(const TString& name, const TString& value);
void ReplaceSubgroup(const TString& name, const TString& value, TIntrusivePtr<TDynamicCounters> subgroup);
Expand Down
26 changes: 26 additions & 0 deletions library/cpp/monlib/dynamic_counters/counters_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,4 +339,30 @@ Y_UNIT_TEST_SUITE(TDynamicCountersTest) {
histogram = rootGroup->FindNamedHistogram("name", "histogram2");
UNIT_ASSERT(histogram);
}

Y_UNIT_TEST(FindSubgroup) {
TDynamicCounterPtr rootGroup(new TDynamicCounters());

auto a = rootGroup->GetSubgroup("a", "1");
auto b1 = rootGroup->GetSubgroup("b", "1");
auto b2 = rootGroup->GetSubgroup("b", "2");
auto c = rootGroup->GetSubgroup("c", "1");
auto e = rootGroup->GetSubgroup("e", "1");

UNIT_ASSERT(a == rootGroup->FindSubgroup("a"));
UNIT_ASSERT(a == rootGroup->FindSubgroup("a", "1"));
UNIT_ASSERT(nullptr == rootGroup->FindSubgroup("a", "2"));

UNIT_ASSERT(nullptr == rootGroup->FindSubgroup("b"));
UNIT_ASSERT(b1 == rootGroup->FindSubgroup("b", "1"));
UNIT_ASSERT(b2 == rootGroup->FindSubgroup("b", "2"));
UNIT_ASSERT(nullptr == rootGroup->FindSubgroup("b", "3"));

UNIT_ASSERT(c == rootGroup->FindSubgroup("c"));
UNIT_ASSERT(c == rootGroup->FindSubgroup("c", "1"));
UNIT_ASSERT(nullptr == rootGroup->FindSubgroup("c", "2"));

UNIT_ASSERT(nullptr == rootGroup->FindSubgroup("d"));
UNIT_ASSERT(nullptr == rootGroup->FindSubgroup("f"));
}
}
10 changes: 7 additions & 3 deletions library/cpp/monlib/service/pages/index_mon_page.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ void TIndexMonPage::Output(IMonHttpRequest& request) {

void TIndexMonPage::OutputIndex(IOutputStream& out, bool pathEndsWithSlash) {
TGuard<TMutex> g(Mtx);
if (SortPagesPending) {
Pages.sort([](const TMonPagePtr& a, const TMonPagePtr& b) {
return AsciiCompareIgnoreCase(a->GetTitle(), b->GetTitle()) < 0;
});
SortPagesPending = false;
}
for (auto& Page : Pages) {
IMonPage* page = Page.Get();
if (page->IsInIndex()) {
Expand Down Expand Up @@ -164,9 +170,7 @@ void TIndexMonPage::OutputBody(IMonHttpRequest& req) {

void TIndexMonPage::SortPages() {
TGuard<TMutex> g(Mtx);
Pages.sort([](const TMonPagePtr& a, const TMonPagePtr& b) {
return AsciiCompareIgnoreCase(a->GetTitle(), b->GetTitle()) < 0;
});
SortPagesPending = true;
}

void TIndexMonPage::ClearPages() {
Expand Down
1 change: 1 addition & 0 deletions library/cpp/monlib/service/pages/index_mon_page.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ namespace NMonitoring {
TPages Pages; // a list of pages to maintain specific order
using TPagesByPath = THashMap<TString, TPages::iterator>;
TPagesByPath PagesByPath;
bool SortPagesPending = false;

TIndexMonPage(const TString& path, const TString& title)
: IMonPage(path, title)
Expand Down
59 changes: 34 additions & 25 deletions ydb/core/base/counters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ static const THashSet<TString> DATABASE_SERVICES
static const THashSet<TString> DATABASE_ATTRIBUTE_SERVICES
= {{ TString("ydb"), TString("datastreams") }};

static const THashSet<TString> DATABASE_ATTRIBUTE_LABELS
static const TVector<TString> DATABASE_ATTRIBUTE_LABELS
= {{ TString("cloud_id"),
TString("folder_id"),
TString("database_id")
Expand All @@ -77,26 +77,20 @@ const THashSet<TString> &GetDatabaseAttributeSensorServices()
return DATABASE_ATTRIBUTE_SERVICES;
}

const THashSet<TString> &GetDatabaseAttributeLabels()
const TVector<TString> &GetDatabaseAttributeLabels()
{
return DATABASE_ATTRIBUTE_LABELS;
}

static TIntrusivePtr<TDynamicCounters> SkipLabels(TIntrusivePtr<TDynamicCounters> counters,
const THashSet<TString> &labels)
const TVector<TString> &labels)
{
TString name, value;
do {
name = "";
counters->EnumerateSubgroups([&name, &value, &labels](const TString &n, const TString &v) {
if (labels.contains(n)) {
name = n;
value = v;
}
});
if (name)
counters = counters->GetSubgroup(name, value);
} while (name);
for (const TString& label : labels) {
auto next = counters->FindSubgroup(label);
if (next) {
counters = next;
}
}

return counters;
}
Expand All @@ -122,17 +116,32 @@ TIntrusivePtr<TDynamicCounters> GetServiceCountersRoot(TIntrusivePtr<TDynamicCou
return root->GetSubgroup("counters", pair.first);
}

static THashSet<TString> MakeServiceCountersExtraLabels() {
THashSet<TString> extraLabels;
extraLabels.insert(DATABASE_LABEL);
extraLabels.insert(SLOT_LABEL);
extraLabels.insert(HOST_LABEL);
extraLabels.insert(DATABASE_ATTRIBUTE_LABELS.begin(),
DATABASE_ATTRIBUTE_LABELS.end());
static TVector<TString> MakeServiceCountersExtraLabels() {
// NOTE: order of labels should match labels maintainer order for efficiency
TVector<TString> extraLabels;
extraLabels.push_back(DATABASE_LABEL);
extraLabels.push_back(SLOT_LABEL);
extraLabels.push_back(HOST_LABEL);
extraLabels.insert(extraLabels.end(),
DATABASE_ATTRIBUTE_LABELS.begin(),
DATABASE_ATTRIBUTE_LABELS.end());
return extraLabels;
}

static const THashSet<TString> SERVICE_COUNTERS_EXTRA_LABELS = MakeServiceCountersExtraLabels();
static const TVector<TString> SERVICE_COUNTERS_EXTRA_LABELS = MakeServiceCountersExtraLabels();

static TIntrusivePtr<TDynamicCounters> SkipExtraLabels(TIntrusivePtr<TDynamicCounters> counters) {
for (;;) {
// Keep trying as long as there is something to skip
auto next = SkipLabels(counters, SERVICE_COUNTERS_EXTRA_LABELS);
if (next == counters) {
break;
}
counters = next;
}

return counters;
}

TIntrusivePtr<TDynamicCounters> GetServiceCounters(TIntrusivePtr<TDynamicCounters> root,
const TString &service, bool skipAddedLabels)
Expand All @@ -145,10 +154,10 @@ TIntrusivePtr<TDynamicCounters> GetServiceCounters(TIntrusivePtr<TDynamicCounter
if (!skipAddedLabels)
return res;

res = SkipLabels(res, SERVICE_COUNTERS_EXTRA_LABELS);
res = SkipExtraLabels(res);

auto utils = root->GetSubgroup("counters", "utils");
utils = SkipLabels(utils, SERVICE_COUNTERS_EXTRA_LABELS);
utils = SkipExtraLabels(utils);
auto lookupCounter = utils->GetSubgroup("component", service)->GetCounter("CounterLookups", true);
res->SetLookupCounter(lookupCounter);
res->SetOnLookup(OnCounterLookup);
Expand Down
4 changes: 3 additions & 1 deletion ydb/core/base/counters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <library/cpp/monlib/dynamic_counters/counters.h>

#include <util/generic/hash_set.h>
#include <util/generic/vector.h>

namespace NKikimr {
constexpr char DATABASE_LABEL[] = "database";
Expand All @@ -22,7 +23,8 @@ namespace NKikimr {
const THashSet<TString> &GetDatabaseSensorServices();
// Get list of services which use top-level database attribute labels for own sensors.
const THashSet<TString> &GetDatabaseAttributeSensorServices();
const THashSet<TString> &GetDatabaseAttributeLabels();
// Get a list of attribute labels (order is important)
const TVector<TString> &GetDatabaseAttributeLabels();
// Drop all extra labels.
void ReplaceSubgroup(TIntrusivePtr<::NMonitoring::TDynamicCounters> root, const TString &service);
} // namespace NKikimr
Loading
Loading