From eaffc4192e94649c0fd3c6bc5f97d4e6116905de Mon Sep 17 00:00:00 2001 From: Yaroslav Dynnikov Date: Fri, 7 Nov 2025 14:54:44 +0300 Subject: [PATCH 1/4] Add capacity metrics to whiteboard --- .../nodewarden/blobstorage_node_warden_ut.cpp | 4 + .../blobstorage/pdisk/blobstorage_pdisk.h | 5 +- .../pdisk/blobstorage_pdisk_chunk_tracker.h | 2 +- .../pdisk/blobstorage_pdisk_impl.cpp | 28 ++++- .../pdisk/blobstorage_pdisk_keeper.h | 18 +++ .../pdisk/blobstorage_pdisk_ut.cpp | 114 +++++++++++++----- .../blobstorage/pdisk/mock/pdisk_mock.cpp | 2 +- .../vdisk/skeleton/skeleton_oos_tracker.cpp | 2 +- ydb/core/mind/bscontroller/bsc.cpp | 10 +- ydb/core/mind/hive/hive_ut.cpp | 4 +- ydb/core/protos/blobstorage_disk.proto | 7 +- ydb/core/protos/node_whiteboard.proto | 6 + 12 files changed, 156 insertions(+), 46 deletions(-) diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index 86abd2d1a292..090f7720f1eb 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -1030,6 +1030,8 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { UNIT_ASSERT(pdiskInfo.HasTotalSize()); UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetExpectedSlotCount(), expectedSlotCount); UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetSlotSizeInUnits(), expectedSlotSizeInUnits); + UNIT_ASSERT(pdiskInfo.HasPDiskFillPercent()); + UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetPDiskFillPercent(), 0.0); break; } @@ -1047,6 +1049,8 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { UNIT_ASSERT(metrics.HasSlotSizeInUnits()); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotCount(), expectedSlotCount); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotSizeInUnits(), expectedSlotSizeInUnits); + UNIT_ASSERT(metrics.HasPDiskFillPercent()); + UNIT_ASSERT_VALUES_EQUAL(metrics.GetPDiskFillPercent(), 0.0); } } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h index 3297b70d43b0..4fc6f11c2ab4 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h @@ -1427,7 +1427,10 @@ struct TEvCheckSpaceResult : TEventLocalPrintHTML(str, SharedQuota.Get(), &ColorBorder, &ColorBorderOccupancy); } - ui32 ColorFlagLimit(TOwner owner, NKikimrBlobStorage::TPDiskSpaceColor::E color) { + ui32 ColorFlagLimit(TOwner owner, NKikimrBlobStorage::TPDiskSpaceColor::E color) const { if (IsOwnerUser(owner)) { return OwnerQuota->ColorFlagLimit(owner, color); } else { diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index cd488c76028c..3701ccabece8 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -1589,7 +1589,7 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { pdiskState.SetSerialNumber(Cfg->ExpectedSerial); const auto& state = static_cast(Mon.PDiskState->Val()); pdiskState.SetState(state); - + // Only report size information when PDisk is not in error state if (*Mon.PDiskBriefState != TPDiskMon::TPDisk::Error) { pdiskState.SetAvailableSize(availableSize); @@ -1624,8 +1624,21 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { vdiskMetrics->SetAvailableSize(ownerFree); vdiskMetrics->SetAllocatedSize(ownerAllocated); double occupancy; - vdiskMetrics->SetStatusFlags(Keeper.GetSpaceStatusFlags(owner, &occupancy)); - vdiskMetrics->SetOccupancy(occupancy); + NPDisk::TStatusFlags statusFlags = Keeper.GetSpaceStatusFlags(owner, &occupancy); + NKikimrBlobStorage::TPDiskSpaceColor::E spaceColor = StatusFlagToSpaceColor(statusFlags); + double vdiskSlotUtilization = Keeper.GetVDiskSlotUtilization(owner); + double vdiskFairPartFillPercent = Keeper.GetVDiskFairPartFillPercent(owner); + vdiskMetrics->SetStatusFlags(statusFlags); + vdiskMetrics->SetNormalizedOccupancy(occupancy); + vdiskMetrics->SetVDiskSlotUtilization(vdiskSlotUtilization); + vdiskMetrics->SetVDiskFairPartFillPercent(vdiskFairPartFillPercent); + vdiskMetrics->SetCapacityAlertLevel(spaceColor); + + vdiskInfo.SetNormalizedOccupancy(occupancy); + vdiskInfo.SetVDiskSlotUtilization(vdiskSlotUtilization); + vdiskInfo.SetVDiskFairPartFillPercent(vdiskFairPartFillPercent); + vdiskInfo.SetCapacityAlertLevel(spaceColor); + auto *vslotId = vdiskMetrics->MutableVSlotId(); vslotId->SetNodeId(PCtx->ActorSystem->NodeId); vslotId->SetPDiskId(PCtx->PDiskId); @@ -1655,6 +1668,10 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { if (ExpectedSlotCount) { pDiskMetrics.SetSlotCount(ExpectedSlotCount); } + + double pdiskFillPercent = Keeper.GetPDiskFillPercent(); + pDiskMetrics.SetPDiskFillPercent(pdiskFillPercent); + pdiskState.SetPDiskFillPercent(pdiskFillPercent); } PCtx->ActorSystem->Send(whiteboardReport.Sender, reportResult); @@ -2182,7 +2199,10 @@ void TPDisk::CheckSpace(TCheckSpace &evCheckSpace) { GetNumActiveSlots(), TString(), GetStatusFlags(OwnerSystem, evCheckSpace.OwnerGroupType)); - result->Occupancy = occupancy; + result->NormalizedOccupancy = occupancy; + result->VDiskSlotUtilization = Keeper.GetVDiskSlotUtilization(evCheckSpace.Owner); + result->VDiskFairPartFillPercent = Keeper.GetVDiskFairPartFillPercent(evCheckSpace.Owner); + result->PDiskFillPercent = Keeper.GetPDiskFillPercent(); PCtx->ActorSystem->Send(evCheckSpace.Sender, result.release()); Mon.CheckSpace.CountResponse(); return; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h index d301325d2f76..5eba0de8c7b5 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h @@ -150,6 +150,24 @@ class TKeeper { return ChunkTracker.EstimateSpaceColor(owner, allocationSize, occupancy); } + double GetPDiskFillPercent() const { + i64 totalUsed = ChunkTracker.GetTotalUsed(); + i64 totalHardLimit = ChunkTracker.GetTotalHardLimit(); + return 100.0 * (totalHardLimit ? (double)totalUsed / totalHardLimit : 1.0); + } + + double GetVDiskSlotUtilization(TOwner owner) const { + i64 used = ChunkTracker.GetOwnerUsed(owner); + ui32 lightYellowLimit = ChunkTracker.ColorFlagLimit(owner, NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_YELLOW); + return 100.0 * (lightYellowLimit ? (double)used / lightYellowLimit : 1.0); + } + + double GetVDiskFairPartFillPercent(TOwner owner) const { + i64 used = ChunkTracker.GetOwnerUsed(owner); + i64 hardLimit = ChunkTracker.GetOwnerHardLimit(owner); + return 100.0 * (hardLimit ? (double)used / hardLimit : 1.0); + } + // // Trimming // diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp index 59ecd2ba841f..ebcb303133fb 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp @@ -1248,15 +1248,25 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { ui32 expectedFreeChunks, ui32 expectedTotalChunks, ui32 expectedUsedChunks, + double expectedNormalizedOccupancy, + double expectedVDiskSlotUtilization, + double expectedPDiskFillPercent, ui32 expectedNumSlots, ui32 expectedNumActiveSlots, NKikimrBlobStorage::TPDiskSpaceColor::E expectedColor ) { + UNIT_ASSERT_GT(expectedTotalChunks, 0); + double expectedVDiskFairPartFillPercent = 100. * ((double)expectedUsedChunks) / expectedTotalChunks; + Cerr << (TStringBuilder() << "... Checking EvCheckSpace" << " VDisk# " << vdisk.VDiskID << " FreeChunks# " << expectedFreeChunks << " TotalChunks# " << expectedTotalChunks << " UsedChunks# " << expectedUsedChunks + << " NormalizedOccupancy# " << expectedNormalizedOccupancy + << " VDiskSlotUtilization# " << expectedVDiskSlotUtilization + << " VDiskFairPartFillPercent# " << expectedVDiskFairPartFillPercent + << " PDiskFillPercent# " << expectedPDiskFillPercent << " NumSlots# " << expectedNumSlots << " NumActiveSlots# " << expectedNumActiveSlots << " Color# " << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(expectedColor) @@ -1264,13 +1274,25 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { auto evCheckSpaceResult = testCtx.TestResponse( new NPDisk::TEvCheckSpace(vdisk.PDiskParams->Owner, vdisk.PDiskParams->OwnerRound), NKikimrProto::OK); - Cerr << (TStringBuilder() << "Got " << evCheckSpaceResult->ToString() << Endl); + Cerr << (TStringBuilder() << "Got " << evCheckSpaceResult->ToString() + << " NormalizedOccupancy# " << evCheckSpaceResult->NormalizedOccupancy + << " VDiskSlotUtilization# " << evCheckSpaceResult->VDiskSlotUtilization + << " VDiskFairPartFillPercent# " << evCheckSpaceResult->VDiskFairPartFillPercent + << " PDiskFillPercent# " << evCheckSpaceResult->PDiskFillPercent + << Endl); UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->FreeChunks, expectedFreeChunks); UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->TotalChunks, expectedTotalChunks); UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->UsedChunks, expectedUsedChunks); UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->NumSlots, expectedNumSlots); UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->NumActiveSlots, expectedNumActiveSlots); UNIT_ASSERT_VALUES_EQUAL(StatusFlagToSpaceColor(evCheckSpaceResult->StatusFlags), expectedColor); + + UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->NormalizedOccupancy, expectedNormalizedOccupancy, 0.01); + UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->VDiskSlotUtilization, expectedVDiskSlotUtilization, 1e-6); + UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->PDiskFillPercent, expectedPDiskFillPercent, 0.1); + + UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->VDiskFairPartFillPercent, expectedVDiskFairPartFillPercent, 0.1); + return evCheckSpaceResult; }; @@ -1281,7 +1303,8 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { using TColor = NKikimrBlobStorage::TPDiskSpaceColor; pdiskConfig->SpaceColorBorder = TColor::ORANGE; testCtx.UpdateConfigRecreatePDisk(pdiskConfig); - // The actual value of SharedQuota.HardLimit internally initialized in TActorTestContext + // The actual value of SharedQuota.HardLimit is initialized in TActorTestContext + // with quite a complex formula. The value used here was obtained experimentally. // Feel free to update if some day it changes const ui32 sharedQuota = 778; @@ -1298,13 +1321,13 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { TVDiskMock vdisk0(&testCtx); vdisk0.InitFull(1u); vdisk0.SendEvLogSync(); - CheckEvCheckSpace(testCtx, vdisk0, sharedQuota, sharedQuota, 0, 1, 1, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk0, sharedQuota, sharedQuota, 0, 0.0, 0.0, 0.0, 1, 1, TColor::GREEN); TVDiskMock vdisk1(&testCtx); vdisk1.InitFull(2u); vdisk1.SendEvLogSync(); - CheckEvCheckSpace(testCtx, vdisk0, sharedQuota, sharedQuota/3*1, 0, 2, 3, TColor::GREEN); - CheckEvCheckSpace(testCtx, vdisk1, sharedQuota, sharedQuota/3*2, 0, 2, 3, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk0, sharedQuota, sharedQuota/3*1, 0, 0.0, 0.0, 0.0, 2, 3, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk1, sharedQuota, sharedQuota/3*2, 0, 0.0, 0.0, 0.0, 2, 3, TColor::GREEN); // State 2: // PDisk.ExpectedSlotCount: 4 @@ -1319,8 +1342,9 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { AwaitAndCheckEvPDiskStateUpdate(testCtx, 0u, 3); ui32 fairQuota = sharedQuota / 4; - CheckEvCheckSpace(testCtx, vdisk0, sharedQuota, fairQuota, 0, 2, 3, TColor::GREEN); - CheckEvCheckSpace(testCtx, vdisk1, sharedQuota, fairQuota*2, 0, 2, 3, TColor::GREEN); + UNIT_ASSERT_VALUES_EQUAL(fairQuota, 194); + CheckEvCheckSpace(testCtx, vdisk0, sharedQuota, fairQuota, 0, 0.0, 0.0, 0.0, 2, 3, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk1, sharedQuota, fairQuota*2, 0, 0.0, 0.0, 0.0, 2, 3, TColor::GREEN); // State 3: // vdisk0 consumes all it's fair quota (1/4 pdisk) @@ -1328,13 +1352,18 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { ui32 vdisk0Used = fairQuota; ui32 sharedFree = sharedQuota - vdisk0Used; + UNIT_ASSERT_VALUES_EQUAL(vdisk0Used, 194); + UNIT_ASSERT_VALUES_EQUAL(sharedFree, 584); for (ui32 i = 0; i < vdisk0Used; ++i) { vdisk0.ReserveChunk(); } vdisk0.CommitReservedChunks(); - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 2, 3, TColor::ORANGE); - CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota*2, 0, 2, 3, TColor::GREEN); + ui32 vdisk0LightYellowLimit = 168; + double vdisk0SlotUtilization = 100. * ((double)vdisk0Used) / vdisk0LightYellowLimit; + UNIT_ASSERT_DOUBLES_EQUAL(vdisk0SlotUtilization, 115.5, 0.1); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 0.97, vdisk0SlotUtilization, 25.0, 2, 3, TColor::ORANGE); + CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota*2, 0, 0.25, 0.0, 25.0, 2, 3, TColor::GREEN); // State 4: // PDisk.ExpectedSlotCount: 2 @@ -1352,13 +1381,18 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { AwaitAndCheckEvPDiskStateUpdate(testCtx, 2u, 2); fairQuota = sharedQuota / 2; + UNIT_ASSERT_VALUES_EQUAL(fairQuota, 389); + UNIT_ASSERT_VALUES_EQUAL(vdisk0Used, 194); UNIT_ASSERT_VALUES_EQUAL(vdisk0Used, fairQuota/2); - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 2, 2, TColor::GREEN); - CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota, 0, 2, 2, TColor::GREEN); + vdisk0LightYellowLimit = 344; + vdisk0SlotUtilization = 100. * ((double)vdisk0Used) / vdisk0LightYellowLimit; + UNIT_ASSERT_DOUBLES_EQUAL(vdisk0SlotUtilization, 56.4, 0.1); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 0.5, vdisk0SlotUtilization, 25.0, 2, 2, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota, 0, 0.25, 0.0, 25.0, 2, 2, TColor::GREEN); // State 5: // Owners.GroupSizeInUnits: [0u, 2u, 4u] - // Owners.GroupSizeInUnits: [1, 1, 2] + // Owners.Weight: [1, 1, 2] Cerr << (TStringBuilder() << "- State 5" << Endl); TVDiskMock vdisk2(&testCtx); @@ -1366,23 +1400,27 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { vdisk2.SendEvLogSync(); fairQuota = sharedQuota / 4; + UNIT_ASSERT_VALUES_EQUAL(fairQuota, 194); UNIT_ASSERT_VALUES_EQUAL(vdisk0Used, fairQuota); - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 3, 4, TColor::ORANGE); - CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota, 0, 3, 4, TColor::GREEN); - CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota*2, 0, 3, 4, TColor::GREEN); + vdisk0LightYellowLimit = 168; + vdisk0SlotUtilization = 100. * ((double)vdisk0Used) / vdisk0LightYellowLimit; + UNIT_ASSERT_DOUBLES_EQUAL(vdisk0SlotUtilization, 115.5, 0.1); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 0.97, vdisk0SlotUtilization, 25.0, 3, 4, TColor::ORANGE); + CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota, 0, 0.25, 0.0, 25.0, 3, 4, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota*2, 0, 0.25, 0.0, 25.0, 3, 4, TColor::GREEN); auto &icb = testCtx.GetRuntime()->GetAppData().Icb; TControlWrapper semiStrictSpaceIsolation(0, 0, 2); TControlBoard::RegisterSharedControl(semiStrictSpaceIsolation, icb->PDiskControls.SemiStrictSpaceIsolation); semiStrictSpaceIsolation = 1; - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 3, 4, TColor::LIGHT_YELLOW); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 0.90, vdisk0SlotUtilization, 25.0, 3, 4, TColor::LIGHT_YELLOW); semiStrictSpaceIsolation = 2; - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 3, 4, TColor::YELLOW); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 0.92, vdisk0SlotUtilization, 25.0, 3, 4, TColor::YELLOW); semiStrictSpaceIsolation = 0; // State 6: // Owners.GroupSizeInUnits: [0u, 2u, 1u] - // Owners.GroupSizeInUnits: [1, 1, 1] + // Owners.Weight: [1, 1, 1] Cerr << (TStringBuilder() << "- State 6" << Endl); testCtx.TestResponse( @@ -1391,9 +1429,15 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { AwaitAndCheckEvPDiskStateUpdate(testCtx, 2u, 3); fairQuota = sharedQuota / 3; - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 3, 3, TColor::GREEN); - CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota, 0, 3, 3, TColor::GREEN); - CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota, 0, 3, 3, TColor::GREEN); + UNIT_ASSERT_VALUES_EQUAL(fairQuota, 259); + vdisk0LightYellowLimit = 227; + vdisk0SlotUtilization = 100. * ((double)vdisk0Used) / vdisk0LightYellowLimit; + double vdisk0FairOccupancy = ((double)vdisk0Used) / fairQuota; + UNIT_ASSERT_DOUBLES_EQUAL(vdisk0FairOccupancy, 0.749, 0.001); + UNIT_ASSERT_DOUBLES_EQUAL(vdisk0SlotUtilization, 85.4, 0.1); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, vdisk0FairOccupancy, vdisk0SlotUtilization, 25.0, 3, 3, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota, 0, 0.25, 0.0, 25.0, 3, 3, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota, 0, 0.25, 0.0, 25.0, 3, 3, TColor::GREEN); // State 7: // PDisk.ExpectedSlotCount: 8 @@ -1411,28 +1455,40 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { AwaitAndCheckEvPDiskStateUpdate(testCtx, 1u, 4); fairQuota = sharedQuota / 8; + UNIT_ASSERT_VALUES_EQUAL(fairQuota, 97); + UNIT_ASSERT_VALUES_EQUAL(vdisk0Used, 194); UNIT_ASSERT_VALUES_EQUAL(vdisk0Used, fairQuota*2); - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 3, 4, TColor::ORANGE); - CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota*2, 0, 3, 4, TColor::GREEN); - CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota, 0, 3, 4, TColor::GREEN); + vdisk0LightYellowLimit = 81; + vdisk0SlotUtilization = 100. * ((double)vdisk0Used) / vdisk0LightYellowLimit; + UNIT_ASSERT_DOUBLES_EQUAL(vdisk0FairOccupancy, 0.749, 0.001); + UNIT_ASSERT_DOUBLES_EQUAL(vdisk0SlotUtilization, 239.5, 0.1); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 0.965, vdisk0SlotUtilization, 25.0, 3, 4, TColor::ORANGE); + CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota*2, 0, 0.25, 0.0, 25.0, 3, 4, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota, 0, 0.25, 0.0, 25.0, 3, 4, TColor::GREEN); // State 8: // vdisk1 makes the whole PDisk Red Cerr << (TStringBuilder() << "- State 8" << Endl); ui32 vdisk1Used = sharedFree - 3; + UNIT_ASSERT_VALUES_EQUAL(vdisk1Used, 581); sharedFree = 3; for (ui32 i = 0; i < vdisk1Used; ++i) { vdisk1.ReserveChunk(); } vdisk1.CommitReservedChunks(); + ui32 vdisk1LightYellowLimit = 168; + UNIT_ASSERT_GE(vdisk1LightYellowLimit, vdisk0LightYellowLimit*2); + double vdisk1SlotUtilization = 100. * ((double)vdisk1Used) / vdisk1LightYellowLimit; + UNIT_ASSERT_DOUBLES_EQUAL(vdisk1SlotUtilization, 345.8, 0.1); + UNIT_ASSERT_VALUES_EQUAL(vdisk0Used, fairQuota*2); UNIT_ASSERT_VALUES_EQUAL(vdisk1Used, fairQuota*6-1); UNIT_ASSERT_VALUES_EQUAL(vdisk0Used + vdisk1Used, sharedQuota - sharedFree); - CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 3, 4, TColor::RED); - CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota*2, vdisk1Used, 3, 4, TColor::RED); - CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota, 0, 3, 4, TColor::RED); + CheckEvCheckSpace(testCtx, vdisk0, sharedFree, fairQuota, vdisk0Used, 0.99, vdisk0SlotUtilization, 99.6, 3, 4, TColor::RED); + CheckEvCheckSpace(testCtx, vdisk1, sharedFree, fairQuota*2, vdisk1Used, 0.99, vdisk1SlotUtilization, 99.6, 3, 4, TColor::RED); + CheckEvCheckSpace(testCtx, vdisk2, sharedFree, fairQuota, 0, 0.99, 0.0, 99.6, 3, 4, TColor::RED); } Y_UNIT_TEST(TestChunkWriteCrossOwner) { @@ -2200,7 +2256,7 @@ Y_UNIT_TEST_SUITE(ReadOnlyPDisk) { NKikimrProto::OK); AwaitAndCheckEvPDiskStateUpdate(testCtx, 0u, 4); - CheckEvCheckSpace(testCtx, vdisk, sharedQuota, sharedQuota, 0, 1, 4, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk, sharedQuota, sharedQuota, 0, 0.0, 0.0, 0.0, 1, 4, TColor::GREEN); testCtx.TestResponse( new NPDisk::TEvChangeExpectedSlotCount(8, 2u), @@ -2210,7 +2266,7 @@ Y_UNIT_TEST_SUITE(ReadOnlyPDisk) { UNIT_ASSERT_VALUES_EQUAL(pdiskConfig->SlotSizeInUnits, 2u); AwaitAndCheckEvPDiskStateUpdate(testCtx, 2u, 2); - CheckEvCheckSpace(testCtx, vdisk, sharedQuota, sharedQuota/8*2, 0, 1, 2, TColor::GREEN); + CheckEvCheckSpace(testCtx, vdisk, sharedQuota, sharedQuota/8*2, 0, 0.0, 0.0, 0.0, 1, 2, TColor::GREEN); } } diff --git a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp index d79868ac1393..928153548957 100644 --- a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp +++ b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp @@ -934,7 +934,7 @@ class TPDiskMockActor : public TActorBootstrapped { auto res = std::make_unique(NKikimrProto::OK, GetStatusFlags(), Impl.GetNumFreeChunks(), Impl.TotalChunks, Impl.TotalChunks - Impl.GetNumFreeChunks(), Impl.Owners.size(), 0u, TString()); - res->Occupancy = GetOccupancy(); + res->NormalizedOccupancy = GetOccupancy(); Impl.FindOwner(msg, res); // to ensure correct owner/round Send(ev->Sender, res.release()); } diff --git a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp index 7895e4e16895..b289d9fd472f 100644 --- a/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp +++ b/ydb/core/blobstorage/vdisk/skeleton/skeleton_oos_tracker.cpp @@ -90,7 +90,7 @@ namespace NKikimr { FreeChunks = msg->FreeChunks; VCtx->OutOfSpaceState.UpdateLocalChunk(msg->StatusFlags); VCtx->OutOfSpaceState.UpdateLocalLog(msg->LogStatusFlags); - VCtx->OutOfSpaceState.UpdateLocalFreeSpaceShare(ui64(1 << 24) * (1.0 - msg->Occupancy)); + VCtx->OutOfSpaceState.UpdateLocalFreeSpaceShare(ui64(1 << 24) * (1.0 - msg->NormalizedOccupancy)); VCtx->OutOfSpaceState.UpdateLocalUsedChunks(msg->UsedChunks); VCtx->OutOfSpaceState.UpdateLocalTotalChunks(msg->TotalChunks); MonGroup.DskTotalBytes() = msg->TotalChunks * PDiskCtx->Dsk->ChunkSize; diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index e34e761a0cd3..4fccdfdbade4 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -188,14 +188,14 @@ bool TBlobStorageController::TGroupInfo::FillInResources( if (metrics.HasMaxWriteThroughput()) { writeThroughput = Min(writeThroughput.value_or(Max()), metrics.GetMaxWriteThroughput() / shareFactor); } - if (const auto& vm = vslot->Metrics; vm.HasOccupancy()) { - occupancy = Max(occupancy.value_or(0), vm.GetOccupancy()); + if (const auto& vm = vslot->Metrics; vm.HasNormalizedOccupancy()) { + occupancy = Max(occupancy.value_or(0), vm.GetNormalizedOccupancy()); } const bool hasAllMetrics = metrics.HasMaxIOPS() && metrics.HasMaxReadThroughput() && metrics.HasMaxWriteThroughput() - && vslot->Metrics.HasOccupancy(); + && vslot->Metrics.HasNormalizedOccupancy(); if (hasAllMetrics) { vdisksWithAllMetrics |= {Topology.get(), vslot->GetShortVDiskId()}; } @@ -238,8 +238,8 @@ bool TBlobStorageController::TGroupInfo::FillInVDiskResources( if (m.HasAllocatedSize()) { pb->SetAllocatedSize(Max(pb->HasAllocatedSize() ? pb->GetAllocatedSize() : 0, f * m.GetAllocatedSize())); } - if (m.HasSpaceColor()) { - pb->SetSpaceColor(pb->HasSpaceColor() ? Max(pb->GetSpaceColor(), m.GetSpaceColor()) : m.GetSpaceColor()); + if (m.HasCapacityAlertLevel()) { + pb->SetSpaceColor(pb->HasSpaceColor() ? Max(pb->GetSpaceColor(), m.GetCapacityAlertLevel()) : m.GetCapacityAlertLevel()); } const bool hasAllMetrics = m.HasAvailableSize() && m.HasAllocatedSize(); diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index feec5c767c61..b2fda3c38829 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -3025,9 +3025,9 @@ Y_UNIT_TEST_SUITE(THiveTest) { vdiskMetrics->MutableVDiskId()->SetVDisk(0); if (tabletGroups.contains(groupId)) { - vdiskMetrics->SetOccupancy(1.0); + vdiskMetrics->SetNormalizedOccupancy(1.0); } else { - vdiskMetrics->SetOccupancy(0.8); + vdiskMetrics->SetNormalizedOccupancy(0.8); } } diff --git a/ydb/core/protos/blobstorage_disk.proto b/ydb/core/protos/blobstorage_disk.proto index 8b894c018bdd..f1eb2101f161 100644 --- a/ydb/core/protos/blobstorage_disk.proto +++ b/ydb/core/protos/blobstorage_disk.proto @@ -62,8 +62,10 @@ message TVDiskMetrics { //optional uint64 WriteThroughput = 6; // bytes per second optional uint32 StatusFlags = 7; optional TVSlotId VSlotId = 8; - optional double Occupancy = 9; - optional NKikimrBlobStorage.TPDiskSpaceColor.E SpaceColor = 10; + optional double NormalizedOccupancy = 9; + optional double VDiskSlotUtilization = 16; // 100.0 * Owner.Used / Owner.LightYellowLimit + optional double VDiskFairPartFillPercent = 17; // 100.0 * Owner.Used / Owner.HardLimit + optional NKikimrBlobStorage.TPDiskSpaceColor.E CapacityAlertLevel = 10; optional NKikimrWhiteboard.EVDiskState State = 11; optional bool Replicated = 12; optional NKikimrWhiteboard.EFlag DiskSpace = 13; @@ -90,4 +92,5 @@ message TPDiskMetrics { optional uint64 UpdateTimestamp = 11; // TInstant::GetValue() optional uint64 SlotCount = 12; optional uint32 SlotSizeInUnits = 13; + optional double PDiskFillPercent = 14; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit } diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index ca8c9dadb6bc..e8b5615d4f21 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -8,6 +8,7 @@ import "ydb/core/protos/memory_stats.proto"; import "ydb/core/protos/bridge.proto"; import "google/protobuf/descriptor.proto"; import "ydb/core/protos/tracing.proto"; +import "ydb/core/protos/blobstorage_disk_color.proto"; package NKikimrWhiteboard; option java_package = "ru.yandex.kikimr.proto"; @@ -150,6 +151,7 @@ message TPDiskStateInfo { optional uint64 EnforcedDynamicSlotSize = 23 [(DefaultField) = true]; optional uint32 NumActiveSlots = 24 [(DefaultField) = true]; optional uint32 SlotSizeInUnits = 25; + optional double PDiskFillPercent = 26; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit } message TEvPDiskStateRequest { @@ -236,6 +238,10 @@ message TVDiskStateInfo { optional uint64 WriteThroughput = 23 [(DefaultField) = true]; optional uint32 GroupSizeInUnits = 32; + optional double VDiskSlotUtilization = 33; // 100.0 * Owner.Used / Owner.LightYellowLimit + optional double NormalizedOccupancy = 34; + optional double VDiskFairPartFillPercent = 35; // 100.0 * Owner.Used / Owner.HardLimit + optional NKikimrBlobStorage.TPDiskSpaceColor.E CapacityAlertLevel = 36; } message TEvVDiskStateRequest { From e6fee8bbc42c83ebcc741c497488d80cc2ff18ea Mon Sep 17 00:00:00 2001 From: Yaroslav Dynnikov Date: Fri, 7 Nov 2025 16:43:58 +0300 Subject: [PATCH 2/4] Use approved naming --- .../nodewarden/blobstorage_node_warden_ut.cpp | 8 +++--- .../blobstorage/pdisk/blobstorage_pdisk.h | 6 ++-- .../pdisk/blobstorage_pdisk_impl.cpp | 28 +++++++++---------- .../pdisk/blobstorage_pdisk_keeper.h | 6 ++-- .../pdisk/blobstorage_pdisk_ut.cpp | 24 ++++++++-------- ydb/core/mind/bscontroller/bsc.cpp | 4 +-- ydb/core/protos/blobstorage_disk.proto | 8 +++--- ydb/core/protos/node_whiteboard.proto | 8 +++--- 8 files changed, 46 insertions(+), 46 deletions(-) diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index 090f7720f1eb..72e91f694fa0 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -1030,8 +1030,8 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { UNIT_ASSERT(pdiskInfo.HasTotalSize()); UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetExpectedSlotCount(), expectedSlotCount); UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetSlotSizeInUnits(), expectedSlotSizeInUnits); - UNIT_ASSERT(pdiskInfo.HasPDiskFillPercent()); - UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetPDiskFillPercent(), 0.0); + UNIT_ASSERT(pdiskInfo.HasPDiskUsage()); + UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetPDiskUsage(), 0.0); break; } @@ -1049,8 +1049,8 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { UNIT_ASSERT(metrics.HasSlotSizeInUnits()); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotCount(), expectedSlotCount); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotSizeInUnits(), expectedSlotSizeInUnits); - UNIT_ASSERT(metrics.HasPDiskFillPercent()); - UNIT_ASSERT_VALUES_EQUAL(metrics.GetPDiskFillPercent(), 0.0); + UNIT_ASSERT(metrics.HasPDiskUsage()); + UNIT_ASSERT_VALUES_EQUAL(metrics.GetPDiskUsage(), 0.0); } } diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h index 4fc6f11c2ab4..9170e72faa81 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk.h @@ -1428,9 +1428,9 @@ struct TEvCheckSpaceResult : TEventLocalSetStatusFlags(statusFlags); vdiskMetrics->SetNormalizedOccupancy(occupancy); - vdiskMetrics->SetVDiskSlotUtilization(vdiskSlotUtilization); - vdiskMetrics->SetVDiskFairPartFillPercent(vdiskFairPartFillPercent); - vdiskMetrics->SetCapacityAlertLevel(spaceColor); + vdiskMetrics->SetVDiskSlotUsage(vdiskSlotUsage); + vdiskMetrics->SetVDiskRawUsage(vdiskRawUsage); + vdiskMetrics->SetCapacityAlert(spaceColor); vdiskInfo.SetNormalizedOccupancy(occupancy); - vdiskInfo.SetVDiskSlotUtilization(vdiskSlotUtilization); - vdiskInfo.SetVDiskFairPartFillPercent(vdiskFairPartFillPercent); - vdiskInfo.SetCapacityAlertLevel(spaceColor); + vdiskInfo.SetVDiskSlotUsage(vdiskSlotUsage); + vdiskInfo.SetVDiskRawUsage(vdiskRawUsage); + vdiskInfo.SetCapacityAlert(spaceColor); auto *vslotId = vdiskMetrics->MutableVSlotId(); vslotId->SetNodeId(PCtx->ActorSystem->NodeId); @@ -1669,9 +1669,9 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { pDiskMetrics.SetSlotCount(ExpectedSlotCount); } - double pdiskFillPercent = Keeper.GetPDiskFillPercent(); - pDiskMetrics.SetPDiskFillPercent(pdiskFillPercent); - pdiskState.SetPDiskFillPercent(pdiskFillPercent); + double pdiskUsage = Keeper.GetPDiskUsage(); + pDiskMetrics.SetPDiskUsage(pdiskUsage); + pdiskState.SetPDiskUsage(pdiskUsage); } PCtx->ActorSystem->Send(whiteboardReport.Sender, reportResult); @@ -2200,9 +2200,9 @@ void TPDisk::CheckSpace(TCheckSpace &evCheckSpace) { TString(), GetStatusFlags(OwnerSystem, evCheckSpace.OwnerGroupType)); result->NormalizedOccupancy = occupancy; - result->VDiskSlotUtilization = Keeper.GetVDiskSlotUtilization(evCheckSpace.Owner); - result->VDiskFairPartFillPercent = Keeper.GetVDiskFairPartFillPercent(evCheckSpace.Owner); - result->PDiskFillPercent = Keeper.GetPDiskFillPercent(); + result->VDiskSlotUsage = Keeper.GetVDiskSlotUsage(evCheckSpace.Owner); + result->VDiskRawUsage = Keeper.GetVDiskRawUsage(evCheckSpace.Owner); + result->PDiskUsage = Keeper.GetPDiskUsage(); PCtx->ActorSystem->Send(evCheckSpace.Sender, result.release()); Mon.CheckSpace.CountResponse(); return; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h index 5eba0de8c7b5..e7cce922c487 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_keeper.h @@ -150,19 +150,19 @@ class TKeeper { return ChunkTracker.EstimateSpaceColor(owner, allocationSize, occupancy); } - double GetPDiskFillPercent() const { + double GetPDiskUsage() const { i64 totalUsed = ChunkTracker.GetTotalUsed(); i64 totalHardLimit = ChunkTracker.GetTotalHardLimit(); return 100.0 * (totalHardLimit ? (double)totalUsed / totalHardLimit : 1.0); } - double GetVDiskSlotUtilization(TOwner owner) const { + double GetVDiskSlotUsage(TOwner owner) const { i64 used = ChunkTracker.GetOwnerUsed(owner); ui32 lightYellowLimit = ChunkTracker.ColorFlagLimit(owner, NKikimrBlobStorage::TPDiskSpaceColor::LIGHT_YELLOW); return 100.0 * (lightYellowLimit ? (double)used / lightYellowLimit : 1.0); } - double GetVDiskFairPartFillPercent(TOwner owner) const { + double GetVDiskRawUsage(TOwner owner) const { i64 used = ChunkTracker.GetOwnerUsed(owner); i64 hardLimit = ChunkTracker.GetOwnerHardLimit(owner); return 100.0 * (hardLimit ? (double)used / hardLimit : 1.0); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp index ebcb303133fb..ea24b08c4623 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp @@ -1249,14 +1249,14 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { ui32 expectedTotalChunks, ui32 expectedUsedChunks, double expectedNormalizedOccupancy, - double expectedVDiskSlotUtilization, - double expectedPDiskFillPercent, + double expectedVDiskSlotUsage, + double expectedPDiskUsage, ui32 expectedNumSlots, ui32 expectedNumActiveSlots, NKikimrBlobStorage::TPDiskSpaceColor::E expectedColor ) { UNIT_ASSERT_GT(expectedTotalChunks, 0); - double expectedVDiskFairPartFillPercent = 100. * ((double)expectedUsedChunks) / expectedTotalChunks; + double expectedVDiskRawUsage = 100. * ((double)expectedUsedChunks) / expectedTotalChunks; Cerr << (TStringBuilder() << "... Checking EvCheckSpace" << " VDisk# " << vdisk.VDiskID @@ -1264,9 +1264,9 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { << " TotalChunks# " << expectedTotalChunks << " UsedChunks# " << expectedUsedChunks << " NormalizedOccupancy# " << expectedNormalizedOccupancy - << " VDiskSlotUtilization# " << expectedVDiskSlotUtilization - << " VDiskFairPartFillPercent# " << expectedVDiskFairPartFillPercent - << " PDiskFillPercent# " << expectedPDiskFillPercent + << " VDiskSlotUsage# " << expectedVDiskSlotUsage + << " VDiskRawUsage# " << expectedVDiskRawUsage + << " PDiskUsage# " << expectedPDiskUsage << " NumSlots# " << expectedNumSlots << " NumActiveSlots# " << expectedNumActiveSlots << " Color# " << NKikimrBlobStorage::TPDiskSpaceColor::E_Name(expectedColor) @@ -1276,9 +1276,9 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { NKikimrProto::OK); Cerr << (TStringBuilder() << "Got " << evCheckSpaceResult->ToString() << " NormalizedOccupancy# " << evCheckSpaceResult->NormalizedOccupancy - << " VDiskSlotUtilization# " << evCheckSpaceResult->VDiskSlotUtilization - << " VDiskFairPartFillPercent# " << evCheckSpaceResult->VDiskFairPartFillPercent - << " PDiskFillPercent# " << evCheckSpaceResult->PDiskFillPercent + << " VDiskSlotUsage# " << evCheckSpaceResult->VDiskSlotUsage + << " VDiskRawUsage# " << evCheckSpaceResult->VDiskRawUsage + << " PDiskUsage# " << evCheckSpaceResult->PDiskUsage << Endl); UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->FreeChunks, expectedFreeChunks); UNIT_ASSERT_VALUES_EQUAL(evCheckSpaceResult->TotalChunks, expectedTotalChunks); @@ -1288,10 +1288,10 @@ Y_UNIT_TEST_SUITE(TPDiskTest) { UNIT_ASSERT_VALUES_EQUAL(StatusFlagToSpaceColor(evCheckSpaceResult->StatusFlags), expectedColor); UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->NormalizedOccupancy, expectedNormalizedOccupancy, 0.01); - UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->VDiskSlotUtilization, expectedVDiskSlotUtilization, 1e-6); - UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->PDiskFillPercent, expectedPDiskFillPercent, 0.1); + UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->VDiskSlotUsage, expectedVDiskSlotUsage, 1e-6); + UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->PDiskUsage, expectedPDiskUsage, 0.1); - UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->VDiskFairPartFillPercent, expectedVDiskFairPartFillPercent, 0.1); + UNIT_ASSERT_DOUBLES_EQUAL(evCheckSpaceResult->VDiskRawUsage, expectedVDiskRawUsage, 0.1); return evCheckSpaceResult; }; diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index 4fccdfdbade4..3550b83b0098 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -238,8 +238,8 @@ bool TBlobStorageController::TGroupInfo::FillInVDiskResources( if (m.HasAllocatedSize()) { pb->SetAllocatedSize(Max(pb->HasAllocatedSize() ? pb->GetAllocatedSize() : 0, f * m.GetAllocatedSize())); } - if (m.HasCapacityAlertLevel()) { - pb->SetSpaceColor(pb->HasSpaceColor() ? Max(pb->GetSpaceColor(), m.GetCapacityAlertLevel()) : m.GetCapacityAlertLevel()); + if (m.HasCapacityAlert()) { + pb->SetSpaceColor(pb->HasSpaceColor() ? Max(pb->GetSpaceColor(), m.GetCapacityAlert()) : m.GetCapacityAlert()); } const bool hasAllMetrics = m.HasAvailableSize() && m.HasAllocatedSize(); diff --git a/ydb/core/protos/blobstorage_disk.proto b/ydb/core/protos/blobstorage_disk.proto index f1eb2101f161..2b3546768e25 100644 --- a/ydb/core/protos/blobstorage_disk.proto +++ b/ydb/core/protos/blobstorage_disk.proto @@ -63,9 +63,9 @@ message TVDiskMetrics { optional uint32 StatusFlags = 7; optional TVSlotId VSlotId = 8; optional double NormalizedOccupancy = 9; - optional double VDiskSlotUtilization = 16; // 100.0 * Owner.Used / Owner.LightYellowLimit - optional double VDiskFairPartFillPercent = 17; // 100.0 * Owner.Used / Owner.HardLimit - optional NKikimrBlobStorage.TPDiskSpaceColor.E CapacityAlertLevel = 10; + optional double VDiskSlotUsage = 16; // 100.0 * Owner.Used / Owner.LightYellowLimit + optional double VDiskRawUsage = 17; // 100.0 * Owner.Used / Owner.HardLimit + optional NKikimrBlobStorage.TPDiskSpaceColor.E CapacityAlert = 10; optional NKikimrWhiteboard.EVDiskState State = 11; optional bool Replicated = 12; optional NKikimrWhiteboard.EFlag DiskSpace = 13; @@ -92,5 +92,5 @@ message TPDiskMetrics { optional uint64 UpdateTimestamp = 11; // TInstant::GetValue() optional uint64 SlotCount = 12; optional uint32 SlotSizeInUnits = 13; - optional double PDiskFillPercent = 14; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit + optional double PDiskUsage = 14; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit } diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index e8b5615d4f21..0b6a54c79d4d 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -151,7 +151,7 @@ message TPDiskStateInfo { optional uint64 EnforcedDynamicSlotSize = 23 [(DefaultField) = true]; optional uint32 NumActiveSlots = 24 [(DefaultField) = true]; optional uint32 SlotSizeInUnits = 25; - optional double PDiskFillPercent = 26; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit + optional double PDiskUsage = 26; // 100.0 * SharedQuota.Used / SharedQuota.HardLimit } message TEvPDiskStateRequest { @@ -238,10 +238,10 @@ message TVDiskStateInfo { optional uint64 WriteThroughput = 23 [(DefaultField) = true]; optional uint32 GroupSizeInUnits = 32; - optional double VDiskSlotUtilization = 33; // 100.0 * Owner.Used / Owner.LightYellowLimit + optional double VDiskSlotUsage = 33; // 100.0 * Owner.Used / Owner.LightYellowLimit optional double NormalizedOccupancy = 34; - optional double VDiskFairPartFillPercent = 35; // 100.0 * Owner.Used / Owner.HardLimit - optional NKikimrBlobStorage.TPDiskSpaceColor.E CapacityAlertLevel = 36; + optional double VDiskRawUsage = 35; // 100.0 * Owner.Used / Owner.HardLimit + optional NKikimrBlobStorage.TPDiskSpaceColor.E CapacityAlert = 36; } message TEvVDiskStateRequest { From c39361a12c49ebdfbd532c96dd91eed17d29c6b7 Mon Sep 17 00:00:00 2001 From: Yaroslav Dynnikov Date: Mon, 10 Nov 2025 00:12:12 +0300 Subject: [PATCH 3/4] Fix node_warden_ut --- .../nodewarden/blobstorage_node_warden_ut.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp index 72e91f694fa0..eae2b000e4db 100644 --- a/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/blobstorage_node_warden_ut.cpp @@ -1014,7 +1014,8 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { void CheckInferredPDiskSettings(TTestBasicRuntime& runtime, TActorId fakeWhiteboard, TActorId fakeNodeWarden, ui32 pdiskId, ui32 expectedSlotCount, ui32 expectedSlotSizeInUnits, TDuration simTimeout = TDuration::Seconds(10)) { - for (int attempt=0; attempt<10; ++attempt) { + const int maxAttempts = 10; + for (int attempt = 1; attempt <= maxAttempts; ++attempt) { // Check EvPDiskStateUpdate sent from PDiskActor to Whiteboard const auto ev = runtime.GrabEdgeEventRethrow(fakeWhiteboard, simTimeout); VERBOSE_COUT(" Got TEvPDiskStateUpdate# " << ev->ToString()); @@ -1022,6 +1023,7 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { NKikimrWhiteboard::TPDiskStateInfo pdiskInfo = ev->Get()->Record; UNIT_ASSERT_VALUES_EQUAL(pdiskInfo.GetPDiskId(), pdiskId); if (pdiskInfo.GetState() != NKikimrBlobStorage::TPDiskState::Normal) { + UNIT_ASSERT_LT_C(attempt, maxAttempts, "last attempt failed"); continue; } UNIT_ASSERT(pdiskInfo.HasExpectedSlotCount()); @@ -1035,7 +1037,7 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { break; } - { + for (int attempt = 1; attempt <= maxAttempts; ++attempt) { // Check EvControllerUpdateDiskStatus sent from PDiskActor to NodeWarden const auto ev = runtime.GrabEdgeEventRethrow(fakeNodeWarden, simTimeout); VERBOSE_COUT(" Got TEvControllerUpdateDiskStatus# " << ev->ToString()); @@ -1045,12 +1047,17 @@ Y_UNIT_TEST_SUITE(TBlobStorageWardenTest) { const NKikimrBlobStorage::TPDiskMetrics &metrics = diskStatus.GetPDisksMetrics(0); UNIT_ASSERT_VALUES_EQUAL(metrics.GetPDiskId(), pdiskId); + if (metrics.GetState() != NKikimrBlobStorage::TPDiskState::Normal) { + UNIT_ASSERT_LT_C(attempt, maxAttempts, "last attempt failed"); + continue; + } UNIT_ASSERT(metrics.HasSlotCount()); UNIT_ASSERT(metrics.HasSlotSizeInUnits()); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotCount(), expectedSlotCount); UNIT_ASSERT_VALUES_EQUAL(metrics.GetSlotSizeInUnits(), expectedSlotSizeInUnits); UNIT_ASSERT(metrics.HasPDiskUsage()); UNIT_ASSERT_VALUES_EQUAL(metrics.GetPDiskUsage(), 0.0); + break; } } From e204220168c44fc5568c4fad1c6b820b902ee1ce Mon Sep 17 00:00:00 2001 From: Yaroslav Dynnikov Date: Tue, 11 Nov 2025 14:24:32 +0300 Subject: [PATCH 4/4] Arrange protobuf fields order --- ydb/core/protos/blobstorage_disk.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/protos/blobstorage_disk.proto b/ydb/core/protos/blobstorage_disk.proto index 2b3546768e25..063ded849f74 100644 --- a/ydb/core/protos/blobstorage_disk.proto +++ b/ydb/core/protos/blobstorage_disk.proto @@ -63,14 +63,14 @@ message TVDiskMetrics { optional uint32 StatusFlags = 7; optional TVSlotId VSlotId = 8; optional double NormalizedOccupancy = 9; - optional double VDiskSlotUsage = 16; // 100.0 * Owner.Used / Owner.LightYellowLimit - optional double VDiskRawUsage = 17; // 100.0 * Owner.Used / Owner.HardLimit optional NKikimrBlobStorage.TPDiskSpaceColor.E CapacityAlert = 10; optional NKikimrWhiteboard.EVDiskState State = 11; optional bool Replicated = 12; optional NKikimrWhiteboard.EFlag DiskSpace = 13; optional bool IsThrottling = 14; optional uint32 ThrottlingRate = 15; + optional double VDiskSlotUsage = 16; // 100.0 * Owner.Used / Owner.LightYellowLimit + optional double VDiskRawUsage = 17; // 100.0 * Owner.Used / Owner.HardLimit } message TPDiskMetrics {