RavenDB-22305 - Missing DirtyMemory from the memory notification

src/Raven.Server/Dashboard/Cluster/Notifications/MemoryUsageNotificationSender.cs

@@ -57,7 +57,7 @@ protected override AbstractClusterDashboardNotification CreateNotification()
                 EncryptionBuffersInUse = encryptionBuffers.CurrentlyInUseSize,
                 EncryptionBuffersPool = encryptionBuffers.TotalPoolSize,
                 MemoryMapped = totalMapping,
-                DirtyMemory = dirtyMemoryState.TotalDirtyInBytes,
+                DirtyMemory = dirtyMemoryState.TotalDirty.GetValue(SizeUnit.Bytes),
                 AvailableMemory = memoryInfo.AvailableMemory.GetValue(SizeUnit.Bytes),
                 AvailableMemoryForProcessing = memoryInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes),
                 TotalSwapUsage = memoryInfo.TotalSwapUsage.GetValue(SizeUnit.Bytes)

src/Raven.Server/Documents/Handlers/Debugging/MemoryDebugHandler.cs

@@ -9,13 +9,11 @@
 using Raven.Server.Routing;
 using Raven.Server.Utils;
 using Raven.Server.Web;
-using Sparrow;
 using Sparrow.Json;
 using Sparrow.Json.Parsing;
 using Sparrow.LowMemory;
 using Sparrow.Platform;
 using Sparrow.Platform.Posix;
-using Sparrow.Server;
 using Sparrow.Server.Platform.Win32;
 using Sparrow.Utils;
 using Voron.Impl;
@@ -331,10 +329,9 @@ private static void WriteMemoryStats(AsyncBlittableJsonTextWriter writer, JsonOp
                 [nameof(MemoryInfo.EncryptionBuffersPool)] = Size.Humane(encryptionBuffers.TotalPoolSize),
                 [nameof(MemoryInfo.EncryptionLockedMemory)] = Size.Humane(Sodium.LockedBytes),
                 [nameof(MemoryInfo.MemoryMapped)] = Size.Humane(totalMapping),
-                [nameof(MemoryInfo.ScratchDirtyMemory)] = memInfo.TotalScratchDirtyMemory.ToString(),
                 [nameof(MemoryInfo.IsHighDirty)] = dirtyMemoryState.IsHighDirty,
-                [nameof(MemoryInfo.DirtyMemory)] = Size.Humane(dirtyMemoryState.TotalDirtyInBytes),
-                [nameof(MemoryInfo.AvailableMemory)] = Size.Humane(memInfo.AvailableMemory.GetValue(SizeUnit.Bytes)),
+                [nameof(MemoryInfo.DirtyMemory)] = dirtyMemoryState.TotalDirty.ToString(),
+                [nameof(MemoryInfo.AvailableMemory)] = memInfo.AvailableMemory.ToString(),
                 [nameof(MemoryInfo.AvailableMemoryForProcessing)] = memInfo.AvailableMemoryForProcessing.ToString(),
             };
             if (memInfo.Remarks != null)

src/Raven.Server/Documents/TransactionOperationsMerger.cs

@@ -882,7 +882,7 @@ private enum PendingOperations
                         $"Operation was cancelled by the transaction merger for transaction #{llt.Id} due to high dirty memory in scratch files." +
                         $" This might be caused by a slow IO storage. Current memory usage: " +
                         $"Total Physical Memory: {MemoryInformation.TotalPhysicalMemory}, " +
-                        $"Total Scratch Allocated Memory: {new Size(dirtyMemoryState.TotalDirtyInBytes, SizeUnit.Bytes)} " +
+                        $"Total Scratch Allocated Memory: {dirtyMemoryState.TotalDirty} " +
                         $"(which is above {_parent.Configuration.Memory.TemporaryDirtyMemoryAllowedPercentage * 100}%)");
                 }
 

src/Raven.Server/Monitoring/MetricsProvider.cs

@@ -154,8 +154,7 @@ private MemoryMetrics GetMemoryMetrics()
         result.TotalSwapUsageInMb = memoryInfoResult.TotalSwapUsage.GetValue(SizeUnit.Megabytes);
         result.WorkingSetSwapUsageInMb = memoryInfoResult.WorkingSetSwapUsage.GetValue(SizeUnit.Megabytes);
 
-        var totalDirtyInBytes = MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes;
-        result.TotalDirtyInMb = new Size(totalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes);
+        result.TotalDirtyInMb = MemoryInformation.GetDirtyMemoryState().TotalDirty.GetValue(SizeUnit.Megabytes);
 
         return result;
     }

src/Raven.Server/Monitoring/Snmp/Objects/Server/1.6/ServerDirtyMemory.cs

@@ -13,8 +13,7 @@ public ServerDirtyMemory() : base(SnmpOids.Server.DirtyMemory)
 
         protected override Gauge32 GetData()
         {
-            var totalDirtyInBytes = MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes;
-            return new Gauge32(new Size(totalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes));
+            return new Gauge32(MemoryInformation.GetDirtyMemoryState().TotalDirty.GetValue(SizeUnit.Megabytes));
         }
     }
 }

src/Raven.Server/NotificationCenter/OutOfMemoryNotifications.cs

@@ -76,7 +76,7 @@ private static MessageDetails OutOfMemoryDetails(Exception exception)
 
             return new MessageDetails
             {
-                Message = $"{MemoryUtils.GetExtendedMemoryInfo(memoryInfo)} {Environment.NewLine}" +
+                Message = $"{MemoryUtils.GetExtendedMemoryInfo(memoryInfo, MemoryInformation.GetDirtyMemoryState())} {Environment.NewLine}" +
                           $"Error: {exception}"
             };
         }

src/Raven.Server/Utils/Cli/RavenCli.cs

@@ -584,13 +584,15 @@ private static bool CommandInfo(List<string> args, RavenCli cli)
         public static string GetInfoText()
         {
             var memoryInfo = MemoryInformation.GetMemoryInformationUsingOneTimeSmapsReader();
+            var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
+
             using (var currentProcess = Process.GetCurrentProcess())
             {
                 return $" Build {ServerVersion.Build}, Version {ServerVersion.Version}, SemVer {ServerVersion.FullVersion}, Commit {ServerVersion.CommitHash}" +
                        Environment.NewLine +
                        $" PID {currentProcess.Id}, {IntPtr.Size * 8} bits, {ProcessorInfo.ProcessorCount} Cores, Arch: {RuntimeInformation.OSArchitecture}" +
                        Environment.NewLine +
-                       $" {memoryInfo.TotalPhysicalMemory} Physical Memory, {memoryInfo.AvailableMemory} Available Memory, {memoryInfo.AvailableMemoryForProcessing} Calculated Available Memory, {memoryInfo.TotalScratchDirtyMemory} Scratch Dirty Memory" +
+                       $" {memoryInfo.TotalPhysicalMemory} Physical Memory, {memoryInfo.AvailableMemory} Available Memory, {memoryInfo.AvailableMemoryForProcessing} Calculated Available Memory, {dirtyMemoryState.TotalDirty} Scratch Dirty Memory" +
                        Environment.NewLine +
                        $" {RuntimeSettings.Describe()}" +
                        Environment.NewLine +
@@ -1118,7 +1120,7 @@ private static bool CommandLowMem(List<string> args, RavenCli cli)
                 SizeClient.Humane(MemoryInformation.GetWorkingSetInBytes()),
                 SizeClient.Humane(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes()),
                 SizeClient.Humane(AbstractLowMemoryMonitor.GetManagedMemoryInBytes()),
-                SizeClient.Humane(MemoryInformation.GetTotalScratchAllocatedMemory()),
+                SizeClient.Humane(MemoryInformation.GetTotalScratchAllocatedMemoryInBytes()),
                 SizeClient.Humane(totalMemoryMapped));
         }
 

src/Sparrow.Server/LowMemory/MemoryInformation.cs

@@ -179,7 +179,7 @@ private static void ThrowInsufficientMemory(MemoryInfoResult memInfo)
             LowMemoryNotification.Instance.SimulateLowMemoryNotification();
 
             throw new EarlyOutOfMemoryException($"The amount of available memory to commit on the system is low. " +
-                                                MemoryUtils.GetExtendedMemoryInfo(memInfo), memInfo);
+                                                MemoryUtils.GetExtendedMemoryInfo(memInfo, GetDirtyMemoryState()), memInfo);
 
         }
 
@@ -423,7 +423,7 @@ internal static MemoryInfoResult GetMemoryInfo(SmapsReader smapsReader = null, b
             }
         }
 
-        public static long GetTotalScratchAllocatedMemory()
+        public static long GetTotalScratchAllocatedMemoryInBytes()
         {
             long totalScratchAllocated = 0;
             foreach (var scratchGetAllocated in DirtyMemoryObjects)
@@ -740,13 +740,13 @@ public static long GetWorkingSetInBytes()
 
         public static DirtyMemoryState GetDirtyMemoryState()
         {
-            var totalScratchMemory = GetTotalScratchAllocatedMemory();
+            var totalScratchMemory = new Size(GetTotalScratchAllocatedMemoryInBytes(), SizeUnit.Bytes);
 
             return new DirtyMemoryState
             {
-                IsHighDirty = totalScratchMemory > TotalPhysicalMemory.GetValue(SizeUnit.Bytes) *
-                              LowMemoryNotification.Instance.TemporaryDirtyMemoryAllowedPercentage,
-                TotalDirtyInBytes = totalScratchMemory
+                IsHighDirty = totalScratchMemory > 
+                              TotalPhysicalMemory * LowMemoryNotification.Instance.TemporaryDirtyMemoryAllowedPercentage,
+                TotalDirty = totalScratchMemory
             };
         }
     }

src/Sparrow/LowMemory/DirtyMemoryState.cs

@@ -4,6 +4,6 @@ public class DirtyMemoryState
     {
         public bool IsHighDirty;
 
-        public long TotalDirtyInBytes;
+        public Size TotalDirty;
     }
 }

src/Sparrow/LowMemory/LowMemoryNotification.cs

@@ -64,7 +64,7 @@ private void RunLowMemoryHandlers(bool isLowMemory, MemoryInfoResult memoryInfo,
                     {
                         _lastLoggedLowMemory = now;
                         _logger.Operations($"Running {_lowMemoryHandlers.Count} low memory handlers with severity: {lowMemorySeverity}. " +
-                                           $"{MemoryUtils.GetExtendedMemoryInfo(memoryInfo)}");
+                                           $"{MemoryUtils.GetExtendedMemoryInfo(memoryInfo, _lowMemoryMonitor.GetDirtyMemoryState())}");
                     }
 
 #if NET7_0_OR_GREATER
@@ -313,14 +313,9 @@ private void SimulateLowMemory()
             if (_lowMemoryMonitor != null)
             {
                 memInfoForLog = _lowMemoryMonitor.GetMemoryInfoOnce();
-                var availableMemForLog = memInfoForLog.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes);
 
                 AddLowMemEvent(LowMemoryState ? LowMemReason.LowMemStateSimulation : LowMemReason.BackToNormalSimulation,
-                    availableMemForLog,
-                    -2,
-                    memInfoForLog.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes),
-                    memInfoForLog.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
-                    memInfoForLog.CurrentCommitCharge.GetValue(SizeUnit.Bytes));
+                    memInfoForLog, totalUnmanaged: -2);
             }
 
             if (_logger.IsInfoEnabled)
@@ -362,12 +357,7 @@ internal int CheckMemoryStatus(AbstractLowMemoryMonitor monitor)
                             _logger.Info("Low memory detected, will try to reduce memory usage...");
 
                         }
-                        AddLowMemEvent(LowMemReason.LowMemOnTimeoutChk,
-                            memoryInfo.AvailableMemory.GetValue(SizeUnit.Bytes),
-                            totalUnmanagedAllocations,
-                            memoryInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes),
-                            memoryInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
-                            memoryInfo.CurrentCommitCharge.GetValue(SizeUnit.Bytes));
+                        AddLowMemEvent(LowMemReason.LowMemOnTimeoutChk, memoryInfo, totalUnmanagedAllocations);
                     }
                     catch (OutOfMemoryException)
                     {
@@ -392,12 +382,8 @@ internal int CheckMemoryStatus(AbstractLowMemoryMonitor monitor)
                 {
                     if (_logger.IsInfoEnabled)
                         _logger.Info("Back to normal memory usage detected");
-                    AddLowMemEvent(LowMemReason.BackToNormal,
-                        memoryInfo.AvailableMemory.GetValue(SizeUnit.Bytes),
-                        totalUnmanagedAllocations,
-                        memoryInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Bytes),
-                        memoryInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
-                        memoryInfo.CurrentCommitCharge.GetValue(SizeUnit.Bytes));
+
+                    AddLowMemEvent(LowMemReason.BackToNormal, memoryInfo, totalUnmanagedAllocations);
                 }
                 LowMemoryState = false;
                 RunLowMemoryHandlers(false, memoryInfo, isLowMemory);
@@ -454,17 +440,17 @@ private LowMemorySeverity IsAvailableMemoryBelowThreshold(MemoryInfoResult memIn
             return LowMemorySeverity.None;
         }
 
-        private void AddLowMemEvent(LowMemReason reason, long availableMem, long totalUnmanaged, long totalScratchDirty, long physicalMem, long currentcommitCharge)
+        private void AddLowMemEvent(LowMemReason reason, MemoryInfoResult memoryInfo, long totalUnmanaged)
         {
             var lowMemEventDetails = new LowMemEventDetails
             {
                 Reason = reason,
-                FreeMem = availableMem,
+                FreeMem = memoryInfo.AvailableMemoryForProcessing.GetValue(SizeUnit.Bytes),
                 TotalUnmanaged = totalUnmanaged,
-                TotalScratchDirty = totalScratchDirty,
-                PhysicalMem = physicalMem,
+                TotalScratchDirty = _lowMemoryMonitor.GetDirtyMemoryState().TotalDirty.GetValue(SizeUnit.Bytes),
+                PhysicalMem = memoryInfo.TotalPhysicalMemory.GetValue(SizeUnit.Bytes),
                 LowMemThreshold = LowMemoryThreshold.GetValue(SizeUnit.Bytes),
-                CurrentCommitCharge = currentcommitCharge,
+                CurrentCommitCharge = memoryInfo.CurrentCommitCharge.GetValue(SizeUnit.Bytes),
                 Time = DateTime.UtcNow
             };
 

src/Sparrow/LowMemory/MemoryInfoResult.cs

@@ -29,7 +29,6 @@ public class MemoryUsageLowHigh
         public Size AvailableMemory;
         public Size AvailableMemoryForProcessing;
         public Size SharedCleanMemory;
-        public Size TotalScratchDirtyMemory;
 
         public Size TotalSwapSize;
         public Size TotalSwapUsage;

src/Sparrow/Utils/MemoryUtils.cs

@@ -11,15 +11,15 @@ public static class MemoryUtils
     private static readonly InvertedComparer InvertedComparerInstance = new InvertedComparer();
     private const int MinAllocatedThresholdInBytes = 10 * 1024 * 1024;
 
-    public static string GetExtendedMemoryInfo(MemoryInfoResult memoryInfo)
+    public static string GetExtendedMemoryInfo(MemoryInfoResult memoryInfo, DirtyMemoryState dirtyState)
     {
         try
         {
             var sb = new StringBuilder();
             TryAppend(() => sb.Append("Commit charge: ").Append(memoryInfo.CurrentCommitCharge).Append(" / ").Append(memoryInfo.TotalCommittableMemory).Append(", "));
             TryAppend(() => sb.Append("Memory: ").Append(memoryInfo.TotalPhysicalMemory - memoryInfo.AvailableMemory).Append(" / ").Append(memoryInfo.TotalPhysicalMemory).Append(", "));
             TryAppend(() => sb.Append("Available memory for processing: ").Append(memoryInfo.AvailableMemoryForProcessing).Append(", "));
-            TryAppend(() => sb.Append("Dirty memory: ").Append(memoryInfo.TotalScratchDirtyMemory).Append(", "));
+            TryAppend(() => sb.Append("Dirty memory: ").Append(dirtyState.TotalDirty).Append(", "));
             TryAppend(() => sb.Append("Managed memory: ").Append(new Size(AbstractLowMemoryMonitor.GetManagedMemoryInBytes(), SizeUnit.Bytes)).Append(", "));
             TryAppend(() => sb.Append("Unmanaged allocations: ").Append(new Size(AbstractLowMemoryMonitor.GetUnmanagedAllocationsInBytes(), SizeUnit.Bytes)).Append(", "));
             TryAppend(() => sb.Append("Lucene managed: ").Append(new Size(NativeMemory.TotalLuceneManagedAllocationsForTermCache, SizeUnit.Bytes)).Append(", "));

test/Tests.Infrastructure/TestMetrics/TestResourceSnapshotWriter.cs

@@ -79,12 +79,13 @@ private TestResourceSnapshot GetTestResourceSnapshot(TestStage testStage, IAssem
             var cpuUsage = _metricCacher.GetCpuUsage();
             var memoryInfo = _metricCacher.GetMemoryInfoExtended();
             var tcpConnections = TcpStatisticsProvider.GetConnections();
+            var dirtyMemoryState = MemoryInformation.GetDirtyMemoryState();
 
             var snapshot = new TestResourceSnapshot
             {
-                TotalScratchAllocatedMemory = new Size(MemoryInformation.GetTotalScratchAllocatedMemory(), SizeUnit.Bytes).GetValue(SizeUnit.Megabytes),
-                TotalDirtyMemory = new Size(MemoryInformation.GetDirtyMemoryState().TotalDirtyInBytes, SizeUnit.Bytes).GetValue(SizeUnit.Megabytes),
-                IsHighDirty = MemoryInformation.GetDirtyMemoryState().IsHighDirty,
+                TotalScratchAllocatedMemory = new Size(MemoryInformation.GetTotalScratchAllocatedMemoryInBytes(), SizeUnit.Bytes).GetValue(SizeUnit.Megabytes),
+                TotalDirtyMemory = dirtyMemoryState.TotalDirty.GetValue(SizeUnit.Megabytes),
+                IsHighDirty = dirtyMemoryState.IsHighDirty,
                 TestStage = testStage,
                 Timestamp = timeStamp.ToString("o"),
                 AssemblyName = assemblyName,
@@ -96,7 +97,7 @@ private TestResourceSnapshot GetTestResourceSnapshot(TestStage testStage, IAssem
                 AvailableMemoryInMb = memoryInfo.AvailableMemory.GetValue(SizeUnit.Megabytes),
                 CurrentCommitChargeInMb = memoryInfo.CurrentCommitCharge.GetValue(SizeUnit.Megabytes),
                 SharedCleanMemoryInMb = memoryInfo.SharedCleanMemory.GetValue(SizeUnit.Megabytes),
-                TotalScratchDirtyMemory = memoryInfo.TotalScratchDirtyMemory.GetValue(SizeUnit.Megabytes),
+                TotalScratchDirtyMemory = MemoryInformation.GetDirtyMemoryState().TotalDirty.GetValue(SizeUnit.Megabytes),
                 CurrentIpv4Connections = tcpConnections.CurrentIpv4,
                 CurrentIpv6Connections = tcpConnections.CurrentIpv6
             };