Simple trim of ArrayPool buffers

src/mscorlib/System.Private.CoreLib.csproj

@@ -126,9 +126,6 @@
     <Compile Include="$(BclSourcesRoot)\System\Runtime\CompilerServices\AsyncMethodBuilder.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Runtime\CompilerServices\TaskAwaiter.cs" />
   </ItemGroup>
-  <ItemGroup>
-    <Compile Include="$(BclSourcesRoot)\System\Runtime\Reliability\CriticalFinalizerObject.cs" />
-  </ItemGroup>
   <ItemGroup>
     <Compile Include="$(BclSourcesRoot)\System\Runtime\MemoryFailPoint.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Runtime\GcSettings.cs" />
@@ -353,6 +350,7 @@
     <Compile Include="$(BclSourcesRoot)\System\MissingFieldException.cs" />
     <Compile Include="$(BclSourcesRoot)\System\MissingMemberException.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Number.CoreCLR.cs" />
+    <Compile Include="$(BclSourcesRoot)\System\PinnableBufferCache.cs" />
     <Compile Include="$(BclSourcesRoot)\System\RtType.cs" />
     <Compile Include="$(BclSourcesRoot)\System\RuntimeArgumentHandle.cs" />
     <Compile Include="$(BclSourcesRoot)\System\RuntimeHandles.cs" />
@@ -473,7 +471,6 @@
     <Compile Include="$(BclSourcesRoot)\System\Threading\Monitor.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Mutex.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Overlapped.cs" />
-    <Compile Include="$(BclSourcesRoot)\System\Threading\PinnableBufferCache.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Semaphore.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\Thread.cs" />
     <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadInterruptedException.cs" />
@@ -631,6 +628,7 @@
     <Compile Include="$(BclSourcesRoot)\mscorlib.Friends.cs" Condition="'$(FeatureCominterop)' == 'true'" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="src\System\PinnableBufferCacheEventSource.cs" />
     <Compile Include="src\System\Runtime\RuntimeImports.cs" />
   </ItemGroup>
   <Import Project="shared\System.Private.CoreLib.Shared.projitems" />
@@ -666,14 +664,10 @@
     <!-- Use a different nativeresource file to avoid conflicts with mscorlib-->
     <Win32Resource Condition="'$(GenerateNativeVersionInfo)'=='true'">$(IntermediateOutputPath)\System.Private.CoreLib.res</Win32Resource>
   </PropertyGroup>
-
   <Import Project="CreateRuntimeRootILLinkDescriptorFile.targets" />
-
   <ItemGroup>
     <EmbeddedResource Include="$(_ILLinkRuntimeRootDescriptorFilePath)" />
   </ItemGroup>
-
   <Import Project="ILLink.targets" />
-
   <Import Project="GenerateCompilerResponseFile.targets" />
-</Project>
+</Project>

src/mscorlib/shared/System.Private.CoreLib.Shared.projitems

@@ -131,6 +131,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\FlagsAttribute.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\FormatException.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\FormattableString.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Gen2GcCallback.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Globalization\BidiCategory.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Globalization\Calendar.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Globalization\CalendarAlgorithmType.cs" />
@@ -425,6 +426,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\CompilerServices\YieldAwaitable.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\ConstrainedExecution\Cer.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\ConstrainedExecution\Consistency.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\ConstrainedExecution\CriticalFinalizerObject.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\ConstrainedExecution\ReliabilityContractAttribute.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\ExceptionServices\ExceptionNotification.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Runtime\ExceptionServices\HandleProcessCorruptedStateExceptionsAttribute.cs" />

src/mscorlib/shared/System/Buffers/ArrayPool.cs

@@ -33,9 +33,7 @@ public abstract class ArrayPool<T>
         /// optimized for very fast access speeds, at the expense of more memory consumption.
         /// The shared pool instance is created lazily on first access.
         /// </remarks>
-        public static ArrayPool<T> Shared { get; } =
-            typeof(T) == typeof(byte) || typeof(T) == typeof(char) ? new TlsOverPerCoreLockedStacksArrayPool<T>() :
-            Create();
+        public static ArrayPool<T> Shared { get; } = new TlsOverPerCoreLockedStacksArrayPool<T>();
 
         /// <summary>
         /// Creates a new <see cref="ArrayPool{T}"/> instance using default configuration options.

src/mscorlib/shared/System/Buffers/ArrayPoolEventSource.cs

@@ -86,5 +86,17 @@ internal unsafe void BufferAllocated(int bufferId, int bufferSize, int poolId, i
         /// </summary>
         [Event(3, Level = EventLevel.Verbose)]
         internal void BufferReturned(int bufferId, int bufferSize, int poolId) => WriteEvent(3, bufferId, bufferSize, poolId);
+
+        /// <summary>
+        /// Event raised when we free a buffer due to inactivity or memory pressure.
+        /// </summary>
+        [Event(4, Level = EventLevel.Informational)]
+        internal void BufferTrimmed(int bufferId, int bufferSize, int poolId) => WriteEvent(4, bufferId, bufferSize, poolId);
+
+        /// <summary>
+        /// Event raised when we check to trim buffers.
+        /// </summary>
+        [Event(5, Level = EventLevel.Informational)]
+        internal void BufferTrimPoll(int milliseconds, int pressure) => WriteEvent(5, milliseconds, pressure);
     }
 }

src/mscorlib/shared/System/Buffers/TlsOverPerCoreLockedStacksArrayPool.cs

@@ -2,11 +2,12 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Microsoft.Win32;
+using System;
+using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Threading;
-
 using Internal.Runtime.Augments;
+using Internal.Runtime.CompilerServices;
 
 namespace System.Buffers
 {
@@ -24,7 +25,6 @@ internal sealed partial class TlsOverPerCoreLockedStacksArrayPool<T> : ArrayPool
     {
         // TODO: #7747: "Investigate optimizing ArrayPool heuristics"
         // - Explore caching in TLS more than one array per size per thread, and moving stale buffers to the global queue.
-        // - Explore dumping stale buffers from the global queue, similar to PinnableBufferCache (maybe merging them).
         // - Explore changing the size of each per-core bucket, potentially dynamically or based on other factors like array size.
         // - Explore changing number of buckets and what sizes of arrays are cached.
         // - Investigate whether false sharing is causing any issues, in particular on LockedStack's count and the contents of its array.
@@ -48,6 +48,15 @@ internal sealed partial class TlsOverPerCoreLockedStacksArrayPool<T> : ArrayPool
         [ThreadStatic]
         private static T[][] t_tlsBuckets;
 
+        private int _callbackCreated;
+
+        private readonly static bool s_TrimBuffers = GetTrimBuffers();
+
+        /// <summary>
+        /// Used to keep track of all thread local buckets for trimming if needed
+        /// </summary>
+        private static readonly ConditionalWeakTable<T[][], object> s_AllTlsBuckets = s_TrimBuffers ? new ConditionalWeakTable<T[][], object>() : null;
+
         /// <summary>Initialize the pool.</summary>
         public TlsOverPerCoreLockedStacksArrayPool()
         {
@@ -182,15 +191,24 @@ public override void Return(T[] array, bool clearArray = false)
                 {
                     t_tlsBuckets = tlsBuckets = new T[NumBuckets][];
                     tlsBuckets[bucketIndex] = array;
+                    if (s_TrimBuffers)
+                    {
+                        s_AllTlsBuckets.Add(tlsBuckets, null);
+                        if (Interlocked.Exchange(ref _callbackCreated, 1) != 1)
+                        {
+                            Gen2GcCallback.Register(Gen2GcCallbackFunc, this);
+                        }
+                    }
                 }
                 else
                 {
                     T[] prev = tlsBuckets[bucketIndex];
                     tlsBuckets[bucketIndex] = array;
+
                     if (prev != null)
                     {
-                        PerCoreLockedStacks bucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex);
-                        bucket.TryPush(prev);
+                        PerCoreLockedStacks stackBucket = _buckets[bucketIndex] ?? CreatePerCoreLockedStacks(bucketIndex);
+                        stackBucket.TryPush(prev);
                     }
                 }
             }
@@ -203,6 +221,92 @@ public override void Return(T[] array, bool clearArray = false)
             }
         }
 
+        public bool Trim()
+        {
+            int milliseconds = Environment.TickCount;
+            MemoryPressure pressure = GetMemoryPressure();
+
+            ArrayPoolEventSource log = ArrayPoolEventSource.Log;
+            if (log.IsEnabled())
+                log.BufferTrimPoll(milliseconds, (int)pressure);
+
+            foreach (PerCoreLockedStacks bucket in _buckets)
+            {
+                bucket?.Trim((uint)milliseconds, Id, pressure, _bucketArraySizes);
+            }
+
+            if (pressure == MemoryPressure.High)
+            {
+                // Under high pressure, release all thread locals
+                foreach (KeyValuePair<T[][], object> tlsBuckets in s_AllTlsBuckets)
+                {
+                    T[][] buckets = tlsBuckets.Key;
+                    for (int i = 0; i < NumBuckets; i++)
+                    {
+                        T[] buffer = buckets[i];
+                        buckets[i] = null;
+
+                        if (log.IsEnabled() && buffer != null)
+                        {
+                            log.BufferTrimmed(buffer.GetHashCode(), buffer.Length, Id);
+                        }
+                    }
+                }
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// This is the static function that is called from the gen2 GC callback.
+        /// The input object is the instance we want the callback on.
+        /// </summary>
+        /// <remarks>
+        /// The reason that we make this function static and take the instance as a parameter is that
+        /// we would otherwise root the instance to the Gen2GcCallback object, leaking the instance even when
+        /// the application no longer needs it.
+        /// </remarks>
+        private static bool Gen2GcCallbackFunc(object target)
+        {
+            return ((TlsOverPerCoreLockedStacksArrayPool<T>)(target)).Trim();
+        }
+
+        private enum MemoryPressure
+        {
+            Low,
+            Medium,
+            High
+        }
+
+        private static MemoryPressure GetMemoryPressure()
+        {
+            const double HighPressureThreshold = .90;                       // Percent of GC memory pressure threshold we consider "high"
+            const double MediumPressureThreshold = .70;                     // Percent of GC memory pressure threshold we consider "medium"
+
+            GC.GetMemoryInfo(out uint threshold, out _, out uint lastLoad, out _, out _);
+            if (lastLoad >= threshold * HighPressureThreshold)
+            {
+                return MemoryPressure.High;
+            }
+            else if (lastLoad >= threshold * MediumPressureThreshold)
+            {
+                return MemoryPressure.Medium;
+            }
+            return MemoryPressure.Low;
+        }
+
+        private static bool GetTrimBuffers()
+        {
+            // Environment uses ArrayPool, so we have to hit the API directly.
+#if !CORECLR
+            // P/Invokes are different for CoreCLR/RT- for RT we'll not allow
+            // enabling/disabling for now.
+            return true;
+#else
+            return CLRConfig.GetBoolValueWithFallbacks("System.Buffers.TrimBuffers", "DOTNET_SYSTEM_BUFFERS_TRIMBUFFERS", defaultValue: true);
+#endif
+        }
+
         /// <summary>
         /// Stores a set of stacks of arrays, with one stack per core.
         /// </summary>
@@ -254,13 +358,24 @@ public T[] TryPop()
                 }
                 return null;
             }
+
+            public bool Trim(uint tickCount, int id, MemoryPressure pressure, int[] bucketSizes)
+            {
+                LockedStack[] stacks = _perCoreStacks;
+                for (int i = 0; i < stacks.Length; i++)
+                {
+                    stacks[i].Trim(tickCount, id, pressure, bucketSizes[i]);
+                }
+                return true;
+            }
         }
 
         /// <summary>Provides a simple stack of arrays, protected by a lock.</summary>
         private sealed class LockedStack
         {
             private readonly T[][] _arrays = new T[MaxBuffersPerArraySizePerCore][];
             private int _count;
+            private uint _firstStackItemMS;
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
             public bool TryPush(T[] array)
@@ -269,6 +384,12 @@ public bool TryPush(T[] array)
                 Monitor.Enter(this);
                 if (_count < MaxBuffersPerArraySizePerCore)
                 {
+                    if (s_TrimBuffers && _count == 0)
+                    {
+                        // Stash the time the bottom of the stack was filled
+                        _firstStackItemMS = (uint)Environment.TickCount;
+                    }
+
                     _arrays[_count++] = array;
                     enqueued = true;
                 }
@@ -289,6 +410,76 @@ public T[] TryPop()
                 Monitor.Exit(this);
                 return arr;
             }
+
+            public void Trim(uint tickCount, int id, MemoryPressure pressure, int bucketSize)
+            {
+                const uint StackTrimAfterMS = 60 * 1000;                        // Trim after 60 seconds for low/moderate pressure
+                const uint StackHighTrimAfterMS = 10 * 1000;                    // Trim after 10 seconds for high pressure
+                const uint StackRefreshMS = StackTrimAfterMS / 4;               // Time bump after trimming (1/4 trim time)
+                const int StackLowTrimCount = 1;                                // Trim one item when pressure is low
+                const int StackMediumTrimCount = 2;                             // Trim two items when pressure is moderate
+                const int StackHighTrimCount = MaxBuffersPerArraySizePerCore;   // Trim all items when pressure is high
+                const int StackLargeBucket = 16384;                             // If the bucket is larger than this we'll trim an extra when under high pressure
+                const int StackModerateTypeSize = 16;                           // If T is larger than this we'll trim an extra when under high pressure
+                const int StackLargeTypeSize = 32;                              // If T is larger than this we'll trim an extra (additional) when under high pressure
+
+                if (_count == 0)
+                    return;
+
+                lock (this)
+                {
+                    uint trimTicks = pressure == MemoryPressure.High ? StackHighTrimAfterMS : StackTrimAfterMS;
+                    if (_count > 0 && _firstStackItemMS > tickCount || (tickCount - _firstStackItemMS) > trimTicks)
+                    {
+                        // We've wrapped the tick count or elapsed enough time since the
+                        // first item went into the stack. Drop the top item so it can
+                        // be collected and make the stack look a little newer.
+
+                        ArrayPoolEventSource log = ArrayPoolEventSource.Log;
+                        int trimCount = StackLowTrimCount;
+                        switch (pressure)
+                        {
+                            case MemoryPressure.High:
+                                trimCount = StackHighTrimCount;
+
+                                // When pressure is high, aggressively trim larger arrays.
+                                if (bucketSize > StackLargeBucket)
+                                {
+                                    trimCount++;
+                                }
+                                if (Unsafe.SizeOf<T>() > StackModerateTypeSize)
+                                {
+                                    trimCount++;
+                                }
+                                if (Unsafe.SizeOf<T>() > StackLargeTypeSize)
+                                {
+                                    trimCount++;
+                                }
+                                break;
+                            case MemoryPressure.Medium:
+                                trimCount = StackMediumTrimCount;
+                                break;
+                        }
+
+                        while (_count > 0 && trimCount-- > 0)
+                        {
+                            T[] array = _arrays[--_count];
+                            _arrays[_count] = null;
+
+                            if (log.IsEnabled())
+                            {
+                                log.BufferTrimmed(array.GetHashCode(), array.Length, id);
+                            }
+                        }
+
+                        if (_count > 0 && _firstStackItemMS < uint.MaxValue - StackRefreshMS)
+                        {
+                            // Give the remaining items a bit more time
+                            _firstStackItemMS += StackRefreshMS;
+                        }
+                    }
+                }
+            }
         }
     }
 }