Make thread pool thread timeouts configurable (#92986)

- Added two config options, one that configures the worker and wait thread timeouts, and another that enables keeping some number of worker threads alive after they are created
- This enables services that take periodic traffic to keep some worker threads around for better latency, while allowing extra threads to time out as appropriate for the service

src/libraries/System.Private.CoreLib/src/System/AppContextConfigHelper.cs

@@ -65,6 +65,45 @@ internal static int GetInt32Config(string configName, int defaultValue, bool all
             }
         }
 
+        internal static int GetInt32Config(string configName, string envVariable, int defaultValue, bool allowNegative = true)
+        {
+            string? str = Environment.GetEnvironmentVariable(envVariable);
+            if (str != null)
+            {
+                try
+                {
+                    int result;
+                    if (str.StartsWith('0'))
+                    {
+                        if (str.Length >= 2 && str[1] == 'x')
+                        {
+                            result = Convert.ToInt32(str, 16);
+                        }
+                        else
+                        {
+                            result = Convert.ToInt32(str, 8);
+                        }
+                    }
+                    else
+                    {
+                        result = int.Parse(str, NumberStyles.AllowLeadingSign, NumberFormatInfo.InvariantInfo);
+                    }
+
+                    if (allowNegative || result >= 0)
+                    {
+                        return result;
+                    }
+                }
+                catch (FormatException)
+                {
+                }
+                catch (OverflowException)
+                {
+                }
+            }
+
+            return GetInt32Config(configName, defaultValue, allowNegative);
+        }
 
         internal static short GetInt16Config(string configName, short defaultValue, bool allowNegative = true)
         {
@@ -112,5 +151,45 @@ internal static short GetInt16Config(string configName, short defaultValue, bool
                 return defaultValue;
             }
         }
+
+        internal static short GetInt16Config(string configName, string envVariable, short defaultValue, bool allowNegative = true)
+        {
+            string? str = Environment.GetEnvironmentVariable(envVariable);
+            if (str != null)
+            {
+                try
+                {
+                    short result;
+                    if (str.StartsWith('0'))
+                    {
+                        if (str.Length >= 2 && str[1] == 'x')
+                        {
+                            result = Convert.ToInt16(str, 16);
+                        }
+                        else
+                        {
+                            result = Convert.ToInt16(str, 8);
+                        }
+                    }
+                    else
+                    {
+                        result = short.Parse(str, NumberStyles.AllowLeadingSign, NumberFormatInfo.InvariantInfo);
+                    }
+
+                    if (allowNegative || result >= 0)
+                    {
+                        return result;
+                    }
+                }
+                catch (FormatException)
+                {
+                }
+                catch (OverflowException)
+                {
+                }
+            }
+
+            return GetInt16Config(configName, defaultValue, allowNegative);
+        }
     }
 }

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.NonBrowser.cs

@@ -7,11 +7,29 @@ namespace System.Threading
 {
     internal sealed partial class PortableThreadPool
     {
+        private int _numThreadsBeingKeptAlive;
+
         /// <summary>
         /// The worker thread infastructure for the CLR thread pool.
         /// </summary>
         private static partial class WorkerThread
         {
+            private static readonly short ThreadsToKeepAlive = DetermineThreadsToKeepAlive();
+
+            private static short DetermineThreadsToKeepAlive()
+            {
+                const short DefaultThreadsToKeepAlive = 0;
+
+                // The number of worker threads to keep alive after they are created. Set to -1 to keep all created worker
+                // threads alive. When the ThreadTimeoutMs config value is also set, for worker threads the timeout applies to
+                // worker threads that are in excess of the number configured for ThreadsToKeepAlive.
+                short threadsToKeepAlive =
+                    AppContextConfigHelper.GetInt16Config(
+                        "System.Threading.ThreadPool.ThreadsToKeepAlive",
+                        "DOTNET_ThreadPool_ThreadsToKeepAlive",
+                        DefaultThreadsToKeepAlive);
+                return threadsToKeepAlive >= -1 ? threadsToKeepAlive : DefaultThreadsToKeepAlive;
+            }
 
             /// <summary>
             /// Semaphore for controlling how many threads are currently working.
@@ -50,10 +68,36 @@ private static void WorkerThreadStart()
                 LowLevelLock threadAdjustmentLock = threadPoolInstance._threadAdjustmentLock;
                 LowLevelLifoSemaphore semaphore = s_semaphore;
 
+                // Determine the idle timeout to use for this thread. Some threads may always be kept alive based on config.
+                int timeoutMs = ThreadPoolThreadTimeoutMs;
+                if (ThreadsToKeepAlive != 0)
+                {
+                    if (ThreadsToKeepAlive < 0)
+                    {
+                        timeoutMs = Timeout.Infinite;
+                    }
+                    else
+                    {
+                        int count = threadPoolInstance._numThreadsBeingKeptAlive;
+                        while (count < ThreadsToKeepAlive)
+                        {
+                            int countBeforeUpdate =
+                                Interlocked.CompareExchange(ref threadPoolInstance._numThreadsBeingKeptAlive, count + 1, count);
+                            if (countBeforeUpdate == count)
+                            {
+                                timeoutMs = Timeout.Infinite;
+                                break;
+                            }
+
+                            count = countBeforeUpdate;
+                        }
+                    }
+                }
+
                 while (true)
                 {
                     bool spinWait = true;
-                    while (semaphore.Wait(ThreadPoolThreadTimeoutMs, spinWait))
+                    while (semaphore.Wait(timeoutMs, spinWait))
                     {
                         WorkerDoWork(threadPoolInstance, ref spinWait);
                     }
@@ -65,7 +109,6 @@ private static void WorkerThreadStart()
                 }
             }
 
-
             private static void CreateWorkerThread()
             {
                 // Thread pool threads must start in the default execution context without transferring the context, so

src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs

@@ -13,7 +13,6 @@ namespace System.Threading
     /// </summary>
     internal sealed partial class PortableThreadPool
     {
-        private const int ThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests.
         private const int SmallStackSizeBytes = 256 * 1024;
 
         private const short MaxPossibleThreadCount = short.MaxValue;
@@ -40,6 +39,23 @@ internal sealed partial class PortableThreadPool
         private static readonly short ForcedMaxWorkerThreads =
             AppContextConfigHelper.GetInt16Config("System.Threading.ThreadPool.MaxThreads", 0, false);
 
+        private static readonly int ThreadPoolThreadTimeoutMs = DetermineThreadPoolThreadTimeoutMs();
+
+        private static int DetermineThreadPoolThreadTimeoutMs()
+        {
+            const int DefaultThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests.
+
+            // The amount of time in milliseconds a thread pool thread waits without having done any work before timing out and
+            // exiting. Set to -1 to disable the timeout. Applies to worker threads and wait threads. Also see the
+            // ThreadsToKeepAlive config value for relevant information.
+            int timeoutMs =
+                AppContextConfigHelper.GetInt32Config(
+                    "System.Threading.ThreadPool.ThreadTimeoutMs",
+                    "DOTNET_ThreadPool_ThreadTimeoutMs",
+                    DefaultThreadPoolThreadTimeoutMs);
+            return timeoutMs >= -1 ? timeoutMs : DefaultThreadPoolThreadTimeoutMs;
+        }
+
         [ThreadStatic]
         private static object? t_completionCountObject;