From 4a648495eec9b63cd5dc40e64080c366b433f1bb Mon Sep 17 00:00:00 2001 From: Malcolm Daigle Date: Tue, 4 Aug 2026 16:26:29 -0700 Subject: [PATCH] Add pool tracing/metrics parity and surface pooled-open timeout cause Instrument ChannelDbConnectionPool with TryPoolerTraceEvent calls across the connection lifecycle so it matches the categories traced by the WaitHandle pool: construction, get, create, return, remove/dispose, clear, startup, shutdown, prune, rate-limit throttle, error state, wait timeout, and the reason a connection was rejected as not live. Fill the two remaining metric gaps in ReplaceConnection, which disposed the old and failed-new connections without counting a hard disconnect. Also address GH#3545: record the last physical-connection-create exception on each pool and attach it as the inner exception of the pooled-open timeout, so callers see why the pool could not produce a connection. Fixes #3545 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../src/Microsoft/Data/Common/AdapterUtil.cs | 13 + .../ConnectionPool/ChannelDbConnectionPool.cs | 159 +++- .../ConnectionPool/IDbConnectionPool.cs | 13 + .../WaitHandleDbConnectionPool.cs | 25 +- .../Data/SqlClient/SqlConnectionFactory.cs | 12 +- ...nnelDbConnectionPoolInstrumentationTest.cs | 679 ++++++++++++++++++ .../TransactedConnectionPoolTest.cs | 1 + ...andleDbConnectionPoolBlockingPeriodTest.cs | 28 + 8 files changed, 926 insertions(+), 4 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolInstrumentationTest.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs index 22aa0360ec..f8c37065a2 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs @@ -1337,6 +1337,19 @@ internal static Exception UndefinedPopulationMechanism(string populationMechanis internal static Exception PooledOpenTimeout() => ADP.InvalidOperation(StringsHelper.GetString(Strings.ADP_PooledOpenTimeout)); + /// + /// Builds the pooled-open timeout exception, attaching (the most + /// recent physical connection creation failure observed by the pool) so a timeout caused by + /// repeated connection failures reports the underlying cause rather than only reporting + /// pool exhaustion. Falls back to the parameterless form when there is no such failure. + /// +#nullable enable + internal static Exception PooledOpenTimeout(Exception? inner) + => inner is null + ? PooledOpenTimeout() + : ADP.InvalidOperation(StringsHelper.GetString(Strings.ADP_PooledOpenTimeout), inner); +#nullable restore + internal static Exception NonPooledOpenTimeout() => ADP.TimeoutException(StringsHelper.GetString(Strings.ADP_NonPooledOpenTimeout)); #endregion diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs index 63d5ff9ca8..b9351f235a 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs @@ -153,6 +153,15 @@ internal sealed class ChannelDbConnectionPool : IDbConnectionPool, IDisposable /// requester to start the loop, and reset to 0 by the loop when it drains. /// private int _warmupLoopRunning; + + /// + /// The exception from the most recent failed physical connection open, retained purely so + /// that a subsequent pooled-open timeout can report it as an inner exception. Cleared on the + /// next successful open. Volatile rather than lock-protected: this is a best-effort + /// diagnostic snapshot, and a torn read across concurrent failures would at worst attach a + /// slightly older failure. See GH#3545. + /// + private volatile Exception? _lastConnectionCreateException; #endregion /// @@ -198,6 +207,12 @@ internal ChannelDbConnectionPool( } State = Running; + + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Constructed. MinPoolSize={1}, MaxPoolSize={2}", + Id, + MinPoolSize, + MaxPoolSize); } #region Properties @@ -218,6 +233,9 @@ public ConcurrentDictionary< /// public bool ErrorOccurred => _errorState?.HasError ?? false; + /// + public Exception? LastConnectionCreateException => _lastConnectionCreateException; + /// public int Id => _instanceId; @@ -383,6 +401,10 @@ public DbConnectionInternal ReplaceConnection( } catch (Exception ex) when (ADP.IsCatchableExceptionType(ex) && ex is not OperationCanceledException) { + // Retain the failure so a caller that ultimately times out waiting for a pooled + // connection can report why creation kept failing. See GH#3545. + _lastConnectionCreateException = ex; + // A failed physical open means the server is unreachable, so enter the blocking // period exactly as OpenNewInternalConnection and WaitHandleDbConnectionPool.CreateObject // do: subsequent opens fast-fail until the period expires. Activation failures in the @@ -426,15 +448,31 @@ public DbConnectionInternal ReplaceConnection( { newConnection.DeactivateConnection(); newConnection.Dispose(); + + // The physical connection was opened (and counted by HardConnectRequest in the + // factory) before activation failed, so balance the counter here. The + // connection never occupied a slot, so the pooled gauge is untouched. + SqlClientDiagnostics.Metrics.HardDisconnectRequest(); throw; } // A successful open clears the blocking period, mirroring OpenNewInternalConnection. + _lastConnectionCreateException = null; _errorState?.Clear(); // Only retire the old connection after the replacement is fully activated and we know we won't fail. oldConnection.DeactivateConnection(); oldConnection.Dispose(); + + // The replacement took over the old connection's slot, so the pooled gauge is + // already correct and only the hard-disconnect counter needs balancing. Traced as a + // destroy so the connection's exit is visible in the pooler trace stream. + SqlClientDiagnostics.Metrics.HardDisconnectRequest(); + + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Disposed.", + Id, + oldConnection.ObjectID); } SqlClientDiagnostics.Metrics.SoftConnectRequest(); @@ -562,6 +600,16 @@ private void DeactivateAndRouteConnection(DbConnectionInternal connection) { RemoveConnection(connection); } + else + { + // The connection was parked in the transacted pool or placed in stasis. Neither + // path returns it to the idle channel, so without this trace the connection simply + // disappears from the pool's trace stream after deactivation. + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Held by a transaction; not returned to the general pool.", + Id, + connection.ObjectID); + } // Ensure the connection was processed by exactly one of the paths above. Debug.Assert(rootTxn || returnToGeneralPool || destroyConnection, @@ -593,6 +641,11 @@ private void PutConnectionInIdleChannel(DbConnectionInternal connection) return; } + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Pushing to general pool.", + Id, + connection.ObjectID); + if (!_idleChannel.TryWrite(connection)) { // The channel has been completed (pool is shutting down). Race window @@ -936,8 +989,17 @@ public bool TryGetConnection( // pool, whose replenishment enters/clears the same error state as user requests. In // practice the warmup loop already stands down before reaching here (its loop condition // checks ErrorOccurred); this covers the narrow race where the state flips in between. + if (ErrorOccurred) + { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Errors are set.", Id); + } + _errorState?.ThrowIfActive(); + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Creating new connection.", Id); + try { // Reserve a pool slot up front so we don't pay the rate-limit cost only to @@ -974,6 +1036,9 @@ public bool TryGetConnection( // TODO: When we fail to acquire a lease, surface the lease metadata // (e.g. RateLimitMetadataName.RetryAfter, ReasonPhrase) in the error // path so the user can identify why the lease was denied. + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Rate limiter saturated; deferring creation to the idle wait.", + Id); faulted = false; return null; } @@ -1048,22 +1113,48 @@ _connectionCreationRateLimiter is not null && if (connection is not null) { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Added to pool.", + Id, + connection.ObjectID); + SqlClientDiagnostics.Metrics.EnterPooledConnection(); // A new connection was added to the pool. If we've grown past MinPoolSize, // start the pruning timer so idle connections can be reclaimed. Pruner?.UpdateTimer(); + // A successful open proves the server is reachable, so a previously recorded + // failure is no longer a useful explanation for a later timeout. See GH#3545. + _lastConnectionCreateException = null; + // A successful creation clears error/backoff state (FR-009). Warmup goes through // this same path and clears the state on success too, mirroring the legacy // WaitHandle pool: a connection that opens proves the server is reachable. _errorState?.Clear(); } + else + { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, No connection created; pool is full or creation is rate limited.", + Id); + } return connection; } catch (Exception ex) when (ADP.IsCatchableExceptionType(ex) && ex is not OperationCanceledException) { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, PoolCreateRequest called CreateConnection which threw an exception: {1}", + Id, + ex); + + // Retain the failure so a caller that ultimately times out waiting for a pooled + // connection can report why creation kept failing. Recorded regardless of whether + // the blocking period is enabled for this pool group, since the timeout can occur + // either way. See GH#3545. + _lastConnectionCreateException = ex; + // Enter the blocking period error state on creation failure if configured. Warmup // goes through this same path (the warmup loop absorbs the rethrow in its own catch), // mirroring the legacy WaitHandle pool, whose replenishment failures also enter the @@ -1112,36 +1203,61 @@ private bool IsLiveConnection(DbConnectionInternal connection) idleTimeout != TimeSpan.Zero && _timeProvider.GetUtcNow().UtcDateTime - connection.ReturnedTime > idleTimeout) { + TraceNotLive(connection, "exceeded the connection idle timeout"); return false; } // Broken physical connection if (!connection.IsConnectionAlive()) { + TraceNotLive(connection, "found dead"); return false; } // Connection has been alive longer than the load balance timeout if (LoadBalanceTimeout != TimeSpan.Zero && DateTime.UtcNow > connection.CreateTime + LoadBalanceTimeout) { + TraceNotLive(connection, "exceeded the load balance timeout"); return false; } // Connection was created before the last Clear, so it's stale. if (connection.ClearGeneration != _clearGeneration) { + TraceNotLive(connection, "was created before the last Clear"); return false; } return true; } + /// + /// Emits the trace for a connection that failed the gate. + /// Split out so each rejection reason is reported individually: the caller only sees that + /// the connection was discarded, which on its own does not explain whether the pool is + /// churning because of idle timeout, load balancing, a Clear, or genuine server failures. + /// + /// The connection that failed the liveness gate. + /// Why the connection was rejected, phrased to read as + /// "Connection {id}, {reason} and removed." + private void TraceNotLive(DbConnectionInternal connection, string reason) => + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, {2} and removed.", + Id, + connection.ObjectID, + reason); + /// /// Closes the provided connection and removes it from the pool. /// /// The connection to be closed. private void RemoveConnection(DbConnectionInternal connection) { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Removing from pool.", + Id, + connection.ObjectID); + // A connection with a delegated transaction cannot be disposed of until the delegated // transaction has actually completed; disposing it would abort the (possibly // distributed) transaction. Leave it alone: when the transaction completes it comes @@ -1157,6 +1273,11 @@ private void RemoveConnection(DbConnectionInternal connection) if (_connectionSlots.TryRemove(connection)) { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Removed from pool.", + Id, + connection.ObjectID); + SqlClientDiagnostics.Metrics.ExitPooledConnection(); } @@ -1168,6 +1289,11 @@ private void RemoveConnection(DbConnectionInternal connection) connection.Dispose(); SqlClientDiagnostics.Metrics.HardDisconnectRequest(); + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Disposed.", + Id, + connection.ObjectID); + // If this removal brought us back to MinPoolSize, disable the pruning timer. Pruner?.UpdateTimer(); @@ -1200,6 +1326,11 @@ private void RemoveConnection(DbConnectionInternal connection) continue; } + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Connection {1}, Popped from general pool.", + Id, + connection.ObjectID); + return connection; } @@ -1234,6 +1365,9 @@ private async Task GetInternalConnection( { DbConnectionInternal? connection = null; + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Getting connection.", Id); + // When automatic enlistment is disabled, the connection must never be bound to the // ambient transaction, so we neither consult the transacted store nor hand the // transaction to activation. HasTransactionAffinity is derived from the connection @@ -1297,10 +1431,18 @@ private async Task GetInternalConnection( } catch (OperationCanceledException) { - throw ADP.PooledOpenTimeout(); + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Wait timed out.", Id); + + // Attach the most recent physical connection failure, if any, so a timeout + // caused by repeatedly failing opens reports that failure instead of only + // reporting pool exhaustion. See GH#3545. + throw ADP.PooledOpenTimeout(_lastConnectionCreateException); } catch (ChannelClosedException) { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Pool is shutting down; abandoning wait.", Id); throw new InvalidOperationException(StringsHelper.GetString(Strings.SQL_ConnectionPoolShutDown)); } @@ -1725,6 +1867,15 @@ private async Task RunWarmupLoopAsync() /// internal void PruneConnections(int count) { + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Pruning up to {1} idle connections. IdleCount={2}, Count={3}", + Id, + count, + IdleCount, + Count); + + int pruned = 0; + while (count > 0 && IsRunning && _connectionSlots.ReservationCount > MinPoolSize @@ -1737,7 +1888,13 @@ internal void PruneConnections(int count) RemoveConnection(connection); count--; + pruned++; } + + SqlClientEventSource.Log.TryPoolerTraceEvent( + " {0}, Pruned {1} idle connections.", + Id, + pruned); } #endregion } diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/IDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/IDbConnectionPool.cs index 3799cf54ea..92cc3cbcfe 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/IDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/IDbConnectionPool.cs @@ -49,6 +49,19 @@ internal interface IDbConnectionPool /// TODO: rename to indicate that this relates to the blocking period bool ErrorOccurred { get; } + /// + /// The exception thrown by the most recent failed attempt to open a physical connection, + /// or null if no attempt has failed since the last successful open. + /// + /// A caller that waits for a pooled connection and ultimately times out cannot otherwise + /// tell whether the pool was merely saturated or whether every creation attempt behind the + /// scenes was failing (e.g. the server refused the TCP connection). This property lets the + /// timeout be reported with the underlying failure attached as an inner exception. It is + /// diagnostic only and is not used to make control-flow decisions. + /// + /// + Exception? LastConnectionCreateException { get; } + /// /// An id that uniqely identifies this connection pool. /// diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs index 00ddb01c26..43aa53dc2c 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs @@ -195,6 +195,15 @@ public void Dispose() private readonly TimeProvider _timeProvider; private readonly BlockingPeriodErrorState _errorState; + /// + /// The exception from the most recent failed physical connection open, retained purely so + /// that a subsequent pooled-open timeout can report it as an inner exception. Cleared on the + /// next successful open. Volatile rather than lock-protected: this is a best-effort + /// diagnostic snapshot, and a torn read across concurrent failures would at worst attach a + /// slightly older failure. See GH#3545. + /// + private volatile Exception _lastConnectionCreateException; + internal Timer _cleanupTimer; private readonly TransactedConnectionPool _transactedConnectionPool; @@ -288,6 +297,9 @@ private int CreationTimeout public bool ErrorOccurred => _errorState.HasError; + /// + public Exception LastConnectionCreateException => _lastConnectionCreateException; + private bool HasTransactionAffinity => PoolGroupOptions.HasTransactionAffinity; public TimeSpan LoadBalanceTimeout => PoolGroupOptions.LoadBalanceTimeout; @@ -551,6 +563,10 @@ private DbConnectionInternal CreateObject(DbConnection owningObject, DbConnectio SqlClientEventSource.Log.TryPoolerTraceEvent(" {0}, Connection {1}, Added to pool.", Id, newObj?.ObjectID); + // A successful open proves the server is reachable, so a previously recorded + // failure is no longer a useful explanation for a later timeout. See GH#3545. + _lastConnectionCreateException = null; + // A successful creation clears any prior error state and resets backoff. _errorState.Clear(); } @@ -558,6 +574,12 @@ private DbConnectionInternal CreateObject(DbConnection owningObject, DbConnectio { ADP.TraceExceptionWithoutRethrow(e); + // Retain the failure so a caller that ultimately times out waiting for a pooled + // connection can report why creation kept failing. Recorded before the + // blocking-period check below so it is captured even when blocking is disabled + // and this method rethrows immediately. See GH#3545. + _lastConnectionCreateException = e; + if (!_connectionPoolGroup.IsBlockingPeriodEnabled()) { throw; @@ -809,7 +831,8 @@ private void WaitForPendingOpen() } else if (timeout) { - next.Completion.TrySetException(ADP.ExceptionWithStackTrace(ADP.PooledOpenTimeout())); + next.Completion.TrySetException( + ADP.ExceptionWithStackTrace(ADP.PooledOpenTimeout(_lastConnectionCreateException))); } else { diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs index 8840864958..b041453269 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlConnectionFactory.cs @@ -337,6 +337,12 @@ internal bool TryGetConnection( int retriesLeft = 10; int timeBetweenRetriesMilliseconds = 1; + // Tracks the most recent physical connection failure observed by the pool we last + // consulted, so a pooled-open timeout can report it as an inner exception rather than + // only reporting pool exhaustion. Hoisted out of the loop because the final give-up + // throw below is outside the pool variable's scope. See GH#3545. + Exception lastConnectionCreateException = null; + do { DbConnectionPoolGroup poolGroup = GetConnectionPoolGroup(owningConnection); @@ -436,12 +442,14 @@ internal bool TryGetConnection( if (connection is null) { + lastConnectionCreateException = connectionPool.LastConnectionCreateException; + // connection creation failed on semaphore waiting or if max pool reached if (connectionPool.IsRunning) { SqlClientEventSource.Log.TryTraceEvent(" {0}, GetConnection failed because a pool timeout occurred.", ObjectId); // If GetConnection failed while the pool is running, the pool timeout occurred. - throw ADP.PooledOpenTimeout(); + throw ADP.PooledOpenTimeout(lastConnectionCreateException); } // We've hit the race condition, where the pool was shut down after we @@ -458,7 +466,7 @@ internal bool TryGetConnection( { SqlClientEventSource.Log.TryTraceEvent(" {0}, GetConnection failed because a pool timeout occurred and all retries were exhausted.", ObjectId); // exhausted all retries or timed out - give up - throw ADP.PooledOpenTimeout(); + throw ADP.PooledOpenTimeout(lastConnectionCreateException); } return true; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolInstrumentationTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolInstrumentationTest.cs new file mode 100644 index 0000000000..a187316925 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolInstrumentationTest.cs @@ -0,0 +1,679 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Data.Common; +using System.Diagnostics.Tracing; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.RateLimiting; +using Microsoft.Data.Common; +using Microsoft.Data.Common.ConnectionString; +using Microsoft.Data.ProviderBase; +using Microsoft.Data.SqlClient.ConnectionPool; +using Microsoft.Data.SqlClient.Diagnostics; +using Microsoft.Data.SqlClient.Tests.Common; +using Xunit; + +using static Microsoft.Data.SqlClient.UnitTests.ConnectionPool.ChannelDbConnectionPoolTest; + +namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool +{ + /// + /// Verifies the diagnostic instrumentation of : the pooler + /// trace events emitted across the connection lifecycle, and the last-connection-create + /// exception that is surfaced as the inner exception of a pooled-open timeout (GH#3545). + /// + public class ChannelDbConnectionPoolInstrumentationTest + { + /// + /// Builds a pool for instrumentation tests. Defaults mirror + /// so behavior is comparable across suites. + /// + /// The factory used to create physical connections. + /// Connection string backing the pool group. Tests override + /// it to control the Pool Blocking Period. + /// Maximum pool size. + /// Minimum pool size. + /// Connection Idle Timeout, in seconds. + /// Optional limiter throttling physical creates. + private static ChannelDbConnectionPool ConstructPool( + SqlConnectionFactory connectionFactory, + string connectionString = "Data Source=localhost;", + int maxPoolSize = 50, + int minPoolSize = 0, + int idleTimeout = 0, + ConcurrencyLimiter? connectionCreationRateLimiter = null) + { + DbConnectionPoolGroupOptions poolGroupOptions = new( + poolByIdentity: false, + minPoolSize: minPoolSize, + maxPoolSize: maxPoolSize, + creationTimeout: 15, + loadBalanceTimeout: 0, + hasTransactionAffinity: true, + idleTimeout: idleTimeout); + + DbConnectionPoolGroup poolGroup = new( + new SqlConnectionOptions(connectionString), + new ConnectionPoolKey("TestDataSource", credential: null, accessToken: null, accessTokenCallback: null, sspiContextProvider: null), + poolGroupOptions); + + return new ChannelDbConnectionPool( + connectionFactory, + poolGroup, + DbConnectionPoolIdentity.NoIdentity, + new DbConnectionPoolProviderInfo(), + connectionCreationRateLimiter); + } + + #region Trace parity + + /// + /// Verifies that the pool traces its own construction, so a trace capture can attribute + /// every later pool-scoped event to a pool whose creation it observed. + /// + [Fact] + public void Construction_EmitsConstructedTrace() + { + // Arrange + using PoolerTraceListener listener = new(); + + // Act + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + + // Assert + TraceAssert.Contains("Constructed.", listener.MessagesForPool(pool.Id)); + } + + /// + /// Verifies that creating a new physical connection is traced, covering Story 1 scenario 2. + /// + [Fact] + public void NewConnection_EmitsCreationTraces() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + using PoolerTraceListener listener = new(); + + // Act + Assert.True(pool.TryGetConnection( + new SqlConnection(), + taskCompletionSource: null, + TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), + out DbConnectionInternal? connection)); + + // Assert + IReadOnlyList messages = listener.MessagesForPool(pool.Id); + Assert.NotNull(connection); + TraceAssert.Contains("Getting connection.", messages); + TraceAssert.Contains("Creating new connection.", messages); + TraceAssert.Contains("Added to pool.", messages); + } + + /// + /// Verifies that retrieving a connection from the idle pool is traced, covering Story 1 + /// scenario 1. + /// + [Fact] + public void IdleConnectionReuse_EmitsPoppedFromGeneralPoolTrace() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + pool.ReturnInternalConnection(connection!, owner); + + using PoolerTraceListener listener = new(); + + // Act + Assert.True(pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? reused)); + + // Assert + Assert.Same(connection, reused); + TraceAssert.Contains("Popped from general pool.", listener.MessagesForPool(pool.Id)); + } + + /// + /// Verifies that returning a connection traces both the deactivation and the routing + /// decision that put it back into the idle pool, covering Story 1 scenario 3. + /// + [Fact] + public void Return_EmitsDeactivateAndPushTraces() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + + using PoolerTraceListener listener = new(); + + // Act + pool.ReturnInternalConnection(connection!, owner); + + // Assert + IReadOnlyList messages = listener.MessagesForPool(pool.Id); + TraceAssert.Contains("Deactivating.", messages); + TraceAssert.Contains("Pushing to general pool.", messages); + } + + /// + /// Verifies that destroying a connection traces the removal and the disposal, covering + /// Story 1 scenario 4. Clear is used as the destruction trigger because it drains the idle + /// channel through the same removal path as every other destroy. + /// + [Fact] + public void Destroy_EmitsRemoveAndDisposeTraces() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + pool.ReturnInternalConnection(connection!, owner); + + using PoolerTraceListener listener = new(); + + // Act + pool.Clear(); + + // Assert + IReadOnlyList messages = listener.MessagesForPool(pool.Id); + TraceAssert.Contains("Clearing.", messages); + TraceAssert.Contains("Removing from pool.", messages); + TraceAssert.Contains("Removed from pool.", messages); + TraceAssert.Contains("Disposed.", messages); + TraceAssert.Contains("Cleared.", messages); + } + + /// + /// Verifies that startup and shutdown are traced with the pool identifier, covering Story 1 + /// scenario 5. + /// + [Fact] + public void StartupAndShutdown_EmitTraces() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + using PoolerTraceListener listener = new(); + + // Act + pool.Startup(); + pool.Shutdown(); + + // Assert + IReadOnlyList messages = listener.MessagesForPool(pool.Id); + Assert.Contains(messages, m => m.IndexOf("Startup", StringComparison.Ordinal) >= 0); + Assert.Contains(messages, m => m.IndexOf("Shutdown", StringComparison.Ordinal) >= 0); + } + + /// + /// Verifies that a failed physical open is traced on the pool's create path, so an operator + /// can see why the pool stopped growing. + /// + [Fact] + public void CreateFailure_EmitsCreateThrewTrace() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new FailingSqlConnectionFactory()); + using PoolerTraceListener listener = new(); + + // Act + Assert.ThrowsAny(() => + pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out _)); + + // Assert + TraceAssert.Contains("which threw an exception", listener.MessagesForPool(pool.Id)); + } + + /// + /// Verifies that a connection discarded for exceeding the Connection Idle Timeout is traced + /// with that specific reason, rather than silently disappearing from the pool. + /// + [Fact] + public void IdleTimeoutEviction_EmitsReasonTrace() + { + // Arrange - idle-timeout eviction is opt-in; the switch defaults to legacy behavior. + using LocalAppContextSwitchesHelper switchesHelper = new(); + switchesHelper.UseLegacyIdleTimeoutBehavior = false; + + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory(), idleTimeout: 1); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + pool.ReturnInternalConnection(connection!, owner); + + // Back-date the return stamp only after the connection is parked in the idle channel: + // the return path re-stamps it so that time spent checked out is not counted as idle. + connection!.SetReturnedTime(DateTime.UtcNow - TimeSpan.FromMinutes(5)); + + using PoolerTraceListener listener = new(); + + // Act - retrieval trips the idle-expiry gate and discards the connection. + Assert.True(pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? replacement)); + + // Assert + Assert.NotNull(replacement); + Assert.NotSame(connection, replacement); + TraceAssert.Contains("exceeded the connection idle timeout and removed.", listener.MessagesForPool(pool.Id)); + } + + /// + /// Verifies that pruning traces each invocation, so idle reclamation is attributable in a + /// trace capture even when it removes nothing (Story 3). + /// + [Fact] + public void Prune_EmitsTracePerInvocation() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory(), maxPoolSize: 4, idleTimeout: 300); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + pool.ReturnInternalConnection(connection!, owner); + + using PoolerTraceListener listener = new(); + + // Act + pool.PruneConnections(1); + + // Assert + IReadOnlyList messages = listener.MessagesForPool(pool.Id); + TraceAssert.Contains("Pruning up to 1 idle connections.", messages); + TraceAssert.Contains("Pruned 1 idle connections.", messages); + } + + #endregion + +#if NET + #region Metric parity + + // The metric counters are process-wide and other suites run in parallel, so these tests + // assert that a counter advanced by at least the expected amount rather than by exactly it. + // The rate counters only ever increase, which makes that assertion stable under concurrency. + + /// + /// Verifies that retrieving an idle connection counts a soft connect, covering Story 2 + /// scenario 3. + /// + [Fact] + public void IdleConnectionReuse_CountsSoftConnect() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + pool.ReturnInternalConnection(connection!, owner); + + long before = MetricReader.Read("_softConnectsRate"); + + // Act + Assert.True(pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? reused)); + + // Assert + Assert.Same(connection, reused); + Assert.True(MetricReader.Read("_softConnectsRate") >= before + 1); + } + + /// + /// Verifies that returning a connection to the idle pool counts a soft disconnect, covering + /// Story 2 scenario 4. + /// + [Fact] + public void Return_CountsSoftDisconnect() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + + long before = MetricReader.Read("_softDisconnectsRate"); + + // Act + pool.ReturnInternalConnection(connection!, owner); + + // Assert + Assert.True(MetricReader.Read("_softDisconnectsRate") >= before + 1); + } + + /// + /// Verifies that destroying a physical connection counts a hard disconnect, covering Story 2 + /// scenario 2. + /// + [Fact] + public void Destroy_CountsHardDisconnect() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + Assert.NotNull(connection); + pool.ReturnInternalConnection(connection!, owner); + + long before = MetricReader.Read("_hardDisconnectsRate"); + + // Act + pool.Clear(); + + // Assert + Assert.True(MetricReader.Read("_hardDisconnectsRate") >= before + 1); + } + + /// + /// Verifies that replacing a connection counts a hard disconnect for the connection it + /// discards. The channel pool swaps the new connection into the old connection's slot, so + /// the pooled-connection gauge is deliberately left untouched. + /// + [Fact] + public void ReplaceConnection_CountsHardDisconnectForDiscardedConnection() + { + // Arrange + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + SqlConnection owner = new(); + Assert.True(pool.TryGetConnection(owner, null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? oldConnection)); + Assert.NotNull(oldConnection); + + long beforeDisconnects = MetricReader.Read("_hardDisconnectsRate"); + long beforePooled = MetricReader.Read("_pooledConnections"); + + // Act + DbConnectionInternal newConnection = pool.ReplaceConnection(owner, oldConnection!, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15))); + + // Assert + Assert.NotSame(oldConnection, newConnection); + Assert.True(MetricReader.Read("_hardDisconnectsRate") >= beforeDisconnects + 1); + Assert.Equal(beforePooled, MetricReader.Read("_pooledConnections")); + } + + #endregion +#endif + + #region Last connection create exception (GH#3545) + + /// + /// Verifies that a pool that has never attempted a physical open reports no create failure, + /// so a timeout from a genuinely saturated pool is not annotated with a stale cause. + /// + [Fact] + public void LastConnectionCreateException_NoAttempts_IsNull() + { + // Arrange / Act + ChannelDbConnectionPool pool = ConstructPool(new SuccessfulSqlConnectionFactory()); + + // Assert + Assert.Null(pool.LastConnectionCreateException); + } + + /// + /// Verifies that a failed physical open is retained on the pool and then discarded once a + /// later open succeeds, so the recorded cause never outlives its relevance. + /// + [Fact] + public void LastConnectionCreateException_RecordedOnFailure_ClearedOnSuccess() + { + // Arrange - NeverBlock keeps the pool out of the blocking period so the second request + // actually attempts another physical open instead of fast-failing on cached state. + ToggleableConnectionFactory factory = new() { ShouldFail = true }; + ChannelDbConnectionPool pool = ConstructPool( + factory, + connectionString: "Data Source=localhost;Pool Blocking Period=NeverBlock;"); + + // Act - a failed open records the cause. + Assert.Throws(() => + pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out _)); + + // Assert + Assert.IsType(pool.LastConnectionCreateException); + + // Act - a successful open proves the server is reachable and clears the cause. + factory.ShouldFail = false; + Assert.True(pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out DbConnectionInternal? connection)); + + // Assert + Assert.NotNull(connection); + Assert.Null(pool.LastConnectionCreateException); + } + + /// + /// Verifies GH#3545 end to end for this pool: when a request waits for a pooled connection + /// and times out, the most recent physical connection failure is attached as the inner + /// exception instead of being lost behind the generic pool-exhaustion message. + /// + [Fact] + public void PooledOpenTimeout_CarriesLastCreateExceptionAsInner() + { + // Arrange - a single-permit limiter lets the test hold the only creation permit, so the + // second request cannot attempt an open and must wait on the idle channel until its + // budget expires. NeverBlock keeps the pool out of the blocking period, which would + // otherwise fast-fail the second request with the cached exception directly. + ToggleableConnectionFactory factory = new() { ShouldFail = true }; + using ConcurrencyLimiter rateLimiter = new( + new ConcurrencyLimiterOptions { PermitLimit = 1, QueueLimit = 0 }); + ChannelDbConnectionPool pool = ConstructPool( + factory, + connectionString: "Data Source=localhost;Pool Blocking Period=NeverBlock;", + maxPoolSize: 4, + connectionCreationRateLimiter: rateLimiter); + + // The first request fails its physical open, which records the cause on the pool. + Assert.Throws(() => + pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromSeconds(15)), out _)); + Assert.IsType(pool.LastConnectionCreateException); + + // Hold the only permit so no further creation can be attempted. + using RateLimitLease lease = rateLimiter.AttemptAcquire(1); + Assert.True(lease.IsAcquired); + + // Act + InvalidOperationException timeout = Assert.Throws(() => + pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromMilliseconds(100)), out _)); + + // Assert + Assert.IsType(timeout.InnerException); + } + + /// + /// Verifies that a timeout with no preceding create failure still reports the plain + /// pool-exhaustion message, so the change does not fabricate a cause. + /// + [Fact] + public void PooledOpenTimeout_NoCreateFailure_HasNoInnerException() + { + // Arrange - hold the only creation permit up front so no open is ever attempted. + using ConcurrencyLimiter rateLimiter = new( + new ConcurrencyLimiterOptions { PermitLimit = 1, QueueLimit = 0 }); + ChannelDbConnectionPool pool = ConstructPool( + new SuccessfulSqlConnectionFactory(), + maxPoolSize: 4, + connectionCreationRateLimiter: rateLimiter); + + using RateLimitLease lease = rateLimiter.AttemptAcquire(1); + Assert.True(lease.IsAcquired); + + // Act + InvalidOperationException timeout = Assert.Throws(() => + pool.TryGetConnection(new SqlConnection(), null, TimeoutTimer.StartNew(TimeSpan.FromMilliseconds(100)), out _)); + + // Assert + Assert.Null(timeout.InnerException); + } + + #endregion + + #region Test classes + + /// + /// Distinctive exception type used to prove that the exact failure recorded by the pool is + /// the one attached to the pooled-open timeout. + /// + internal sealed class TestConnectionCreateException : Exception + { + internal TestConnectionCreateException() + : base("Simulated physical connection failure.") + { + } + } + + /// + /// Connection factory whose success or failure can be flipped between requests, so a single + /// pool can be driven through a failure and a subsequent recovery. + /// + internal sealed class ToggleableConnectionFactory : SqlConnectionFactory + { + /// + /// When true, the next creation attempt throws . + /// + internal volatile bool ShouldFail; + + /// + protected override DbConnectionInternal CreateConnection( + SqlConnectionOptions options, + ConnectionPoolKey poolKey, + DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, + IDbConnectionPool pool, + DbConnection owningConnection, + TimeoutTimer timeout) + { + if (ShouldFail) + { + throw new TestConnectionCreateException(); + } + + return new StubDbConnectionInternal(); + } + } + + /// + /// Connection factory that always fails with . + /// + internal sealed class FailingSqlConnectionFactory : SqlConnectionFactory + { + /// + protected override DbConnectionInternal CreateConnection( + SqlConnectionOptions options, + ConnectionPoolKey poolKey, + DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, + IDbConnectionPool pool, + DbConnection owningConnection, + TimeoutTimer timeout) + => throw new TestConnectionCreateException(); + } + + /// + /// Captures PoolerTrace events from the SqlClient event source. + /// + /// Tests filter captured messages by pool id (see ) because the + /// event source is process-wide: xUnit runs test classes in parallel, so traces from other + /// pools are expected to appear in the same capture. + /// + /// + internal sealed class PoolerTraceListener : EventListener + { + private const string SqlClientEventSourceName = "Microsoft.Data.SqlClient.EventSource"; + + // Mirrors SqlClientEventSource.Keywords.PoolerTrace. Duplicated as a literal because + // that type is not visible to this assembly. + private const EventKeywords PoolerTraceKeyword = (EventKeywords)32; + + // Lazily initialized: EventListener's base constructor invokes OnEventSourceCreated, + // which enables events, before this class's field initializers have run. Traces can + // therefore arrive on another thread before a plain field initializer would have + // assigned the queue. + private ConcurrentQueue? _messages; + + private ConcurrentQueue Messages => + LazyInitializer.EnsureInitialized(ref _messages)!; + + /// + protected override void OnEventSourceCreated(EventSource eventSource) + { + if (eventSource.Name == SqlClientEventSourceName) + { + EnableEvents(eventSource, EventLevel.Informational, PoolerTraceKeyword); + } + } + + /// + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (eventData.Payload is null) + { + return; + } + + foreach (object? payload in eventData.Payload) + { + if (payload is string message) + { + Messages.Enqueue(message); + } + } + } + + /// + /// Returns the captured messages emitted for the given pool. + /// + /// The to filter on. + internal IReadOnlyList MessagesForPool(int poolId) + { + // Pool-scoped traces render the id as the first substituted argument, immediately + // after the "|CPOOL> " marker, e.g. + // " 7, Clearing." + // " 7" + // The trailing boundary keeps pool 7 from matching pool 70. + Regex pattern = new( + @"CPOOL> " + Regex.Escape(poolId.ToString(CultureInfo.InvariantCulture)) + @"(\D|$)", + RegexOptions.CultureInvariant); + + return Messages.Where(m => pattern.IsMatch(m)).ToList(); + } + } + + #endregion + } + +#if NET + /// + /// Reads the private counter fields of the process-wide instance. + /// The counters are not otherwise observable without an EventCounter listener and its polling + /// interval, which would make these tests slow and timing dependent. + /// + internal static class MetricReader + { + /// + /// Reads the current value of the named counter field. + /// + /// Private field name declared on . + internal static long Read(string fieldName) + { + FieldInfo? field = typeof(SqlClientMetrics).GetField(fieldName, BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + return (long)field!.GetValue(SqlClientDiagnostics.Metrics)!; + } + } +#endif + + /// + /// xUnit assertion helper for substring matching over a captured trace stream. + /// + internal static class TraceAssert + { + /// + /// Asserts that at least one captured message contains . + /// + internal static void Contains(string fragment, IReadOnlyList messages) => + Assert.True( + messages.Any(m => m.IndexOf(fragment, StringComparison.Ordinal) >= 0), + $"Expected a trace containing \"{fragment}\". Captured:{Environment.NewLine}{string.Join(Environment.NewLine, messages)}"); + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/TransactedConnectionPoolTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/TransactedConnectionPoolTest.cs index 588418ac03..99645c6979 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/TransactedConnectionPoolTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/TransactedConnectionPoolTest.cs @@ -660,6 +660,7 @@ internal class MockDbConnectionPool : IDbConnectionPool public SqlConnectionFactory ConnectionFactory => throw new NotImplementedException(); public int Count => throw new NotImplementedException(); public bool ErrorOccurred => throw new NotImplementedException(); + public Exception? LastConnectionCreateException => null; public int Id { get; } = 1; public int IdleCount => throw new NotImplementedException(); public DbConnectionPoolIdentity Identity => throw new NotImplementedException(); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs index e84f040caa..ca7bdf1bde 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/WaitHandleDbConnectionPoolBlockingPeriodTest.cs @@ -200,6 +200,34 @@ public void TryGetConnection_WhenFactorySucceeds_DoesNotEnterBlockingPeriod() Assert.Equal(1, factory.CreateConnectionCallCount); } + /// + /// Verifies that the pool records the exception from a failed physical create and clears it + /// once a create succeeds, so a later pooled-open timeout reports the most recent cause rather + /// than a stale one (GH#3545). + /// + [Fact] + public void LastConnectionCreateException_RecordedOnFailure_ClearedOnSuccess() + { + // Arrange - NeverBlock keeps the error state from fast-failing the second request. + bool shouldFail = true; + var factory = new ConfigurableSqlConnectionFactory(_ => + shouldFail ? throw SqlExceptionHelper.CreateSqlException("server unreachable") : new MockDbConnectionInternal()); + var pool = CreatePool(factory, "Data Source=localhost;Pool Blocking Period=NeverBlock;"); + using var owner = new SqlConnection(); + + Assert.Null(pool.LastConnectionCreateException); + + // Act & Assert - the failure is recorded. + Assert.Throws(() => TryGetConnectionSync(pool, owner, out _)); + Assert.IsType(pool.LastConnectionCreateException); + + // Act & Assert - a subsequent success clears it. + shouldFail = false; + Assert.True(TryGetConnectionSync(pool, owner, out DbConnectionInternal? connection)); + Assert.NotNull(connection); + Assert.Null(pool.LastConnectionCreateException); + } + /// /// Verifies that once the blocking period's exit timer fires, the next request retries the /// factory and a successful create recovers the pool: