Enable TCP states and ThreadPool stats

docs/client-concepts/connection/configuration-options.asciidoc

@@ -136,7 +136,7 @@ Sets the default ping timeout in milliseconds for ping requests, which are used
 
 `PrettyJson`::
 
-Forces all requests to have ?pretty=true querystring parameter appended, causing Elasticsearch to return formatted JSON. Also forces the client to send out formatted JSON. Defaults to `false`
+Forces all requests to have ?pretty=true querystring parameter appended, causing Elasticsearch to return formatted JSON. Defaults to `false`
 
 `Proxy`::
 

docs/client-concepts/troubleshooting/debug-information.asciidoc

@@ -31,10 +31,17 @@ response.DebugInformation.Should().Contain("Valid NEST response");
 ----
 
 This can be useful in tracking down numerous problems and can also be useful when filing an
-{github}/issues[issue] on our github repository.
+{github}/issues[issue] on the GitHub repository.
+
+==== Request and response bytes
 
 By default, the request and response bytes are not available within the debug information, but
-can be enabled globally on Connection Settings
+can be enabled globally on Connection Settings by setting `DisableDirectStreaming`. This
+disables direct streaming of
+
+. the serialized request type to the request stream
+
+. the response stream to a deserialized response type
 
 [source,csharp]
 ----
@@ -59,8 +66,116 @@ var response = client.Search<Project>(s => s
         .MatchAll()
     )
 );
+----
+<1> disable direct streaming for *this* request only
+
+Configuring `DisableDirectStreaming` on an individual request takes precedence over
+any global configuration.
+
+There is typically a performance and allocation cost associated with disabling direct streaming
+since both the request and response bytes must be buffered in memory, to allow them to be
+exposed on the response call details.
+
+==== TCP statistics
+
+It can often be useful to see the statistics for active TCP connections, particularly when
+trying to diagnose issues with the client. The client can collect the states of active TCP
+connections just before making a request, and expose these on the response and in the debug
+information.
+
+Similarly to `DisableDirectStreaming`, TCP statistics can be collected for every request
+by configuring on `ConnectionSettings`
+
+[source,csharp]
+----
+var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
+
+var settings = new ConnectionSettings(connectionPool)
+    .EnableTcpStats(); <1>
+
+var client = new ElasticClient(settings);
+----
+<1> collect TCP statistics for *all* requests
+
+or on a _per request_ basis
+
+[source,csharp]
+----
+var response = client.Search<Project>(s => s
+    .RequestConfiguration(r => r
+        .EnableTcpStats() <1>
+    )
+    .Query(q => q
+        .MatchAll()
+    )
+);
+
+var debugInformation = response.DebugInformation;
+----
+<1> collect TCP statistics for *this* request only
+
+With `EnableTcpStats` set, the states of active TCP connections will now be included
+on the response and in the debug information.
+
+The client includes a `TcpStats`
+class to help with retrieving more detail about active TCP connections should it be
+required
+
+[source,csharp]
+----
+var tcpStatistics = TcpStats.GetActiveTcpConnections(); <1>
+var ipv4Stats = TcpStats.GetTcpStatistics(NetworkInterfaceComponent.IPv4); <2>
+var ipv6Stats = TcpStats.GetTcpStatistics(NetworkInterfaceComponent.IPv6); <3>
+
+var response = client.Search<Project>(s => s
+    .Query(q => q
+        .MatchAll()
+    )
+);
+----
+<1> Retrieve details about active TCP connections, including local and remote addresses and ports
+<2> Retrieve statistics about IPv4
+<3> Retrieve statistics about IPv6
+
+==== ThreadPool statistics
+
+It can often be useful to see the statistics for thread pool threads, particularly when
+trying to diagnose issues with the client. The client can collect statistics for both
+worker threads and asynchronous I/O threads, and expose these on the response and
+in debug information.
+
+Similar to collecting TCP statistics, ThreadPool statistics can be collected for all requests
+by configuring `EnableThreadPoolStats` on `ConnectionSettings`
 
-response.DebugInformation.Should().Contain("\"match_all\":");
+[source,csharp]
+----
+var connectionPool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
+
+var settings = new ConnectionSettings(connectionPool)
+     .EnableThreadPoolStats(); <1>
+
+var client = new ElasticClient(settings);
 ----
-<1> disable direct streaming for *this* request
+<1> collect thread pool statistics for *all* requests
+
+or on a _per request_ basis
+
+[source,csharp]
+----
+var response = client.Search<Project>(s => s
+     .RequestConfiguration(r => r
+             .EnableThreadPoolStats() <1>
+     )
+     .Query(q => q
+         .MatchAll()
+     )
+ );
+
+var debugInformation = response.DebugInformation; <2>
+----
+<1> collect thread pool statistics for *this* request only
+<2> contains thread pool statistics
+
+With `EnableThreadPoolStats` set, the statistics of thread pool threads will now be included
+on the response and in the debug information.
 

docs/client-concepts/troubleshooting/debug-mode.asciidoc

@@ -0,0 +1,73 @@
+:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/master
+
+:github: https://github.com/elastic/elasticsearch-net
+
+:nuget: https://www.nuget.org/packages
+
+////
+IMPORTANT NOTE
+==============
+This file has been generated from https://github.com/elastic/elasticsearch-net/tree/master/src/Tests/Tests/ClientConcepts/Troubleshooting/DebugMode.doc.cs. 
+If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
+please modify the original csharp file found at the link and submit the PR with that change. Thanks!
+////
+
+[[debug-mode]]
+=== Debug mode
+
+The <<debug-information, Debug information>> explains that every response from Elasticsearch.Net
+and NEST contains a `DebugInformation` property, and properties on `ConnectionSettings` and
+`RequestConfiguration` can control which additional information is included in debug information,
+for all requests or on a per request basis, respectively.
+
+During development, it can be useful to enable the most verbose debug information, to help
+identify and troubleshoot problems, or simply ensure that the client is behaving as expected.
+The `EnableDebugMode` setting on `ConnectionSettings` is a convenient shorthand for enabling
+verbose debug information, configuring a number of settings like
+
+* disabling direct streaming to capture request and response bytes
+
+* prettyfying JSON responses from Elasticsearch
+
+* collecting TCP statistics when a request is made
+
+* collecting thread pool statistics when a request is made
+
+* including the Elasticsearch stack trace in the response if there is a an error on the server side
+
+[source,csharp]
+----
+IConnectionPool pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
+
+var settings = new ConnectionSettings(pool)
+    .EnableDebugMode(); <1>
+
+var client = new ElasticClient(settings);
+
+var response = client.Search<Project>(s => s
+    .Query(q => q
+        .MatchAll()
+    )
+);
+
+var debugInformation = response.DebugInformation; <2>
+----
+<1> configure debug mode
+<2> verbose debug information
+
+In addition to exposing debug information on the response, debug mode will also cause the debug
+information to be written to the trace listeners in the `System.Diagnostics.Debug.Listeners` collection
+by default, when the request has completed. A delegate can be passed when enabling debug mode to perform
+a different action when a request has completed, using <<logging-with-on-request-completed, `OnRequestCompleted`>>
+
+[source,csharp]
+----
+var pool = new SingleNodeConnectionPool(new Uri("http://localhost:9200"));
+var client = new ElasticClient(new ConnectionSettings(pool)
+    .EnableDebugMode(apiCallDetails =>
+    {
+        // do something with the call details e.g. send with logging framework
+    })
+);
+----
+

docs/troubleshooting.asciidoc

@@ -61,3 +61,5 @@ include::client-concepts/troubleshooting/audit-trail.asciidoc[]
 
 include::client-concepts/troubleshooting/debug-information.asciidoc[]
 
+include::client-concepts/troubleshooting/debug-mode.asciidoc[]
+

src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs

@@ -201,6 +201,8 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private bool _throwExceptions;
 		private bool _transferEncodingChunked;
 		private IMemoryStreamFactory _memoryStreamFactory = RecyclableMemoryStreamFactory.Default;
+		private bool _enableTcpStats;
+		private bool _enableThreadPoolStats;
 
 		private string _userAgent = ConnectionConfiguration.DefaultUserAgent;
 		private Func<HttpMethod, int, bool> _statusCodeToResponseSuccess;
@@ -280,6 +282,8 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 		string IConnectionConfigurationValues.UserAgent => _userAgent;
 		Func<HttpMethod, int, bool> IConnectionConfigurationValues.StatusCodeToResponseSuccess => _statusCodeToResponseSuccess;
 		bool IConnectionConfigurationValues.TransferEncodingChunked => _transferEncodingChunked;
+		bool IConnectionConfigurationValues.EnableTcpStats => _enableTcpStats;
+		bool IConnectionConfigurationValues.EnableThreadPoolStats => _enableThreadPoolStats;
 
 		void IDisposable.Dispose() => DisposeManagedResources();
 
@@ -453,7 +457,7 @@ public T Proxy(Uri proxyAddress, string username, SecureString password) =>
 		/// <summary>
 		/// Forces all requests to have ?pretty=true querystring parameter appended,
 		/// causing Elasticsearch to return formatted JSON.
-		/// Also forces the client to send out formatted JSON. Defaults to <c>false</c>
+		/// Defaults to <c>false</c>
 		/// </summary>
 		public T PrettyJson(bool b = true) => Assign(b, (a, v) =>
 		{
@@ -562,16 +566,10 @@ public T EnableDebugMode(Action<IApiCallDetails> onRequestCompleted = null) =>
 			PrettyJson()
 				.IncludeServerStackTraceOnError()
 				.DisableDirectStreaming()
+				.EnableTcpStats()
+				.EnableThreadPoolStats()
 				.Assign(onRequestCompleted, (a, v) =>
-				{
-					var originalCompletedRequestHandler = _completedRequestHandler;
-					var debugCompletedRequestHandler = v ?? (d => Debug.WriteLine(d.DebugInformation));
-					_completedRequestHandler = d =>
-					{
-						originalCompletedRequestHandler?.Invoke(d);
-						debugCompletedRequestHandler.Invoke(d);
-					};
-				});
+					_completedRequestHandler += v ?? (d => Debug.WriteLine(d.DebugInformation)));
 
 		/// <summary>
 		/// Register a ServerCertificateValidationCallback, this is called per endpoint until it returns true.
@@ -623,6 +621,10 @@ public T SkipDeserializationForStatusCodes(params int[] statusCodes) =>
 		/// </summary>
 		public T MemoryStreamFactory(IMemoryStreamFactory memoryStreamFactory) => Assign(memoryStreamFactory, (a, v) => a._memoryStreamFactory = v);
 
+		public T EnableTcpStats(bool enableTcpStats = true) => Assign(enableTcpStats, (a, v) => a._enableTcpStats = v);
+
+		public T EnableThreadPoolStats(bool enableThreadPoolStats = true) => Assign(enableThreadPoolStats, (a, v) => a._enableThreadPoolStats = v);
+
 		protected virtual void DisposeManagedResources()
 		{
 			_connectionPool?.Dispose();

src/Elasticsearch.Net/Configuration/IConnectionConfigurationValues.cs

@@ -263,5 +263,15 @@ public interface IConnectionConfigurationValues : IDisposable
 		#endif
 		/// </summary>
 		TimeSpan DnsRefreshTimeout { get; }
+
+		/// <summary>
+		/// Enable statistics about TCP connections to be collected when making a request
+		/// </summary>
+		bool EnableTcpStats { get; }
+
+		/// <summary>
+		/// Enable statistics about thread pools to be collected when making a request
+		/// </summary>
+		bool EnableThreadPoolStats { get; }
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfiguration.cs

@@ -120,6 +120,12 @@ public interface IRequestConfiguration
 		/// Try to send these headers for this single request
 		/// </summary>
 		NameValueCollection Headers { get; set; }
+
+		/// <inheritdoc cref="IConnectionConfigurationValues.EnableTcpStats"/>
+		bool? EnableTcpStats { get; set; }
+
+		/// <inheritdoc cref="IConnectionConfigurationValues.EnableThreadPoolStats"/>
+		bool? EnableThreadPoolStats { get; set; }
 	}
 
 	public class RequestConfiguration : IRequestConfiguration
@@ -162,6 +168,10 @@ public class RequestConfiguration : IRequestConfiguration
 		public bool? TransferEncodingChunked { get; set; }
 		/// <inheritdoc />
 		public NameValueCollection Headers { get; set; }
+		/// <inheritdoc />
+		public bool? EnableTcpStats { get; set; }
+		/// <inheritdoc />
+		public bool? EnableThreadPoolStats { get; set; }
 	}
 
 	public class RequestConfigurationDescriptor : IRequestConfiguration
@@ -209,6 +219,8 @@ public RequestConfigurationDescriptor(IRequestConfiguration config)
 		bool? IRequestConfiguration.ThrowExceptions { get; set; }
 		bool? IRequestConfiguration.TransferEncodingChunked { get; set; }
 		NameValueCollection IRequestConfiguration.Headers { get; set; }
+		bool? IRequestConfiguration.EnableTcpStats { get; set; }
+		bool? IRequestConfiguration.EnableThreadPoolStats { get; set; }
 
 		/// <summary>
 		/// Submit the request on behalf in the context of a different shield user
@@ -379,5 +391,18 @@ public RequestConfigurationDescriptor GlobalHeaders(NameValueCollection headers)
 			return this;
 		}
 
+		/// <inheritdoc cref="IRequestConfiguration.EnableTcpStats" />
+		public RequestConfigurationDescriptor EnableTcpStats(bool? enableTcpStats = true)
+		{
+			Self.EnableTcpStats = enableTcpStats;
+			return this;
+		}
+
+		/// <inheritdoc cref="IRequestConfiguration.EnableThreadPoolStats" />
+		public RequestConfigurationDescriptor EnableThreadPoolStats(bool? enableThreadPoolStats = true)
+		{
+			Self.EnableThreadPoolStats = enableThreadPoolStats;
+			return this;
+		}
 	}
 }