Send notification about HTTP response without content (netfx)

Author: Liudmila Molkova

src/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/HttpHandlerDiagnosticListener.cs

@@ -539,6 +539,31 @@ public override void RemoveAt(int index)
                     {
                         s_instance.RaiseResponseEvent(request, response);
                     }
+                    else
+                    {
+                        // In case reponse content length is 0 and request is async, 
+                        // we won't have a HttpWebResponse set on request object when this method is called
+                        // http://referencesource.microsoft.com/#System/net/System/Net/HttpWebResponse.cs,525
+
+                        // But we there will be CoreResponseData object that is either exception 
+                        // or the internal HTTP reponse representation having status, content and headers
+
+                        var coreResponse = s_coreResponseAccessor(request);
+                        if (coreResponse != null && s_coreResponseDataType.IsInstanceOfType(coreResponse))
+                        {
+                            HttpStatusCode status = s_coreStatusCodeAccessor(coreResponse);
+                            WebHeaderCollection headers = s_coreHeadersAccessor(coreResponse);
+
+                            // Manual creation of HttpWebResponse here is not possible as this method is eventually called from the 
+                            // HttpWebResponse ctor. So we will send Stop event with the Status and Headers payload
+                            // to notify listeners about response;
+                            // We use two different names for Stop events since one event with payload type that varies creates
+                            // complications for efficient payload parsing and is not supported by DiagnosicSource helper 
+                            // libraries (e.g. Microsoft.Extensions.DiagnosticAdapter)
+
+                            s_instance.RaiseResponseEvent(request, status, headers);
+                        }
+                    }
                 }
 
                 base.RemoveAt(index);
@@ -606,22 +631,33 @@ private void RaiseResponseEvent(HttpWebRequest request, HttpWebResponse response
             // Response event could be received several times for the same request in case it was redirected
             // IsLastResponse checks if response is the last one (no more redirects will happen)
             // based on response StatusCode and number or redirects done so far
-            if (request.Headers[RequestIdHeaderName] != null && IsLastResponse(request, response))
+            if (request.Headers[RequestIdHeaderName] != null && IsLastResponse(request, response.StatusCode))
             {
                 // only send Stop if request was instrumented
                 this.Write(RequestStopName, new { Request = request, Response = response });
             }
         }
 
-        private bool IsLastResponse(HttpWebRequest request, HttpWebResponse response)
+        private void RaiseResponseEvent(HttpWebRequest request, HttpStatusCode statusCode, WebHeaderCollection headers)
+        {
+            // Response event could be received several times for the same request in case it was redirected
+            // IsLastResponse checks if response is the last one (no more redirects will happen)
+            // based on response StatusCode and number or redirects done so far
+            if (request.Headers[RequestIdHeaderName] != null && IsLastResponse(request, statusCode))
+            {
+                this.Write(RequestStopExName, new { Request = request, StatusCode = statusCode, Headers = headers });
+            }
+        }
+
+        private bool IsLastResponse(HttpWebRequest request, HttpStatusCode statusCode)
         {
             if (request.AllowAutoRedirect)
             {
-                if (response.StatusCode == HttpStatusCode.Ambiguous       ||  // 300
-                    response.StatusCode == HttpStatusCode.Moved           ||  // 301
-                    response.StatusCode == HttpStatusCode.Redirect        ||  // 302
-                    response.StatusCode == HttpStatusCode.RedirectMethod  ||  // 303
-                    response.StatusCode == HttpStatusCode.RedirectKeepVerb)   // 307
+                if (statusCode == HttpStatusCode.Ambiguous       ||  // 300
+                    statusCode == HttpStatusCode.Moved           ||  // 301
+                    statusCode == HttpStatusCode.Redirect        ||  // 302
+                    statusCode == HttpStatusCode.RedirectMethod  ||  // 303
+                    statusCode == HttpStatusCode.RedirectKeepVerb)   // 307
                 {
                     return s_autoRedirectsAccessor(request) >= request.MaximumAutomaticRedirections;
                 }
@@ -642,40 +678,27 @@ private static void PrepareReflectionObjects()
             s_connectionType = systemNetHttpAssembly?.GetType("System.Net.Connection");
             s_writeListField = s_connectionType?.GetField("m_WriteList", BindingFlags.Instance | BindingFlags.NonPublic);
 
-            // Second step: Generate an accessor for HttpWebRequest._HttpResponse
-            FieldInfo responseField = typeof(HttpWebRequest).GetField("_HttpResponse", BindingFlags.NonPublic | BindingFlags.Instance);
-            if (responseField != null)
-            {
-                string methodName = responseField.ReflectedType.FullName + ".get_" + responseField.Name;
-                DynamicMethod getterMethod = new DynamicMethod(methodName, typeof(HttpWebResponse), new Type[] { typeof(HttpWebRequest) }, true);
-                ILGenerator generator = getterMethod.GetILGenerator();
-                generator.Emit(OpCodes.Ldarg_0);
-                generator.Emit(OpCodes.Ldfld, responseField);
-                generator.Emit(OpCodes.Ret);
-                s_httpResponseAccessor = (Func<HttpWebRequest, HttpWebResponse>)getterMethod.CreateDelegate(typeof(Func<HttpWebRequest, HttpWebResponse>));
-            }
+            s_httpResponseAccessor = CreateFieldGetter<HttpWebRequest, HttpWebResponse>("_HttpResponse", BindingFlags.NonPublic | BindingFlags.Instance);
+            s_autoRedirectsAccessor = CreateFieldGetter<HttpWebRequest, int>("_AutoRedirects", BindingFlags.NonPublic | BindingFlags.Instance);
+            s_coreResponseAccessor = CreateFieldGetter<HttpWebRequest, object>("_CoreResponse", BindingFlags.NonPublic | BindingFlags.Instance);
 
-            // Third step: Generate an accessor for HttpWebRequest._AutoRedirects
-            FieldInfo redirectsField = typeof(HttpWebRequest).GetField("_AutoRedirects", BindingFlags.NonPublic | BindingFlags.Instance);
-            if (redirectsField != null)
+            s_coreResponseDataType = systemNetHttpAssembly?.GetType("System.Net.CoreResponseData");
+            if (s_coreResponseDataType != null)
             {
-                string methodName = redirectsField.ReflectedType.FullName + ".get_" + redirectsField.Name;
-                DynamicMethod getterMethod = new DynamicMethod(methodName, typeof(int), new Type[] { typeof(HttpWebRequest) }, true);
-                ILGenerator generator = getterMethod.GetILGenerator();
-                generator.Emit(OpCodes.Ldarg_0);
-                generator.Emit(OpCodes.Ldfld, redirectsField);
-                generator.Emit(OpCodes.Ret);
-                s_autoRedirectsAccessor = (Func<HttpWebRequest, int>)getterMethod.CreateDelegate(typeof(Func<HttpWebRequest, int>));
+                s_coreStatusCodeAccessor = CreateFieldGetter<HttpStatusCode>(s_coreResponseDataType, "m_StatusCode", BindingFlags.Public | BindingFlags.Instance);
+                s_coreHeadersAccessor = CreateFieldGetter<WebHeaderCollection>(s_coreResponseDataType, "m_ResponseHeaders", BindingFlags.Public | BindingFlags.Instance);
             }
-
             // Double checking to make sure we have all the pieces initialized
             if (s_connectionGroupListField == null ||
                 s_connectionGroupType == null ||
                 s_connectionListField == null ||
                 s_connectionType == null ||
                 s_writeListField == null ||
                 s_httpResponseAccessor == null ||
-                s_autoRedirectsAccessor == null)
+                s_autoRedirectsAccessor == null || 
+                s_coreResponseDataType == null ||
+                s_coreStatusCodeAccessor == null ||
+                s_coreHeadersAccessor == null)
             {
                 // If anything went wrong here, just return false. There is nothing we can do.
                 throw new InvalidOperationException("Unable to initialize all required reflection objects");
@@ -697,7 +720,48 @@ private static void PerformInjection()
             servicePointTableField.SetValue(null, newTable);
         }
 
-#endregion
+        private static Func<TClass, TField> CreateFieldGetter<TClass, TField>(string fieldName, BindingFlags flags) where TClass : class
+        {
+            FieldInfo field = typeof(TClass).GetField(fieldName, flags);
+            if (field != null)
+            {
+                string methodName = field.ReflectedType.FullName + ".get_" + field.Name;
+                DynamicMethod getterMethod = new DynamicMethod(methodName, typeof(TField), new [] { typeof(TClass) }, true);
+                ILGenerator generator = getterMethod.GetILGenerator();
+                generator.Emit(OpCodes.Ldarg_0);
+                generator.Emit(OpCodes.Ldfld, field);
+                generator.Emit(OpCodes.Ret);
+                return (Func<TClass, TField>)getterMethod.CreateDelegate(typeof(Func<TClass, TField>));
+            }
+
+            return null;
+        }
+
+
+        /// <summary>
+        /// Creates getter for a field defined in private or internal type
+        /// repesented with classType variable
+        /// </summary>
+        private static Func<object, TField> CreateFieldGetter<TField>(Type classType, string fieldName, BindingFlags flags)
+        {
+            FieldInfo field = classType.GetField(fieldName, flags);
+            if (field != null)
+            {
+                string methodName = classType.FullName + ".get_" + field.Name;
+                DynamicMethod getterMethod = new DynamicMethod(methodName, typeof(TField), new [] { typeof(object) }, true);
+                ILGenerator generator = getterMethod.GetILGenerator();
+                generator.Emit(OpCodes.Ldarg_0);
+                generator.Emit(OpCodes.Castclass, classType);
+                generator.Emit(OpCodes.Ldfld, field);
+                generator.Emit(OpCodes.Ret);
+
+                return (Func<object, TField>)getterMethod.CreateDelegate(typeof(Func<object, TField>));
+            }
+
+            return null;
+        }
+
+        #endregion
 
         internal static HttpHandlerDiagnosticListener s_instance = new HttpHandlerDiagnosticListener();
 
@@ -706,6 +770,7 @@ private static void PerformInjection()
         private const string ActivityName = "System.Net.Http.Desktop.HttpRequestOut";
         private const string RequestStartName = "System.Net.Http.Desktop.HttpRequestOut.Start";
         private const string RequestStopName = "System.Net.Http.Desktop.HttpRequestOut.Stop";
+        private const string RequestStopExName = "System.Net.Http.Desktop.HttpRequestOut.Ex.Stop";
         private const string InitializationFailed = "System.Net.Http.InitializationFailed";
         private const string RequestIdHeaderName = "Request-Id";
         private const string CorrelationContextHeaderName = "Correlation-Context";
@@ -721,7 +786,11 @@ private static void PerformInjection()
         private static FieldInfo s_writeListField;
         private static Func<HttpWebRequest, HttpWebResponse> s_httpResponseAccessor;
         private static Func<HttpWebRequest, int> s_autoRedirectsAccessor;
+        private static Func<HttpWebRequest, object> s_coreResponseAccessor;
+        private static Func<object, HttpStatusCode> s_coreStatusCodeAccessor;
+        private static Func<object, WebHeaderCollection> s_coreHeadersAccessor;
+        private static Type s_coreResponseDataType;
 
-#endregion
+        #endregion
     }
 }

src/System.Diagnostics.DiagnosticSource/tests/HttpHandlerDiagnosticListenerTests.cs

@@ -134,6 +134,45 @@ public async Task TestBasicReceiveAndResponseEvents()
             }
         }
 
+        /// <summary>
+        /// Test to make sure we get both request and response events.
+        /// </summary>
+        [Fact]
+        public async Task TestResponseWithoutContentEvents()
+        {
+            using (var eventRecords = new EventObserverAndRecorder())
+            {
+                // Send a random Http request to generate some events
+                using (var client = new HttpClient())
+                {
+                    (await client.GetAsync(Configuration.Http.RemoteEmptyContentServer)).Dispose();
+                }
+
+                // We should have exactly one Start and one Stop event
+                Assert.Equal(1, eventRecords.Records.Count(rec => rec.Key.EndsWith("Start")));
+                Assert.Equal(1, eventRecords.Records.Count(rec => rec.Key.EndsWith("Stop")));
+                Assert.Equal(2, eventRecords.Records.Count);
+
+                // Check to make sure: The first record must be a request, the next record must be a response. 
+                KeyValuePair<string, object> startEvent;
+                Assert.True(eventRecords.Records.TryDequeue(out startEvent));
+                Assert.Equal("System.Net.Http.Desktop.HttpRequestOut.Start", startEvent.Key);
+                HttpWebRequest startRequest = ReadPublicProperty<HttpWebRequest>(startEvent.Value, "Request");
+                Assert.NotNull(startRequest);
+                Assert.NotNull(startRequest.Headers["Request-Id"]);
+
+                KeyValuePair<string, object> stopEvent;
+                Assert.True(eventRecords.Records.TryDequeue(out stopEvent));
+                Assert.Equal("System.Net.Http.Desktop.HttpRequestOut.Ex.Stop", stopEvent.Key);
+                HttpWebRequest stopRequest = ReadPublicProperty<HttpWebRequest>(stopEvent.Value, "Request");
+                Assert.Equal(startRequest, stopRequest);
+                HttpStatusCode status = ReadPublicProperty<HttpStatusCode>(stopEvent.Value, "StatusCode");
+                Assert.NotNull(status);
+
+                WebHeaderCollection headers = ReadPublicProperty<WebHeaderCollection>(stopEvent.Value, "Headers");
+                Assert.NotNull(headers);
+            }
+        }
         /// <summary>
         /// Test that if request is redirected, it gets only one Start and one Stop event
         /// </summary>