Write EventSource events with metadata to EventPipe (#11577)

* Fix EventPipe initialization.

* rebase code with master, add prototype of event source metadata

* fix define event

* add null ptr checking

* move WriteEvent metadata serialization to managed side

* Fix enabled keywords for COMPlus_PerformanceTracing=1.

* put parameter name into metadata, use WriteToBuffer

* Write event metadata to the file.

* allocate buffer in EventPipeEvent ctor

* prevent double free

* refactor code

* AddEvent params reorder

Author: Xiangyang (Mark) Guo

src/mscorlib/shared/System/Diagnostics/Tracing/EventProvider.cs

@@ -54,6 +54,7 @@ internal partial class EventProvider : IDisposable
         // subclasses of EventProvider use when creating efficient (but unsafe) version of
         // EventWrite.   We do make it a nested type because we really don't expect anyone to use 
         // it except subclasses (and then only rarely).  
+        [StructLayout(LayoutKind.Sequential)]
         public struct EventData
         {
             internal unsafe ulong Ptr;
@@ -78,7 +79,7 @@ internal SessionInfo(int sessionIdBit_, int etwSessionId_)
 
         private static bool m_setInformationMissing;
 
-        private IEventProvider m_eventProvider;          // The interface that implements the specific logging mechanism functions.
+        internal IEventProvider m_eventProvider;         // The interface that implements the specific logging mechanism functions.
         UnsafeNativeMethods.ManifestEtw.EtwEnableCallback m_etwCallback;     // Trace Callback function
         private long m_regHandle;                        // Trace Registration Handle
         private byte m_level;                            // Tracing Level
@@ -936,7 +937,7 @@ to fill the passed in ETW data descriptor.
         // </SecurityKernel>
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Maintainability", "CA1502:AvoidExcessiveComplexity", Justification = "Performance-critical code")]
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1045:DoNotPassTypesByReference")]
-        internal unsafe bool WriteEvent(ref EventDescriptor eventDescriptor, Guid* activityID, Guid* childActivityID, params object[] eventPayload)
+        internal unsafe bool WriteEvent(ref EventDescriptor eventDescriptor, IntPtr eventHandle, Guid* activityID, Guid* childActivityID, params object[] eventPayload)
         {
             int status = 0;
 
@@ -1064,7 +1065,7 @@ internal unsafe bool WriteEvent(ref EventDescriptor eventDescriptor, Guid* activ
                                 userDataPtr[refObjPosition[7]].Ptr = (ulong)v7;
                             }
 
-                            status = m_eventProvider.EventWriteTransferWrapper(m_regHandle, ref eventDescriptor, activityID, childActivityID, argCount, userData);
+                            status = m_eventProvider.EventWriteTransferWrapper(m_regHandle, ref eventDescriptor, eventHandle, activityID, childActivityID, argCount, userData);
                         }
                     }
                     else
@@ -1090,7 +1091,7 @@ internal unsafe bool WriteEvent(ref EventDescriptor eventDescriptor, Guid* activ
                             }
                         }
 
-                        status = m_eventProvider.EventWriteTransferWrapper(m_regHandle, ref eventDescriptor, activityID, childActivityID, argCount, userData);
+                        status = m_eventProvider.EventWriteTransferWrapper(m_regHandle, ref eventDescriptor, eventHandle, activityID, childActivityID, argCount, userData);
 
                         for (int i = 0; i < refObjIndex; ++i)
                         {
@@ -1132,7 +1133,7 @@ internal unsafe bool WriteEvent(ref EventDescriptor eventDescriptor, Guid* activ
         // <CallsSuppressUnmanagedCode Name="UnsafeNativeMethods.ManifestEtw.EventWrite(System.Int64,EventDescriptor&,System.UInt32,System.Void*):System.UInt32" />
         // </SecurityKernel>
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1045:DoNotPassTypesByReference")]
-        internal unsafe protected bool WriteEvent(ref EventDescriptor eventDescriptor, Guid* activityID, Guid* childActivityID, int dataCount, IntPtr data)
+        internal unsafe protected bool WriteEvent(ref EventDescriptor eventDescriptor, IntPtr eventHandle, Guid* activityID, Guid* childActivityID, int dataCount, IntPtr data)
         {
             if (childActivityID != null)
             {
@@ -1143,7 +1144,7 @@ internal unsafe protected bool WriteEvent(ref EventDescriptor eventDescriptor, G
                                 (EventOpcode)eventDescriptor.Opcode == EventOpcode.Stop);
             }
 
-            int status = m_eventProvider.EventWriteTransferWrapper(m_regHandle, ref eventDescriptor, activityID, childActivityID, dataCount, (EventData*)data);
+            int status = m_eventProvider.EventWriteTransferWrapper(m_regHandle, ref eventDescriptor, eventHandle, activityID, childActivityID, dataCount, (EventData*)data);
 
             if (status != 0)
             {
@@ -1166,6 +1167,7 @@ internal unsafe bool WriteEventRaw(
             status = m_eventProvider.EventWriteTransferWrapper(
                 m_regHandle,
                 ref eventDescriptor,
+                IntPtr.Zero,
                 activityID,
                 relatedActivityID,
                 dataCount,
@@ -1241,6 +1243,7 @@ uint IEventProvider.EventUnregister(long registrationHandle)
         unsafe int IEventProvider.EventWriteTransferWrapper(
             long registrationHandle,
             ref EventDescriptor eventDescriptor,
+            IntPtr eventHandle,
             Guid* activityId,
             Guid* relatedActivityId,
             int userDataCount,
@@ -1262,6 +1265,12 @@ int IEventProvider.EventActivityIdControl(UnsafeNativeMethods.ManifestEtw.Activi
                 ControlCode,
                 ref ActivityId);
         }
+
+        // Define an EventPipeEvent handle.
+        unsafe IntPtr IEventProvider.DefineEventHandle(uint eventID, string eventName, Int64 keywords, uint eventVersion, uint level, byte *pMetadata, uint metadataLength)
+        {
+            throw new System.NotSupportedException();
+        }
     }
 
 #elif !FEATURE_PERFTRACING
@@ -1285,6 +1294,7 @@ uint IEventProvider.EventUnregister(long registrationHandle)
         unsafe int IEventProvider.EventWriteTransferWrapper(
             long registrationHandle,
             ref EventDescriptor eventDescriptor,
+            IntPtr eventHandle,
             Guid* activityId,
             Guid* relatedActivityId,
             int userDataCount,
@@ -1297,6 +1307,12 @@ int IEventProvider.EventActivityIdControl(UnsafeNativeMethods.ManifestEtw.Activi
         {
             return 0;
         }
+
+        // Define an EventPipeEvent handle.
+        unsafe IntPtr IEventProvider.DefineEventHandle(uint eventID, string eventName, Int64 keywords, uint eventVersion, uint level, byte *pMetadata, uint metadataLength)
+        {
+            throw new System.NotSupportedException();
+        }
     }
 
 #endif

src/mscorlib/shared/System/Diagnostics/Tracing/EventSource.cs

@@ -686,6 +686,106 @@ protected EventSource(EventSourceSettings settings, params string[] traits)
             Initialize(eventSourceGuid, eventSourceName, traits);
         }
 
+#if FEATURE_PERFTRACING
+        // Generate the serialized blobs that describe events for all strongly typed events (that is events that define strongly
+        // typed event methods. Dynamically defined events (that use Write) hare defined on the fly and are handled elsewhere.
+        private unsafe void DefineEventPipeEvents()
+        {
+            Debug.Assert(m_eventData != null);
+            Debug.Assert(m_provider != null);
+            int cnt = m_eventData.Length;
+            for (int i = 0; i < cnt; i++)
+            {
+                uint eventID = (uint)m_eventData[i].Descriptor.EventId;
+                if (eventID == 0)
+                    continue;
+
+                string eventName = m_eventData[i].Name;
+                Int64 keywords = m_eventData[i].Descriptor.Keywords;
+                uint eventVersion = m_eventData[i].Descriptor.Version;
+                uint level = m_eventData[i].Descriptor.Level;
+
+                // evnetID          : 4 bytes
+                // eventName        : (eventName.Length + 1) * 2 bytes
+                // keywords         : 8 bytes
+                // eventVersion     : 4 bytes
+                // level            : 4 bytes
+                // parameterCount   : 4 bytes
+                uint metadataLength = 24 + ((uint)eventName.Length + 1) * 2;
+
+                // Increase the metadataLength for the types of all parameters.
+                metadataLength += (uint)m_eventData[i].Parameters.Length * 4;
+
+                // Increase the metadataLength for the names of all parameters.
+                foreach (var parameter in m_eventData[i].Parameters)
+                {
+                    string parameterName = parameter.Name;
+                    metadataLength = metadataLength + ((uint)parameterName.Length + 1) * 2;
+                }
+
+                byte[] metadata = new byte[metadataLength];
+
+                // Write metadata: evnetID, eventName, keywords, eventVersion, level, parameterCount, param1 type, param1 name...
+                fixed (byte *pMetadata = metadata)
+                {
+                    uint offset = 0;
+                    WriteToBuffer(pMetadata, metadataLength, ref offset, eventID);
+                    fixed(char *pEventName = eventName)
+                    {
+                        WriteToBuffer(pMetadata, metadataLength, ref offset, (byte *)pEventName, ((uint)eventName.Length + 1) * 2);
+                    }
+                    WriteToBuffer(pMetadata, metadataLength, ref offset, keywords);
+                    WriteToBuffer(pMetadata, metadataLength, ref offset, eventVersion);
+                    WriteToBuffer(pMetadata, metadataLength, ref offset, level);
+                    WriteToBuffer(pMetadata, metadataLength, ref offset, (uint)m_eventData[i].Parameters.Length);
+                    foreach (var parameter in m_eventData[i].Parameters)
+                    {
+                        // Write parameter type.
+                        WriteToBuffer(pMetadata, metadataLength, ref offset, (uint)Type.GetTypeCode(parameter.ParameterType));
+                        
+                        // Write parameter name.
+                        string parameterName = parameter.Name;
+                        fixed (char *pParameterName = parameterName)
+                        {
+                            WriteToBuffer(pMetadata, metadataLength, ref offset, (byte *)pParameterName, ((uint)parameterName.Length + 1) * 2);
+                        }
+                    }
+                    Debug.Assert(metadataLength == offset);
+                    IntPtr eventHandle = m_provider.m_eventProvider.DefineEventHandle(eventID, eventName, keywords, eventVersion, level, pMetadata, metadataLength);
+                    m_eventData[i].EventHandle = eventHandle; 
+                }
+            }
+        }
+
+        // Copy src to buffer and modify the offset.
+        // Note: We know the buffer size ahead of time to make sure no buffer overflow.
+        private static unsafe void WriteToBuffer(byte *buffer, uint bufferLength, ref uint offset, byte *src, uint srcLength)
+        {
+            Debug.Assert(bufferLength >= (offset + srcLength));
+            for (int i = 0; i < srcLength; i++)
+            {
+                *(byte *)(buffer + offset + i) = *(byte *)(src + i);
+            }
+            offset += srcLength;
+        }
+
+        // Copy uint value to buffer.
+        private static unsafe void WriteToBuffer(byte *buffer, uint bufferLength, ref uint offset, uint value)
+        {
+            Debug.Assert(bufferLength >= (offset + 4));
+            *(uint *)(buffer + offset) = value;
+            offset += 4;
+        }
+
+        // Copy long value to buffer.
+        private static unsafe void WriteToBuffer(byte *buffer, uint bufferLength, ref uint offset, long value)
+        {
+            Debug.Assert(bufferLength >= (offset + 8));
+            *(long *)(buffer + offset) = value;
+            offset += 8;
+        }           
+#endif
+
         internal virtual void GetMetadata(out Guid eventSourceGuid, out string eventSourceName, out EventMetadata[] eventData, out byte[] manifestBytes)
         {
             //
@@ -1179,7 +1279,7 @@ protected unsafe void WriteEventWithRelatedActivityIdCore(int eventId, Guid* rel
                                     // by default the Descriptor.Keyword will have the perEventSourceSessionId bit 
                                     // mask set to 0x0f so, when all ETW sessions want the event we don't need to 
                                     // synthesize a new one
-                                    if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, pActivityId, relatedActivityId, eventDataCount, (IntPtr)data))
+                                    if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, m_eventData[eventId].EventHandle, pActivityId, relatedActivityId, eventDataCount, (IntPtr)data))
                                         ThrowEventSourceException(m_eventData[eventId].Name);
                                 }
                                 else
@@ -1199,7 +1299,7 @@ protected unsafe void WriteEventWithRelatedActivityIdCore(int eventId, Guid* rel
                                         m_eventData[eventId].Descriptor.Task,
                                         unchecked((long)etwSessions.ToEventKeywords() | origKwd));
 
-                                    if (!m_provider.WriteEvent(ref desc, pActivityId, relatedActivityId, eventDataCount, (IntPtr)data))
+                                    if (!m_provider.WriteEvent(ref desc, m_eventData[eventId].EventHandle, pActivityId, relatedActivityId, eventDataCount, (IntPtr)data))
                                         ThrowEventSourceException(m_eventData[eventId].Name);
                                 }
                             }
@@ -1229,7 +1329,7 @@ protected unsafe void WriteEventWithRelatedActivityIdCore(int eventId, Guid* rel
 #else
                         if (!SelfDescribingEvents)
                         {
-                            if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, pActivityId, relatedActivityId, eventDataCount, (IntPtr)data))
+                            if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, m_eventData[eventId].EventHandle, pActivityId, relatedActivityId, eventDataCount, (IntPtr)data))
                                 ThrowEventSourceException(m_eventData[eventId].Name);
                         }
                         else
@@ -1489,7 +1589,6 @@ private unsafe void Initialize(Guid eventSourceGuid, string eventSourceName, str
                 }
 #endif // PLATFORM_WINDOWS
 #endif // FEATURE_MANAGED_ETW
-
                 Debug.Assert(!m_eventSourceEnabled);     // We can't be enabled until we are completely initted.  
                 // We are logically completely initialized at this point.  
                 m_completelyInited = true;
@@ -1941,7 +2040,7 @@ private unsafe void WriteEventVarargs(int eventId, Guid* childActivityID, object
                                     // by default the Descriptor.Keyword will have the perEventSourceSessionId bit 
                                     // mask set to 0x0f so, when all ETW sessions want the event we don't need to 
                                     // synthesize a new one
-                                    if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, pActivityId, childActivityID, args))
+                                    if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, m_eventData[eventId].EventHandle, pActivityId, childActivityID, args))
                                         ThrowEventSourceException(m_eventData[eventId].Name);
                                 }
                                 else
@@ -1958,7 +2057,7 @@ private unsafe void WriteEventVarargs(int eventId, Guid* childActivityID, object
                                         m_eventData[eventId].Descriptor.Task,
                                         unchecked((long)etwSessions.ToEventKeywords() | origKwd));
 
-                                    if (!m_provider.WriteEvent(ref desc, pActivityId, childActivityID, args))
+                                    if (!m_provider.WriteEvent(ref desc, m_eventData[eventId].EventHandle, pActivityId, childActivityID, args))
                                         ThrowEventSourceException(m_eventData[eventId].Name);
                                 }
                             }
@@ -1988,7 +2087,7 @@ private unsafe void WriteEventVarargs(int eventId, Guid* childActivityID, object
 #else
                         if (!SelfDescribingEvents)
                         {
-                            if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, pActivityId, childActivityID, args))
+                            if (!m_provider.WriteEvent(ref m_eventData[eventId].Descriptor, m_eventData[eventId].EventHandle, pActivityId, childActivityID, args))
                                 ThrowEventSourceException(m_eventData[eventId].Name);
                         }
                         else
@@ -2232,7 +2331,7 @@ private unsafe void WriteEventString(EventLevel level, long keywords, string msg
                         data.Ptr = (ulong)msgStringPtr;
                         data.Size = (uint)(2 * (msgString.Length + 1));
                         data.Reserved = 0;
-                        m_provider.WriteEvent(ref descr, null, null, 1, (IntPtr)((void*)&data));
+                        m_provider.WriteEvent(ref descr, m_eventData[0].EventHandle, null, null, 1, (IntPtr)((void*)&data));
                     }
                 }
             }
@@ -2517,6 +2616,7 @@ root them and modify shared library definition to force export them.
         partial struct EventMetadata
         {
             public EventDescriptor Descriptor;
+            public IntPtr EventHandle;              // EventPipeEvent handle.
             public EventTags Tags;
             public bool EnabledForAnyListener;      // true if any dispatcher has this event turned on
             public bool EnabledForETW;              // is this event on for the OS ETW data dispatcher?
@@ -3065,6 +3165,10 @@ private void EnsureDescriptorsInitialized()
                         dispatcher.m_EventEnabled = new bool[m_eventData.Length];
                     dispatcher = dispatcher.m_Next;
                 }
+#if FEATURE_PERFTRACING
+                // Initialize the EventPipe event handles.
+                DefineEventPipeEvents();
+#endif                
             }
             if (s_currentPid == 0)
             {
@@ -3119,7 +3223,7 @@ private unsafe bool SendManifest(byte[] rawManifest)
                     dataDescrs[1].Size = (uint)Math.Min(dataLeft, chunkSize);
                     if (m_provider != null)
                     {
-                        if (!m_provider.WriteEvent(ref manifestDescr, null, null, 2, (IntPtr)dataDescrs))
+                        if (!m_provider.WriteEvent(ref manifestDescr, IntPtr.Zero, null, null, 2, (IntPtr)dataDescrs))
                         {
                             // Turns out that if users set the BufferSize to something less than 64K then WriteEvent
                             // can fail.   If we get this failure on the first chunk try again with something smaller
@@ -3597,7 +3701,11 @@ private static byte[] CreateManifestAndDescriptors(Type eventSourceType, string
                 throw new ArgumentException(msg, exception);
             }
 
+#if FEATURE_PERFTRACING
+            return null;
+#else
             return bNeedsManifest ? res : null;
+#endif
         }
 
         private static bool RemoveFirstArgIfRelatedActivityId(ref ParameterInfo[] args)
@@ -3686,6 +3794,7 @@ private static void AddEventDescriptor(ref EventMetadata[] eventData, string eve
             eventData[eventAttribute.EventId].Message = eventAttribute.Message;
             eventData[eventAttribute.EventId].ActivityOptions = eventAttribute.ActivityOptions;
             eventData[eventAttribute.EventId].HasRelatedActivityID = hasRelatedActivityID;
+            eventData[eventAttribute.EventId].EventHandle = IntPtr.Zero;
         }
 
         // Helper used by code:CreateManifestAndDescriptors that trims the m_eventData array to the correct