Skip to content

Allow user to specify field(s) used to import XEvents #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arvindshmicrosoft merged 1 commit into from
Feb 21, 2022
Merged

Allow user to specify field(s) used to import XEvents #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions Engine/StackResolver.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ namespace Microsoft.SqlServer.Utils.Misc.SQLCallStackResolver {
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;

public class StackResolver : IDisposable {
Expand All @@ -37,9 +38,13 @@ public void CancelRunningTasks() {
this.cancelRequested = true;
}

public Task<Tuple<List<string>, List<string>>> GetDistinctXELFieldsAsync(string[] xelFiles, int eventsToSample) {
return XELHelper.GetDistinctXELActionsFieldsAsync(xelFiles, eventsToSample);
}

/// Public method which to help import XEL files
public Tuple<int, string> ExtractFromXEL(string[] xelFiles, bool bucketize) {
return XELHelper.ExtractFromXEL(this, xelFiles, bucketize);
public async Task<Tuple<int, string>> ExtractFromXEL(string[] xelFiles, bool groupEvents, List<string> relevantFields) {
return await XELHelper.ExtractFromXELAsync(this, xelFiles, groupEvents, relevantFields);
}

/// Convert virtual-address only type frames to their module+offset format
Expand Down
2 changes: 1 addition & 1 deletion Engine/SymSrvHelpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ namespace Microsoft.SqlServer.Utils.Misc.SQLCallStackResolver {
using System.Text;

public static class SymSrvHelpers {
static int processId = Process.GetCurrentProcess().Id;
static readonly int processId = Process.GetCurrentProcess().Id;

/// Wrapper around the symsrv.dll functionality to initialize the symbol load handler for this process.
private static bool InitSymSrv(string symPath) {
Expand Down
79 changes: 34 additions & 45 deletions Engine/XELHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace Microsoft.SqlServer.Utils.Misc.SQLCallStackResolver {
using Microsoft.SqlServer.XEvent.XELite;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Globalization;
Expand All @@ -14,70 +15,35 @@ namespace Microsoft.SqlServer.Utils.Misc.SQLCallStackResolver {

internal class XELHelper {
/// Read a XEL file, consume all callstacks, optionally bucketize them, and in all cases, return the information as equivalent XML
internal static Tuple<int, string> ExtractFromXEL(StackResolver parent, string[] xelFiles, bool bucketize) {
internal async static Task<Tuple<int, string>> ExtractFromXELAsync(StackResolver parent, string[] xelFiles, bool groupEvents, List<string> fieldsToGroupOn) {
Contract.Requires(xelFiles != null);
parent.cancelRequested = false;
var callstackSlots = new Dictionary<string, long>();
var callstackRaw = new Dictionary<string, string>();
var callstackSlots = new ConcurrentDictionary<string, long>();
var callstackRaw = new ConcurrentDictionary<string, string>();
var xmlEquivalent = new StringBuilder();

// the below feels quite hacky. Unfortunately till such time that we have strong typing in XELite I believe this is unavoidable
var relevantKeyNames = new string[] { "callstack", "call_stack", "stack_frames" };
foreach (var xelFileName in xelFiles) {
if (File.Exists(xelFileName)) {
parent.StatusMessage = $@"Reading {xelFileName}...";
var xeStream = new XEFileEventStreamer(xelFileName);
xeStream.ReadEventStream(
await xeStream.ReadEventStream(
() => {
return Task.CompletedTask;
},
evt => {
var allStacks = (from actTmp in evt.Actions
from keyName in relevantKeyNames
where actTmp.Key.ToLower(CultureInfo.CurrentCulture).StartsWith(keyName)
select actTmp.Value as string)
.Union(
from fldTmp in evt.Fields
from keyName in relevantKeyNames
where fldTmp.Key.ToLower(CultureInfo.CurrentCulture).StartsWith(keyName)
select fldTmp.Value as string);

foreach (var callStackString in allStacks) {
if (string.IsNullOrEmpty(callStackString)) {
continue;
}

if (bucketize) {
lock (callstackSlots) {
if (!callstackSlots.ContainsKey(callStackString)) {
callstackSlots.Add(callStackString, 1);
}
else {
callstackSlots[callStackString]++;
}
}
}
else {
var evtId = string.Format(CultureInfo.CurrentCulture, "File: {0}, Timestamp: {1}, UUID: {2}:", xelFileName, evt.Timestamp.ToLocalTime().ToString("yyyy-MM-dd HH:mm:ss.fffffff", CultureInfo.CurrentCulture), evt.UUID);
lock (callstackRaw) {
if (!callstackRaw.ContainsKey(evtId)) {
callstackRaw.Add(evtId, callStackString);
}
else {
callstackRaw[evtId] += $"{Environment.NewLine}{callStackString}";
}
}
}
var eventKey = string.Join(Environment.NewLine, evt.Actions.Union(evt.Fields).Join(fieldsToGroupOn, l => l.Key, r => r, (l, r) => new { val = l.Value.ToString() }).Select(v => v.val)).Trim();
if (!string.IsNullOrWhiteSpace(eventKey)) {
if (groupEvents) callstackSlots.AddOrUpdate(eventKey, 1, (k, v) => v + 1);
else callstackRaw.AddOrUpdate(string.Format(CultureInfo.CurrentCulture, "File: {0}, Timestamp: {1}, UUID: {2}:", xelFileName, evt.Timestamp.ToLocalTime().ToString("yyyy-MM-dd HH:mm:ss.fffffff", CultureInfo.CurrentCulture), evt.UUID), eventKey, (k, v) => v + $"{Environment.NewLine}{eventKey}");
}
return Task.CompletedTask;
},
CancellationToken.None).Wait();
CancellationToken.None);
}
}

parent.StatusMessage = "Finished reading file(s), finalizing output...";
int finalEventCount;
if (bucketize) {
if (groupEvents) {
xmlEquivalent.AppendLine("<HistogramTarget>");
parent.globalCounter = 0;
foreach (var item in callstackSlots.OrderByDescending(key => key.Value)) {
Expand Down Expand Up @@ -117,5 +83,28 @@ where fldTmp.Key.ToLower(CultureInfo.CurrentCulture).StartsWith(keyName)
parent.StatusMessage = $@"Finished processing {xelFiles.Length} XEL files";
return new Tuple<int, string>(finalEventCount, xmlEquivalent.ToString());
}

internal async static Task<Tuple<List<string>, List<string>>> GetDistinctXELActionsFieldsAsync(string[] xelFiles, int eventsToSampleFromEachFile) {
Contract.Requires(xelFiles != null && eventsToSampleFromEachFile > 0);
var allActions = new HashSet<string>();
var allFields = new HashSet<string>();
foreach (var file in xelFiles) {
var numEvents = 0;
var cts = new CancellationTokenSource();
var xeStream = new XEFileEventStreamer(file);
try {
await xeStream.ReadEventStream(
evt => {
if (Interlocked.Increment(ref numEvents) > eventsToSampleFromEachFile) cts.Cancel();
lock (allActions) evt.Actions.Select(action => allActions.Add(action.Key)).Count();
lock (allFields) evt.Fields.Select(field => allFields.Add(field.Key)).Count();
return Task.CompletedTask;
}, cts.Token);
} catch (OperationCanceledException) {
}
}

return new Tuple<List<string>, List<string>>(allActions.OrderBy(k => k).ToList(), allFields.OrderBy(k => k).ToList());
}
}
}
96 changes: 96 additions & 0 deletions GUI/FieldSelection.Designer.cs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

53 changes: 53 additions & 0 deletions GUI/FieldSelection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License - see LICENSE file in this repo.
namespace Microsoft.SqlServer.Utils.Misc.SQLCallStackResolver {
using System;
using System.Collections.Generic;
using System.Linq;
using System.Windows.Forms;
public partial class FieldSelection : Form {
public List<string> AllActions = new List<string>();
public List<string> AllFields = new List<string>();
public List<string> SelectedEventItems = new List<string>();
private int newItemOrdinal = 1;
public FieldSelection() {
InitializeComponent();
}

private void OKButton_Click(object sender, EventArgs e) {
this.SelectedEventItems = listViewActionsFields.Items.Cast<ListViewItem>().Where(f => f.Checked).Select(f => f.Text).ToList();
}

private void ListSelection_Load(object sender, EventArgs e) {
var grpActions = listViewActionsFields.Groups.Add("Actions", "Actions");
var grpFields = listViewActionsFields.Groups.Add("Fields", "Fields");
listViewActionsFields.Items.AddRange(this.AllActions.Select(a => new ListViewItem() { Text = a, Group = grpActions, Checked = IsCallStackName(a) }).ToArray());
listViewActionsFields.Items.AddRange(this.AllFields.Select(f => new ListViewItem() { Text = f, Group = grpFields, Checked = IsCallStackName(f) }).ToArray());
listViewActionsFields.Items[0].Selected = true;
}

private bool IsCallStackName(string a) {
return (new List<string>() { "callstack", "call_stack", "stack_frames" }).Where(v => a.StartsWith(v)).Any();
}

private void AddItemButton_Click(object sender, EventArgs e) {
if (listViewActionsFields.SelectedItems.Count > 0) {
var selectedGroup = listViewActionsFields.SelectedItems[0].Group;
listViewActionsFields.SelectedItems.Clear();
var newItem = listViewActionsFields.Items.Add(new ListViewItem() { Text = $"New item {newItemOrdinal}", Group = selectedGroup });
newItemOrdinal++;
newItem.Selected = true;
listViewActionsFields.EnsureVisible(newItem.Index);
newItem.BeginEdit();
}
}

private void DelItemButton_Click(object sender, EventArgs e) {
if (listViewActionsFields.SelectedItems.Count > 0) {
foreach (ListViewItem item in listViewActionsFields.SelectedItems) {
listViewActionsFields.Items.Remove(item);
}
}
}
}
}
1 change: 1 addition & 0 deletions GUI/FieldSelection.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<?xml version="1.0" encoding="utf-8"?><root><xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata"><xsd:import namespace="http://www.w3.org/XML/1998/namespace" /><xsd:element name="root" msdata:IsDataSet="true"><xsd:complexType><xsd:choice maxOccurs="unbounded"><xsd:element name="metadata"><xsd:complexType><xsd:sequence><xsd:element name="value" type="xsd:string" minOccurs="0" /></xsd:sequence><xsd:attribute name="name" use="required" type="xsd:string" /><xsd:attribute name="type" type="xsd:string" /><xsd:attribute name="mimetype" type="xsd:string" /><xsd:attribute ref="xml:space" /></xsd:complexType></xsd:element><xsd:element name="assembly"><xsd:complexType><xsd:attribute name="alias" type="xsd:string" /><xsd:attribute name="name" type="xsd:string" /></xsd:complexType></xsd:element><xsd:element name="data"><xsd:complexType><xsd:sequence><xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" /><xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" /></xsd:sequence><xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" /><xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" /><xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" /><xsd:attribute ref="xml:space" /></xsd:complexType></xsd:element><xsd:element name="resheader"><xsd:complexType><xsd:sequence><xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" /></xsd:sequence><xsd:attribute name="name" type="xsd:string" use="required" /></xsd:complexType></xsd:element></xsd:choice></xsd:complexType></xsd:element></xsd:schema><resheader name="resmimetype"><value>text/microsoft-resx</value></resheader><resheader name="version"><value>2.0</value></resheader><resheader name="reader"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader><resheader name="writer"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader><metadata name="DataItem.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"><value>False</value></metadata></root>
Loading