Skip to content

Commit

Permalink
Reduced memory footprint
Browse files Browse the repository at this point in the history
  • Loading branch information
@spaghettidba
spaghettidba committed Aug 2, 2018
1 parent 0a598c0 commit 547463a
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 34 deletions.
29 changes: 25 additions & 4 deletions WorkloadTools/Consumer/Analysis/WorkloadAnalyzer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using WorkloadTools.Util;

namespace WorkloadTools.Consumer.Analysis
Expand Down Expand Up @@ -39,7 +40,7 @@ public class WorkloadAnalyzer

private static int MAX_WRITE_RETRIES = Properties.Settings.Default.WorkloadAnalyzer_MAX_WRITE_RETRIES;

struct NormalizedQuery
private class NormalizedQuery
{
public long Hash { get; set; }
public string NormalizedText { get; set; }
Expand Down Expand Up @@ -365,7 +366,8 @@ select new

logger.Info("Wait stats written");
}
waitsData.Rows.Clear();
waitsData.Dispose();
waitsData = null;
}
}

Expand Down Expand Up @@ -411,7 +413,8 @@ select new
}
logger.Info("Performance counters written");
}
counterData.Rows.Clear();
counterData.Dispose();
counterData = null;
}
}

Expand Down Expand Up @@ -578,7 +581,7 @@ private void WriteNormalizedQueries(Dictionary<long, NormalizedQuery> values, Sq
bulkCopy.DestinationTableName = "#NormalizedQueries";
bulkCopy.BatchSize = 1000;
bulkCopy.BulkCopyTimeout = 300;
bulkCopy.WriteToServer(DataUtils.ToDataTable(from t in values select new { t.Value.Hash, t.Value.NormalizedText, t.Value.ExampleText }));
bulkCopy.WriteToServer(DataUtils.ToDataTable(from t in values where (t.Value != null) select new { t.Value.Hash, t.Value.NormalizedText, t.Value.ExampleText }));

}

Expand All @@ -602,6 +605,24 @@ private void WriteNormalizedQueries(Dictionary<long, NormalizedQuery> values, Sq
cmd.CommandText = sql;
cmd.ExecuteNonQuery();
}


// Erase from memory all the normalized queries
// already written to the database. This should reduce
// the memory footprint quite a lot
foreach(var hash in values.Keys.ToList())
{
values[hash] = null;
}
// Run the Garbage Collector in a separate task
Task.Factory.StartNew(() => InvokeGC());
}


private void InvokeGC()
{
GC.Collect();
GC.WaitForPendingFinalizers();
}


Expand Down
90 changes: 60 additions & 30 deletions WorkloadTools/Listener/ExtendedEvents/ExtendedEventsWorkloadListener.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ public class ExtendedEventsWorkloadListener : WorkloadListener
{
private static Logger logger = LogManager.GetCurrentClassLogger();

private enum evntType
{
Action,
Field
}

public ExtendedEventsWorkloadListener()
{
Expand Down Expand Up @@ -124,9 +129,6 @@ FROM sys.dm_xe_sessions
ALTER EVENT SESSION [sqlworkload] ON SERVER STATE = STOP;
DROP EVENT SESSION [sqlworkload] ON SERVER;
END
";
SqlCommand cmd = conn.CreateCommand();
cmd.CommandText = sql;
Expand Down Expand Up @@ -154,17 +156,17 @@ private void ReadEventsFromStream()
string commandText = String.Empty;
if (evt.Name == "rpc_completed")
{
commandText = evt.Fields["statement"].Value.ToString();
commandText = (string)TryGetValue(evt, evntType.Field, "statement");
evnt.Type = WorkloadEvent.EventType.RPCCompleted;
}
else if (evt.Name == "sql_batch_completed")
{
commandText = evt.Fields["batch_text"].Value.ToString();
commandText = (string)TryGetValue(evt, evntType.Field, "batch_text");
evnt.Type = WorkloadEvent.EventType.BatchCompleted;
}
else if (evt.Name == "attention")
{
commandText = evt.Actions["sql_text"].Value.ToString();
commandText = (string)TryGetValue(evt, evntType.Action, "sql_text");
evnt.Type = WorkloadEvent.EventType.Timeout;
}
else
Expand All @@ -173,33 +175,39 @@ private void ReadEventsFromStream()
continue;
}

if (evt.Actions["client_app_name"].Value != null)
evnt.ApplicationName = (string)evt.Actions["client_app_name"].Value;
if (evt.Actions["database_name"].Value != null)
evnt.DatabaseName = (string)evt.Actions["database_name"].Value;
if (evt.Actions["client_hostname"].Value != null)
evnt.HostName = (string)evt.Actions["client_hostname"].Value;
if (evt.Actions["server_principal_name"].Value != null)
evnt.LoginName = (string)evt.Actions["server_principal_name"].Value;
if (evt.Actions["session_id"].Value != null)
evnt.SPID = Convert.ToInt32(evt.Actions["session_id"].Value);
if (commandText != null)
evnt.Text = commandText;


evnt.StartTime = evt.Timestamp.LocalDateTime;

if (evnt.Type == WorkloadEvent.EventType.Timeout)
try
{
evnt.Duration = Convert.ToInt64(evt.Fields["duration"].Value);
evnt.CPU = Convert.ToInt32(evnt.Duration / 1000);
evnt.ApplicationName = (string)TryGetValue(evt, evntType.Action, "client_app_name");
evnt.DatabaseName = (string)TryGetValue(evt, evntType.Action, "database_name");
evnt.HostName = (string)TryGetValue(evt, evntType.Action, "client_hostname");
evnt.LoginName = (string)TryGetValue(evt, evntType.Action, "server_principal_name");
object oSession = TryGetValue(evt, evntType.Action, "session_id");
if (oSession != null)
evnt.SPID = Convert.ToInt32(oSession);
if (commandText != null)
evnt.Text = commandText;


evnt.StartTime = evt.Timestamp.LocalDateTime;

if (evnt.Type == WorkloadEvent.EventType.Timeout)
{
evnt.Duration = Convert.ToInt64(evt.Fields["duration"].Value);
evnt.CPU = Convert.ToInt32(evnt.Duration / 1000);
}
else
{
evnt.Reads = Convert.ToInt64(evt.Fields["logical_reads"].Value);
evnt.Writes = Convert.ToInt64(evt.Fields["writes"].Value);
evnt.CPU = Convert.ToInt32(evt.Fields["cpu_time"].Value);
evnt.Duration = Convert.ToInt64(evt.Fields["duration"].Value);
}

}
else
catch(Exception e)
{
evnt.Reads = Convert.ToInt64(evt.Fields["logical_reads"].Value);
evnt.Writes = Convert.ToInt64(evt.Fields["writes"].Value);
evnt.CPU = Convert.ToInt32(evt.Fields["cpu_time"].Value);
evnt.Duration = Convert.ToInt64(evt.Fields["duration"].Value);
logger.Error(e, "Error converting XE data from the stream.");
throw;
}

if (transformer.Skip(evnt.Text))
Expand All @@ -226,5 +234,27 @@ private void ReadEventsFromStream()
Dispose();
}
}

private object TryGetValue(PublishedEvent evt, evntType t, string name)
{
object result = null;
if(t == evntType.Action)
{
PublishedAction act;
if(evt.Actions.TryGetValue(name, out act))
{
result = act.Value;
}
}
else
{
PublishedEventField fld;
if (evt.Fields.TryGetValue(name, out fld))
{
result = fld.Value;
}
}
return result;
}
}
}

0 comments on commit 547463a

Please sign in to comment.