Skip to content

Commit

Permalink
Merge pull request #12 from calvinhsia/dev/calvinh/worksb3
Browse files Browse the repository at this point in the history 
Dev/calvinh/worksb3
  • Loading branch information
@calvinhsia
calvinhsia committed Jun 15, 2022
2 parents be86416 + c4b6c99 commit 4849483
Show file tree
Hide file tree
Showing 54 changed files with 3,820 additions and 823 deletions.
148 changes: 83 additions & 65 deletions Microsoft.Test.Stress/Microsoft.Test.Stress/DumpAnalyzer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public DumpDataAnalysisResult AnalyzeDump(string dumpFile, string typesToReportS
var dumpDataAnalysisResult = new DumpDataAnalysisResult();

Regex typesToReportStatisticsOnRegex = null;
if (!string.IsNullOrEmpty( typesToReportStatisticsOn))
if (!string.IsNullOrEmpty(typesToReportStatisticsOn))
{
typesToReportStatisticsOnRegex = new Regex(typesToReportStatisticsOn, RegexOptions.Compiled);
dumpDataAnalysisResult.typeStatistics = new TypeStatistics();
Expand All @@ -88,39 +88,87 @@ public DumpDataAnalysisResult AnalyzeDump(string dumpFile, string typesToReportS
try
{
// logger.LogMessage($"in {nameof(AnalyzeDump)} {dumpFile}");
using (var dataTarget = DataTarget.LoadCrashDump(dumpFile))
using (var dataTarget = DataTarget.LoadDump(dumpFile))
{
if (dataTarget.ClrVersions.Count != 1)
if (dataTarget.ClrVersions.Length != 1)
{
throw new InvalidOperationException($"Expected 1 ClrVersion in process. Found {dataTarget.ClrVersions.Count} ");
throw new InvalidOperationException($"Expected 1 ClrVersion in process. Found {dataTarget.ClrVersions.Length} ");
}
var dacLocation = dataTarget.ClrVersions[0].LocalMatchingDac;
logger?.LogMessage($"Got Dac {dacLocation}");
var runtime = dataTarget.ClrVersions[0].CreateRuntime();
logger?.LogMessage($"Got runtime {runtime}");
var nObjCount = 0;
var lstStrings = new List<ClrObject>();
var markedObjects = new HashSet<ulong>();
foreach (var obj in EnumerateRootedObjects(runtime.Heap))
var clrver = dataTarget.ClrVersions[0];
var dacFileName = string.Empty;
try
{
dacFileName = dataTarget.BinaryLocator.FindBinary(clrver.DacInfo.PlatformSpecificFileName, clrver.DacInfo.IndexTimeStamp, clrver.DacInfo.IndexFileSize);
if (string.IsNullOrEmpty(dacFileName))
{
dacFileName = clrver.DacInfo.LocalDacPath;
}
}
catch (Exception ex)
{
var typ = obj.Type?.Name;
if (typ == "System.String")
IOException ioException = null;
if (ex is AggregateException aggregateException)
{
lstStrings.Add(obj);
if (ex.InnerException is IOException exception)
{
ioException = exception;
}
}
if (!dumpDataAnalysisResult.dictTypes.ContainsKey(typ))
if (ex is IOException ioexception)
{
dumpDataAnalysisResult.dictTypes[typ] = 1;
ioException = ioexception;
}
if (ioException == null)
{
throw;
}
// System.IO.IOException: The process cannot access the file 'C:\Users\calvinh\AppData\Local\Temp\symbols\mscordacwks_x86_x86_4.6.26.00.dll\54c2e0d969b000\mscordacwks_x86_x86_4.6.26.00.dll' because it is being used by another process.
var m = Regex.Match(ioException.Message, @".*'(.*)'.*");
if (m.Success)
{
dacFileName = m.Groups[1].Value;
}
else
{
dumpDataAnalysisResult.dictTypes[typ]++;
throw;
}
nObjCount++;

if (typesToReportStatisticsOnRegex?.IsMatch(typ) == true)
}
if (string.IsNullOrEmpty(dacFileName))
{
throw new InvalidOperationException($"Could not create or find dacFile");
}
logger?.LogMessage($"Got Dac {dacFileName}");
var runtime = dataTarget.ClrVersions[0].CreateRuntime();
logger?.LogMessage($"Got runtime {runtime}");
var nObjCount = 0;
var lstStrings = new List<string>();
var markedObjects = new HashSet<ulong>();
foreach (var obj in EnumerateRootedObjects(runtime.Heap))
{
if (obj.Type != null)
{
CalculateTypeStatisticsPhase1(obj, typesToReportStatisticsOnRegex, markedObjects, dumpDataAnalysisResult.typeStatistics);
var typ = obj.Type?.Name;
if (typ == "System.String")
{
if (!string.IsNullOrEmpty(obj.AsString()))
{
lstStrings.Add(obj.AsString());
}
}
if (!dumpDataAnalysisResult.dictTypes.ContainsKey(typ))
{
dumpDataAnalysisResult.dictTypes[typ] = 1;
}
else
{
dumpDataAnalysisResult.dictTypes[typ]++;
}
nObjCount++;

if (typesToReportStatisticsOnRegex?.IsMatch(typ) == true)
{
CalculateTypeStatisticsPhase1(obj, typesToReportStatisticsOnRegex, markedObjects, dumpDataAnalysisResult.typeStatistics);
}
}
}
if (typesToReportStatisticsOnRegex != null)
Expand All @@ -131,46 +179,15 @@ public DumpDataAnalysisResult AnalyzeDump(string dumpFile, string typesToReportS
}

logger?.LogMessage($"Total Object Count = {nObjCount:n0} TypeCnt = {dumpDataAnalysisResult.dictTypes.Count} {dumpFile}");
var maxLength = 1024;
var strValue = string.Empty;
foreach (var str in lstStrings)
foreach (var strValue in lstStrings)
{
if (str.Type.IsString)
if (!dumpDataAnalysisResult.dictStrings.ContainsKey(strValue))
{
var addrToUse = str.Address + (uint)IntPtr.Size; // skip clsid
byte[] buff = new byte[IntPtr.Size];
if (runtime.ReadMemory(
addrToUse,
buff,
IntPtr.Size,
out var bytesRead
))
{
var len = BitConverter.ToUInt32(buff, 0);
if (maxLength > 0)
{
len = Math.Min(len, (uint)maxLength);
}
buff = new byte[len * 2];
if (runtime.ReadMemory(
addrToUse + (uint)IntPtr.Size, // skip clsid, len
buff,
buff.Length,
out bytesRead
))
{
var enc = new UnicodeEncoding();
strValue = enc.GetString(buff, 0, buff.Length);
if (!dumpDataAnalysisResult.dictStrings.ContainsKey(strValue))
{
dumpDataAnalysisResult.dictStrings[strValue] = 1;
}
else
{
dumpDataAnalysisResult.dictStrings[strValue]++;
}
}
}
dumpDataAnalysisResult.dictStrings[strValue] = 1;
}
else
{
dumpDataAnalysisResult.dictStrings[strValue]++;
}
// logger.LogMessage($"STR {strValue}");
}
Expand All @@ -189,6 +206,7 @@ out bytesRead
catch (Exception ex)
{
logger?.LogMessage($"Exception analyzing dump {ex}");
throw; // we want the test to fail if didn't get analysis
}
return dumpDataAnalysisResult;
}
Expand Down Expand Up @@ -329,7 +347,7 @@ private IEnumerable<ClrObject> EnumerateRootedObjects(ClrHeap heap)
HashSet<ulong> visitedObjects = new HashSet<ulong>();
Queue<ClrObject> objectQueue = new Queue<ClrObject>();

foreach (ClrRoot root in heap.EnumerateRoots())
foreach (IClrRoot root in heap.EnumerateRoots())
{
if (!visitedObjects.Contains(root.Object))
{
Expand All @@ -349,7 +367,7 @@ private IEnumerable<ClrObject> EnumerateRootedObjects(ClrHeap heap)
continue;
}
// Follow all references.
foreach (var reference in obj.EnumerateObjectReferences())
foreach (var reference in obj.EnumerateReferences())
{
if (!reference.IsNull && !visitedObjects.Contains(reference.Address))
{
Expand Down Expand Up @@ -380,7 +398,7 @@ private void CalculateTypeStatisticsPhase1(ClrObject rootObj, Regex typesToRepor
ClrObject obj = objectQueue.Dequeue();
typeStatistics.InclusiveRetainedBytes += obj.Size;

foreach (var reference in obj.EnumerateObjectReferences())
foreach (var reference in obj.EnumerateReferences())
{
if (!reference.IsNull && !markedObjects.Contains(reference.Address))
{
Expand Down Expand Up @@ -411,7 +429,7 @@ private void CalculateTypeStatisticsPhase2(ClrHeap heap, Regex typesToReportStat
// Start with exclusive = inclusive and walk the heap from roots looking for objects to subtract from this number.
typeStatistics.ExclusiveRetainedBytes = typeStatistics.InclusiveRetainedBytes;

foreach (ClrRoot root in heap.EnumerateRoots())
foreach (IClrRoot root in heap.EnumerateRoots())
{
// Interested only in roots outside of our marked inclusive graph.
if (!markedObjects.Contains(root.Object))
Expand Down Expand Up @@ -440,7 +458,7 @@ private void CalculateTypeStatisticsPhase2(ClrHeap heap, Regex typesToReportStat
}

// Follow all references.
foreach (var reference in obj.EnumerateObjectReferences())
foreach (var reference in obj.EnumerateReferences())
{
if (!reference.IsNull && !visitedObjects.Contains(reference.Address))
{
Expand Down
88 changes: 56 additions & 32 deletions Microsoft.Test.Stress/Microsoft.Test.Stress/MeasurementHolder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public class FileResultsData
{
public string filename;
public string description;
public override string ToString() => $"{filename} {description}";
}
public class MeasurementHolder : IDisposable
{
Expand Down Expand Up @@ -337,42 +338,61 @@ public async Task CheckIfNeedToTakeSnapshotsAsync()
var currentDumpFile = await DoCreateDumpAsync($"Taking final snapshot dump at iteration {nSamplesTaken}");
if (!string.IsNullOrEmpty(baseDumpFileName))
{
_oDumpAnalyzer = new DumpAnalyzer(Logger);
var sb = new StringBuilder();
sb.AppendLine($"'{TestName}' Leaks Found");
foreach (var leak in lstLeakResults)
try
{
sb.AppendLine($"Leak Detected: {leak}");
_oDumpAnalyzer = new DumpAnalyzer(Logger);
var sb = new StringBuilder();
sb.AppendLine($"'{TestName}' Leaks Found");
foreach (var leak in lstLeakResults)
{
sb.AppendLine($"Leak Detected: {leak}");
}
sb.AppendLine();
_oDumpAnalyzer.GetDiff(sb,
baseDumpFileName,
currentDumpFile,
stressUtilOptions.NumIterations,
stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot,
stressUtilOptions.TypesToReportStatisticsOn,
out DumpAnalyzer.TypeStatistics baselineTypeStatistics,
out DumpAnalyzer.TypeStatistics currentTypeStatistics);

if (baselineTypeStatistics != null)
{
dictTelemetryProperties["TypeStatsExclusiveRetainedBytes_Base"] = baselineTypeStatistics.ExclusiveRetainedBytes;
dictTelemetryProperties["TypeStatsInclusiveRetainedBytes_Base"] = baselineTypeStatistics.InclusiveRetainedBytes;
}
if (currentTypeStatistics != null)
{
dictTelemetryProperties["TypeStatsExclusiveRetainedBytes_Final"] = currentTypeStatistics.ExclusiveRetainedBytes;
dictTelemetryProperties["TypeStatsInclusiveRetainedBytes_Final"] = currentTypeStatistics.InclusiveRetainedBytes;
}

var fname = Path.Combine(ResultsFolder, $"{DiffFileName}_{nSamplesTaken}.txt");
File.WriteAllText(fname, sb.ToString());
if (stressUtilOptions.ShowUI)
{
Process.Start(fname);
}
lstFileResults.Add(new FileResultsData() { filename = fname, description = $"Differences for Type and String counts at iter {nSamplesTaken}" });
Logger.LogMessage("DumpDiff Analysis " + fname);
}
sb.AppendLine();
_oDumpAnalyzer.GetDiff(sb,
baseDumpFileName,
currentDumpFile,
stressUtilOptions.NumIterations,
stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot,
stressUtilOptions.TypesToReportStatisticsOn,
out DumpAnalyzer.TypeStatistics baselineTypeStatistics,
out DumpAnalyzer.TypeStatistics currentTypeStatistics);

if (baselineTypeStatistics != null)
catch (FileNotFoundException ex)
{
dictTelemetryProperties["TypeStatsExclusiveRetainedBytes_Base"] = baselineTypeStatistics.ExclusiveRetainedBytes;
dictTelemetryProperties["TypeStatsInclusiveRetainedBytes_Base"] = baselineTypeStatistics.InclusiveRetainedBytes;
Logger.LogMessage($"{ex}");
void DumpDir(string dir)
{
Logger.LogMessage($" Dumping folder contents {dir}");
foreach (var file in Directory.EnumerateFiles(dir, "System.Runtime.CompilerServices.Unsafe.*", SearchOption.AllDirectories))
{
var finfo = new FileInfo(file);
var verinfo = FileVersionInfo.GetVersionInfo(file);
Logger.LogMessage($" {finfo.Length,20:n0} {verinfo.FileVersion,30} {file} ");
}
}
DumpDir(Environment.CurrentDirectory);
throw;
}
if (currentTypeStatistics != null)
{
dictTelemetryProperties["TypeStatsExclusiveRetainedBytes_Final"] = currentTypeStatistics.ExclusiveRetainedBytes;
dictTelemetryProperties["TypeStatsInclusiveRetainedBytes_Final"] = currentTypeStatistics.InclusiveRetainedBytes;
}

var fname = Path.Combine(ResultsFolder, $"{DiffFileName}_{nSamplesTaken}.txt");
File.WriteAllText(fname, sb.ToString());
if (stressUtilOptions.ShowUI)
{
Process.Start(fname);
}
lstFileResults.Add(new FileResultsData() { filename = fname, description = $"Differences for Type and String counts at iter {nSamplesTaken}" });
Logger.LogMessage("DumpDiff Analysis " + fname);
}
else
{
Expand Down Expand Up @@ -812,6 +832,10 @@ public void Dispose()
{
if (Logger is Logger myLogger)
{
lstFileResults.ForEach(f =>
{
myLogger.LogMessage($"Attaching Test Result {f}");
});
var sb = new StringBuilder();
foreach (var str in myLogger._lstLoggedStrings)
{
Expand Down
3 changes: 2 additions & 1 deletion Microsoft.Test.Stress/Microsoft.Test.Stress/Microsoft.Test.Stress.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Diagnostics.Runtime">
<Version>1.1.126102</Version>
<Version>2.0.226801</Version>
</PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Telemetry">
<Version>16.4.37</Version>
Expand Down Expand Up @@ -136,6 +136,7 @@
<EmbeddedResource Include="Properties\Resources.resx">
<Generator>ResXFileCodeGenerator</Generator>
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
<SubType>Designer</SubType>
</EmbeddedResource>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
Expand Down
Loading

0 comments on commit 4849483

Please sign in to comment.