Expand PerfMap format to support metadata for symbol indexation (dotn…

…et#53792)

I have expanded the PerfMap format produced by Crossgen2 and
R2RDump to produce metadata in form of pseudo-symbol records with
high addresses. In this version I have implemented four metadata
entries - output GUID, target OS, target architecture and perfmap
format version number.  I have verified for System.Private.CoreLib
and for the composite framework that Crossgen2 and R2RDump
produce identical metadata.

To facilitate a smooth transition to the new perfmap format, in
accordance with Juan's suggestion I have introduced a new command-line
option to explicitly specify the perfmap format revision. As of today,
0 corresponds to the legacy Crossgen1-style output where the
perfmap file name includes the {MVID} section, perfmap format #1
corresponds to current Crossgen2 with its new naming scheme.
As of today there are no differences in the file content.

Thanks

Tomas

src/coreclr/tools/aot/ILCompiler.Diagnostics/AssemblyInfo.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace ILCompiler.Diagnostics
+{
+    public struct AssemblyInfo
+    {
+        public readonly string Name;
+        public readonly Guid Mvid;
+
+        public AssemblyInfo(string name, Guid mvid)
+        {
+            Name = name;
+            Mvid = mvid;
+        }
+    }
+}

src/coreclr/tools/aot/ILCompiler.Diagnostics/ILCompiler.Diagnostics.csproj

@@ -17,4 +17,8 @@
     <Configurations>Debug;Release;Checked</Configurations>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\ILCompiler.TypeSystem.ReadyToRun\ILCompiler.TypeSystem.ReadyToRun.csproj" />
+  </ItemGroup>
+
 </Project>

src/coreclr/tools/aot/ILCompiler.Diagnostics/PerfMapWriter.cs

@@ -4,23 +4,60 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+
+using Internal.TypeSystem;
 
 namespace ILCompiler.Diagnostics
 {
     public class PerfMapWriter
     {
+        public const int LegacyCrossgen1FormatVersion = 0;
+
+        public const int CurrentFormatVersion = 1;
+
+        public enum PseudoRVA : uint
+        {
+            OutputGuid         = 0xFFFFFFFF,
+            TargetOS           = 0xFFFFFFFE,
+            TargetArchitecture = 0xFFFFFFFD,
+            FormatVersion      = 0xFFFFFFFC,
+        }
+
         private TextWriter _writer;
 
         private PerfMapWriter(TextWriter writer)
         {
             _writer = writer;
         }
 
-        public static void Write(string perfMapFileName, IEnumerable<MethodInfo> methods)
+        public static void Write(string perfMapFileName, int perfMapFormatVersion, IEnumerable<MethodInfo> methods, IEnumerable<AssemblyInfo> inputAssemblies, TargetOS targetOS, TargetArchitecture targetArch)
         {
+            if (perfMapFormatVersion > CurrentFormatVersion)
+            {
+                throw new NotSupportedException(perfMapFormatVersion.ToString());
+            }
+
             using (TextWriter writer = new StreamWriter(perfMapFileName))
             {
+                IEnumerable<AssemblyInfo> orderedInputs = inputAssemblies.OrderBy(asm => asm.Name, StringComparer.OrdinalIgnoreCase);
+
                 PerfMapWriter perfMapWriter = new PerfMapWriter(writer);
+
+                List<byte> inputHash = new List<byte>();
+                foreach (AssemblyInfo inputAssembly in orderedInputs)
+                {
+                    inputHash.AddRange(inputAssembly.Mvid.ToByteArray());
+                }
+                inputHash.Add((byte)targetOS);
+                inputHash.Add((byte)targetArch);
+                Guid outputGuid = new Guid(MD5.HashData(inputHash.ToArray()));
+                perfMapWriter.WriteLine(outputGuid.ToString(), (uint)PseudoRVA.OutputGuid, 0);
+                perfMapWriter.WriteLine(targetOS.ToString(), (uint)PseudoRVA.TargetOS, 0);
+                perfMapWriter.WriteLine(targetArch.ToString(), (uint)PseudoRVA.TargetArchitecture, 0);
+                perfMapWriter.WriteLine(CurrentFormatVersion.ToString(), (uint)PseudoRVA.FormatVersion, 0);
+
                 foreach (MethodInfo methodInfo in methods)
                 {
                     if (methodInfo.HotRVA != 0 && methodInfo.HotLength != 0)

src/coreclr/tools/aot/ILCompiler.ReadyToRun/CodeGen/ReadyToRunObjectWriter.cs

@@ -45,6 +45,12 @@ internal class ReadyToRunObjectWriter
         /// </summary>
         private readonly EcmaModule _componentModule;
 
+        /// <summary>
+        /// Compilation input files. Input files are emitted as perfmap entries and used
+        /// to calculate the output GUID of the ReadyToRun executable for symbol indexation.
+        /// </summary>
+        private readonly IEnumerable<string> _inputFiles;
+
         /// <summary>
         /// Nodes to emit into the output executable as collected by the dependency analysis.
         /// </summary>
@@ -101,9 +107,9 @@ internal class ReadyToRunObjectWriter
         private string _perfMapPath;
 
         /// <summary>
-        /// MVID of the input managed module to embed in the perfmap file name.
+        /// Requested version of the perfmap file format
         /// </summary>
-        private Guid? _perfMapMvid;
+        private int _perfMapFormatVersion;
 
         /// <summary>
         /// If non-zero, the PE file will be laid out such that it can naturally be mapped with a higher alignment than 4KB.
@@ -132,6 +138,7 @@ public NodeInfo(ISymbolNode node, int nodeIndex, int symbolIndex)
         public ReadyToRunObjectWriter(
             string objectFilePath,
             EcmaModule componentModule,
+            IEnumerable<string> inputFiles,
             IEnumerable<DependencyNode> nodes,
             NodeFactory factory,
             bool generateMapFile,
@@ -140,13 +147,14 @@ public NodeInfo(ISymbolNode node, int nodeIndex, int symbolIndex)
             string pdbPath,
             bool generatePerfMapFile,
             string perfMapPath,
-            Guid? perfMapMvid,
+            int perfMapFormatVersion,
             bool generateProfileFile,
             CallChainProfile callChainProfile,
             int customPESectionAlignment)
         {
             _objectFilePath = objectFilePath;
             _componentModule = componentModule;
+            _inputFiles = inputFiles;
             _nodes = nodes;
             _nodeFactory = factory;
             _customPESectionAlignment = customPESectionAlignment;
@@ -156,7 +164,7 @@ public NodeInfo(ISymbolNode node, int nodeIndex, int symbolIndex)
             _pdbPath = pdbPath;
             _generatePerfMapFile = generatePerfMapFile;
             _perfMapPath = perfMapPath;
-            _perfMapMvid = perfMapMvid;
+            _perfMapFormatVersion = perfMapFormatVersion;
 
             bool generateMap = (generateMapFile || generateMapCsvFile);
             bool generateSymbols = (generatePdbFile || generatePerfMapFile);
@@ -329,6 +337,11 @@ public void EmitPortableExecutable()
 
                 if (_outputInfoBuilder != null)
                 {
+                    foreach (string inputFile in _inputFiles)
+                    {
+                        _outputInfoBuilder.AddInputModule(_nodeFactory.TypeSystemContext.GetModuleFromPath(inputFile));
+                    }
+
                     r2rPeBuilder.AddSections(_outputInfoBuilder);
 
                     if (_generateMapFile)
@@ -361,7 +374,7 @@ public void EmitPortableExecutable()
                         {
                             path = Path.GetDirectoryName(_objectFilePath);
                         }
-                        _symbolFileBuilder.SavePerfMap(path, _objectFilePath, _perfMapMvid);
+                        _symbolFileBuilder.SavePerfMap(path, _perfMapFormatVersion, _objectFilePath, _nodeFactory.Target.OperatingSystem, _nodeFactory.Target.Architecture);
                     }
 
                     if (_profileFileBuilder != null)
@@ -430,6 +443,7 @@ private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int node
         public static void EmitObject(
             string objectFilePath,
             EcmaModule componentModule,
+            IEnumerable<string> inputFiles,
             IEnumerable<DependencyNode> nodes,
             NodeFactory factory,
             bool generateMapFile,
@@ -438,7 +452,7 @@ private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int node
             string pdbPath,
             bool generatePerfMapFile,
             string perfMapPath,
-            Guid? perfMapMvid,
+            int perfMapFormatVersion,
             bool generateProfileFile,
             CallChainProfile callChainProfile,
             int customPESectionAlignment)
@@ -447,6 +461,7 @@ private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int node
             ReadyToRunObjectWriter objectWriter = new ReadyToRunObjectWriter(
                 objectFilePath,
                 componentModule,
+                inputFiles,
                 nodes,
                 factory,
                 generateMapFile: generateMapFile,
@@ -455,7 +470,7 @@ private void EmitObjectData(R2RPEBuilder r2rPeBuilder, ObjectData data, int node
                 pdbPath: pdbPath,
                 generatePerfMapFile: generatePerfMapFile,
                 perfMapPath: perfMapPath,
-                perfMapMvid: perfMapMvid,
+                perfMapFormatVersion: perfMapFormatVersion,
                 generateProfileFile: generateProfileFile,
                 callChainProfile,
                 customPESectionAlignment);

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilation.cs

@@ -247,7 +247,7 @@ public sealed class ReadyToRunCodegenCompilation : Compilation
         private readonly string _pdbPath;
         private readonly bool _generatePerfMapFile;
         private readonly string _perfMapPath;
-        private readonly Guid? _perfMapMvid;
+        private readonly int _perfMapFormatVersion;
         private readonly bool _generateProfileFile;
         private readonly Func<MethodDesc, string> _printReproInstructions;
 
@@ -283,7 +283,7 @@ public sealed class ReadyToRunCodegenCompilation : Compilation
             string pdbPath,
             bool generatePerfMapFile,
             string perfMapPath,
-            Guid? perfMapMvid,
+            int perfMapFormatVersion,
             bool generateProfileFile,
             int parallelism,
             ProfileDataManager profileData,
@@ -309,7 +309,7 @@ public sealed class ReadyToRunCodegenCompilation : Compilation
             _pdbPath = pdbPath;
             _generatePerfMapFile = generatePerfMapFile;
             _perfMapPath = perfMapPath;
-            _perfMapMvid = perfMapMvid;
+            _perfMapFormatVersion = perfMapFormatVersion;
             _generateProfileFile = generateProfileFile;
             _customPESectionAlignment = customPESectionAlignment;
             SymbolNodeFactory = new ReadyToRunSymbolNodeFactory(nodeFactory, verifyTypeAndFieldLayout);
@@ -347,6 +347,7 @@ public override void Compile(string outputFile)
                 ReadyToRunObjectWriter.EmitObject(
                     outputFile,
                     componentModule: null,
+                    inputFiles: _inputFiles,
                     nodes,
                     NodeFactory,
                     generateMapFile: _generateMapFile,
@@ -355,7 +356,7 @@ public override void Compile(string outputFile)
                     pdbPath: _pdbPath,
                     generatePerfMapFile: _generatePerfMapFile,
                     perfMapPath: _perfMapPath,
-                    perfMapMvid: _perfMapMvid,
+                    perfMapFormatVersion: _perfMapFormatVersion,
                     generateProfileFile: _generateProfileFile,
                     callChainProfile: _profileData.CallChainProfile,
                     _customPESectionAlignment);
@@ -427,6 +428,7 @@ private void RewriteComponentFile(string inputFile, string outputFile, string ow
             ReadyToRunObjectWriter.EmitObject(
                 outputFile,
                 componentModule: inputModule,
+                inputFiles: new string[] { inputFile },
                 componentGraph.MarkedNodeList,
                 componentFactory,
                 generateMapFile: false,
@@ -435,7 +437,7 @@ private void RewriteComponentFile(string inputFile, string outputFile, string ow
                 pdbPath: null,
                 generatePerfMapFile: false,
                 perfMapPath: null,
-                perfMapMvid: null,
+                perfMapFormatVersion: _perfMapFormatVersion,
                 generateProfileFile: false,
                 _profileData.CallChainProfile,
                 customPESectionAlignment: 0);

src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs

@@ -31,7 +31,7 @@ public sealed class ReadyToRunCodegenCompilationBuilder : CompilationBuilder
         private string _pdbPath;
         private bool _generatePerfMapFile;
         private string _perfMapPath;
-        private Guid? _perfMapMvid;
+        private int _perfMapFormatVersion;
         private bool _generateProfileFile;
         private int _parallelism;
         Func<MethodDesc, string> _printReproInstructions;
@@ -156,11 +156,11 @@ public ReadyToRunCodegenCompilationBuilder UsePdbFile(bool generatePdbFile, stri
             return this;
         }
 
-        public ReadyToRunCodegenCompilationBuilder UsePerfMapFile(bool generatePerfMapFile, string perfMapPath, Guid? inputModuleMvid)
+        public ReadyToRunCodegenCompilationBuilder UsePerfMapFile(bool generatePerfMapFile, string perfMapPath, int perfMapFormatVersion)
         {
             _generatePerfMapFile = generatePerfMapFile;
             _perfMapPath = perfMapPath;
-            _perfMapMvid = inputModuleMvid;
+            _perfMapFormatVersion = perfMapFormatVersion;
             return this;
         }
 
@@ -312,7 +312,7 @@ public override ICompilation ToCompilation()
                 pdbPath: _pdbPath,
                 generatePerfMapFile: _generatePerfMapFile,
                 perfMapPath: _perfMapPath,
-                perfMapMvid: _perfMapMvid,
+                perfMapFormatVersion: _perfMapFormatVersion,
                 generateProfileFile: _generateProfileFile,
                 _parallelism,
                 _profileData,

src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/OutputInfoBuilder.cs

@@ -106,6 +106,7 @@ public OutputSymbol(int sectionIndex, int offset, string name)
     /// </summary>
     public class OutputInfoBuilder
     {
+        private readonly List<EcmaModule> _inputModules;
         private readonly List<OutputNode> _nodes;
         private readonly List<OutputSymbol> _symbols;
         private readonly List<Section> _sections;
@@ -117,6 +118,7 @@ public class OutputInfoBuilder
 
         public OutputInfoBuilder()
         {
+            _inputModules = new List<EcmaModule>();
             _nodes = new List<OutputNode>();
             _symbols = new List<OutputSymbol>();
             _sections = new List<Section>();
@@ -127,6 +129,11 @@ public OutputInfoBuilder()
             _relocCounts = new Dictionary<RelocType, int>();
         }
 
+        public void AddInputModule(EcmaModule module)
+        {
+            _inputModules.Add(module);
+        }
+
         public void AddNode(OutputNode node, ISymbolDefinitionNode symbol)
         {
             _nodes.Add(node);
@@ -197,6 +204,16 @@ public IEnumerable<MethodInfo> EnumerateMethods()
             }
         }
 
+        public IEnumerable<AssemblyInfo> EnumerateInputAssemblies()
+        {
+            foreach (EcmaModule inputModule in _inputModules)
+            {
+                yield return new AssemblyInfo(
+                    inputModule.Assembly.GetName().Name,
+                    inputModule.MetadataReader.GetGuid(inputModule.MetadataReader.GetModuleDefinition().Mvid));
+            }
+        }
+
         private string FormatMethodName(MethodDesc method, TypeNameFormatter typeNameFormatter)
         {
             StringBuilder output = new StringBuilder();

src/coreclr/tools/aot/ILCompiler.ReadyToRun/ObjectWriter/SymbolFileBuilder.cs

@@ -8,6 +8,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
+using Internal.TypeSystem;
 using ILCompiler.Diagnostics;
 
 namespace ILCompiler.PEWriter
@@ -28,12 +29,34 @@ public void SavePdb(string pdbPath, string dllFileName)
             new PdbWriter(pdbPath, PDBExtraData.None).WritePDBData(dllFileName, _outputInfoBuilder.EnumerateMethods());
         }
 
-        public void SavePerfMap(string perfMapPath, string dllFileName, Guid? perfMapMvid)
+        public void SavePerfMap(string perfMapPath, int perfMapFormatVersion, string dllFileName, TargetOS targetOS, TargetArchitecture targetArch)
         {
-            string mvidComponent = (perfMapMvid.HasValue ? perfMapMvid.Value.ToString() : "composite");
-            string perfMapFileName = Path.Combine(perfMapPath, Path.GetFileNameWithoutExtension(dllFileName) + ".ni.{" + mvidComponent + "}.map");
+            string perfMapExtension;
+            if (perfMapFormatVersion == PerfMapWriter.LegacyCrossgen1FormatVersion)
+            {
+                string mvidComponent = null;
+                foreach (AssemblyInfo inputAssembly in _outputInfoBuilder.EnumerateInputAssemblies())
+                {
+                    if (mvidComponent == null)
+                    {
+                        mvidComponent = inputAssembly.Mvid.ToString();
+                    }
+                    else
+                    {
+                        mvidComponent = "composite";
+                        break;
+                    }
+                }
+                perfMapExtension = ".ni.{" + mvidComponent + "}.map";
+            }
+            else
+            {
+                perfMapExtension = ".ni.r2rmap";
+            }
+
+            string perfMapFileName = Path.Combine(perfMapPath, Path.GetFileNameWithoutExtension(dllFileName) + perfMapExtension);
             Console.WriteLine("Emitting PerfMap file: {0}", perfMapFileName);
-            PerfMapWriter.Write(perfMapFileName, _outputInfoBuilder.EnumerateMethods());
+            PerfMapWriter.Write(perfMapFileName, perfMapFormatVersion, _outputInfoBuilder.EnumerateMethods(), _outputInfoBuilder.EnumerateInputAssemblies(), targetOS, targetArch);
         }
     }
 }