From a478cd481c4d996353a2009ce3eb3ebe475447b3 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 18 May 2022 15:06:29 -0600 Subject: [PATCH] Share MetadataIndex instances across multiple generators This should drastically cut down on memory demands (from 1.2GB to store 128 copies of MetadataIndex to just 1 of them). It should presumably improve warmup perf as well, since we only create one index instead of 128. --- src/Microsoft.Windows.CsWin32/Generator.cs | 21 +- .../MetadataIndex.cs | 274 +++++++++--------- .../NamespaceMetadata.cs | 6 + 3 files changed, 147 insertions(+), 154 deletions(-) diff --git a/src/Microsoft.Windows.CsWin32/Generator.cs b/src/Microsoft.Windows.CsWin32/Generator.cs index 146ad125..0e3bce7e 100644 --- a/src/Microsoft.Windows.CsWin32/Generator.cs +++ b/src/Microsoft.Windows.CsWin32/Generator.cs @@ -2,12 +2,12 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Immutable; -using System.Collections.ObjectModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Reflection; using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -312,6 +312,7 @@ public class Generator : IDisposable private readonly TypeSyntaxSettings functionPointerTypeSettings; private readonly TypeSyntaxSettings errorMessageTypeSettings; + private readonly PEReader peReader; private readonly GeneratorOptions options; private readonly CSharpCompilation? compilation; private readonly CSharpParseOptions? parseOptions; @@ -351,6 +352,8 @@ public Generator(string metadataLibraryPath, Docs? docs, GeneratorOptions option this.InputAssemblyName = Path.GetFileNameWithoutExtension(metadataLibraryPath); this.MetadataIndex = MetadataIndex.Get(metadataLibraryPath, compilation?.Options.Platform); this.ApiDocs = docs; + this.peReader = new PEReader(MetadataIndex.CreateFileView(metadataLibraryPath)); + this.Reader = this.peReader.GetMetadataReader(); this.options = options; this.options.Validate(); @@ -419,9 +422,7 @@ private enum FriendlyOverloadOf internal Docs? ApiDocs { get; } - internal ReadOnlyCollection Apis => this.MetadataIndex.Apis; - - internal MetadataReader Reader => this.MetadataIndex.Reader; + internal MetadataReader Reader { get; } internal LanguageVersion LanguageVersion => this.parseOptions?.LanguageVersion ?? LanguageVersion.CSharp9; @@ -713,7 +714,7 @@ public bool TryGetEnumName(string enumValueName, [NotNullWhen(true)] out string? /// A cancellation token. public void GenerateAllExternMethods(CancellationToken cancellationToken) { - foreach (MethodDefinitionHandle methodHandle in this.Apis.SelectMany(api => api.GetMethods())) + foreach (MethodDefinitionHandle methodHandle in this.MetadataIndex.Apis.SelectMany(api => this.Reader.GetTypeDefinition(api).GetMethods())) { cancellationToken.ThrowIfCancellationRequested(); @@ -741,7 +742,7 @@ public void GenerateAllExternMethods(CancellationToken cancellationToken) /// A cancellation token. public void GenerateAllConstants(CancellationToken cancellationToken) { - foreach (FieldDefinitionHandle fieldDefHandle in this.Apis.SelectMany(api => api.GetFields())) + foreach (FieldDefinitionHandle fieldDefHandle in this.MetadataIndex.Apis.SelectMany(api => this.Reader.GetTypeDefinition(api).GetFields())) { cancellationToken.ThrowIfCancellationRequested(); @@ -772,7 +773,7 @@ public void GenerateAllConstants(CancellationToken cancellationToken) public bool TryGenerateAllExternMethods(string moduleName, CancellationToken cancellationToken) { bool successful = false; - foreach (MethodDefinitionHandle methodHandle in this.Apis.SelectMany(api => api.GetMethods())) + foreach (MethodDefinitionHandle methodHandle in this.MetadataIndex.Apis.SelectMany(api => this.Reader.GetTypeDefinition(api).GetMethods())) { cancellationToken.ThrowIfCancellationRequested(); @@ -1885,7 +1886,7 @@ internal bool TryGetTypeDefHandle(TypeReferenceHandle typeRefHandle, out Qualifi return this.SuperGenerator.TryGetTypeDefinitionHandle(new QualifiedTypeReferenceHandle(this, typeRefHandle), out typeDefHandle); } - if (this.MetadataIndex.TryGetTypeDefHandle(typeRefHandle, out TypeDefinitionHandle localTypeDefHandle)) + if (this.MetadataIndex.TryGetTypeDefHandle(this.Reader, typeRefHandle, out TypeDefinitionHandle localTypeDefHandle)) { typeDefHandle = new QualifiedTypeDefinitionHandle(this, localTypeDefHandle); return true; @@ -1895,7 +1896,7 @@ internal bool TryGetTypeDefHandle(TypeReferenceHandle typeRefHandle, out Qualifi return false; } - internal bool TryGetTypeDefHandle(TypeReferenceHandle typeRefHandle, out TypeDefinitionHandle typeDefHandle) => this.MetadataIndex.TryGetTypeDefHandle(typeRefHandle, out typeDefHandle); + internal bool TryGetTypeDefHandle(TypeReferenceHandle typeRefHandle, out TypeDefinitionHandle typeDefHandle) => this.MetadataIndex.TryGetTypeDefHandle(this.Reader, typeRefHandle, out typeDefHandle); internal bool TryGetTypeDefHandle(TypeReference typeRef, out TypeDefinitionHandle typeDefHandle) => this.TryGetTypeDefHandle(typeRef.Namespace, typeRef.Name, out typeDefHandle); @@ -2048,7 +2049,7 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - MetadataIndex.Return(this.MetadataIndex); + this.peReader.Dispose(); } } diff --git a/src/Microsoft.Windows.CsWin32/MetadataIndex.cs b/src/Microsoft.Windows.CsWin32/MetadataIndex.cs index e21c1b98..354528b9 100644 --- a/src/Microsoft.Windows.CsWin32/MetadataIndex.cs +++ b/src/Microsoft.Windows.CsWin32/MetadataIndex.cs @@ -12,10 +12,22 @@ namespace Microsoft.Windows.CsWin32; +/// +/// A cached, shareable index into a particular metadata file. +/// +/// +/// This class must not store anything to do with a , +/// since that is attached to a stream which will not allow for concurrent use. +/// This means we cannot store definitions (e.g. ) +/// because they store the as a field. +/// We can store handles though (e.g. , since +/// the only thing they store is an index into the metadata, which is constant across +/// instances for a given metadata file. +/// [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] -internal class MetadataIndex : IDisposable +internal class MetadataIndex { - private static readonly Dictionary> Cache = new(); + private static readonly Dictionary Cache = new(); /// /// A cache of metadata files read. @@ -27,13 +39,7 @@ internal class MetadataIndex : IDisposable private readonly Platform? platform; - private readonly Stream metadataStream; - - private readonly PEReader peReader; - - private readonly MetadataReader mr; - - private readonly List apis = new(); + private readonly List apis = new(); private readonly HashSet releaseMethods = new HashSet(StringComparer.Ordinal); @@ -50,160 +56,153 @@ internal class MetadataIndex : IDisposable /// private readonly Dictionary handleTypeReleaseMethod = new(); - private MetadataIndex(string metadataPath, Stream metadataStream, Platform? platform) + /// + /// Initializes a new instance of the class. + /// + /// The path to the metadata that this index will represent. + /// The platform filter to apply when reading the metadata. + private MetadataIndex(string metadataPath, Platform? platform) { this.metadataPath = metadataPath; this.platform = platform; - try - { - this.metadataStream = metadataStream; - this.peReader = new PEReader(this.metadataStream); - this.mr = this.peReader.GetMetadataReader(); + using PEReader peReader = new PEReader(CreateFileView(metadataPath)); + MetadataReader mr = peReader.GetMetadataReader(); - foreach (MemberReferenceHandle memberRefHandle in this.mr.MemberReferences) + foreach (MemberReferenceHandle memberRefHandle in mr.MemberReferences) + { + MemberReference memberReference = mr.GetMemberReference(memberRefHandle); + if (memberReference.GetKind() == MemberReferenceKind.Method) { - MemberReference memberReference = this.mr.GetMemberReference(memberRefHandle); - if (memberReference.GetKind() == MemberReferenceKind.Method) + if (memberReference.Parent.Kind == HandleKind.TypeReference) { - if (memberReference.Parent.Kind == HandleKind.TypeReference) + if (mr.StringComparer.Equals(memberReference.Name, ".ctor")) { - if (this.mr.StringComparer.Equals(memberReference.Name, ".ctor")) + var trh = (TypeReferenceHandle)memberReference.Parent; + TypeReference tr = mr.GetTypeReference(trh); + if (mr.StringComparer.Equals(tr.Name, "SupportedArchitectureAttribute") && + mr.StringComparer.Equals(tr.Namespace, "Windows.Win32.Interop")) { - var trh = (TypeReferenceHandle)memberReference.Parent; - TypeReference tr = this.mr.GetTypeReference(trh); - if (this.mr.StringComparer.Equals(tr.Name, "SupportedArchitectureAttribute") && - this.mr.StringComparer.Equals(tr.Namespace, "Windows.Win32.Interop")) - { - this.SupportedArchitectureAttributeCtor = memberRefHandle; - break; - } + this.SupportedArchitectureAttributeCtor = memberRefHandle; + break; } } } } + } - void PopulateNamespace(NamespaceDefinition ns, string? parentNamespace) - { - string nsLeafName = this.mr.GetString(ns.Name); - string nsFullName = string.IsNullOrEmpty(parentNamespace) ? nsLeafName : $"{parentNamespace}.{nsLeafName}"; + void PopulateNamespace(NamespaceDefinition ns, string? parentNamespace) + { + string nsLeafName = mr.GetString(ns.Name); + string nsFullName = string.IsNullOrEmpty(parentNamespace) ? nsLeafName : $"{parentNamespace}.{nsLeafName}"; - var nsMetadata = new NamespaceMetadata(nsFullName); + var nsMetadata = new NamespaceMetadata(nsFullName); - foreach (TypeDefinitionHandle tdh in ns.TypeDefinitions) + foreach (TypeDefinitionHandle tdh in ns.TypeDefinitions) + { + TypeDefinition td = mr.GetTypeDefinition(tdh); + string typeName = mr.GetString(td.Name); + if (typeName == "Apis") { - TypeDefinition td = this.mr.GetTypeDefinition(tdh); - string typeName = this.mr.GetString(td.Name); - if (typeName == "Apis") + this.apis.Add(tdh); + foreach (MethodDefinitionHandle methodDefHandle in td.GetMethods()) { - this.apis.Add(td); - foreach (MethodDefinitionHandle methodDefHandle in td.GetMethods()) + MethodDefinition methodDef = mr.GetMethodDefinition(methodDefHandle); + string methodName = mr.GetString(methodDef.Name); + if (MetadataUtilities.IsCompatibleWithPlatform(mr, this, platform, methodDef.GetCustomAttributes())) { - MethodDefinition methodDef = this.mr.GetMethodDefinition(methodDefHandle); - string methodName = this.mr.GetString(methodDef.Name); - if (MetadataUtilities.IsCompatibleWithPlatform(this.mr, this, platform, methodDef.GetCustomAttributes())) - { - nsMetadata.Methods.Add(methodName, methodDefHandle); - } - else - { - nsMetadata.MethodsForOtherPlatform.Add(methodName); - } + nsMetadata.Methods.Add(methodName, methodDefHandle); } - - foreach (FieldDefinitionHandle fieldDefHandle in td.GetFields()) + else { - FieldDefinition fieldDef = this.mr.GetFieldDefinition(fieldDefHandle); - const FieldAttributes expectedFlags = FieldAttributes.Static | FieldAttributes.Public; - if ((fieldDef.Attributes & expectedFlags) == expectedFlags) - { - string fieldName = this.mr.GetString(fieldDef.Name); - nsMetadata.Fields.Add(fieldName, fieldDefHandle); - } + nsMetadata.MethodsForOtherPlatform.Add(methodName); } } - else if (typeName == "") + + foreach (FieldDefinitionHandle fieldDefHandle in td.GetFields()) { + FieldDefinition fieldDef = mr.GetFieldDefinition(fieldDefHandle); + const FieldAttributes expectedFlags = FieldAttributes.Static | FieldAttributes.Public; + if ((fieldDef.Attributes & expectedFlags) == expectedFlags) + { + string fieldName = mr.GetString(fieldDef.Name); + nsMetadata.Fields.Add(fieldName, fieldDefHandle); + } } - else if (MetadataUtilities.IsCompatibleWithPlatform(this.mr, this, platform, td.GetCustomAttributes())) - { - nsMetadata.Types.Add(typeName, tdh); + } + else if (typeName == "") + { + } + else if (MetadataUtilities.IsCompatibleWithPlatform(mr, this, platform, td.GetCustomAttributes())) + { + nsMetadata.Types.Add(typeName, tdh); - // Detect if this is a struct representing a native handle. - if (td.GetFields().Count == 1 && td.BaseType.Kind == HandleKind.TypeReference) + // Detect if this is a struct representing a native handle. + if (td.GetFields().Count == 1 && td.BaseType.Kind == HandleKind.TypeReference) + { + TypeReference baseType = mr.GetTypeReference((TypeReferenceHandle)td.BaseType); + if (mr.StringComparer.Equals(baseType.Name, nameof(ValueType)) && mr.StringComparer.Equals(baseType.Namespace, nameof(System))) { - TypeReference baseType = this.mr.GetTypeReference((TypeReferenceHandle)td.BaseType); - if (this.mr.StringComparer.Equals(baseType.Name, nameof(ValueType)) && this.mr.StringComparer.Equals(baseType.Namespace, nameof(System))) + foreach (CustomAttributeHandle h in td.GetCustomAttributes()) { - foreach (CustomAttributeHandle h in td.GetCustomAttributes()) + CustomAttribute att = mr.GetCustomAttribute(h); + if (MetadataUtilities.IsAttribute(mr, att, Generator.InteropDecorationNamespace, Generator.RAIIFreeAttribute)) { - CustomAttribute att = this.mr.GetCustomAttribute(h); - if (MetadataUtilities.IsAttribute(this.mr, att, Generator.InteropDecorationNamespace, Generator.RAIIFreeAttribute)) + CustomAttributeValue args = att.DecodeValue(CustomAttributeTypeProvider.Instance); + if (args.FixedArguments[0].Value is string freeMethodName) { - CustomAttributeValue args = att.DecodeValue(CustomAttributeTypeProvider.Instance); - if (args.FixedArguments[0].Value is string freeMethodName) + this.handleTypeReleaseMethod.Add(tdh, freeMethodName); + this.releaseMethods.Add(freeMethodName); + + using FieldDefinitionHandleCollection.Enumerator fieldEnum = td.GetFields().GetEnumerator(); + fieldEnum.MoveNext(); + FieldDefinitionHandle fieldHandle = fieldEnum.Current; + FieldDefinition fieldDef = mr.GetFieldDefinition(fieldHandle); + if (fieldDef.DecodeSignature(SignatureHandleProvider.Instance, null) is PrimitiveTypeHandleInfo { PrimitiveTypeCode: PrimitiveTypeCode.IntPtr or PrimitiveTypeCode.UIntPtr }) { - this.handleTypeReleaseMethod.Add(tdh, freeMethodName); - this.releaseMethods.Add(freeMethodName); - - using FieldDefinitionHandleCollection.Enumerator fieldEnum = td.GetFields().GetEnumerator(); - fieldEnum.MoveNext(); - FieldDefinitionHandle fieldHandle = fieldEnum.Current; - FieldDefinition fieldDef = this.mr.GetFieldDefinition(fieldHandle); - if (fieldDef.DecodeSignature(SignatureHandleProvider.Instance, null) is PrimitiveTypeHandleInfo { PrimitiveTypeCode: PrimitiveTypeCode.IntPtr or PrimitiveTypeCode.UIntPtr }) - { - this.handleTypeStructsWithIntPtrSizeFields.Add(typeName); - } + this.handleTypeStructsWithIntPtrSizeFields.Add(typeName); } - - break; } + + break; } } } } - else - { - nsMetadata.TypesForOtherPlatform.Add(typeName); - } - } - - if (!nsMetadata.IsEmpty) - { - this.MetadataByNamespace.Add(nsFullName, nsMetadata); } - - foreach (NamespaceDefinitionHandle childNsHandle in ns.NamespaceDefinitions) + else { - PopulateNamespace(this.mr.GetNamespaceDefinition(childNsHandle), nsFullName); + nsMetadata.TypesForOtherPlatform.Add(typeName); } } - foreach (NamespaceDefinitionHandle childNsHandle in this.mr.GetNamespaceDefinitionRoot().NamespaceDefinitions) + if (!nsMetadata.IsEmpty) { - PopulateNamespace(this.mr.GetNamespaceDefinitionRoot(), parentNamespace: null); + this.MetadataByNamespace.Add(nsFullName, nsMetadata); } - this.CommonNamespace = CommonPrefix(this.MetadataByNamespace.Keys.ToList()); - if (this.CommonNamespace[this.CommonNamespace.Length - 1] == '.') - { - this.CommonNamespaceDot = this.CommonNamespace; - this.CommonNamespace = this.CommonNamespace.Substring(0, this.CommonNamespace.Length - 1); - } - else + foreach (NamespaceDefinitionHandle childNsHandle in ns.NamespaceDefinitions) { - this.CommonNamespaceDot = this.CommonNamespace + "."; + PopulateNamespace(mr.GetNamespaceDefinition(childNsHandle), nsFullName); } } - catch + + foreach (NamespaceDefinitionHandle childNsHandle in mr.GetNamespaceDefinitionRoot().NamespaceDefinitions) { - this.peReader?.Dispose(); - this.metadataStream?.Dispose(); - throw; + PopulateNamespace(mr.GetNamespaceDefinitionRoot(), parentNamespace: null); } - } - internal MetadataReader Reader => this.mr; + this.CommonNamespace = CommonPrefix(this.MetadataByNamespace.Keys.ToList()); + if (this.CommonNamespace[this.CommonNamespace.Length - 1] == '.') + { + this.CommonNamespaceDot = this.CommonNamespace; + this.CommonNamespace = this.CommonNamespace.Substring(0, this.CommonNamespace.Length - 1); + } + else + { + this.CommonNamespaceDot = this.CommonNamespace + "."; + } + } /// /// Gets the ref handle to the constructor on the SupportedArchitectureAttribute, if there is one. @@ -213,7 +212,7 @@ void PopulateNamespace(NamespaceDefinition ns, string? parentNamespace) /// /// Gets the "Apis" classes across all namespaces. /// - internal ReadOnlyCollection Apis => new(this.apis); + internal ReadOnlyCollection Apis => new(this.apis); /// /// Gets a dictionary of namespace metadata, indexed by the string handle to their namespace. @@ -230,77 +229,64 @@ void PopulateNamespace(NamespaceDefinition ns, string? parentNamespace) private string DebuggerDisplay => $"{this.metadataPath} ({this.platform})"; - public void Dispose() - { - this.peReader.Dispose(); - this.metadataStream.Dispose(); - } - internal static MetadataIndex Get(string metadataPath, Platform? platform) { metadataPath = Path.GetFullPath(metadataPath); - var key = new CacheKey(metadataPath, platform); - MemoryMappedViewStream metadataBytes; + CacheKey key = new(metadataPath, platform); lock (Cache) { - if (Cache.TryGetValue(key, out Stack stack) && stack.Count > 0) + if (!Cache.TryGetValue(key, out MetadataIndex index)) { - return stack.Pop(); + Cache.Add(key, index = new MetadataIndex(metadataPath, platform)); } - // Read the entire metadata file exactly once so that many MemoryStreams can share the memory. - if (!MetadataFiles.TryGetValue(metadataPath, out MemoryMappedFile? file)) - { - var metadataStream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read); - file = MemoryMappedFile.CreateFromFile(metadataStream, mapName: null, capacity: 0, MemoryMappedFileAccess.Read, HandleInheritability.None, leaveOpen: false); - MetadataFiles.Add(metadataPath, file); - } - - metadataBytes = file.CreateViewStream(offset: 0, size: 0, MemoryMappedFileAccess.Read); + return index; } - - return new MetadataIndex(metadataPath, metadataBytes, platform); } - internal static void Return(MetadataIndex index) + internal static MemoryMappedViewStream CreateFileView(string metadataPath) { - var key = new CacheKey(index.metadataPath, index.platform); lock (Cache) { - if (!Cache.TryGetValue(key, out Stack stack)) + // We use a memory mapped file so that many threads can perform random access on it concurrently, + // only mapping the file into memory once. + if (!MetadataFiles.TryGetValue(metadataPath, out MemoryMappedFile? file)) { - Cache.Add(key, stack = new Stack()); + var metadataStream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read); + file = MemoryMappedFile.CreateFromFile(metadataStream, mapName: null, capacity: 0, MemoryMappedFileAccess.Read, HandleInheritability.None, leaveOpen: false); + MetadataFiles.Add(metadataPath, file); } - stack.Push(index); + return file.CreateViewStream(offset: 0, size: 0, MemoryMappedFileAccess.Read); } } /// /// Attempts to translate a to a . /// + /// The metadata reader to use. /// The reference handle. /// Receives the type def handle, if one was discovered. /// if a TypeDefinition was found; otherwise . - internal bool TryGetTypeDefHandle(TypeReferenceHandle typeRefHandle, out TypeDefinitionHandle typeDefHandle) + internal bool TryGetTypeDefHandle(MetadataReader reader, TypeReferenceHandle typeRefHandle, out TypeDefinitionHandle typeDefHandle) { if (this.refToDefCache.TryGetValue(typeRefHandle, out typeDefHandle)) { return !typeDefHandle.IsNil; } - TypeReference typeRef = this.Reader.GetTypeReference(typeRefHandle); + TypeReference typeRef = reader.GetTypeReference(typeRefHandle); if (typeRef.ResolutionScope.Kind != HandleKind.AssemblyReference) { - foreach (TypeDefinitionHandle tdh in this.Reader.TypeDefinitions) + foreach (TypeDefinitionHandle tdh in reader.TypeDefinitions) { - TypeDefinition typeDef = this.Reader.GetTypeDefinition(tdh); + TypeDefinition typeDef = reader.GetTypeDefinition(tdh); if (typeDef.Name == typeRef.Name && typeDef.Namespace == typeRef.Namespace) { if (typeRef.ResolutionScope.Kind == HandleKind.TypeReference) { // The ref is nested. Verify that the type we found is nested in the same type as well. - if (this.TryGetTypeDefHandle((TypeReferenceHandle)typeRef.ResolutionScope, out TypeDefinitionHandle nestingTypeDef) && nestingTypeDef == typeDef.GetDeclaringType()) + if (this.TryGetTypeDefHandle(reader, (TypeReferenceHandle)typeRef.ResolutionScope, out TypeDefinitionHandle nestingTypeDef) && nestingTypeDef == typeDef.GetDeclaringType()) { typeDefHandle = tdh; break; diff --git a/src/Microsoft.Windows.CsWin32/NamespaceMetadata.cs b/src/Microsoft.Windows.CsWin32/NamespaceMetadata.cs index 9ebe9047..c04dbb98 100644 --- a/src/Microsoft.Windows.CsWin32/NamespaceMetadata.cs +++ b/src/Microsoft.Windows.CsWin32/NamespaceMetadata.cs @@ -6,6 +6,12 @@ namespace Microsoft.Windows.CsWin32; +/// +/// An immutable index into metadata. +/// +/// +/// This class must not contain definitions. It may contain handles. See devremarks for details. +/// [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] internal class NamespaceMetadata {