Linker to NativeAOT sync (#71485)

This brings changes from the linker repo to ilc roughly as of a week ago. This required some non-trivial changes to the code either already in ilc or the one coming from linker:
* Port of CompilerGeneratedState
  * Modify it to use lock free hashtable (since it's a global cache and thus needs to be thread safe)
  * In ILC it requires to operate on generic definitions of everything - so add asserts and code to guarantee this
  * It's not a static class anymore, so pass the instance around as necessary
  * Requires IL access, so needs ILProvider on input
* Cyclic dependency between Logger and CompilerGeneratedState, one needs the instance of the other to work. For now solved by setting it through a property on CompilerGeneratedState
* Split of ReflectionMethodBodyScanner into AttributeDataFlow and GenericParameterDataFlow - mostly direct port from linker
  * The AttributeDataFlow works slightly differently in ilc so adopted the code there
* Changes to DiagnosticContext
  * Can selectively disable Trim, AOT, Single-File diagnostics - used to correctly apply RUC, RDC and RAF suppressions
* Improve handling of RUC and similar warnings in ReflectionMarker
* Added ILOffset to MessageOrigin
  * Even though we already store file/column, the ILOffset is necessary since we use the MessageOrigin as a key in dictionary for recorded patterns and thus we need a way to make the origin unique for each callsite (and file/line/column might not be unique or if there's no PDB those will be null anyway).
* Enable Equality/HashCode implementation on MessageOrigin, still no CompareTo since we don't need it for anything right now.

eng/Versions.props

@@ -50,6 +50,7 @@
       TODO: Remove pinned version once arcade supplies a compiler that enables the repo to compile.
     -->
     <MicrosoftNetCompilersToolsetVersion>4.4.0-1.22358.14</MicrosoftNetCompilersToolsetVersion>
+    <StaticCsVersion>0.1.0</StaticCsVersion>
     <!-- SDK dependencies -->
     <MicrosoftDotNetCompatibilityVersion>2.0.0-preview.4.22252.4</MicrosoftDotNetCompatibilityVersion>
     <!-- Arcade dependencies -->

eng/testing/tests.singlefile.targets

@@ -26,7 +26,7 @@
     <IlcBuildTasksPath>$(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll</IlcBuildTasksPath>
     <IlcSdkPath>$(CoreCLRAotSdkDir)</IlcSdkPath>
     <IlcFrameworkPath>$(NetCoreAppCurrentTestHostSharedFrameworkPath)</IlcFrameworkPath>
-    <NoWarn>$(NoWarn);IL3050;IL3051;IL3052;IL3055;IL1005</NoWarn>
+    <NoWarn>$(NoWarn);IL3050;IL3051;IL3052;IL3055;IL1005;IL3002</NoWarn>
     <TrimmerSingleWarn>false</TrimmerSingleWarn>
     <SuppressTrimAnalysisWarnings>true</SuppressTrimAnalysisWarnings>
 

src/coreclr/tools/aot/ILCompiler.Compiler.Tests/DependencyGraphTests.cs

@@ -3,7 +3,7 @@
 
 using System;
 using System.Collections.Generic;
-
+using ILCompiler.Dataflow;
 using Internal.IL;
 using Internal.TypeSystem;
 using Internal.TypeSystem.Ecma;
@@ -67,11 +67,12 @@ public void TestDependencyGraphInvariants(EcmaMethod method)
             CompilationModuleGroup compilationGroup = new SingleFileCompilationModuleGroup();
 
             NativeAotILProvider ilProvider = new NativeAotILProvider();
+            CompilerGeneratedState compilerGeneratedState = new CompilerGeneratedState(ilProvider, Logger.Null);
 
             UsageBasedMetadataManager metadataManager = new UsageBasedMetadataManager(compilationGroup, context,
                 new FullyBlockedMetadataBlockingPolicy(), new FullyBlockedManifestResourceBlockingPolicy(),
                 null, new NoStackTraceEmissionPolicy(), new NoDynamicInvokeThunkGenerationPolicy(),
-                new ILLink.Shared.TrimAnalysis.FlowAnnotations(Logger.Null, ilProvider), UsageBasedMetadataGenerationOptions.None,
+                new ILLink.Shared.TrimAnalysis.FlowAnnotations(Logger.Null, ilProvider, compilerGeneratedState), UsageBasedMetadataGenerationOptions.None,
                 Logger.Null, Array.Empty<KeyValuePair<string, bool>>(), Array.Empty<string>(), Array.Empty<string>());
 
             CompilationBuilder builder = new RyuJitCompilationBuilder(context, compilationGroup)

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/Dataflow/ArrayValue.cs

@@ -27,9 +27,9 @@ public static MultiValue Create(MultiValue size, TypeDesc elementType)
             return result;
         }
 
-        public static MultiValue Create(int size, TypeDesc elementType)
+        public static ArrayValue Create(int size, TypeDesc elementType)
         {
-            return new MultiValue(new ArrayValue(new ConstIntValue(size), elementType));
+            return new ArrayValue(new ConstIntValue(size), elementType);
         }
 
         /// <summary>

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/Dataflow/AttributeDataFlow.cs

@@ -0,0 +1,140 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Reflection.Metadata;
+
+using ILCompiler.DependencyAnalysis;
+using ILCompiler.Logging;
+
+using ILLink.Shared.TrimAnalysis;
+
+using Internal.TypeSystem;
+
+using CustomAttributeValue = System.Reflection.Metadata.CustomAttributeValue<Internal.TypeSystem.TypeDesc>;
+using DependencyList = ILCompiler.DependencyAnalysisFramework.DependencyNodeCore<ILCompiler.DependencyAnalysis.NodeFactory>.DependencyList;
+using MultiValue = ILLink.Shared.DataFlow.ValueSet<ILLink.Shared.DataFlow.SingleValue>;
+
+#nullable enable
+
+namespace ILCompiler.Dataflow
+{
+    public readonly struct AttributeDataFlow
+    {
+        readonly Logger _logger;
+        readonly NodeFactory _factory;
+        readonly FlowAnnotations _annotations;
+        readonly MessageOrigin _origin;
+
+        public AttributeDataFlow(Logger logger, NodeFactory factory, FlowAnnotations annotations, in MessageOrigin origin)
+        {
+            _annotations = annotations;
+            _factory = factory;
+            _logger = logger;
+            _origin = origin;
+        }
+
+        public DependencyList? ProcessAttributeDataflow(MethodDesc method, CustomAttributeValue arguments)
+        {
+            DependencyList? result = null;
+
+            // First do the dataflow for the constructor parameters if necessary.
+            if (_annotations.RequiresDataflowAnalysis(method))
+            {
+                var builder = ImmutableArray.CreateBuilder<object?>(arguments.FixedArguments.Length);
+                foreach (var argument in arguments.FixedArguments)
+                {
+                    builder.Add(argument.Value);
+                }
+
+                ProcessAttributeDataflow(method, builder.ToImmutableArray(), ref result);
+            }
+
+            // Named arguments next
+            TypeDesc attributeType = method.OwningType;
+            foreach (var namedArgument in arguments.NamedArguments)
+            {
+                if (namedArgument.Kind == CustomAttributeNamedArgumentKind.Field)
+                {
+                    FieldDesc field = attributeType.GetField(namedArgument.Name);
+                    if (field != null)
+                    {
+                        ProcessAttributeDataflow(field, namedArgument.Value, ref result);
+                    }
+                }
+                else
+                {
+                    Debug.Assert(namedArgument.Kind == CustomAttributeNamedArgumentKind.Property);
+                    PropertyPseudoDesc property = ((MetadataType)attributeType).GetProperty(namedArgument.Name, null);
+                    MethodDesc setter = property.SetMethod;
+                    if (setter != null && setter.Signature.Length > 0 && !setter.Signature.IsStatic)
+                    {
+                        ProcessAttributeDataflow(setter, ImmutableArray.Create(namedArgument.Value), ref result);
+                    }
+                }
+            }
+
+            return result;
+        }
+
+        void ProcessAttributeDataflow(MethodDesc method, ImmutableArray<object?> arguments, ref DependencyList? result)
+        {
+            for (int i = 0; i < method.Signature.Length; i++)
+            {
+                var parameterValue = _annotations.GetMethodParameterValue(method, i);
+                if (parameterValue.DynamicallyAccessedMemberTypes != DynamicallyAccessedMemberTypes.None)
+                {
+                    MultiValue value = GetValueForCustomAttributeArgument(arguments[i]);
+                    var diagnosticContext = new DiagnosticContext(_origin, diagnosticsEnabled: true, _logger);
+                    RequireDynamicallyAccessedMembers(diagnosticContext, value, parameterValue, parameterValue.ParameterOrigin, ref result);
+                }
+            }
+        }
+
+        public void ProcessAttributeDataflow(FieldDesc field, object? value, ref DependencyList? result)
+        {
+            var fieldValueCandidate = _annotations.GetFieldValue(field);
+            if (fieldValueCandidate is ValueWithDynamicallyAccessedMembers fieldValue
+                && fieldValue.DynamicallyAccessedMemberTypes != DynamicallyAccessedMemberTypes.None)
+            {
+                MultiValue valueNode = GetValueForCustomAttributeArgument(value);
+                var diagnosticContext = new DiagnosticContext(_origin, diagnosticsEnabled: true, _logger);
+                RequireDynamicallyAccessedMembers(diagnosticContext, valueNode, fieldValue, new FieldOrigin(field), ref result);
+            }
+        }
+
+        MultiValue GetValueForCustomAttributeArgument(object? argument)
+            => argument switch
+            {
+                TypeDesc td => new SystemTypeValue(td),
+                string str => new KnownStringValue(str),
+                null => NullValue.Instance,
+                // We shouldn't have gotten a None annotation from flow annotations since only string/Type can have annotations
+                _ => throw new InvalidOperationException()
+            };
+
+        void RequireDynamicallyAccessedMembers(
+            in DiagnosticContext diagnosticContext,
+            in MultiValue value,
+            ValueWithDynamicallyAccessedMembers targetValue,
+            Origin memberWithRequirements,
+            ref DependencyList? result)
+        {
+            var reflectionMarker = new ReflectionMarker(_logger, _factory, _annotations, typeHierarchyDataFlow: false, enabled: true);
+            var requireDynamicallyAccessedMembersAction = new RequireDynamicallyAccessedMembersAction(reflectionMarker, diagnosticContext, memberWithRequirements);
+            requireDynamicallyAccessedMembersAction.Invoke(value, targetValue);
+
+            if (result == null)
+            {
+                result = reflectionMarker.Dependencies;
+            }
+            else
+            {
+                result.AddRange(reflectionMarker.Dependencies);
+            }
+        }
+    }
+}

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/Dataflow/CompilerGeneratedCallGraph.cs

@@ -0,0 +1,77 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+
+using Internal.TypeSystem;
+using Internal.TypeSystem.Ecma;
+
+#nullable enable
+
+namespace ILCompiler.Dataflow
+{
+    sealed class CompilerGeneratedCallGraph
+    {
+        readonly Dictionary<TypeSystemEntity, HashSet<TypeSystemEntity>> _callGraph;
+
+        public CompilerGeneratedCallGraph() => _callGraph = new Dictionary<TypeSystemEntity, HashSet<TypeSystemEntity>>();
+
+        void TrackCallInternal(TypeSystemEntity fromMember, TypeSystemEntity toMember)
+        {
+            if (!_callGraph.TryGetValue(fromMember, out HashSet<TypeSystemEntity>? toMembers))
+            {
+                toMembers = new HashSet<TypeSystemEntity>();
+                _callGraph.Add(fromMember, toMembers);
+            }
+            toMembers.Add(toMember);
+        }
+
+        public void TrackCall(MethodDesc fromMethod, MethodDesc toMethod)
+        {
+            Debug.Assert(fromMethod.IsTypicalMethodDefinition);
+            Debug.Assert(toMethod.IsTypicalMethodDefinition);
+            Debug.Assert(CompilerGeneratedNames.IsLambdaOrLocalFunction(toMethod.Name));
+            TrackCallInternal(fromMethod, toMethod);
+        }
+
+        public void TrackCall(MethodDesc fromMethod, DefType toType)
+        {
+            Debug.Assert(fromMethod.IsTypicalMethodDefinition);
+            Debug.Assert(toType.IsTypeDefinition);
+            Debug.Assert(CompilerGeneratedNames.IsStateMachineType(toType.Name));
+            TrackCallInternal(fromMethod, toType);
+        }
+
+        public void TrackCall(DefType fromType, MethodDesc toMethod)
+        {
+            Debug.Assert(fromType.IsTypeDefinition);
+            Debug.Assert(toMethod.IsTypicalMethodDefinition);
+            Debug.Assert(CompilerGeneratedNames.IsStateMachineType(fromType.Name));
+            Debug.Assert(CompilerGeneratedNames.IsLambdaOrLocalFunction(toMethod.Name));
+            TrackCallInternal(fromType, toMethod);
+        }
+
+        public IEnumerable<TypeSystemEntity> GetReachableMembers(MethodDesc start)
+        {
+            Queue<TypeSystemEntity> queue = new();
+            HashSet<TypeSystemEntity> visited = new();
+            visited.Add(start);
+            queue.Enqueue(start);
+            while (queue.TryDequeue(out TypeSystemEntity? method))
+            {
+                if (!_callGraph.TryGetValue(method, out HashSet<TypeSystemEntity>? callees))
+                    continue;
+
+                foreach (var callee in callees)
+                {
+                    if (visited.Add(callee))
+                    {
+                        queue.Enqueue(callee);
+                        yield return callee;
+                    }
+                }
+            }
+        }
+    }
+}

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/Dataflow/CompilerGeneratedNames.cs

@@ -0,0 +1,74 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#nullable enable
+
+namespace ILCompiler.Dataflow
+{
+    sealed class CompilerGeneratedNames
+    {
+        internal static bool IsGeneratedMemberName(string memberName)
+        {
+            return memberName.Length > 0 && memberName[0] == '<';
+        }
+
+        internal static bool IsLambdaDisplayClass(string className)
+        {
+            if (!IsGeneratedMemberName(className))
+                return false;
+
+            // This is true for static lambdas (which are emitted into a class like <>c)
+            // and for instance lambdas (which are emitted into a class like <>c__DisplayClass1_0)
+            return className.StartsWith("<>c");
+        }
+
+        internal static bool IsStateMachineType(string typeName)
+        {
+            if (!IsGeneratedMemberName(typeName))
+                return false;
+
+            // State machines are generated into types with names like <OwnerMethodName>d__0
+            // Or if its nested in a local function the name will look like <<OwnerMethodName>g__Local>d and so on
+            int i = typeName.LastIndexOf('>');
+            if (i == -1)
+                return false;
+
+            return typeName.Length > i + 1 && typeName[i + 1] == 'd';
+        }
+
+        internal static bool IsGeneratedType(string name) => IsStateMachineType(name) || IsLambdaDisplayClass(name);
+
+        internal static bool IsLambdaOrLocalFunction(string methodName) => IsLambdaMethod(methodName) || IsLocalFunction(methodName);
+
+        // Lambda methods have generated names like "<UserMethod>b__0_1" where "UserMethod" is the name
+        // of the original user code that contains the lambda method declaration.
+        internal static bool IsLambdaMethod(string methodName)
+        {
+            if (!IsGeneratedMemberName(methodName))
+                return false;
+
+            int i = methodName.IndexOf('>', 1);
+            if (i == -1)
+                return false;
+
+            // Ignore the method ordinal/generation and lambda ordinal/generation.
+            return methodName.Length > i + 1 && methodName[i + 1] == 'b';
+        }
+
+        // Local functions have generated names like "<UserMethod>g__LocalFunction|0_1" where "UserMethod" is the name
+        // of the original user code that contains the lambda method declaration, and "LocalFunction" is the name of
+        // the local function.
+        internal static bool IsLocalFunction(string methodName)
+        {
+            if (!IsGeneratedMemberName(methodName))
+                return false;
+
+            int i = methodName.IndexOf('>', 1);
+            if (i == -1)
+                return false;
+
+            // Ignore the method ordinal/generation and local function ordinal/generation.
+            return methodName.Length > i + 1 && methodName[i + 1] == 'g';
+        }
+    }
+}