Adjust IL scanner for generic inlining

src/ILCompiler.Compiler/src/IL/ILImporter.Scanner.cs

@@ -568,6 +568,33 @@ private void ImportCall(ILOpcode opcode, int token)
                             _dependencies.Add(instParam, reason);
                         }
 
+                        if (instParam == null
+                            && !targetMethod.OwningType.IsValueType
+                            && !_factory.TypeSystemContext.IsSpecialUnboxingThunk(_canonMethod))
+                        {
+                            // We have a call to a shared instance method and we're already in a shared context.
+                            // e.g. this is a call to Foo<T>.Method() and we're about to add Foo<__Canon>.Method()
+                            // to the dependency graph).
+                            //
+                            // We will pretend the runtime determined owning type (Foo<T>) got allocated as well.
+                            // This is because RyuJIT might end up inlining the shared method body, making it concrete again,
+                            // without actually having to go through a dictionary.
+                            // (This would require inlining across two generic contexts, but RyuJIT does that.)
+                            //
+                            // If we didn't have a constructed type for this at the scanning time, we wouldn't
+                            // know the dictionary dependencies at the inlined site, leading to a compile failure.
+                            // (Remember that dictionary dependencies of instance methods on generic reference types
+                            // are tied to the owning type.)
+                            //
+                            // This is not ideal, because if e.g. Foo<string> never got allocated otherwise, this code is
+                            // unreachable and we're making the scanner scan more of it.
+                            //
+                            // Technically, we could get away with injecting a RuntimeDeterminedMethodNode here
+                            // but that introduces more complexities and doesn't seem worth it at this time.
+                            Debug.Assert(targetMethod.AcquiresInstMethodTableFromThis());
+                            _dependencies.Add(GetGenericLookupHelper(ReadyToRunHelperId.TypeHandle, runtimeDeterminedMethod.OwningType), reason + " - inlining protection");
+                        }
+
                         _dependencies.Add(_factory.CanonicalEntrypoint(targetMethod), reason);
                     }
                     else
@@ -609,6 +636,32 @@ private void ImportCall(ILOpcode opcode, int token)
                         _dependencies.Add(instParam, reason);
                     }
 
+                    if (instParam == null
+                        && concreteMethod != targetMethod
+                        && targetMethod.OwningType.NormalizeInstantiation() == targetMethod.OwningType
+                        && !targetMethod.OwningType.IsValueType)
+                    {
+                        // We have a call to a shared instance method and we still know the concrete
+                        // type of the generic instance (e.g. this is a call to Foo<string>.Method()
+                        // and we're about to add Foo<__Canon>.Method() to the dependency graph).
+                        //
+                        // We will pretend the concrete type got allocated as well. This is because RyuJIT might
+                        // end up inlining the shared method body, making it concrete again.
+                        //
+                        // If we didn't have a constructed type for this at the scanning time, we wouldn't
+                        // know the dictionary dependencies at the inlined site, leading to a compile failure.
+                        // (Remember that dictionary dependencies of instance methods on generic reference types
+                        // are tied to the owning type.)
+                        //
+                        // This is not ideal, because if Foo<string> never got allocated otherwise, this code is
+                        // unreachable and we're making the scanner scan more of it.
+                        //
+                        // Technically, we could get away with injecting a ShadowConcreteMethod for the concrete
+                        // method, but that's more complex and doesn't seem worth it at this time.
+                        Debug.Assert(targetMethod.AcquiresInstMethodTableFromThis());
+                        _dependencies.Add(_compilation.NodeFactory.MaximallyConstructableType(concreteMethod.OwningType), reason + " - inlining protection");
+                    }
+
                     _dependencies.Add(_compilation.NodeFactory.MethodEntrypoint(targetMethod), reason);
                 }
             }

tests/src/Simple/Generics/Generics.cs

@@ -32,6 +32,7 @@ static int Main()
         TestReflectionInvoke.Run();
         TestFieldAccess.Run();
         TestDevirtualization.Run();
+        TestGenericInlining.Run();
 #if !CODEGEN_CPP
         TestNullableCasting.Run();
         TestMDArrayAddressMethod.Run();
@@ -2367,4 +2368,41 @@ public static void Run()
             DoGenericDevirtBoxedShared<string>();
         }
     }
+
+    class TestGenericInlining
+    {
+        class NeverSeenInstantiated<T> { }
+
+        class AnotherNeverSeenInstantiated<T> { }
+
+        class NeverAllocatedIndirection<T, U>
+        {
+            public string GetString() => new AnotherNeverSeenInstantiated<T>().ToString();
+        }
+
+        class NeverAllocated<T>
+        {
+            static NeverAllocatedIndirection<T, object> s_indirection = null;
+
+            public string GetString() => new NeverSeenInstantiated<T>().ToString();
+            public string GetStringIndirect() => s_indirection.GetString();
+        }
+
+        class Dummy { }
+
+        static NeverAllocated<Dummy> s_neverAllocated = null;
+
+        public static void Run()
+        {
+            // We're just making sure the compiler doesn't crash.
+            // Both of the calls below are expected to get inlined by an optimized codegen,
+            // triggering interesting behaviors in the dependency analysis of the scanner
+            // that runs before compilation.
+            if (s_neverAllocated != null)
+            {
+                Console.WriteLine(s_neverAllocated.GetString());
+                Console.WriteLine(s_neverAllocated.GetStringIndirect());
+            }
+        }
+    }
 }