Introduce detection of generic cycles

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/Compilation.cs

@@ -92,6 +92,11 @@ public virtual MethodIL GetMethodIL(MethodDesc method)
 
         protected abstract void CompileInternal(string outputFile, ObjectDumper dumper);
 
+        public void DetectGenericCycles(MethodDesc caller, MethodDesc callee)
+        {
+            _nodeFactory.TypeSystemContext.DetectGenericCycles(caller, callee);
+        }
+
         public bool CanInline(MethodDesc caller, MethodDesc callee)
         {
             return _inliningPolicy.CanInline(caller, callee);

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.Aot.cs

@@ -168,6 +168,13 @@ protected override DelegateInfo CreateDelegateInfo(TypeDesc delegateType)
         {
             return new DelegateInfo(delegateType, _delegateFeatures);
         }
+
+        private readonly LazyGenericsSupport.GenericCycleDetector _genericCycleDetector = new LazyGenericsSupport.GenericCycleDetector();
+
+        public void DetectGenericCycles(TypeSystemEntity owner, TypeSystemEntity referent)
+        {
+            _genericCycleDetector.DetectCycle(owner, referent);
+        }
     }
 
     public class SharedGenericsConfiguration

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/GenericLookupResult.cs

@@ -145,8 +145,21 @@ public sealed override int GetHashCode()
 
         public virtual void EmitDictionaryEntry(ref ObjectDataBuilder builder, NodeFactory factory, GenericLookupResultContext dictionary, GenericDictionaryNode dictionaryNode)
         {
-            ISymbolNode target = GetTarget(factory, dictionary);
-            if (LookupResultReferenceType(factory) == GenericLookupResultReferenceType.ConditionalIndirect)
+            ISymbolNode target;
+            try
+            {
+                target = GetTarget(factory, dictionary);
+            }
+            catch (TypeSystemException)
+            {
+                target = null;
+            }
+
+            if (target == null)
+            {
+                builder.EmitZeroPointer();
+            }
+            else if (LookupResultReferenceType(factory) == GenericLookupResultReferenceType.ConditionalIndirect)
             {
                 builder.EmitPointerRelocOrIndirectionReference(target);
             }
@@ -228,6 +241,9 @@ public override ISymbolNode GetTarget(NodeFactory factory, GenericLookupResultCo
         {
             // We are getting a maximally constructable type symbol because this might be something passed to newobj.
             TypeDesc instantiatedType = _type.GetNonRuntimeDeterminedTypeFromRuntimeDeterminedSubtypeViaSubstitution(dictionary.TypeInstantiation, dictionary.MethodInstantiation);
+
+            factory.TypeSystemContext.DetectGenericCycles(dictionary.Context, instantiatedType);
+
             return factory.MaximallyConstructableType(instantiatedType);
         }
 
@@ -619,6 +635,9 @@ public MethodDictionaryGenericLookupResult(MethodDesc method)
         public override ISymbolNode GetTarget(NodeFactory factory, GenericLookupResultContext dictionary)
         {
             MethodDesc instantiatedMethod = _method.GetNonRuntimeDeterminedMethodFromRuntimeDeterminedMethodViaSubstitution(dictionary.TypeInstantiation, dictionary.MethodInstantiation);
+
+            factory.TypeSystemContext.DetectGenericCycles(dictionary.Context, instantiatedMethod);
+
             return factory.MethodGenericDictionary(instantiatedMethod);
         }
 

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ReadyToRunGenericHelperNode.cs

@@ -143,10 +143,19 @@ public IEnumerable<DependencyListEntry> InstantiateDependencies(NodeFactory fact
                     break;
             }
 
-            // All generic lookups depend on the thing they point to
-            result.Add(new DependencyListEntry(
-                        _lookupSignature.GetTarget(factory, lookupContext),
-                        "Dictionary dependency"));
+            try
+            {
+                // All generic lookups depend on the thing they point to
+                result.Add(new DependencyListEntry(
+                            _lookupSignature.GetTarget(factory, lookupContext),
+                            "Dictionary dependency"));
+            }
+            catch (TypeSystemException)
+            {
+                // TODO: we need to mark this ReadyToRun helper as "injured". We're now in a situation
+                // when a runtime lookup could produce an error result. The helper should check for that
+                // situation and throw exception if result of the lookup produces an injured result.
+            }
 
             return result.ToArray();
         }