Enable recursive processing of dynamic assemblies

This introduces the ability to add assemblies to LinkContext lazily and have them
processed in MarkStep. For now, the only additional processing is to
load references and run the TypeMap logic. Later, embedded XML processing and constant propagation will also happen for lazy assemblies

Fixes dotnet#943
Fixes dotnet#1079

src/linker/Linker.Dataflow/ReflectionMethodBodyScanner.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -807,6 +807,7 @@ public override bool HandleCall (MethodBody callingMethodBody, MethodReference c
 									// Intentionally ignore - it's not wrong for code to call Type.GetType on non-existing name, the code might expect null/exception back.
 									reflectionContext.RecordHandledPattern ();
 								} else {
+									_context.ProcessReferenceClosure (foundType.Module.Assembly);
 									reflectionContext.RecordRecognizedPattern (foundType, () => _markStep.MarkTypeVisibleToReflection (foundTypeRef, new DependencyInfo (DependencyKind.AccessedViaReflection, callingMethodDefinition), callingMethodDefinition));
 									methodReturnValue = MergePointValue.MergeValues (methodReturnValue, new SystemTypeValue (foundType));
 								}
@@ -1339,6 +1340,7 @@ void ProcessCreateInstanceByName (ref ReflectionPatternContext reflectionContext
 								reflectionContext.RecordUnrecognizedPattern (2061, $"The assembly name '{assemblyNameStringValue.Contents}' passed to method '{calledMethod.GetDisplayName ()}' references assembly which is not available.");
 								continue;
 							}
+							_context.ProcessReferenceClosure (resolvedAssembly);
 
 							var typeRef = _context.TypeNameResolver.ResolveTypeName (resolvedAssembly, typeNameStringValue.Contents);
 							var resolvedType = typeRef?.Resolve ();
@@ -1579,6 +1581,7 @@ void RequireDynamicallyAccessedMembers (ref ReflectionPatternContext reflectionC
 						// Intentionally ignore - it's not wrong for code to call Type.GetType on non-existing name, the code might expect null/exception back.
 						reflectionContext.RecordHandledPattern ();
 					} else {
+						_context.ProcessReferenceClosure (foundType.Module.Assembly);
 						MarkType (ref reflectionContext, typeRef);
 						MarkTypeForDynamicallyAccessedMembers (ref reflectionContext, foundType, requiredMemberTypes);
 					}

src/linker/Linker.Steps/DynamicDependencyLookupStep.cs

@@ -4,14 +4,13 @@
 
 using System;
 using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
 using Mono.Cecil;
 
 #nullable enable
 
 namespace Mono.Linker.Steps
 {
-	public class DynamicDependencyLookupStep : LoadReferencesStep
+	public class DynamicDependencyLookupStep : BaseStep
 	{
 		protected override void ProcessAssembly (AssemblyDefinition assembly)
 		{
@@ -74,7 +73,7 @@ void ProcessDynamicDependencyAttributes (IMemberDefinition member)
 					var assembly = Context.Resolve (new AssemblyNameReference (assemblyName, new Version ()));
 					if (assembly == null)
 						continue;
-					ProcessReferences (assembly);
+					Context.ProcessReferenceClosure (assembly);
 				}
 			}
 
@@ -90,8 +89,7 @@ void ProcessDynamicDependencyAttributes (IMemberDefinition member)
 					Context.LogWarning ($"Unresolved assembly '{dynamicDependency.AssemblyName}' in 'DynamicDependencyAttribute'", 2035, member);
 					continue;
 				}
-
-				ProcessReferences (assembly);
+				Context.ProcessReferenceClosure (assembly);
 			}
 		}
 

src/linker/Linker.Steps/IAssemblyStep.cs

@@ -0,0 +1,15 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Mono.Cecil;
+using System.Collections.Generic;
+
+namespace Mono.Linker.Steps
+{
+	public interface IAssemblyStep
+	{
+		void Initialize (LinkContext context);
+		void ProcessAssemblies (HashSet<AssemblyDefinition> assembly);
+	}
+}

src/linker/Linker.Steps/LinkAttributesStep.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -173,6 +173,8 @@ bool GetAttributeType (XPathNodeIterator iterator, string attributeFullName, out
 				}
 
 				attributeType = Context.TypeNameResolver.ResolveTypeName (assembly, attributeFullName)?.Resolve ();
+				if (attributeType != null)
+					Context.ProcessReferenceClosure (attributeType.Module.Assembly);
 			}
 
 			if (attributeType == null) {
@@ -341,8 +343,7 @@ protected override AssemblyDefinition GetAssembly (LinkContext context, Assembly
 		{
 			var assembly = context.Resolve (assemblyName);
 			if (assembly != null)
-				ProcessReferences (assembly);
-
+				context.ProcessReferenceClosure (assembly);
 			return assembly;
 		}
 	}

src/linker/Linker.Steps/LoadReferencesStep.cs

@@ -32,20 +32,42 @@
 
 namespace Mono.Linker.Steps
 {
-	public class LoadReferencesStep : BaseStep
+	public class LoadReferencesStep : IAssemblyStep
 	{
+		LinkContext _context;
+
+		LinkContext Context => _context;
+
 		readonly HashSet<AssemblyNameDefinition> references = new HashSet<AssemblyNameDefinition> ();
 
-		protected override void ProcessAssembly (AssemblyDefinition assembly)
+		HashSet<AssemblyDefinition> newReferences = new HashSet<AssemblyDefinition> ();
+
+		public void Initialize (LinkContext context)
 		{
-			ProcessReferences (assembly);
+			_context = context;
+		}
+
+		public virtual void ProcessAssemblies (HashSet<AssemblyDefinition> assemblies)
+		{
+			newReferences.Clear ();
+
+			foreach (var assembly in assemblies)
+				ProcessReferences (assembly);
+
+			// Ensure that subsequent IAssemblySteps only process assemblies
+			// which have not already been processed.
+			assemblies.Clear ();
+			foreach (var assembly in newReferences)
+				assemblies.Add (assembly);
 		}
 
 		protected void ProcessReferences (AssemblyDefinition assembly)
 		{
 			if (!references.Add (assembly.Name))
 				return;
 
+			newReferences.Add (assembly);
+
 			Context.RegisterAssembly (assembly);
 
 			foreach (AssemblyDefinition referenceDefinition in Context.ResolveReferences (assembly)) {

src/linker/Linker.Steps/MarkStep.cs

@@ -347,6 +347,9 @@ void Process ()
 					if (!assembly.MainModule.HasExportedTypes)
 						continue;
 
+					// We may have resolved new type forwarder assemblies which have not been processed.
+					_context.ProcessReferenceClosure (assembly);
+
 					foreach (var exported in assembly.MainModule.ExportedTypes) {
 						bool isForwarder = exported.IsForwarder;
 						var declaringType = exported.DeclaringType;
@@ -633,11 +636,12 @@ void MarkDynamicDependency (DynamicDependency dynamicDependency, IMemberDefiniti
 			Debug.Assert (context is MethodDefinition || context is FieldDefinition);
 			AssemblyDefinition assembly;
 			if (dynamicDependency.AssemblyName != null) {
-				assembly = _context.GetLoadedAssembly (dynamicDependency.AssemblyName);
+				assembly = _context.Resolve (dynamicDependency.AssemblyName);
 				if (assembly == null) {
 					_context.LogWarning ($"Unresolved assembly '{dynamicDependency.AssemblyName}' in 'DynamicDependencyAttribute'", 2035, context);
 					return;
 				}
+				_context.ProcessReferenceClosure (assembly);
 			} else {
 				assembly = context.DeclaringType.Module.Assembly;
 				Debug.Assert (assembly != null);
@@ -741,12 +745,13 @@ protected virtual void MarkUserDependency (MemberReference context, CustomAttrib
 			AssemblyDefinition assembly;
 			var args = ca.ConstructorArguments;
 			if (args.Count >= 3 && args[2].Value is string assemblyName) {
-				assembly = _context.GetLoadedAssembly (assemblyName);
+				assembly = _context.Resolve (assemblyName);
 				if (assembly == null) {
 					_context.LogWarning (
 						$"Could not resolve dependency assembly '{assemblyName}' specified in a 'PreserveDependency' attribute", 2003, context.Resolve ());
 					return;
 				}
+				_context.ProcessReferenceClosure (assembly);
 			} else {
 				assembly = null;
 			}
@@ -1567,12 +1572,19 @@ TypeDefinition GetDebuggerAttributeTargetType (CustomAttribute ca, AssemblyDefin
 				if (property.Name == "TargetTypeName") {
 					string targetTypeName = (string) property.Argument.Value;
 					TypeName typeName = TypeParser.ParseTypeName (targetTypeName);
+					TypeDefinition typeDef = null;
 					if (typeName is AssemblyQualifiedTypeName assemblyQualifiedTypeName) {
 						AssemblyDefinition assembly = _context.GetLoadedAssembly (assemblyQualifiedTypeName.AssemblyName.Name);
-						return _context.TypeNameResolver.ResolveTypeName (assembly, targetTypeName)?.Resolve ();
+						typeDef = _context.TypeNameResolver.ResolveTypeName (assembly, targetTypeName)?.Resolve ();
+						if (typeDef != null)
+							_context.ProcessReferenceClosure (typeDef.Module.Assembly);
+						return typeDef;
 					}
 
-					return _context.TypeNameResolver.ResolveTypeName (asm, targetTypeName)?.Resolve ();
+					typeDef = _context.TypeNameResolver.ResolveTypeName (asm, targetTypeName)?.Resolve ();
+					if (typeDef != null)
+						_context.ProcessReferenceClosure (typeDef.Module.Assembly);
+					return typeDef;
 				}
 			}
 
@@ -1666,6 +1678,9 @@ protected virtual void MarkTypeConverterLikeDependency (CustomAttribute attribut
 			switch (attribute.ConstructorArguments[0].Value) {
 			case string s:
 				tdef = _context.TypeNameResolver.ResolveTypeName (s)?.Resolve ();
+				if (tdef != null)
+					_context.ProcessReferenceClosure (tdef.Module.Assembly);
+				// ResolveTypeName might resolve an assembly (or multiple assemblies for generic arguments...)
 				break;
 			case TypeReference type:
 				tdef = type.Resolve ();

src/linker/Linker.Steps/ProcessLinkerXmlStepBase.cs

@@ -7,7 +7,7 @@
 
 namespace Mono.Linker.Steps
 {
-	public abstract class ProcessLinkerXmlStepBase : LoadReferencesStep
+	public abstract class ProcessLinkerXmlStepBase : BaseStep
 	{
 		const string FullNameAttributeName = "fullname";
 		const string LinkerElementName = "linker";

src/linker/Linker.Steps/ResolveFromAssemblyStep.cs

@@ -67,6 +67,7 @@ protected override void Process ()
 			Context.Resolver.IgnoreUnresolved = false;
 			AssemblyDefinition assembly = _assembly ?? Context.Resolve (_file);
 			Context.Resolver.IgnoreUnresolved = ignoreUnresolved;
+			Context.ProcessReferenceClosure (assembly);
 
 			if (_rootVisibility != RootVisibility.Any && HasInternalsVisibleTo (assembly))
 				_rootVisibility = RootVisibility.PublicAndFamilyAndAssembly;

src/linker/Linker.Steps/ResolveFromXmlStep.cs

@@ -320,8 +320,7 @@ protected override AssemblyDefinition GetAssembly (LinkContext context, Assembly
 		{
 			var assembly = context.Resolve (assemblyName);
 			if (assembly != null)
-				ProcessReferences (assembly);
-
+				context.ProcessReferenceClosure (assembly);
 			return assembly;
 		}
 

src/linker/Linker.Steps/TypeMapStep.cs

@@ -32,13 +32,22 @@
 namespace Mono.Linker.Steps
 {
 
-	public class TypeMapStep : BaseStep
+	public class TypeMapStep : IAssemblyStep
 	{
+		LinkContext _context;
+		AnnotationStore Annotations => _context.Annotations;
 
-		protected override void ProcessAssembly (AssemblyDefinition assembly)
+		public void Initialize (LinkContext context)
 		{
-			foreach (TypeDefinition type in assembly.MainModule.Types)
-				MapType (type);
+			_context = context;
+		}
+
+		public void ProcessAssemblies (HashSet<AssemblyDefinition> assemblies)
+		{
+			foreach (var assembly in assemblies) {
+				foreach (TypeDefinition type in assembly.MainModule.Types)
+					MapType (type);
+			}
 		}
 
 		protected virtual void MapType (TypeDefinition type)

src/linker/Linker/Driver.cs

@@ -701,7 +701,6 @@ protected int SetupContext (ILogger customLogger = null)
 			// ResolveFromAssemblyStep [optional, possibly many]
 			// ResolveFromXmlStep [optional, possibly many]
 			// [mono only] ResolveFromXApiStep [optional, possibly many]
-			// LoadReferencesStep
 			// [mono only] LoadI18nAssemblies
 			// BlacklistStep
 			//   dynamically adds steps:
@@ -711,7 +710,6 @@ protected int SetupContext (ILogger customLogger = null)
 			// LinkAttributesStep [optional, possibly many]
 			// DynamicDependencyLookupStep
 			// [mono only] PreserveCalendarsStep [optional]
-			// TypeMapStep
 			// BodySubstituterStep [optional]
 			// RemoveSecurityStep [optional]
 			// [mono only] RemoveFeaturesStep [optional]
@@ -726,6 +724,12 @@ protected int SetupContext (ILogger customLogger = null)
 			// RegenerateGuidStep [optional]
 			// SealerStep
 			// OutputStep
+
+			//
+			// Pipeline Steps which run when new assemblies are processed
+			//
+			// LoadReferencesStep
+			// TypeMapStep
 			//
 
 			foreach (string custom_step in custom_steps) {
@@ -1202,10 +1206,8 @@ static void About ()
 		static Pipeline GetStandardPipeline ()
 		{
 			Pipeline p = new Pipeline ();
-			p.AppendStep (new LoadReferencesStep ());
 			p.AppendStep (new BlacklistStep ());
 			p.AppendStep (new DynamicDependencyLookupStep ());
-			p.AppendStep (new TypeMapStep ());
 			p.AppendStep (new MarkStep ());
 			p.AppendStep (new ValidateVirtualMethodAnnotationsStep ());
 			p.AppendStep (new ProcessWarningsStep ());
@@ -1214,6 +1216,8 @@ static Pipeline GetStandardPipeline ()
 			p.AppendStep (new CleanStep ());
 			p.AppendStep (new RegenerateGuidStep ());
 			p.AppendStep (new OutputStep ());
+			p.AppendAssemblyStep (new LoadReferencesStep ());
+			p.AppendAssemblyStep (new TypeMapStep ());
 			return p;
 		}
 

src/linker/Linker/LinkContext.cs

@@ -304,6 +304,15 @@ public TypeDefinition GetType (string fullName)
 			return assembly.MainModule.GetType (fullName);
 		}
 
+		// Run the per-assembly logic on a new assembly.
+		// Each step may modify the context of assemblies passed to the next.
+		public void ProcessReferenceClosure (AssemblyDefinition assembly)
+		{
+			var assemblies = new HashSet<AssemblyDefinition> { assembly };
+			foreach (var step in Pipeline.GetAssemblySteps ())
+				step.ProcessAssemblies (assemblies);
+		}
+
 		public AssemblyDefinition Resolve (string name)
 		{
 			if (File.Exists (name)) {

src/linker/Linker/Pipeline.cs

@@ -38,10 +38,12 @@ public class Pipeline
 	{
 
 		readonly List<IStep> _steps;
+		readonly List<IAssemblyStep> _assemblySteps;
 
 		public Pipeline ()
 		{
 			_steps = new List<IStep> ();
+			_assemblySteps = new List<IAssemblyStep> ();
 		}
 
 		public void PrependStep (IStep step)
@@ -54,6 +56,11 @@ public void AppendStep (IStep step)
 			_steps.Add (step);
 		}
 
+		public void AppendAssemblyStep (IAssemblyStep step)
+		{
+			_assemblySteps.Add (step);
+		}
+
 		public void AddStepBefore (Type target, IStep step)
 		{
 			for (int i = 0; i < _steps.Count; i++) {
@@ -123,6 +130,9 @@ public void RemoveStep (Type target)
 
 		public void Process (LinkContext context)
 		{
+			foreach (var step in _assemblySteps)
+				step.Initialize (context);
+
 			while (_steps.Count > 0) {
 				IStep step = _steps[0];
 				ProcessStep (context, step);
@@ -140,6 +150,11 @@ public IStep[] GetSteps ()
 			return _steps.ToArray ();
 		}
 
+		public IAssemblyStep[] GetAssemblySteps ()
+		{
+			return _assemblySteps.ToArray ();
+		}
+
 		public bool ContainsStep (Type type)
 		{
 			foreach (IStep step in _steps)

test/Mono.Linker.Tests.Cases/Attributes/CoreLibraryAssemblyAttributesAreKept.cs

@@ -7,6 +7,9 @@ namespace Mono.Linker.Tests.Cases.Attributes
 {
 	[Reference ("System.dll")]
 	[SetupLinkerCoreAction ("link")]
+	// IComponent has TypeConverterAttribute referencing System.dll, which has
+	// unresolved references to Win32 assemblies.
+	[SetupLinkerArgument ("--skip-unresolved", "true")]
 	[KeptAttributeInAssembly (PlatformAssemblies.CoreLib, typeof (AssemblyDescriptionAttribute))]
 	[KeptAttributeInAssembly (PlatformAssemblies.CoreLib, typeof (AssemblyCompanyAttribute))]
 #if !NETCOREAPP