Marshal methods runtime support

Context: 903ba37
Context: e1af958
Context: dotnet/java-interop#1027

Commit 903ba37 mentioned a TODO:
    
> Update/rewrite infrastructure to focus on implementing the runtime
> side of marshal methods, making it possible to actually run
> applications which use marshal methods.

Implement the necessary runtime elements to enable running of
applications with marshal methods.  It is now possible, if
LLVM marshal methods are enabled/`ENABLE_MARSHAL_METHODS` is defined,
to run both plain .NET SDK for Android and MAUI apps.

Update `src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Runtime.InteropServices.xml`
so that `System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute`
is always preserved, as it is required by LLVM Marshal Methods.

The `[UnmanagedCallersOnly]` attribute used by marshal methods
requires that the invoked method have [blittable types][0] for the
method return type and all parameter types.  Unfortunately, `bool`
is *not* a blittable type.  Implement generation of wrapper methods
which replace `bool` with `byte` and convert the values appropriately
before calling the actual target method.  In a "hello world" MAUI
test app there are 133 such methods (out of around 180 total).

Implement code that enables us to show error messages with the proper
assembly, class and method names on failure to look up or obtain
pointer to native callback methods.

TODO:

  * Process *all* assemblies, including `Mono.Android.dll`, for
    Java Callable Wrapper generation.  This is necessary so that we
    can find and emit LLVM marshal methods for types defined within
    `Mono.Android.dll`.

  * Remove the `ENABLE_MARSHAL_METHODS` define, and enable
    LLVM marshal methods for everyone.

  * Update `<GenerateJavaStubs/>` to rewrite all assemblies for all
    Supported ABIs.  Currently, we don't support `Java.Lang.Object` &
    `Java.Lang.Throwable` subclasses being located in per-ABI
    assemblies.

  * How do `Java_…` native functions interact with
    `JNIEnv::RegisterNatives()`?
    dotnet#7285 (comment)

  * *Can* JNI `native` methods contain "non-printable" characters
    such as `\n`, or "non-representable in ELF symbols" characters
    such as `-` (e.g. Kotlin mangled methods)?
    dotnet#7285 (comment)

  * Cleanup, cleanup, cleanup

[0]: https://docs.microsoft.com/en-us/dotnet/framework/interop/blittable-and-non-blittable-types

src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Runtime.InteropServices.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<linker>
+	<assembly fullname="System.Runtime.InteropServices">
+		<type fullname="System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute" />
+	</assembly>
+</linker>

src/Mono.Android/Android.Runtime/AndroidRuntime.cs

@@ -473,8 +473,28 @@ static bool CallRegisterMethodByIndex (JniNativeMethodRegistrationArguments argu
 		public override void RegisterNativeMembers (JniType nativeClass, Type type, string? methods) =>
 			RegisterNativeMembers (nativeClass, type, methods.AsSpan ());
 
+#if ENABLE_MARSHAL_METHODS
+		// Temporary hack, see comments in RegisterNativeMembers below
+		static readonly Dictionary<string, string[]> dynamicRegistrationMethods = new Dictionary<string, string[]> (StringComparer.Ordinal) {
+			{"Android.Views.View+IOnLayoutChangeListenerImplementor",             new string[] { "GetOnLayoutChange_Landroid_view_View_IIIIIIIIHandler" }},
+			{"Android.Views.View+IOnLayoutChangeListenerInvoker",                 new string[] { "GetOnLayoutChange_Landroid_view_View_IIIIIIIIHandler" }},
+			{"Java.Interop.TypeManager+JavaTypeManager",                          new string[] { "GetActivateHandler" }},
+		};
+#endif
+
 		public void RegisterNativeMembers (JniType nativeClass, Type type, ReadOnlySpan<char> methods)
 		{
+#if ENABLE_MARSHAL_METHODS
+			Logger.Log (LogLevel.Info, "monodroid-mm", $"RegisterNativeMembers ('{nativeClass?.Name}', '{type?.FullName}', '{methods.ToString ()}')");
+			Logger.Log (LogLevel.Info, "monodroid-mm", "RegisterNativeMembers called from:");
+			var st = new StackTrace (true);
+			Logger.Log (LogLevel.Info, "monodroid-mm", st.ToString ());
+
+			if (methods.IsEmpty) {
+				Logger.Log (LogLevel.Info, "monodroid-mm", "No methods to register, returning");
+				return;
+			}
+#endif
 			try {
 				if (FastRegisterNativeMembers (nativeClass, type, methods))
 					return;
@@ -497,6 +517,9 @@ public void RegisterNativeMembers (JniType nativeClass, Type type, ReadOnlySpan<
 				MethodInfo []? typeMethods = null;
 
 				ReadOnlySpan<char> methodsSpan = methods;
+#if ENABLE_MARSHAL_METHODS
+				bool needToRegisterNatives = false;
+#endif
 				while (!methodsSpan.IsEmpty) {
 					int newLineIndex = methodsSpan.IndexOf ('\n');
 
@@ -508,7 +531,7 @@ public void RegisterNativeMembers (JniType nativeClass, Type type, ReadOnlySpan<
 							out ReadOnlySpan<char> callbackString,
 							out ReadOnlySpan<char> callbackDeclaringTypeString);
 
-						Delegate callback;
+						Delegate? callback = null;
 						if (callbackString.SequenceEqual ("__export__")) {
 							var mname = name.Slice (2);
 							MethodInfo? minfo = null;
@@ -522,6 +545,9 @@ public void RegisterNativeMembers (JniType nativeClass, Type type, ReadOnlySpan<
 							if (minfo == null)
 								throw new InvalidOperationException (String.Format ("Specified managed method '{0}' was not found. Signature: {1}", mname.ToString (), signature.ToString ()));
 							callback = CreateDynamicCallback (minfo);
+#if ENABLE_MARSHAL_METHODS
+							needToRegisterNatives = true;
+#endif
 						} else {
 							Type callbackDeclaringType = type;
 							if (!callbackDeclaringTypeString.IsEmpty) {
@@ -530,20 +556,73 @@ public void RegisterNativeMembers (JniType nativeClass, Type type, ReadOnlySpan<
 							while (callbackDeclaringType.ContainsGenericParameters) {
 								callbackDeclaringType = callbackDeclaringType.BaseType!;
 							}
-							GetCallbackHandler connector = (GetCallbackHandler) Delegate.CreateDelegate (typeof (GetCallbackHandler),
-								callbackDeclaringType, callbackString.ToString ());
-							callback = connector ();
+#if ENABLE_MARSHAL_METHODS
+							// TODO: this is temporary hack, it needs a full fledged registration mechanism for methods like these (that is, ones which
+							// aren't registered with [Register] but are baked into Mono.Android's managed and Java code)
+							bool createCallback = false;
+							string declaringTypeName = callbackDeclaringType.FullName;
+							string callbackName = callbackString.ToString ();
+
+							foreach (var kvp in dynamicRegistrationMethods) {
+								string dynamicTypeName = kvp.Key;
+
+								foreach (string dynamicCallbackMethodName in kvp.Value) {
+									if (ShouldRegisterDynamically (declaringTypeName, callbackName, dynamicTypeName, dynamicCallbackMethodName)) {
+										createCallback = true;
+										break;
+									}
+								}
+
+								if (createCallback) {
+									break;
+								}
+							}
+
+							if (createCallback) {
+								Logger.Log (LogLevel.Info, "monodroid-mm", $"  creating delegate for: '{callbackString.ToString()}' in type {callbackDeclaringType.FullName}");
+#endif
+								GetCallbackHandler connector = (GetCallbackHandler) Delegate.CreateDelegate (typeof (GetCallbackHandler),
+								                                                                             callbackDeclaringType, callbackString.ToString ());
+								callback = connector ();
+#if ENABLE_MARSHAL_METHODS
+							} else {
+								Logger.Log (LogLevel.Warn, "monodroid-mm", $"  would try to create delegate for: '{callbackString.ToString()}' in type {callbackDeclaringType.FullName}");
+							}
+#endif
+						}
+
+						if (callback != null) {
+#if ENABLE_MARSHAL_METHODS
+							needToRegisterNatives = true;
+#endif
+							natives [nativesIndex++] = new JniNativeMethodRegistration (name.ToString (), signature.ToString (), callback);
 						}
-						natives [nativesIndex++] = new JniNativeMethodRegistration (name.ToString (), signature.ToString (), callback);
 					}
 
 					methodsSpan = newLineIndex != -1 ? methodsSpan.Slice (newLineIndex + 1) : default;
 				}
 
-				JniEnvironment.Types.RegisterNatives (nativeClass.PeerReference, natives, natives.Length);
+#if ENABLE_MARSHAL_METHODS
+				if (needToRegisterNatives) {
+#endif
+					JniEnvironment.Types.RegisterNatives (nativeClass.PeerReference, natives, nativesIndex);
+#if ENABLE_MARSHAL_METHODS
+				}
+#endif
 			} catch (Exception e) {
 				JniEnvironment.Runtime.RaisePendingException (e);
 			}
+
+#if ENABLE_MARSHAL_METHODS
+			bool ShouldRegisterDynamically (string callbackTypeName, string callbackString, string typeName, string callbackName)
+			{
+				if (String.Compare (typeName, callbackTypeName, StringComparison.Ordinal) != 0) {
+					return false;
+				}
+
+				return String.Compare (callbackName, callbackString, StringComparison.Ordinal) == 0;
+			}
+#endif
 		}
 
 		static int CountMethods (ReadOnlySpan<char> methodsSpan)

src/Mono.Android/Mono.Android.csproj

@@ -49,6 +49,10 @@
     <JavaCallableWrapperAbsAssembly>$([System.IO.Path]::GetFullPath ('$(OutputPath)$(AssemblyName).dll'))</JavaCallableWrapperAbsAssembly>
   </PropertyGroup>
 
+  <PropertyGroup Condition=" '$(_EnableMarshalMethods)' == 'YesPlease' ">
+    <DefineConstants>$(DefineConstants);ENABLE_MARSHAL_METHODS</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup Condition=" '$(TargetFramework)' == 'monoandroid10' ">
     <Reference Include="mscorlib">
       <HintPath>$(OutputPath)..\v1.0\mscorlib.dll</HintPath>

src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs

@@ -217,6 +217,9 @@ void Run (DirectoryAssemblyResolver res)
 
 #if ENABLE_MARSHAL_METHODS
 			if (!Debug) {
+				// TODO: we must rewrite assemblies for all SupportedAbis. Alternatively, we need to copy the ones that are identical
+				// Cecil does **not** guarantee that the same assembly modified twice in the same will yield the same result - tokens may differ, so can
+				// MVID.
 				var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, marshalMethodsAssemblyPaths, Log);
 				rewriter.Rewrite (res);
 			}
@@ -349,6 +352,11 @@ void Run (DirectoryAssemblyResolver res)
 			regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first.");
 			foreach (var type in javaTypes) {
 				if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) {
+#if ENABLE_MARSHAL_METHODS
+					if (!classifier.FoundDynamicallyRegisteredMethods (type)) {
+						continue;
+					}
+#endif
 					string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.');
 					regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);",
 						type.GetAssemblyQualifiedName (cache), javaKey);
@@ -362,12 +370,25 @@ void Run (DirectoryAssemblyResolver res)
 				template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ()));
 
 #if ENABLE_MARSHAL_METHODS
+			if (!Debug) {
+				Log.LogDebugMessage ($"Number of generated marshal methods: {classifier.MarshalMethods.Count}");
+
+				if (classifier.RejectedMethodCount > 0) {
+					Log.LogWarning ($"Number of methods in the project that will be registered dynamically: {classifier.RejectedMethodCount}");
+				}
+
+				if (classifier.WrappedMethodCount > 0) {
+					Log.LogWarning ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}");
+				}
+			}
+
 			void StoreMarshalAssemblyPath (string name, ITaskItem asm)
 			{
 				if (Debug) {
 					return;
 				}
 
+				// TODO: we need to keep paths to ALL the assemblies, we need to rewrite them for all RIDs eventually. Right now we rewrite them just for one RID
 				if (!marshalMethodsAssemblyPaths.TryGetValue (name, out HashSet<string> assemblyPaths)) {
 					assemblyPaths = new HashSet<string> ();
 					marshalMethodsAssemblyPaths.Add (name, assemblyPaths);
@@ -406,7 +427,7 @@ bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCac
 						jti.Generate (writer);
 #if ENABLE_MARSHAL_METHODS
 						if (!Debug) {
-							if (classifier.FoundDynamicallyRegisteredMethods) {
+							if (classifier.FoundDynamicallyRegisteredMethods (t)) {
 								Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName ()}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods.");
 							}
 						}

src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs

@@ -444,11 +444,12 @@ void AddEnvironment ()
 #if ENABLE_MARSHAL_METHODS
 			var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<MarshalMethodsState> (GenerateJavaStubs.MarshalMethodsRegisterTaskKey, RegisteredTaskObjectLifetime.Build);
 
-			var marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator {
-				NumberOfAssembliesInApk = assemblyCount,
-				UniqueAssemblyNames = uniqueAssemblyNames,
-				MarshalMethods = marshalMethodsState?.MarshalMethods,
-			};
+			var marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator (
+				assemblyCount,
+				uniqueAssemblyNames,
+				marshalMethodsState?.MarshalMethods,
+				Log
+			);
 			marshalMethodsAsmGen.Init ();
 #endif
 			foreach (string abi in SupportedAbis) {

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs

@@ -91,6 +91,7 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto
 					"System.Console.dll",
 					"System.Private.CoreLib.dll",
 					"System.Runtime.dll",
+					"System.Runtime.InteropServices.dll",
 					"System.Linq.dll",
 					"UnnamedProject.dll",
 					//NOTE: appeared in .NET 7.0.100-rc.1.22423.7

src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs

@@ -588,10 +588,10 @@ public void WriteStructureArray<T> (StructureInfo<T> info, IList<StructureInstan
 			WriteStructureArray<T> (info, instances, LlvmIrVariableOptions.Default, symbolName, writeFieldComment, initialComment);
 		}
 
-		public void WriteArray (IList<string> values, string symbolName)
+		public void WriteArray (IList<string> values, string symbolName, string? initialComment = null)
 		{
 			WriteEOL ();
-			WriteEOL (symbolName);
+			WriteEOL (initialComment ?? symbolName);
 
 			ulong arrayStringCounter = 0;
 			var strings = new List<StringSymbolInfo> ();