[One .NET] support $(EnableLLVM) with AOT (#6157)

Fixed some issues that allows `$(EnableLLVM)` to work:

* `$(_LLVMPath)` needs to point to Mono's AOT pack that contains `opt`
  and `llc`.
* We need to fill out `ld-name` and `ld-flags`.
* `ld-flags` needs some special workarounds in order for them to be
  passed in correctly on .NET 6.
    * Use space as a delimiter
    * Escape spaces in paths
* Added `temp-path` parameter. This prevents the warning on Windows:

    'rm' is not recognized as an internal or external command

I enabled tests around .NET 6 and LLVM.

~~ Results ~~

All tests:

 1. Were running on a [Google Pixel 5][0], and
 2. Enabled two architectures, arm64 and x86, and
 3. **AOT time** was average of 10 runs with `-c Release
    -p:RunAOTCompilation=true`, with the `Activity: Displayed` time
 4. **AOT+LLVM time** was average of 10 runs with `-c Release
    -p:RunAOTCompilation=true -p:EnableLLVM=true` with the
    `Activity: Displayed` time.

| Test                |      AOT time | AOT+LLVM time |  AOT apk size |  AOT+LLVM apk size |
| ------------------- | ------------: | ------------: | ------------: | -----------------: |
| [HelloAndroid][1]   |  00:00:00.238 |  00:00:00.226 |    12,073,931 |         12,602,315 |
| [HelloMaui][2]      |  00:00:00.624 |  00:00:00.596 |    43,167,801 |         47,546,425 |

[0]: https://store.google.com/us/product/pixel_5_specs?hl=en-US
[1]: https://github.com/dotnet/maui-samples/tree/714460431541f40570e91225e8ba4bc1fe08025f/HelloAndroid
[2]: https://github.com/dotnet/maui-samples/tree/714460431541f40570e91225e8ba4bc1fe08025f/HelloMaui

build-tools/automation/azure-pipelines.yaml

@@ -49,7 +49,7 @@ variables:
   # - This is a non-fork branch with name containing "mono-" (for Mono bumps)
   IsMonoBranch: $[and(ne(variables['System.PullRequest.IsFork'], 'True'), or(contains(variables['Build.SourceBranchName'], 'mono-'), contains(variables['System.PullRequest.SourceBranch'], 'mono-')))]
   RunAllTests: $[or(eq(variables['XA.RunAllTests'], true), eq(variables['IsMonoBranch'], true))]
-  DotNetNUnitCategories: '& TestCategory != DotNetIgnore & TestCategory != HybridAOT & TestCategory != ProfiledAOT & TestCategory != LLVM & TestCategory != MkBundle & TestCategory != MonoSymbolicate & TestCategory != PackagesConfig & TestCategory != StaticProject & TestCategory != Debugger & TestCategory != SystemApplication'
+  DotNetNUnitCategories: '& TestCategory != DotNetIgnore & TestCategory != HybridAOT & TestCategory != ProfiledAOT & TestCategory != MkBundle & TestCategory != MonoSymbolicate & TestCategory != PackagesConfig & TestCategory != StaticProject & TestCategory != Debugger & TestCategory != SystemApplication'
   NUnit.NumberOfTestWorkers: 4
   GitHub.Token: $(github--pat--vs-mobiletools-engineering-service2)
   CONVERT_JAVADOC_TO_XMLDOC: $[ne(variables['Build.DefinitionName'], 'Xamarin.Android-PR')]
@@ -809,6 +809,17 @@ stages:
         artifactFolder: net6-aot
         useDotNet: true
 
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(XA.Build.Configuration)
+        testName: Mono.Android.NET_Tests-AotLlvm
+        project: tests/Mono.Android-Tests/Runtime-Microsoft.Android.Sdk/Mono.Android.NET-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android.NET_Tests-$(XA.Build.Configuration)AotLlvm.xml
+        extraBuildArgs: /p:TestsFlavor=AotLlvm /p:RunAOTCompilation=true /p:EnableLlvm=true
+        artifactSource: bin/Test$(XA.Build.Configuration)/net6.0-android/Mono.Android.NET_Tests-Signed.apk
+        artifactFolder: net6-aotllvm
+        useDotNet: true
+
     - task: MSBuild@1
       displayName: shut down emulator
       inputs:

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Aot.targets

@@ -54,17 +54,26 @@ They run in a context of an inner build with a single $(RuntimeIdentifier).
         AotOutputDirectory="$(_AndroidAotBinDirectory)"
         RuntimeIdentifier="$(RuntimeIdentifier)"
         EnableLLVM="$(EnableLLVM)"
+        UsingAndroidNETSdk="true"
         Profiles="@(_AotProfiles)">
       <Output PropertyName="_AotArguments" TaskParameter="Arguments" />
-      <Output PropertyName="_LLVMPath" TaskParameter="LLVMPath" />
+      <Output PropertyName="_AotOutputDirectory" TaskParameter="OutputDirectory" />
     </GetAotArguments>
+    <PropertyGroup>
+      <_MonoAOTCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier', '$(RuntimeIdentifier)'))</_MonoAOTCompilerPath>
+      <_LLVMPath Condition=" '$(EnableLLVM)' == 'true' ">$([System.IO.Path]::GetDirectoryName ('$(_MonoAOTCompilerPath)'))</_LLVMPath>
+    </PropertyGroup>
     <ItemGroup>
-      <_MonoAOTAssemblies Include="@(_AndroidAotInputs->'%(FullPath)')" AotArguments="$(_AotArguments)" />
+      <_MonoAOTAssemblies
+          Include="@(_AndroidAotInputs->'%(FullPath)')"
+          TempDirectory="$([MSBuild]::EnsureTrailingSlash($(_AotOutputDirectory)))%(FileName)"
+          AotArguments="$(_AotArguments),temp-path=$([System.IO.Path]::GetFullPath(%(_MonoAOTAssemblies.TempDirectory)))"
+      />
     </ItemGroup>
-    <MakeDir Directories="$(IntermediateOutputPath)aot\" />
+    <MakeDir Directories="$(IntermediateOutputPath)aot\;@(_MonoAOTAssemblies->'%(TempDirectory)')" />
     <MonoAOTCompiler
         Assemblies="@(_MonoAOTAssemblies)"
-        CompilerBinaryPath="@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier', '$(RuntimeIdentifier)'))"
+        CompilerBinaryPath="$(_MonoAOTCompilerPath)"
         DisableParallelAot="$(_DisableParallelAot)"
         LibraryFormat="So"
         Mode="$(AndroidAotMode)"

src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs

@@ -2,15 +2,12 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Text;
 using System.Threading;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
 
-using Java.Interop.Tools.Diagnostics;
 using Xamarin.Android.Tools;
-using Xamarin.Build;
 using Microsoft.Android.Build.Tasks;
 
 namespace Xamarin.Android.Tasks
@@ -54,44 +51,6 @@ public class Aot : GetAotArguments
 		[Output]
 		public string[] NativeLibrariesReferences { get; set; }
 
-		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, string archDir = null)
-		{
-			var baseDir = Path.GetFullPath(Path.Combine(binDir, ".."));
-
-			string libDir = Path.Combine (baseDir, "lib", "gcc");
-			if (!String.IsNullOrEmpty (archDir))
-				libDir = Path.Combine (libDir, archDir);
-
-			var gccLibDir = Directory.EnumerateDirectories (libDir).ToList();
-			gccLibDir.Sort();
-
-			var libPath = gccLibDir.LastOrDefault();
-			if (libPath == null) {
-				goto no_toolchain_error;
-			}
-
-			if (ndk.UsesClang)
-				return libPath;
-
-			gccLibDir = Directory.EnumerateDirectories(libPath).ToList();
-			gccLibDir.Sort();
-
-			libPath = gccLibDir.LastOrDefault();
-			if (libPath == null) {
-				goto no_toolchain_error;
-			}
-
-			return libPath;
-
-		  no_toolchain_error:
-			throw new Exception("Could not find a valid NDK compiler toolchain library path");
-		}
-
-		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, AndroidTargetArch arch)
-		{
-			return GetNdkToolchainLibraryDir (ndk, binDir, ndk.GetArchDirName (arch));
-		}
-
 		static string QuoteFileName(string fileName)
 		{
 			var builder = new CommandLineBuilder();
@@ -171,59 +130,6 @@ IEnumerable<Config> GetAotConfigs (NdkTools ndk)
 				}
 
 				string toolPrefix = GetToolPrefix (ndk, arch, out int level);
-				var toolchainPath = toolPrefix.Substring(0, toolPrefix.LastIndexOf(Path.DirectorySeparatorChar));
-				var ldFlags = string.Empty;
-				if (EnableLLVM) {
-					if (string.IsNullOrEmpty (AndroidNdkDirectory)) {
-						yield return Config.Invalid;
-						yield break;
-					}
-
-					string androidLibPath = string.Empty;
-					try {
-						androidLibPath = ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level);
-					} catch (InvalidOperationException ex) {
-						Diagnostic.Error (5101, ex.Message);
-					}
-
-					string toolchainLibDir;
-					if (ndk.UsesClang)
-						toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath, arch);
-					else
-						toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath);
-
-					var libs = new List<string>();
-					if (ndk.UsesClang) {
-						libs.Add ($"-L{toolchainLibDir.TrimEnd ('\\')}");
-						libs.Add ($"-L{androidLibPath.TrimEnd ('\\')}");
-
-						if (arch == AndroidTargetArch.Arm) {
-							// Needed for -lunwind to work
-							string compilerLibDir = Path.Combine (toolchainPath, "..", "sysroot", "usr", "lib", ndk.GetArchDirName (arch));
-							libs.Add ($"-L{compilerLibDir.TrimEnd ('\\')}");
-						}
-					}
-
-					libs.Add (Path.Combine (toolchainLibDir, "libgcc.a"));
-					libs.Add (Path.Combine (androidLibPath, "libc.so"));
-					libs.Add (Path.Combine (androidLibPath, "libm.so"));
-
-					ldFlags = $"\\\"{string.Join("\\\";\\\"", libs)}\\\"";
-				}
-
-				string ldName = String.Empty;
-				if (EnableLLVM) {
-					ldName = ndk.GetToolPath (NdkToolKind.Linker, arch, level);
-					if (!String.IsNullOrEmpty (ldName)) {
-						ldName = Path.GetFileName (ldName);
-						if (ldName.IndexOf ('-') >= 0) {
-							ldName = ldName.Substring (ldName.LastIndexOf ("-") + 1);
-						}
-					}
-				} else {
-					ldName = "ld";
-				}
-
 				foreach (var assembly in ResolvedAssemblies) {
 					string outputFile = Path.Combine(outdir, string.Format ("libaot-{0}.so",
 						Path.GetFileName (assembly.ItemSpec)));
@@ -232,20 +138,13 @@ IEnumerable<Config> GetAotConfigs (NdkTools ndk)
 						Path.GetFileName (assembly.ItemSpec)));
 
 					string tempDir = Path.Combine (outdir, Path.GetFileName (assembly.ItemSpec));
-					if (!Directory.Exists (tempDir))
-						Directory.CreateDirectory (tempDir);
-
-					var aotOptions = GetAotOptions (outdir, mtriple, toolPrefix);
-					aotOptions.Add ($"outfile={outputFile}");
-					aotOptions.Add ($"llvm-path={SdkBinDirectory}");
-					aotOptions.Add ($"temp-path={tempDir}");
-
-					if (!String.IsNullOrEmpty (ldName)) {
-						// MUST be before `ld-flags`, otherwise Mono fails to parse it on Windows
-						aotOptions.Add ($"ld-name={ldName}");
-					}
+					Directory.CreateDirectory (tempDir);
 
-					aotOptions.Add ($"ld-flags={ldFlags}");
+					var aotOptions = GetAotOptions (ndk, arch, level, outdir, mtriple, toolPrefix);
+					// NOTE: ordering seems to matter on Windows
+					aotOptions.Insert (0, $"outfile={outputFile}");
+					aotOptions.Insert (0, $"llvm-path={SdkBinDirectory}");
+					aotOptions.Insert (0, $"temp-path={tempDir}");
 
 					// we need to quote the entire --aot arguments here to make sure it is parsed
 					// on windows as one argument. Otherwise it will be split up into multiple

src/Xamarin.Android.Build.Tasks/Tasks/GetAotArguments.cs

@@ -1,7 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
+using Java.Interop.Tools.Diagnostics;
 using Microsoft.Android.Build.Tasks;
 using Microsoft.Build.Framework;
 using Xamarin.Android.Tools;
@@ -48,13 +50,15 @@ public class GetAotArguments : AndroidAsyncTask
 
 		public ITaskItem [] Profiles { get; set; }
 
+		public bool UsingAndroidNETSdk { get; set; }
+
 		public string AotAdditionalArguments { get; set; }
 
 		[Output]
 		public string Arguments { get; set; }
 
 		[Output]
-		public string LLVMPath { get; set; }
+		public string OutputDirectory { get; set; }
 
 		protected AotMode AotMode;
 		protected SequencePointsMode SequencePointsMode;
@@ -135,11 +139,10 @@ public override Task RunTaskAsync ()
 			}
 
 			(_, string outdir, string mtriple, AndroidTargetArch arch) = GetAbiSettings (abi);
-			string toolPrefix = GetToolPrefix (ndk, arch, out _);
+			string toolPrefix = GetToolPrefix (ndk, arch, out int level);
 
-			Arguments = string.Join (",", GetAotOptions (outdir, mtriple, toolPrefix));
-			if (EnableLLVM)
-				LLVMPath = Path.GetDirectoryName (toolPrefix);
+			Arguments = string.Join (",", GetAotOptions (ndk, arch, level, outdir, mtriple, toolPrefix));
+			OutputDirectory = outdir;
 			return Task.CompletedTask;
 		}
 
@@ -245,7 +248,7 @@ int GetNdkApiLevel (NdkTools ndk, AndroidTargetArch arch)
 		/// <summary>
 		/// Returns a list of parameters to pass to the --aot switch
 		/// </summary>
-		protected List<string> GetAotOptions (string outdir, string mtriple, string toolPrefix)
+		protected List<string> GetAotOptions (NdkTools ndk, AndroidTargetArch arch, int level, string outdir, string mtriple, string toolPrefix)
 		{
 			List<string> aotOptions = new List<string> ();
 
@@ -266,7 +269,117 @@ protected List<string> GetAotOptions (string outdir, string mtriple, string tool
 			aotOptions.Add ("asmwriter");
 			aotOptions.Add ($"mtriple={mtriple}");
 			aotOptions.Add ($"tool-prefix={toolPrefix}");
+
+			string ldName;
+			if (EnableLLVM) {
+				ldName = ndk.GetToolPath (NdkToolKind.Linker, arch, level);
+				if (!string.IsNullOrEmpty (ldName)) {
+					ldName = Path.GetFileName (ldName);
+					if (ldName.IndexOf ('-') >= 0) {
+						ldName = ldName.Substring (ldName.LastIndexOf ("-") + 1);
+					}
+				}
+			} else {
+				ldName = "ld";
+			}
+			string ldFlags = GetLdFlags (ndk, arch, level, toolPrefix);
+
+			// MUST be before `ld-flags`, otherwise Mono fails to parse it on Windows
+			if (!string.IsNullOrEmpty (ldName)) {
+				aotOptions.Add ($"ld-name={ldName}");
+			}
+			if (!string.IsNullOrEmpty (ldFlags)) {
+				aotOptions.Add ($"ld-flags={ldFlags}");
+			}
+
 			return aotOptions;
 		}
+
+		string GetLdFlags(NdkTools ndk, AndroidTargetArch arch, int level, string toolPrefix)
+		{
+			var toolchainPath = toolPrefix.Substring (0, toolPrefix.LastIndexOf (Path.DirectorySeparatorChar));
+			var ldFlags = string.Empty;
+			if (EnableLLVM) {
+				if (string.IsNullOrEmpty (AndroidNdkDirectory)) {
+					return null;
+				}
+
+				string androidLibPath = string.Empty;
+				try {
+					androidLibPath = ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level);
+				} catch (InvalidOperationException ex) {
+					Diagnostic.Error (5101, ex.Message);
+				}
+
+				string toolchainLibDir;
+				if (ndk.UsesClang)
+					toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath, arch);
+				else
+					toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath);
+
+				var libs = new List<string> ();
+				if (ndk.UsesClang) {
+					libs.Add ($"-L{toolchainLibDir.TrimEnd ('\\')}");
+					libs.Add ($"-L{androidLibPath.TrimEnd ('\\')}");
+
+					if (arch == AndroidTargetArch.Arm) {
+						// Needed for -lunwind to work
+						string compilerLibDir = Path.Combine (toolchainPath, "..", "sysroot", "usr", "lib", ndk.GetArchDirName (arch));
+						libs.Add ($"-L{compilerLibDir.TrimEnd ('\\')}");
+					}
+				}
+
+				libs.Add (Path.Combine (toolchainLibDir, "libgcc.a"));
+				libs.Add (Path.Combine (androidLibPath, "libc.so"));
+				libs.Add (Path.Combine (androidLibPath, "libm.so"));
+
+				if (UsingAndroidNETSdk) {
+					// NOTE: in .NET 6+ use space for the delimiter and escape spaces in paths
+					var escaped = libs.Select (l => l.Replace (" ", "\\ "));
+					ldFlags = string.Join (" ", escaped);
+				} else {
+					ldFlags = $"\\\"{string.Join ("\\\";\\\"", libs)}\\\"";
+				}
+			}
+			return ldFlags;
+		}
+
+		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, string archDir = null)
+		{
+			var baseDir = Path.GetFullPath (Path.Combine (binDir, ".."));
+
+			string libDir = Path.Combine (baseDir, "lib", "gcc");
+			if (!String.IsNullOrEmpty (archDir))
+				libDir = Path.Combine (libDir, archDir);
+
+			var gccLibDir = Directory.EnumerateDirectories (libDir).ToList ();
+			gccLibDir.Sort ();
+
+			var libPath = gccLibDir.LastOrDefault ();
+			if (libPath == null) {
+				goto no_toolchain_error;
+			}
+
+			if (ndk.UsesClang)
+				return libPath;
+
+			gccLibDir = Directory.EnumerateDirectories (libPath).ToList ();
+			gccLibDir.Sort ();
+
+			libPath = gccLibDir.LastOrDefault ();
+			if (libPath == null) {
+				goto no_toolchain_error;
+			}
+
+			return libPath;
+
+		no_toolchain_error:
+			throw new Exception ("Could not find a valid NDK compiler toolchain library path");
+		}
+
+		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, AndroidTargetArch arch)
+		{
+			return GetNdkToolchainLibraryDir (ndk, binDir, ndk.GetArchDirName (arch));
+		}
 	}
 }