[release/6.0.1xx-preview7] [dotnet/msbuild] Add support for using LLV…

…M to build .NET apps. Fixes #11379. (#12176) * [xharness] Add test case for running monotouch-test with LLVM on .NET on device. * [dotnet] Parse MtouchUseLlvm. * [tests] Add helper makefiles for building monotouch-test and MySimpleApp in .NET from the command line. * [tools] Pass the AOTCompiler property to the ComputeAOTArguments linker steps. So that the ComputeAOTArguments can compute the llvm-path value to pass to the AOT compiler (the llvm-path value states where the opt and llc command-line tools are, and they're next to the AOT compiler). * [dotnet] Remove unused AOTCompileTaskBase.AOTData property. * [msbuild/dotnet] Use properties on the _AssembliesToAOT item group to specify output paths. This deduplicates a little bit code to compute the output path. * [tools] Compute the llvm output file and pass it to the AOT compiler / native linker * [msbuild] Remove unused CompileNativeCodeTaskBase.ObjectFiles property. * [msbuild/dotnet] Make sure target Outputs show up in a task Output to make the build work correctly on windows. Co-authored-by: Rolf Bjarne Kvinge <rolf@xamarin.com>
xamarin · Jul 22, 2021 · ad40441 · ad40441 · vs-mobiletools-engineering-service2 · Jul 23, 2021
1 parent 29e512d
commit ad40441
Show file tree

Hide file tree

Showing 12 changed files with 129 additions and 32 deletions.
diff --git a/dotnet/targets/Xamarin.Shared.Sdk.targets b/dotnet/targets/Xamarin.Shared.Sdk.targets
@@ -357,6 +357,7 @@
 
 			<_CustomLinkerOptions>
 				AssemblyName=$(AssemblyName).dll
+				AOTCompiler=$(_AOTCompiler)
 				AOTOutputDirectory=$(_AOTOutputDirectory)
 				CacheDirectory=$(_LinkerCacheDirectory)
 				Debug=$(_BundlerDebug)
@@ -383,6 +384,7 @@
 				SdkVersion=$(_SdkVersion)
 				TargetArchitectures=$(TargetArchitectures)
 				TargetFramework=$(_ComputedTargetFrameworkMoniker)
+				UseLlvm=$(MtouchUseLlvm)
 				Verbosity=$(_BundlerVerbosity)
 				XamarinRuntime=$(_XamarinRuntime)
 			</_CustomLinkerOptions>
@@ -703,7 +705,7 @@
 			Condition="'$(_SdkIsSimulator)' != 'true' And '$(_PlatformName)' != 'macOS'"
 			DependsOnTargets="_ComputeVariables"
 			Inputs="@(_AssembliesToAOT)"
-			Outputs="@(_AssembliesToAOT -> '$(_AOTOutputDirectory)%(Arch)\%(Filename)%(Extension).o')">
+			Outputs="@(_AssembliesToAOT -> '%(ObjectFile)');@(_AssembliesToAOT -> '%(LLVMFile)');">
 
 		<Error Text="The AOT compiler '$(_AOTCompiler)' does not exist." Condition="!Exists ($(_AOTCompiler))" />
 
@@ -719,7 +721,7 @@
 			TargetFrameworkMoniker="$(_ComputedTargetFrameworkMoniker)"
 		>
 			<Output TaskParameter="AssemblyFiles" ItemName="_AOTAssemblyFiles" />
-			<Output TaskParameter="AOTData" ItemName="_AOTData" />
+			<Output TaskParameter="FileWrites" ItemName="FileWrites" />
 		</AOTCompile>
 
 		<CompileNativeCode
@@ -732,12 +734,12 @@
 			SdkRoot="$(_SdkRoot)"
 			TargetFrameworkMoniker="$(_ComputedTargetFrameworkMoniker)"
 		>
-			<Output TaskParameter="ObjectFiles" ItemName="_AOTObjectFiles" />
 		</CompileNativeCode>
 
 		<ItemGroup Condition="'$(IsMacEnabled)' == 'true'">
 			<!-- Add the AOT-compiled output to the main executable -->
-			<_XamarinMainLibraries Include="@(_AOTObjectFiles)" />
+			<_XamarinMainLibraries Include="@(_AssembliesToAOT -> '%(ObjectFile)')" />
+			<_XamarinMainLibraries Include="@(_AssembliesToAOT -> '%(LLVMFile)')" />
 
 			<!-- copy the aotdata files to the .app -->
 			<ResolvedFileToPublish Include="%(_AssembliesToAOT.AOTData)" >

diff --git a/msbuild/Xamarin.MacDev.Tasks.Core/Tasks/AOTCompileTaskBase.cs b/msbuild/Xamarin.MacDev.Tasks.Core/Tasks/AOTCompileTaskBase.cs
@@ -36,7 +36,7 @@ public abstract class AOTCompileTaskBase : XamarinTask {
 		public ITaskItem[] AssemblyFiles { get; set; }
 
 		[Output]
-		public ITaskItem[] AOTData { get; set; }
+		public ITaskItem[] FileWrites { get; set; }
 #endregion
 
 		public override bool Execute ()
@@ -64,9 +64,7 @@ public override bool Execute ()
 			Directory.CreateDirectory (OutputDirectory);
 
 			var aotAssemblyFiles = new List<ITaskItem> ();
-			var aotDataFiles = new List<ITaskItem> ();
 			var processes = new Task<Execution> [Assemblies.Length];
-			var objectFiles = new List<ITaskItem> ();
 
 			var environment = new Dictionary<string, string> {
 				{ "MONO_PATH", Path.GetFullPath (InputDirectory) },
@@ -85,7 +83,6 @@ public override bool Execute ()
 				aotAssemblyItem.SetMetadata ("Arguments", "-Xlinker -rpath -Xlinker @executable_path/ -Qunused-arguments -x assembler -D DEBUG");
 				aotAssemblyItem.SetMetadata ("Arch", arch);
 				aotAssemblyFiles.Add (aotAssemblyItem);
-				aotDataFiles.Add (new TaskItem (aotData));
 
 				var arguments = new List<string> ();
 				if (!StringUtils.TryParseArguments (aotArguments, out var parsedArguments, out var ex)) {
@@ -111,9 +108,14 @@ public override bool Execute ()
 
 			System.Threading.Tasks.Task.WaitAll (processes);
 
-			AOTData = aotDataFiles.ToArray ();
 			AssemblyFiles = aotAssemblyFiles.ToArray ();
 
+			// For Windows support it's necessary to have the files we're going to create as an Output parameter, so that the files are
+			// created on the windows side too, which makes the Inputs/Outputs logic work properly when working from Windows.
+			var objectFiles = Assemblies.Select (v => v.GetMetadata ("ObjectFile")).Where (v => !string.IsNullOrEmpty (v));
+			var llvmFiles = Assemblies.Select (v => v.GetMetadata ("LLVMFile")).Where (v => !string.IsNullOrEmpty (v));
+			FileWrites = objectFiles.Union (llvmFiles).Select (v => new TaskItem (v)).ToArray ();
+
 			return !Log.HasLoggedErrors;
 
 		}

diff --git a/msbuild/Xamarin.MacDev.Tasks.Core/Tasks/CompileNativeCodeTaskBase.cs b/msbuild/Xamarin.MacDev.Tasks.Core/Tasks/CompileNativeCodeTaskBase.cs
@@ -21,9 +21,6 @@ public abstract class CompileNativeCodeTaskBase : XamarinTask {
 		[Required]
 		public string MinimumOSVersion { get; set; }
 
-		[Output]
-		public ITaskItem [] ObjectFiles { get; set; }
-
 		[Required]
 		public string SdkDevPath { get; set; }
 
@@ -37,10 +34,6 @@ public abstract class CompileNativeCodeTaskBase : XamarinTask {
 		public override bool Execute ()
 		{
 			var processes = new Task<Execution> [CompileInfo.Length];
-			var objectFiles = new List<ITaskItem> ();
-
-			if (ObjectFiles != null)
-				objectFiles.AddRange (ObjectFiles);
 
 			for (var i = 0; i < CompileInfo.Length; i++) {
 				var info = CompileInfo [i];
@@ -99,7 +92,6 @@ public override bool Execute ()
 				outputFile = Path.GetFullPath (outputFile);
 				arguments.Add ("-o");
 				arguments.Add (outputFile);
-				objectFiles.Add (new TaskItem (outputFile));
 
 				arguments.Add ("-c");
 				arguments.Add (src);
@@ -109,8 +101,6 @@ public override bool Execute ()
 
 			System.Threading.Tasks.Task.WaitAll (processes);
 
-			ObjectFiles = objectFiles.ToArray ();
-
 			return !Log.HasLoggedErrors;
 		}
 	}

diff --git a/tests/dotnet/MySimpleApp/iOS/Makefile b/tests/dotnet/MySimpleApp/iOS/Makefile
@@ -1,9 +1,13 @@
-TOP=../../../..
+include ../shared.mk
 
-include $(TOP)/Make.config
+dev:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64"
 
-build:
-	$(DOTNET6) build /bl *.csproj $(MSBUILD_VERBOSITY)
+run-dev:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64 /t:Run"
 
-run:
-	$(DOTNET6) build /bl *.csproj $(MSBUILD_VERBOSITY) -t:Run
+llvm:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64 /p:MtouchUseLlvm=true /p:MtouchLink=SdkOnly /p:Configuration=Release-llvm /bl:@.binlog"
+
+norm:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64                       /p:MtouchLink=SdkOnly /p:Configuration=Release-norm /bl:@.binlog"
diff --git a/tests/dotnet/MySimpleApp/shared.mk b/tests/dotnet/MySimpleApp/shared.mk
@@ -0,0 +1,28 @@
+TOP=../../../..
+include $(TOP)/Make.config
+
+prepare:
+	$(Q) $(MAKE) -C $(TOP)/tests/dotnet copy-dotnet-config
+
+reload:
+	$(Q) rm -Rf $(TOP)/tests/dotnet/packages
+	$(Q) $(MAKE) -C $(TOP) -j8 all
+	$(Q) $(MAKE) -C $(TOP) -j8 install
+	$(Q) git clean -xfdq
+
+reload-and-build:
+	$(Q) $(MAKE) reload
+	$(Q) $(MAKE) build
+
+reload-and-run:
+	$(Q) $(MAKE) reload
+	$(Q) $(MAKE) run
+
+build: prepare
+	$(Q) $(DOTNET6) build /bl *.csproj $(MSBUILD_VERBOSITY) $(BUILD_ARGUMENTS)
+
+run: prepare
+	$(Q) $(DOTNET6) build /bl *.csproj $(MSBUILD_VERBOSITY) $(BUILD_ARGUMENTS) -t:Run
+
+diag: prepare
+	$(Q) $(DOTNET6) build /v:diag msbuild.binlog
diff --git a/tests/monotouch-test/dotnet/iOS/Makefile b/tests/monotouch-test/dotnet/iOS/Makefile
@@ -0,0 +1,16 @@
+include ../shared.mk
+
+dev:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64"
+
+llvm:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64 /p:MtouchUseLlvm=true /p:MtouchLink=SdkOnly /p:Configuration=Release-llvm /bl:$@.binlog"
+
+run-llvm:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64 /p:MtouchUseLlvm=true /p:MtouchLink=SdkOnly /p:Configuration=Release-llvm /bl:$@.binlog /t:Run"
+
+norm:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64                       /p:MtouchLink=SdkOnly /p:Configuration=Release-norm /bl:$@.binlog"
+
+run-norm:
+	$(Q) $(MAKE) build BUILD_ARGUMENTS="/p:RuntimeIdentifier=ios-arm64                       /p:MtouchLink=SdkOnly /p:Configuration=Release-norm /bl:$@.binlog /t:Run"
diff --git a/tests/monotouch-test/dotnet/shared.mk b/tests/monotouch-test/dotnet/shared.mk
@@ -0,0 +1,28 @@
+TOP=../../../..
+include $(TOP)/Make.config
+
+prepare:
+	$(Q) $(MAKE) -C $(TOP)/tests/dotnet copy-dotnet-config
+
+reload:
+	$(Q) rm -Rf $(TOP)/tests/dotnet/packages
+	$(Q) $(MAKE) -C $(TOP) -j8 all
+	$(Q) $(MAKE) -C $(TOP) -j8 install
+	$(Q) git clean -xfdq
+
+reload-and-build:
+	$(Q) $(MAKE) reload
+	$(Q) $(MAKE) build
+
+reload-and-run:
+	$(Q) $(MAKE) reload
+	$(Q) $(MAKE) run
+
+build: prepare
+	$(Q) $(DOTNET6) build /bl *.csproj $(MSBUILD_VERBOSITY) $(BUILD_ARGUMENTS)
+
+run: prepare
+	$(Q) $(DOTNET6) build /bl *.csproj $(MSBUILD_VERBOSITY) $(BUILD_ARGUMENTS) -t:Run
+
+diag: prepare
+	$(Q) $(DOTNET6) build /v:diag msbuild.binlog
diff --git a/tests/xharness/Jenkins/TestData.cs b/tests/xharness/Jenkins/TestData.cs
@@ -18,6 +18,7 @@ class TestData
 		public bool? Ignored;
 		public bool EnableSGenConc;
 		public bool UseThumb;
+		public bool UseLlvm;
 		public bool? UseMonoRuntime;
 		public MonoNativeLinkMode MonoNativeLinkMode;
 		public IEnumerable<IDevice> Candidates;

diff --git a/tests/xharness/Jenkins/TestVariationsFactory.cs b/tests/xharness/Jenkins/TestVariationsFactory.cs
@@ -83,6 +83,8 @@ IEnumerable<TestData> GetTestData (RunTestTask test)
 						}
 						yield return new TestData { Variation = "Release (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = false, Profiling = false, Undefines = "FULL_AOT_RUNTIME", Ignored = ignore };
 					}
+					if (test.TestProject.IsDotNetProject)
+						yield return new TestData { Variation = "Release (LLVM)", Debug = false, UseLlvm = true, Ignored = ignore };
 					break;
 				case  string name when name.StartsWith ("mscorlib", StringComparison.Ordinal):
 					if (supports_debug)
@@ -182,6 +184,7 @@ public IEnumerable<T> CreateTestVariations<T> (IEnumerable<T> tests, Func<MSBuil
 					var use_mono_runtime = test_data.UseMonoRuntime;
 					var xammac_arch = test_data.XamMacArch;
 					var runtime_identifer = test_data.RuntimeIdentifier;
+					var use_llvm = test_data.UseLlvm;
 
 					if (task.TestProject.IsDotNetProject)
 						variation += " [dotnet]";
@@ -232,6 +235,8 @@ public IEnumerable<T> CreateTestVariations<T> (IEnumerable<T> tests, Func<MSBuil
 							clone.Xml.SetNode ("MtouchEnableSGenConc", "true", task.ProjectPlatform, configuration);
 						if (test_data.UseThumb) // no need to check the platform, already done at the data iterator
 							clone.Xml.SetNode ("MtouchUseThumb", "true", task.ProjectPlatform, configuration);
+						if (use_llvm)
+							clone.Xml.SetTopLevelPropertyGroupValue ("MtouchUseLlvm", "true");
 
 						if (!debug && !isMac)
 							clone.Xml.SetMtouchUseLlvm (true, task.ProjectPlatform, configuration);

diff --git a/tools/common/Application.cs b/tools/common/Application.cs
@@ -1408,7 +1408,7 @@ public IList<string> GetAotArguments (string filename, Abi abi, string outputDir
 			return processArguments;
 		}
 
-		public void GetAotArguments (string filename, Abi abi, string outputDir, string outputFile, string llvmOutputFile, string dataFile, out List<string> processArguments, out List<string> aotArguments)
+		public void GetAotArguments (string filename, Abi abi, string outputDir, string outputFile, string llvmOutputFile, string dataFile, out List<string> processArguments, out List<string> aotArguments, string llvm_path = null)
 		{
 			string fname = Path.GetFileName (filename);
 			processArguments = new List<string> ();
@@ -1472,8 +1472,13 @@ public void GetAotArguments (string filename, Abi abi, string outputDir, string
 				aotArguments.Add ($"msym-dir={msymdir}");
 			}
 
-			if (enable_llvm)
-				aotArguments.Add ($"llvm-path={Driver.GetFrameworkCurrentDirectory (app)}/LLVM/bin/");
+			if (enable_llvm) {
+				if (!string.IsNullOrEmpty (llvm_path)) {
+					aotArguments.Add ($"llvm-path={llvm_path}");
+				} else {
+					aotArguments.Add ($"llvm-path={Driver.GetFrameworkCurrentDirectory (app)}/LLVM/bin/");
+				}
+			}
 
 			aotArguments.Add ($"outfile={outputFile}");
 			if (enable_llvm)

diff --git a/tools/dotnet-linker/LinkerConfiguration.cs b/tools/dotnet-linker/LinkerConfiguration.cs
@@ -17,6 +17,7 @@
 namespace Xamarin.Linker {
 	public class LinkerConfiguration {
 		public List<Abi> Abis;
+		public string AOTCompiler;
 		public string AOTOutputDirectory;
 		public string CacheDirectory { get; private set; }
 		public Version DeploymentTarget { get; private set; }
@@ -79,6 +80,7 @@ public static LinkerConfiguration GetInstance (LinkContext context, bool createI
 			Target = new Target (Application);
 			CompilerFlags = new CompilerFlags (Target);
 
+			var use_llvm = false;
 			var lines = File.ReadAllLines (linker_file);
 			var significantLines = new List<string> (); // This is the input the cache uses to verify if the cache is still valid
 			for (var i = 0; i < lines.Length; i++) {
@@ -103,6 +105,9 @@ public static LinkerConfiguration GetInstance (LinkContext context, bool createI
 					// This is the AssemblyName MSBuild property for the main project (which is also the root/entry assembly)
 					Application.RootAssemblies.Add (value);
 					break;
+				case "AOTCompiler":
+					AOTCompiler = value;
+					break;
 				case "AOTOutputDirectory":
 					AOTOutputDirectory = value;
 					break;
@@ -223,6 +228,9 @@ public static LinkerConfiguration GetInstance (LinkContext context, bool createI
 						throw new InvalidOperationException ($"Invalid TargetFramework '{value}' in {linker_file}");
 					Driver.TargetFramework = TargetFramework.Parse (value);
 					break;
+				case "UseLlvm":
+					use_llvm = string.Equals ("true", value, StringComparison.OrdinalIgnoreCase);
+					break;
 				case "Verbosity":
 					if (!int.TryParse (value, out var verbosity))
 						throw new InvalidOperationException ($"Invalid Verbosity '{value}' in {linker_file}");
@@ -253,6 +261,12 @@ public static LinkerConfiguration GetInstance (LinkContext context, bool createI
 				ErrorHelper.Show (messages);
 			}
 
+			if (use_llvm) {
+				for (var i = 0; i < Abis.Count; i++) {
+					Abis [i] |= Abi.LLVM;
+				}
+			}
+
 			Application.CreateCache (significantLines.ToArray ());
 			Application.Cache.Location = CacheDirectory;
 			Application.DeploymentTarget = DeploymentTarget;
@@ -314,6 +328,7 @@ public void Write ()
 				Console.WriteLine ($"    SdkRootDirectory: {SdkRootDirectory}");
 				Console.WriteLine ($"    SdkVersion: {SdkVersion}");
 				Console.WriteLine ($"    UseInterpreter: {Application.UseInterpreter}");
+				Console.WriteLine ($"    UseLlvm: {Application.IsLLVM}");
 				Console.WriteLine ($"    Verbosity: {Verbosity}");
 				Console.WriteLine ($"    XamarinRuntime: {Application.XamarinRuntime}");
 			}

diff --git a/tools/dotnet-linker/Steps/ComputeAOTArguments.cs b/tools/dotnet-linker/Steps/ComputeAOTArguments.cs
@@ -36,16 +36,17 @@ protected override void TryEndProcess ()
 					var arch = abi.AsArchString ();
 					var aotAssembly = Path.Combine (outputDirectory, arch, Path.GetFileName (input) + ".s");
 					var aotData = Path.Combine (outputDirectory, arch, Path.GetFileNameWithoutExtension (input) + ".aotdata");
-					var llvmFile = string.Empty;
-					if ((abi & Abi.LLVM) == Abi.LLVM)
-						throw ErrorHelper.CreateError (99, $"Support for LLVM hasn't been implemented yet.");
-					app.GetAotArguments (asm.FullPath, abi, outputDirectory, aotAssembly, llvmFile, aotData, out var processArguments, out var aotArguments);
+					var llvmFile = Configuration.Application.IsLLVM ? Path.Combine (outputDirectory, arch, Path.GetFileName (input) + ".llvm.o") : string.Empty;
+					var objectFile = Path.Combine (outputDirectory, arch, Path.GetFileName (input) + ".o");
+					app.GetAotArguments (asm.FullPath, abi, outputDirectory, aotAssembly, llvmFile, aotData, out var processArguments, out var aotArguments, Path.GetDirectoryName (Configuration.AOTCompiler));
 					item.Metadata.Add ("Arguments", StringUtils.FormatArguments (aotArguments));
 					item.Metadata.Add ("ProcessArguments", StringUtils.FormatArguments (processArguments));
 					item.Metadata.Add ("Abi", abiString);
 					item.Metadata.Add ("Arch", arch);
 					item.Metadata.Add ("AOTData", aotData);
 					item.Metadata.Add ("AOTAssembly", aotAssembly);
+					item.Metadata.Add ("LLVMFile", llvmFile);
+					item.Metadata.Add ("ObjectFile", objectFile);
 				}
 
 				assembliesToAOT.Add (item);