Skip to content

Commit

Permalink
[Xamarin.Android.Build.Tasks] Generate LLVM IR, not native asm (dotne…
Browse files Browse the repository at this point in the history 
…t#6853)

Instead of continuing to generate native assembler code, as we've
been doing for a while now, switch to generating [LLVM IR][0] sources.
This has the advantage that LLVM takes care for us of generating valid
native assembler code, making sure it adheres to all the ABI
requirements of the target platform.  It also makes it easier, in the
future, to generate actual executable code instead of just data as we
do now.

Once the LLVM IR is generated, the `llc` utility from LLVM is used to
generate the actual native assembler code.  LLVM IR files are larger
than the native assembler ones we've been generating so far, because
of the language's verbosity, but we generate less of them.

This PR covers generation of compressed assemblies, application
environment, and typemap sources.

[0]: https://llvm.org/docs/LangRef.html
  • Loading branch information
@grendello
grendello committed Mar 29, 2022
1 parent 4ce67f5 commit 5271f3e
Show file tree
Hide file tree
Showing 62 changed files with 3,793 additions and 2,556 deletions.
1 change: 1 addition & 0 deletions build-tools/installers/create-installers.targets
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,7 @@
<_MSBuildFilesWin Include="$(MSBuildSrcDir)\aapt2.exe" />
<_MSBuildFilesWin Include="$(MSBuildSrcDir)\binutils\bin\as.exe" />
<_MSBuildFilesWin Include="$(MSBuildSrcDir)\binutils\bin\ld.exe" />
<_MSBuildFilesWin Include="$(MSBuildSrcDir)\binutils\bin\llc.exe" />
<_MSBuildFilesWin Include="$(MSBuildSrcDir)\binutils\bin\llvm-mc.exe" />
<_MSBuildFilesWin Include="$(MSBuildSrcDir)\binutils\bin\llvm-strip.exe" />
<_MSBuildFilesWin Include="$(MSBuildSrcDir)\binutils\bin\aarch64-linux-android-as.cmd" />
Expand Down
1 change: 1 addition & 0 deletions build-tools/installers/unix-binutils.projitems
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<_MSBuildFilesUnixSignAndHarden Include="$(MSBuildSrcDir)\$(HostOS)\binutils\bin\i686-linux-android-ld" />
<_MSBuildFilesUnixSignAndHarden Include="$(MSBuildSrcDir)\$(HostOS)\binutils\bin\i686-linux-android-strip" />
<_MSBuildFilesUnixSignAndHarden Include="$(MSBuildSrcDir)\$(HostOS)\binutils\bin\ld" />
<_MSBuildFilesUnixSignAndHarden Include="$(MSBuildSrcDir)\$(HostOS)\binutils\bin\llc" />
<_MSBuildFilesUnixSignAndHarden Include="$(MSBuildSrcDir)\$(HostOS)\binutils\bin\llvm-mc" />
<_MSBuildFilesUnixSignAndHarden Include="$(MSBuildSrcDir)\$(HostOS)\binutils\bin\llvm-strip" />
<_MSBuildFilesUnixSignAndHarden Include="$(MSBuildSrcDir)\$(HostOS)\binutils\bin\x86_64-linux-android-as" />
Expand Down
1 change: 1 addition & 0 deletions build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ public static partial class Defaults
// Unprefixed tools
new NDKTool (name: "as"),
new NDKTool (name: "ld"),
new NDKTool (name: "llc"),
new NDKTool (name: "llvm-mc"),
new NDKTool (name: "llvm-strip"),
};
Expand Down
67 changes: 16 additions & 51 deletions src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ sealed class Config
public bool DebugBuild { get; set; }

[Required]
public string WorkingDirectory { get; set; }
public new string WorkingDirectory { get; set; }

[Required]
public string AndroidBinUtilsDirectory { get; set; }
Expand All @@ -57,7 +57,7 @@ void RunAssembler (Config config)
};

string assemblerName = Path.GetFileName (config.AssemblerPath);
LogDebugMessage ($"[Native Assembler] {psi.FileName} {psi.Arguments}");
LogDebugMessage ($"[LLVM llc] {psi.FileName} {psi.Arguments}");
using (var proc = new Process ()) {
proc.OutputDataReceived += (s, e) => {
if (e.Data != null)
Expand Down Expand Up @@ -95,61 +95,26 @@ void RunAssembler (Config config)

IEnumerable<Config> GetAssemblerConfigs ()
{
foreach (ITaskItem item in Sources) {
string abi = item.GetMetadata ("abi")?.ToLowerInvariant ();
string prefix = String.Empty;
AndroidTargetArch arch;

switch (abi) {
case "armeabi-v7a":
prefix = Path.Combine (AndroidBinUtilsDirectory, "arm-linux-androideabi");
arch = AndroidTargetArch.Arm;
break;

case "arm64":
case "arm64-v8a":
case "aarch64":
prefix = Path.Combine (AndroidBinUtilsDirectory, "aarch64-linux-android");
arch = AndroidTargetArch.Arm64;
break;

case "x86":
prefix = Path.Combine (AndroidBinUtilsDirectory, "i686-linux-android");
arch = AndroidTargetArch.X86;
break;

case "x86_64":
prefix = Path.Combine (AndroidBinUtilsDirectory, "x86_64-linux-android");
arch = AndroidTargetArch.X86_64;
break;

default:
throw new NotSupportedException ($"Unsupported Android target architecture ABI: {abi}");
}
const string assemblerOptions =
"-O2 " +
"--debugger-tune=lldb " + // NDK uses lldb now
"--debugify-level=location+variables " +
"--fatal-warnings " +
"--filetype=obj " +
"--relocation-model=pic";
string llcPath = Path.Combine (AndroidBinUtilsDirectory, "llc");

// We don't need the directory since our WorkingDirectory is (and must be) where all the
// sources are (because of the typemap.inc file being included by the other sources with
// a relative path of `.`)
foreach (ITaskItem item in Sources) {
// We don't need the directory since our WorkingDirectory is where all the sources are
string sourceFile = Path.GetFileName (item.ItemSpec);
var assemblerOptions = new List<string> {
"--warn",
"-o",
QuoteFileName (sourceFile.Replace (".s", ".o"))
};

if (DebugBuild)
assemblerOptions.Add ("-g");

assemblerOptions.Add (QuoteFileName (sourceFile));

string baseExecutablePath = $"{prefix}-as";
string executableDir = Path.GetDirectoryName (baseExecutablePath);
string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (baseExecutablePath));
string outputFile = QuoteFileName (sourceFile.Replace (".ll", ".o"));
string executableDir = Path.GetDirectoryName (llcPath);
string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (llcPath));

yield return new Config {
InputSource = item.ItemSpec,
AssemblerPath = Path.Combine (executableDir, executableName),
AssemblerOptions = String.Join (" ", assemblerOptions),
AssemblerOptions = $"{assemblerOptions} -o={outputFile} {QuoteFileName (sourceFile)}",
};
}
}
Expand Down
12 changes: 7 additions & 5 deletions src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,16 +74,18 @@ void GenerateCompressedAssemblySources ()

void Generate (IDictionary<string, CompressedAssemblyInfo> dict)
{
var llvmAsmgen = new CompressedAssembliesNativeAssemblyGenerator (dict);
llvmAsmgen.Init ();

foreach (string abi in SupportedAbis) {
string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"compressed_assemblies.{abi.ToLowerInvariant ()}");
string asmFilePath = $"{baseAsmFilePath}.s";
var asmgen = new CompressedAssembliesNativeAssemblyGenerator (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), dict, baseAsmFilePath);
string llvmIrFilePath = $"{baseAsmFilePath}.ll";

using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
asmgen.Write (sw, asmFilePath);
llvmAsmgen.Write (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llvmIrFilePath);
sw.Flush ();
if (Files.CopyIfStreamChanged (sw.BaseStream, asmFilePath)) {
Log.LogDebugMessage ($"File {asmFilePath} was regenerated");
if (Files.CopyIfStreamChanged (sw.BaseStream, llvmIrFilePath)) {
Log.LogDebugMessage ($"File {llvmIrFilePath} was regenerated");
}
}
}
Expand Down
54 changes: 27 additions & 27 deletions src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -383,39 +383,39 @@ void AddEnvironment ()

bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath);
var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<ApplicationConfigTaskState> (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build);
var appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) {
IsBundledApp = IsBundledApplication,
UsesMonoAOT = usesMonoAOT,
UsesMonoLLVM = EnableLLVM,
UsesAssemblyPreload = usesAssemblyPreload,
MonoAOTMode = aotMode.ToString ().ToLowerInvariant (),
AndroidPackageName = AndroidPackageName,
BrokenExceptionTransitions = brokenExceptionTransitions,
PackageNamingPolicy = pnp,
BoundExceptionType = boundExceptionType,
InstantRunEnabled = InstantRunEnabled,
JniAddNativeMethodRegistrationAttributePresent = appConfState != null ? appConfState.JniAddNativeMethodRegistrationAttributePresent : false,
HaveRuntimeConfigBlob = haveRuntimeConfigBlob,
NumberOfAssembliesInApk = assemblyCount,
BundledAssemblyNameWidth = assemblyNameWidth,
NumberOfAssemblyStoresInApks = 2, // Until feature APKs are a thing, we're going to have just two stores in each app - one for arch-agnostic
// and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app
// runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names
// and in the same order.
MonoComponents = monoComponents,
NativeLibraries = uniqueNativeLibraries,
HaveAssemblyStore = UseAssemblyStore,
};
appConfigAsmGen.Init ();

foreach (string abi in SupportedAbis) {
string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}");
string asmFilePath = $"{baseAsmFilePath}.s";

var asmgen = new ApplicationConfigNativeAssemblyGenerator (GetAndroidTargetArchForAbi (abi), environmentVariables, systemProperties, Log) {
IsBundledApp = IsBundledApplication,
UsesMonoAOT = usesMonoAOT,
UsesMonoLLVM = EnableLLVM,
UsesAssemblyPreload = usesAssemblyPreload,
MonoAOTMode = aotMode.ToString ().ToLowerInvariant (),
AndroidPackageName = AndroidPackageName,
BrokenExceptionTransitions = brokenExceptionTransitions,
PackageNamingPolicy = pnp,
BoundExceptionType = boundExceptionType,
InstantRunEnabled = InstantRunEnabled,
JniAddNativeMethodRegistrationAttributePresent = appConfState != null ? appConfState.JniAddNativeMethodRegistrationAttributePresent : false,
HaveRuntimeConfigBlob = haveRuntimeConfigBlob,
NumberOfAssembliesInApk = assemblyCount,
BundledAssemblyNameWidth = assemblyNameWidth,
NumberOfAssemblyStoresInApks = 2, // Until feature APKs are a thing, we're going to have just two stores in each app - one for arch-agnostic
// and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app
// runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names
// and in the same order.
MonoComponents = monoComponents,
NativeLibraries = uniqueNativeLibraries,
HaveAssemblyStore = UseAssemblyStore,
};
string llFilePath = $"{baseAsmFilePath}.ll";

using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
asmgen.Write (sw, asmFilePath);
appConfigAsmGen.Write (GetAndroidTargetArchForAbi (abi), sw, llFilePath);
sw.Flush ();
Files.CopyIfStreamChanged (sw.BaseStream, asmFilePath);
Files.CopyIfStreamChanged (sw.BaseStream, llFilePath);
}
}

Expand Down
57 changes: 29 additions & 28 deletions src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,67 +111,68 @@ IEnumerable<Config> GetLinkerConfigs ()
abis [abi] = GatherFilesForABI(item.ItemSpec, abi, ObjectFiles);
}

const string commonLinkerArgs =
"--unresolved-symbols=ignore-in-shared-libs " +
"--export-dynamic " +
"-soname libxamarin-app.so " +
"-z relro " +
"-z noexecstack " +
"--enable-new-dtags " +
"--eh-frame-hdr " +
"-shared " +
"--build-id " +
"--warn-shared-textrel " +
"--fatal-warnings";

string stripSymbolsArg = DebugBuild ? String.Empty : " -s";

string ld = Path.Combine (AndroidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, "ld"));
var targetLinkerArgs = new List<string> ();
foreach (var kvp in abis) {
string abi = kvp.Key;
InputFiles inputs = kvp.Value;
AndroidTargetArch arch;

var linkerArgs = new List <string> {
"--unresolved-symbols=ignore-in-shared-libs",
"--export-dynamic",
"-soname", "libxamarin-app.so",
"-z", "relro",
"-z", "noexecstack",
"--enable-new-dtags",
"--eh-frame-hdr",
"-shared",
"--build-id",
"--warn-shared-textrel",
"--fatal-warnings",
"-o", QuoteFileName (inputs.OutputSharedLibrary),
};

targetLinkerArgs.Clear ();
string elf_arch;
switch (abi) {
case "armeabi-v7a":
arch = AndroidTargetArch.Arm;
linkerArgs.Add ("-X");
targetLinkerArgs.Add ("-X");
elf_arch = "armelf_linux_eabi";
break;

case "arm64":
case "arm64-v8a":
case "aarch64":
arch = AndroidTargetArch.Arm64;
linkerArgs.Add ("--fix-cortex-a53-843419");
targetLinkerArgs.Add ("--fix-cortex-a53-843419");
elf_arch = "aarch64linux";
break;

case "x86":
arch = AndroidTargetArch.X86;
elf_arch = "elf_i386";
break;

case "x86_64":
arch = AndroidTargetArch.X86_64;
elf_arch = "elf_x86_64";
break;

default:
throw new NotSupportedException ($"Unsupported Android target architecture ABI: {abi}");
}

linkerArgs.Add ("-m");
linkerArgs.Add (elf_arch);
targetLinkerArgs.Add ("-m");
targetLinkerArgs.Add (elf_arch);

foreach (string file in inputs.ObjectFiles) {
linkerArgs.Add (QuoteFileName (file));
targetLinkerArgs.Add (QuoteFileName (file));
}

string ld = MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, $"{NdkTools.GetBinutilsToolchainPrefix (arch)}ld");
targetLinkerArgs.Add ("-o");
targetLinkerArgs.Add (QuoteFileName (inputs.OutputSharedLibrary));

string targetArgs = String.Join (" ", targetLinkerArgs);
yield return new Config {
LinkerPath = Path.Combine (AndroidBinUtilsDirectory, ld),
LinkerOptions = String.Join (" ", linkerArgs),
LinkerPath = ld,
LinkerOptions = $"{commonLinkerArgs}{stripSymbolsArg} {targetArgs}",
OutputSharedLibrary = inputs.OutputSharedLibrary,
};
}
Expand Down
33 changes: 1 addition & 32 deletions src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,10 @@ public override bool RunTask ()
{
var sources = new List<ITaskItem> ();
var includes = new List<ITaskItem> ();
bool haveSharedSource = false;
bool haveArmV7SharedSource = false;
bool typeMapMode = false;
string baseName;

if (String.Compare ("typemap", Mode, StringComparison.OrdinalIgnoreCase) == 0) {
baseName = TypeMapBase;
typeMapMode = true;
} else if (String.Compare ("environment", Mode, StringComparison.OrdinalIgnoreCase) == 0) {
baseName = EnvBase;
} else if (String.Compare ("compressed", Mode, StringComparison.OrdinalIgnoreCase) == 0) {
Expand All @@ -61,36 +57,9 @@ public override bool RunTask ()

TaskItem item;
foreach (string abi in BuildTargetAbis) {
if (typeMapMode) {
if (String.Compare (ArmV7a, abi, StringComparison.Ordinal) == 0)
haveArmV7SharedSource = true;
else
haveSharedSource = true;
}

item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.{abi}.s"));
item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.{abi}.ll"));
item.SetMetadata ("abi", abi);
sources.Add (item);

if (!typeMapMode)
continue;

if (!InstantRunEnabled && !Debug) {
item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.{abi}-managed.inc"));
item.SetMetadata ("abi", abi);
includes.Add (item);
}
}

if (haveArmV7SharedSource) {
item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.{ArmV7a}-shared.inc"));
item.SetMetadata ("abi", ArmV7a);
includes.Add (item);
}

if (haveSharedSource) {
item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.shared.inc"));
includes.Add (item);
}

AssemblySources = sources.ToArray ();
Expand Down
6 changes: 3 additions & 3 deletions src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void CheckMonoComponentsMask (bool enableProfiler, bool useInterpreter, b
Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
string objPath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath);

List<string> envFiles = EnvironmentHelper.GatherEnvironmentFiles (objPath, String.Join (";", abis), true);
List<EnvironmentHelper.EnvironmentFile> envFiles = EnvironmentHelper.GatherEnvironmentFiles (objPath, String.Join (";", abis), true);
EnvironmentHelper.ApplicationConfig app_config = EnvironmentHelper.ReadApplicationConfig (envFiles);
Assert.That (app_config, Is.Not.Null, "application_config must be present in the environment files");
Assert.IsTrue (app_config.mono_components_mask == expectedMask, "Expected Mono Components mask 0x{expectedMask:x}, got 0x{app_config.mono_components_mask:x}");
Expand Down Expand Up @@ -126,13 +126,13 @@ public void CheckAssemblyCounts (bool isRelease, bool aot)
Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
string objPath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath);

List<string> envFiles = EnvironmentHelper.GatherEnvironmentFiles (objPath, String.Join (";", abis), true);
List<EnvironmentHelper.EnvironmentFile> envFiles = EnvironmentHelper.GatherEnvironmentFiles (objPath, String.Join (";", abis), true);
EnvironmentHelper.ApplicationConfig app_config = EnvironmentHelper.ReadApplicationConfig (envFiles);
Assert.That (app_config, Is.Not.Null, "application_config must be present in the environment files");

if (aot) {
foreach (var env in envFiles) {
StringAssert.Contains ("libaot-Mono.Android.dll.so", File.ReadAllText (env));
StringAssert.Contains ("libaot-Mono.Android.dll.so", File.ReadAllText (env.Path));
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ public void DSOPageAlignment ()
public void EnvironmentFileContents ()
{
string intermediateOutputDir = Path.Combine (testProjectPath, "obj", XABuildPaths.Configuration);
List<string> envFiles = EnvironmentHelper.GatherEnvironmentFiles (intermediateOutputDir, Xamarin.Android.Tools.XABuildConfig.SupportedABIs, true);
List<EnvironmentHelper.EnvironmentFile> envFiles = EnvironmentHelper.GatherEnvironmentFiles (intermediateOutputDir, Xamarin.Android.Tools.XABuildConfig.SupportedABIs, true);
EnvironmentHelper.ApplicationConfig app_config = EnvironmentHelper.ReadApplicationConfig (envFiles);
Assert.That (app_config, Is.Not.Null, "application_config must be present in the environment files");
}
Expand Down

0 comments on commit 5271f3e

Please sign in to comment.