[build] Add OSS Azure Pipeline Linux Build

README.md

@@ -16,8 +16,8 @@ Xamarin.Android provides open-source bindings of the Android SDK for use with
 
 [oss-macOS-x86_64-icon]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android/badge/icon
 [oss-macOS-x86_64-status]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android/
-[oss-ubuntu-x86_64-icon]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android-linux/badge/icon
-[oss-ubuntu-x86_64-status]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android-linux/
+[oss-ubuntu-x86_64-icon]: https://dev.azure.com/xamarin/public/_apis/build/status/Xamarin.Android-OSS?branchName=master
+[oss-ubuntu-x86_64-status]: https://dev.azure.com/xamarin/public/_build/latest?definitionId=48
 [oss-windows-x86_64-icon]: https://devdiv.visualstudio.com/_apis/public/build/definitions/0bdbc590-a062-4c3f-b0f6-9383f67865ee/7212/badge
 [oss-windows-x86_64-status]: https://devdiv.visualstudio.com/DevDiv/Default/_build/index?definitionId=7212
 

build-tools/automation/azure-pipelines-oss.yaml

@@ -0,0 +1,67 @@
+# Xamarin.Android-OSS Pipeline
+
+name: $(Build.SourceBranchName)-$(Build.SourceVersion)-$(Rev:r)
+
+trigger:
+  - master
+  - d16-*
+
+pr:
+  - master
+  - d16-*
+
+# Global variables
+variables:
+  OSSBuildConfiguration: Release
+
+jobs:
+- job: linux_build_package
+  displayName: Linux Build
+  pool:
+    vmImage: ubuntu-18.04
+  timeoutInMinutes: 180
+  cancelTimeoutInMinutes: 5
+  workspace:
+    clean: all
+  steps:
+  - checkout: self
+    submodules: recursive
+
+  - task: NuGetToolInstaller@1
+    displayName: 'Use NuGet 5.x'
+    inputs:
+      versionSpec: 5.x
+
+  - script: |
+      sudo apt-get install -y gnupg ca-certificates
+      sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+      echo "deb https://download.mono-project.com/repo/ubuntu preview-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-preview.list
+      sudo apt-get update
+      sudo apt-get install -y mono-devel
+      sudo apt-get install -y ca-certificates-mono
+    displayName: install mono preview
+
+  - script: make jenkins V=1 PREPARE_CI_PR=1 PREPARE_AUTOPROVISION=1 CONFIGURATION=$(OSSBuildConfiguration)
+    displayName: make jenkins
+
+  - script: make package-deb V=1 CONFIGURATION=$(OSSBuildConfiguration)
+    displayName: make package-deb
+
+  - script: |
+      mkdir -p $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts
+      cp $(System.DefaultWorkingDirectory)/*xamarin.android*.tar.bz2 $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts
+      cp $(System.DefaultWorkingDirectory)/*.changes $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts
+      cp $(System.DefaultWorkingDirectory)/*.dsc $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts
+      cp $(System.DefaultWorkingDirectory)/*.deb $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts
+    displayName: copy linux artifacts
+
+  - task: PublishPipelineArtifact@1
+    displayName: publish linux artifacts
+    inputs:
+      artifactName: Linux Packages
+      targetPath: $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts
+
+  - template: yaml-templates/upload-results.yaml
+    parameters:
+      configuration: $(OSSBuildConfiguration)
+      artifactName: OSS Build Results - Linux

build-tools/automation/build.linux.groovy

@@ -34,7 +34,6 @@ def chRootPackages = '''
     libtool
     libtool-bin
     libz-mingw-w64-dev
-    libzip-dev
     linux-libc-dev:i386
     mono-csharp-shell
     mono-devel

build-tools/debian-metadata/control

@@ -2,7 +2,7 @@ Source: xamarin.android-oss
 Section: unknown
 Priority: optional
 Maintainer: Xamarin <hello@xamarin.com>
-Build-Depends: debhelper (>=9), cli-common-dev (>= 0.9~), libzip-dev
+Build-Depends: debhelper (>=9), cli-common-dev (>= 0.9~)
 Standards-Version: 3.9.6
 Homepage: https://www.xamarin.com/platform
 Vcs-Git: https://github.com/xamarin/xamarin-android.git

build-tools/debian-metadata/rules

@@ -40,6 +40,7 @@ override_dh_clideps:
 		--exclude-moduleref=java-interop \
 		--exclude-moduleref=libfam.so.0 \
 		--exclude-moduleref=libgamin-1.so.0 \
+		--exclude-moduleref=libmono-btls-shared \
 		--exclude-moduleref=lzo.dll \
 		--exclude-moduleref=mscoree.dll \
 		--exclude-moduleref=mscorwks.dll \

build-tools/scripts/BuildEverything.mk

@@ -10,8 +10,14 @@
 # The other targets depended upon by leeroy also require rules.mk to be present and thus they
 # are invoked in the same way framework-assemblies is
 #
+# Local `make jenkins` invocations should build everything by default. We need to ensure `-a` is passed to xaprepare when no CI flags are set.
+#
 jenkins:
-	$(MAKE) PREPARE_CI=1 prepare
+ifeq ($(PREPARE_CI_PR)$(PREPARE_CI),00)
+	$(MAKE) PREPARE_ARGS=-a prepare
+else
+	$(MAKE) prepare
+endif
 	$(MAKE) leeroy $(ZIP_OUTPUT)
 ifeq ($(XA_INVERTED_COMMERCIAL_BUILD),true)
 	$(MAKE) commercial

build-tools/xaprepare/xaprepare/Application/Context.cs

@@ -340,6 +340,21 @@ public string DebugFileExtension {
 		/// </summary>
 		public bool MonoArchiveDownloaded { get; set; }
 
+		/// <summary>
+		///   Determines whether or not we are running on a hosted azure pipelines agent.
+		///   These agents have certain limitations, the most pressing being the amount of available storage.
+		///   https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops#capabilities-and-limitations.
+		/// </summary>
+		public bool IsRunningOnHostedAzureAgent {
+			get {
+				string agentNameValue = Environment.GetEnvironmentVariable ("AGENT_NAME");
+				bool hasHostedAgentName = !string.IsNullOrEmpty (agentNameValue) && agentNameValue.ToUpperInvariant ().Contains ("AZURE PIPELINES");
+				string serverTypeValue = Environment.GetEnvironmentVariable ("SYSTEM_SERVERTYPE");
+				bool isHostedServerType = !string.IsNullOrEmpty (serverTypeValue) && serverTypeValue.ToUpperInvariant ().Contains ("HOSTED");
+				return hasHostedAgentName || isHostedServerType;
+			}
+		}
+
 		static Context ()
 		{
 			Instance = new Context ();

build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 
 namespace Xamarin.Android.Prepare
 {
@@ -87,26 +88,30 @@ void Generate (Context context, StreamWriter sw)
 					stableApiLevels.Add (api);
 			}
 
+			var enabledJitAbis = AbiNames.AllJitAbis.Where (a => context.IsTargetJitAbiEnabled (a));
+			var enabledHostAbis = AbiNames.AllHostAbis.Where (a => context.IsHostJitAbiEnabled (a));
+			var enabledAotAbis = AbiNames.AllAotAbis.Where (a => context.IsTargetAotAbiEnabled (a));
+
 			WriteVariable ("ALL_API_LEVELS",    ToValue (allApiLevels));
 			WriteVariable ("ALL_PLATFORM_IDS",  ToValue (allPlatformIDs));
 			WriteVariable ("ALL_FRAMEWORKS",    ToValue (allFrameworks));
 			WriteVariable ("API_LEVELS",        ToValue (apiLevels));
 			WriteVariable ("STABLE_API_LEVELS", ToValue (stableApiLevels));
 			WriteVariable ("FRAMEWORKS",        ToValue (frameworks));
 			WriteVariable ("STABLE_FRAMEWORKS", ToValue (stableFrameworks));
-			WriteVariable ("ALL_JIT_ABIS",      ToValue (AbiNames.AllJitAbis));
-			WriteVariable ("ALL_HOST_ABIS",     ToValue (AbiNames.AllHostAbis));
-			WriteVariable ("ALL_AOT_ABIS",      ToValue (AbiNames.AllAotAbis));
+			WriteVariable ("ALL_JIT_ABIS",      ToValue (enabledJitAbis.ToList()));
+			WriteVariable ("ALL_HOST_ABIS",     ToValue (enabledHostAbis.ToList ()));
+			WriteVariable ("ALL_AOT_ABIS",      ToValue (enabledAotAbis.ToList ()));
 			WriteVariable ("ANDROID_TOOLCHAIN_DIR", context.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory));
 			if (context.MonoOptions != null && context.MonoOptions.Count > 0) {
 				WriteVariable ("MONO_OPTIONS", ToValue (context.MonoOptions));
 				sw.WriteLine ("export MONO_OPTIONS");
 			}
 
 			sw.WriteLine ("_MSBUILD_ARGS = \\");
-			sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetJitAbis}={Utilities.ToXamarinAndroidPropertyValue (AbiNames.AllJitAbis)} \\");
-			sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedHostJitAbis}={Utilities.ToXamarinAndroidPropertyValue (AbiNames.AllHostAbis)} \\");
-			sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetAotAbis}={Utilities.ToXamarinAndroidPropertyValue (AbiNames.AllAotAbis)}");
+			sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetJitAbis}={context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetJitAbis)} \\");
+			sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedHostJitAbis}={context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedHostJitAbis)} \\");
+			sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetAotAbis}={context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetAotAbis)}");
 
 			OutputOSVariables (context, sw);
 

build-tools/xaprepare/xaprepare/Application/Program.DebianLinux.cs

@@ -38,7 +38,7 @@ protected override bool CheckWhetherInstalled ()
 
 		public override async Task<bool> Install ()
 		{
-			var runner = new ProcessRunner ("sudo", "apt-get", "-f", "-u", "install", PackageName) {
+			var runner = new ProcessRunner ("sudo", "apt-get", "-y", "-f", "-u", "install", PackageName) {
 				EchoStandardOutput = true,
 				EchoStandardError = true,
 				ProcessTimeout = TimeSpan.FromMinutes (30),

build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.Debian.cs

@@ -6,7 +6,6 @@ namespace Xamarin.Android.Prepare
 	class LinuxDebian : LinuxDebianCommon
 	{
 		static readonly List<DebianLinuxProgram> packages = new List<DebianLinuxProgram> {
-			new DebianLinuxProgram ("zlib1g-dev"),
 			new DebianLinuxProgram ("libtool-bin", "libtool"),
 		};
 

build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.DebianCommon.cs

@@ -10,7 +10,9 @@ abstract class LinuxDebianCommon : Linux
 			new DebianLinuxProgram ("automake"),
 			new DebianLinuxProgram ("cmake"),
 			new DebianLinuxProgram ("build-essential"),
+			new DebianLinuxProgram ("cli-common-dev"),
 			new DebianLinuxProgram ("curl"),
+			new DebianLinuxProgram ("devscripts"),
 			new DebianLinuxProgram ("gcc"),
 			new DebianLinuxProgram ("g++"),
 			new DebianLinuxProgram ("g++-mingw-w64"),
@@ -22,6 +24,7 @@ abstract class LinuxDebianCommon : Linux
 			new DebianLinuxProgram ("linux-libc-dev"),
 			new DebianLinuxProgram ("make"),
 			new DebianLinuxProgram ("ninja-build", "ninja"),
+			new DebianLinuxProgram ("nuget"),
 			new DebianLinuxProgram ("p7zip-full", "7z"),
 			new DebianLinuxProgram ("sqlite3"),
 			new DebianLinuxProgram ("vim-common"),

build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.UbuntuCommon.cs

@@ -5,11 +5,6 @@ namespace Xamarin.Android.Prepare
 {
 	abstract partial class LinuxUbuntuCommon : LinuxDebianCommon
 	{
-		static readonly List<DebianLinuxProgram> commonPackages64bit = new List<DebianLinuxProgram> {
-			new DebianLinuxProgram ("linux-libc-dev:i386"),
-			new DebianLinuxProgram ("zlib1g-dev:i386"),
-		};
-
 		static readonly List<DebianLinuxProgram> libtoolPackages = new List<DebianLinuxProgram> {
 			new DebianLinuxProgram ("libtool-bin", "libtool"),
 		};
@@ -20,14 +15,6 @@ protected LinuxUbuntuCommon (Context context)
 			: base (context)
 		{}
 
-		protected override void InitializeDependencies ()
-		{
-			base.InitializeDependencies ();
-
-			if (Is64Bit)
-				Dependencies.AddRange (commonPackages64bit);
-		}
-
 		protected override bool InitOS ()
 		{
 			if (!base.InitOS ())

build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs

@@ -163,6 +163,10 @@ async Task Unpack (Context context, string tempDirRoot, AndroidPackage pkg)
 				throw new InvalidOperationException ($"Failed to unpack {pkg.LocalPackagePath}");
 			}
 
+			// Clean up zip after extraction if running on a hosted azure pipelines agent.
+			if (context.IsRunningOnHostedAzureAgent)
+				Utilities.DeleteFileSilent (pkg.LocalPackagePath);
+
 			if (pkg.Component.NoSubdirectory) {
 				Utilities.MoveDirectoryContentsRecursively (tempDir, pkg.DestinationDir);
 				return;

build-tools/xaprepare/xaprepare/Steps/Step_DownloadMonoArchive.cs

@@ -126,6 +126,9 @@ async Task<bool> DownloadAndUpackIfNeeded (Context context, string name, string
 				Utilities.MoveDirectoryContentsRecursively (tempDir, destinationDirectory, resetFileTimestamp: true);
 			} finally {
 				Utilities.DeleteDirectorySilent (tempDir);
+				// Clean up zip after extraction if running on a hosted azure pipelines agent.
+				if (context.IsRunningOnHostedAzureAgent)
+					Utilities.DeleteFileSilent (localPath);
 			}
 
 			return true;

build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs

@@ -61,6 +61,9 @@ protected override async Task<bool> Execute (Context context)
 				Utilities.MoveDirectoryContentsRecursively (rootDir, corettoInstallDir);
 			} finally {
 				Utilities.DeleteDirectorySilent (tempDir);
+				// Clean up zip after extraction if running on a hosted azure pipelines agent.
+				if (context.IsRunningOnHostedAzureAgent)
+					Utilities.DeleteFileSilent (localPackagePath);
 			}
 
 			return true;