diff --git a/README.md b/README.md index 28c545e665e..93d922bf240 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ Xamarin.Android provides open-source bindings of the Android SDK for use with [oss-macOS-x86_64-icon]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android/badge/icon [oss-macOS-x86_64-status]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android/ -[oss-ubuntu-x86_64-icon]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android-linux/badge/icon -[oss-ubuntu-x86_64-status]: https://jenkins.mono-project.com/view/Xamarin.Android/job/xamarin-android-linux/ +[oss-ubuntu-x86_64-icon]: https://dev.azure.com/xamarin/public/_apis/build/status/Xamarin.Android-OSS?branchName=master +[oss-ubuntu-x86_64-status]: https://dev.azure.com/xamarin/public/_build/latest?definitionId=48 [oss-windows-x86_64-icon]: https://devdiv.visualstudio.com/_apis/public/build/definitions/0bdbc590-a062-4c3f-b0f6-9383f67865ee/7212/badge [oss-windows-x86_64-status]: https://devdiv.visualstudio.com/DevDiv/Default/_build/index?definitionId=7212 diff --git a/build-tools/automation/azure-pipelines-oss.yaml b/build-tools/automation/azure-pipelines-oss.yaml new file mode 100644 index 00000000000..c6a0b5cbfc3 --- /dev/null +++ b/build-tools/automation/azure-pipelines-oss.yaml @@ -0,0 +1,67 @@ +# Xamarin.Android-OSS Pipeline + +name: $(Build.SourceBranchName)-$(Build.SourceVersion)-$(Rev:r) + +trigger: + - master + - d16-* + +pr: + - master + - d16-* + +# Global variables +variables: + OSSBuildConfiguration: Release + +jobs: +- job: linux_build_package + displayName: Linux Build + pool: + vmImage: ubuntu-18.04 + timeoutInMinutes: 180 + cancelTimeoutInMinutes: 5 + workspace: + clean: all + steps: + - checkout: self + submodules: recursive + + - task: NuGetToolInstaller@1 + displayName: 'Use NuGet 5.x' + inputs: + versionSpec: 5.x + + - script: | + sudo apt-get install -y gnupg ca-certificates + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + echo "deb https://download.mono-project.com/repo/ubuntu preview-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-preview.list + sudo apt-get update + sudo apt-get install -y mono-devel + sudo apt-get install -y ca-certificates-mono + displayName: install mono preview + + - script: make jenkins V=1 PREPARE_CI_PR=1 PREPARE_AUTOPROVISION=1 CONFIGURATION=$(OSSBuildConfiguration) + displayName: make jenkins + + - script: make package-deb V=1 CONFIGURATION=$(OSSBuildConfiguration) + displayName: make package-deb + + - script: | + mkdir -p $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts + cp $(System.DefaultWorkingDirectory)/*xamarin.android*.tar.bz2 $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts + cp $(System.DefaultWorkingDirectory)/*.changes $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts + cp $(System.DefaultWorkingDirectory)/*.dsc $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts + cp $(System.DefaultWorkingDirectory)/*.deb $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts + displayName: copy linux artifacts + + - task: PublishPipelineArtifact@1 + displayName: publish linux artifacts + inputs: + artifactName: Linux Packages + targetPath: $(System.DefaultWorkingDirectory)/bin/Build$(OSSBuildConfiguration)/linux-artifacts + + - template: yaml-templates/upload-results.yaml + parameters: + configuration: $(OSSBuildConfiguration) + artifactName: OSS Build Results - Linux diff --git a/build-tools/automation/build.linux.groovy b/build-tools/automation/build.linux.groovy index 41798d79bff..ffe138e7690 100644 --- a/build-tools/automation/build.linux.groovy +++ b/build-tools/automation/build.linux.groovy @@ -34,7 +34,6 @@ def chRootPackages = ''' libtool libtool-bin libz-mingw-w64-dev - libzip-dev linux-libc-dev:i386 mono-csharp-shell mono-devel diff --git a/build-tools/debian-metadata/control b/build-tools/debian-metadata/control index f6421ca14cb..753b1f60406 100644 --- a/build-tools/debian-metadata/control +++ b/build-tools/debian-metadata/control @@ -2,7 +2,7 @@ Source: xamarin.android-oss Section: unknown Priority: optional Maintainer: Xamarin -Build-Depends: debhelper (>=9), cli-common-dev (>= 0.9~), libzip-dev +Build-Depends: debhelper (>=9), cli-common-dev (>= 0.9~) Standards-Version: 3.9.6 Homepage: https://www.xamarin.com/platform Vcs-Git: https://github.com/xamarin/xamarin-android.git diff --git a/build-tools/debian-metadata/rules b/build-tools/debian-metadata/rules index 0823a8ce7cb..cf380523ba9 100755 --- a/build-tools/debian-metadata/rules +++ b/build-tools/debian-metadata/rules @@ -40,6 +40,7 @@ override_dh_clideps: --exclude-moduleref=java-interop \ --exclude-moduleref=libfam.so.0 \ --exclude-moduleref=libgamin-1.so.0 \ + --exclude-moduleref=libmono-btls-shared \ --exclude-moduleref=lzo.dll \ --exclude-moduleref=mscoree.dll \ --exclude-moduleref=mscorwks.dll \ diff --git a/build-tools/scripts/BuildEverything.mk b/build-tools/scripts/BuildEverything.mk index 1dcf266ad28..4b96b1fba68 100644 --- a/build-tools/scripts/BuildEverything.mk +++ b/build-tools/scripts/BuildEverything.mk @@ -10,8 +10,14 @@ # The other targets depended upon by leeroy also require rules.mk to be present and thus they # are invoked in the same way framework-assemblies is # +# Local `make jenkins` invocations should build everything by default. We need to ensure `-a` is passed to xaprepare when no CI flags are set. +# jenkins: - $(MAKE) PREPARE_CI=1 prepare +ifeq ($(PREPARE_CI_PR)$(PREPARE_CI),00) + $(MAKE) PREPARE_ARGS=-a prepare +else + $(MAKE) prepare +endif $(MAKE) leeroy $(ZIP_OUTPUT) ifeq ($(XA_INVERTED_COMMERCIAL_BUILD),true) $(MAKE) commercial diff --git a/build-tools/xaprepare/xaprepare/Application/Context.cs b/build-tools/xaprepare/xaprepare/Application/Context.cs index 8205be94df8..eb0822bdeda 100644 --- a/build-tools/xaprepare/xaprepare/Application/Context.cs +++ b/build-tools/xaprepare/xaprepare/Application/Context.cs @@ -340,6 +340,21 @@ public string DebugFileExtension { /// public bool MonoArchiveDownloaded { get; set; } + /// + /// Determines whether or not we are running on a hosted azure pipelines agent. + /// These agents have certain limitations, the most pressing being the amount of available storage. + /// https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops#capabilities-and-limitations. + /// + public bool IsRunningOnHostedAzureAgent { + get { + string agentNameValue = Environment.GetEnvironmentVariable ("AGENT_NAME"); + bool hasHostedAgentName = !string.IsNullOrEmpty (agentNameValue) && agentNameValue.ToUpperInvariant ().Contains ("AZURE PIPELINES"); + string serverTypeValue = Environment.GetEnvironmentVariable ("SYSTEM_SERVERTYPE"); + bool isHostedServerType = !string.IsNullOrEmpty (serverTypeValue) && serverTypeValue.ToUpperInvariant ().Contains ("HOSTED"); + return hasHostedAgentName || isHostedServerType; + } + } + static Context () { Instance = new Context (); diff --git a/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs b/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs index 31fd8b9cb03..7e6f06564d2 100644 --- a/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs +++ b/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace Xamarin.Android.Prepare { @@ -87,6 +88,10 @@ void Generate (Context context, StreamWriter sw) stableApiLevels.Add (api); } + var enabledJitAbis = AbiNames.AllJitAbis.Where (a => context.IsTargetJitAbiEnabled (a)); + var enabledHostAbis = AbiNames.AllHostAbis.Where (a => context.IsHostJitAbiEnabled (a)); + var enabledAotAbis = AbiNames.AllAotAbis.Where (a => context.IsTargetAotAbiEnabled (a)); + WriteVariable ("ALL_API_LEVELS", ToValue (allApiLevels)); WriteVariable ("ALL_PLATFORM_IDS", ToValue (allPlatformIDs)); WriteVariable ("ALL_FRAMEWORKS", ToValue (allFrameworks)); @@ -94,9 +99,9 @@ void Generate (Context context, StreamWriter sw) WriteVariable ("STABLE_API_LEVELS", ToValue (stableApiLevels)); WriteVariable ("FRAMEWORKS", ToValue (frameworks)); WriteVariable ("STABLE_FRAMEWORKS", ToValue (stableFrameworks)); - WriteVariable ("ALL_JIT_ABIS", ToValue (AbiNames.AllJitAbis)); - WriteVariable ("ALL_HOST_ABIS", ToValue (AbiNames.AllHostAbis)); - WriteVariable ("ALL_AOT_ABIS", ToValue (AbiNames.AllAotAbis)); + WriteVariable ("ALL_JIT_ABIS", ToValue (enabledJitAbis.ToList())); + WriteVariable ("ALL_HOST_ABIS", ToValue (enabledHostAbis.ToList ())); + WriteVariable ("ALL_AOT_ABIS", ToValue (enabledAotAbis.ToList ())); WriteVariable ("ANDROID_TOOLCHAIN_DIR", context.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory)); if (context.MonoOptions != null && context.MonoOptions.Count > 0) { WriteVariable ("MONO_OPTIONS", ToValue (context.MonoOptions)); @@ -104,9 +109,9 @@ void Generate (Context context, StreamWriter sw) } sw.WriteLine ("_MSBUILD_ARGS = \\"); - sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetJitAbis}={Utilities.ToXamarinAndroidPropertyValue (AbiNames.AllJitAbis)} \\"); - sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedHostJitAbis}={Utilities.ToXamarinAndroidPropertyValue (AbiNames.AllHostAbis)} \\"); - sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetAotAbis}={Utilities.ToXamarinAndroidPropertyValue (AbiNames.AllAotAbis)}"); + sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetJitAbis}={context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetJitAbis)} \\"); + sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedHostJitAbis}={context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedHostJitAbis)} \\"); + sw.WriteLine ($"\t/p:{KnownProperties.AndroidSupportedTargetAotAbis}={context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetAotAbis)}"); OutputOSVariables (context, sw); diff --git a/build-tools/xaprepare/xaprepare/Application/Program.DebianLinux.cs b/build-tools/xaprepare/xaprepare/Application/Program.DebianLinux.cs index fe0e5e70844..9ef10de5d20 100644 --- a/build-tools/xaprepare/xaprepare/Application/Program.DebianLinux.cs +++ b/build-tools/xaprepare/xaprepare/Application/Program.DebianLinux.cs @@ -38,7 +38,7 @@ protected override bool CheckWhetherInstalled () public override async Task Install () { - var runner = new ProcessRunner ("sudo", "apt-get", "-f", "-u", "install", PackageName) { + var runner = new ProcessRunner ("sudo", "apt-get", "-y", "-f", "-u", "install", PackageName) { EchoStandardOutput = true, EchoStandardError = true, ProcessTimeout = TimeSpan.FromMinutes (30), diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.Debian.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.Debian.cs index c21e9c880c0..165d348259a 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.Debian.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.Debian.cs @@ -6,7 +6,6 @@ namespace Xamarin.Android.Prepare class LinuxDebian : LinuxDebianCommon { static readonly List packages = new List { - new DebianLinuxProgram ("zlib1g-dev"), new DebianLinuxProgram ("libtool-bin", "libtool"), }; diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.DebianCommon.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.DebianCommon.cs index da6c79854ad..3028d6f2e29 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.DebianCommon.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.DebianCommon.cs @@ -10,7 +10,9 @@ abstract class LinuxDebianCommon : Linux new DebianLinuxProgram ("automake"), new DebianLinuxProgram ("cmake"), new DebianLinuxProgram ("build-essential"), + new DebianLinuxProgram ("cli-common-dev"), new DebianLinuxProgram ("curl"), + new DebianLinuxProgram ("devscripts"), new DebianLinuxProgram ("gcc"), new DebianLinuxProgram ("g++"), new DebianLinuxProgram ("g++-mingw-w64"), @@ -22,6 +24,7 @@ abstract class LinuxDebianCommon : Linux new DebianLinuxProgram ("linux-libc-dev"), new DebianLinuxProgram ("make"), new DebianLinuxProgram ("ninja-build", "ninja"), + new DebianLinuxProgram ("nuget"), new DebianLinuxProgram ("p7zip-full", "7z"), new DebianLinuxProgram ("sqlite3"), new DebianLinuxProgram ("vim-common"), diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.UbuntuCommon.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.UbuntuCommon.cs index fcdd32b3898..2fb35695b6e 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.UbuntuCommon.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/Linux.UbuntuCommon.cs @@ -5,11 +5,6 @@ namespace Xamarin.Android.Prepare { abstract partial class LinuxUbuntuCommon : LinuxDebianCommon { - static readonly List commonPackages64bit = new List { - new DebianLinuxProgram ("linux-libc-dev:i386"), - new DebianLinuxProgram ("zlib1g-dev:i386"), - }; - static readonly List libtoolPackages = new List { new DebianLinuxProgram ("libtool-bin", "libtool"), }; @@ -20,14 +15,6 @@ protected LinuxUbuntuCommon (Context context) : base (context) {} - protected override void InitializeDependencies () - { - base.InitializeDependencies (); - - if (Is64Bit) - Dependencies.AddRange (commonPackages64bit); - } - protected override bool InitOS () { if (!base.InitOS ()) diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs index d5b1d729a88..109a1a44fd5 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs @@ -163,6 +163,10 @@ async Task Unpack (Context context, string tempDirRoot, AndroidPackage pkg) throw new InvalidOperationException ($"Failed to unpack {pkg.LocalPackagePath}"); } + // Clean up zip after extraction if running on a hosted azure pipelines agent. + if (context.IsRunningOnHostedAzureAgent) + Utilities.DeleteFileSilent (pkg.LocalPackagePath); + if (pkg.Component.NoSubdirectory) { Utilities.MoveDirectoryContentsRecursively (tempDir, pkg.DestinationDir); return; diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_DownloadMonoArchive.cs b/build-tools/xaprepare/xaprepare/Steps/Step_DownloadMonoArchive.cs index 9e357ad64c4..0a4f55af786 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_DownloadMonoArchive.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_DownloadMonoArchive.cs @@ -126,6 +126,9 @@ async Task DownloadAndUpackIfNeeded (Context context, string name, string Utilities.MoveDirectoryContentsRecursively (tempDir, destinationDirectory, resetFileTimestamp: true); } finally { Utilities.DeleteDirectorySilent (tempDir); + // Clean up zip after extraction if running on a hosted azure pipelines agent. + if (context.IsRunningOnHostedAzureAgent) + Utilities.DeleteFileSilent (localPath); } return true; diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs index a3e04d01bf7..45d1b3983f3 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs @@ -61,6 +61,9 @@ protected override async Task Execute (Context context) Utilities.MoveDirectoryContentsRecursively (rootDir, corettoInstallDir); } finally { Utilities.DeleteDirectorySilent (tempDir); + // Clean up zip after extraction if running on a hosted azure pipelines agent. + if (context.IsRunningOnHostedAzureAgent) + Utilities.DeleteFileSilent (localPackagePath); } return true;