[7.17] [build] Cross compile docker images (elastic#128272)

* [build] Cross compile docker images

* typo

* debug

* Revert "[build] Cross compile docker images"

This reverts commit 621780e.

* revert

* support docker-cross-compile flag

* fix types/tests

* fix more tests

* download cloud dependencies based on cross compile flag

* fix array

* fix more tests

src/dev/build/args.test.ts

@@ -36,6 +36,7 @@ it('build default and oss dist for current platform, without packages, by defaul
         "createGenericFolders": true,
         "createPlatformFolders": true,
         "createRpmPackage": false,
+        "dockerCrossCompile": false,
         "downloadFreshNode": true,
         "initialize": true,
         "isRelease": false,
@@ -63,6 +64,7 @@ it('builds packages if --all-platforms is passed', () => {
         "createGenericFolders": true,
         "createPlatformFolders": true,
         "createRpmPackage": true,
+        "dockerCrossCompile": false,
         "downloadFreshNode": true,
         "initialize": true,
         "isRelease": false,
@@ -90,6 +92,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "createGenericFolders": true,
         "createPlatformFolders": true,
         "createRpmPackage": true,
+        "dockerCrossCompile": false,
         "downloadFreshNode": true,
         "initialize": true,
         "isRelease": false,
@@ -117,6 +120,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "createGenericFolders": true,
         "createPlatformFolders": true,
         "createRpmPackage": false,
+        "dockerCrossCompile": false,
         "downloadFreshNode": true,
         "initialize": true,
         "isRelease": false,
@@ -145,6 +149,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "createGenericFolders": true,
         "createPlatformFolders": true,
         "createRpmPackage": false,
+        "dockerCrossCompile": false,
         "downloadFreshNode": true,
         "initialize": true,
         "isRelease": false,
@@ -180,6 +185,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
         "createGenericFolders": true,
         "createPlatformFolders": true,
         "createRpmPackage": false,
+        "dockerCrossCompile": false,
         "downloadFreshNode": true,
         "initialize": true,
         "isRelease": false,
@@ -208,6 +214,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () =>
         "createGenericFolders": true,
         "createPlatformFolders": true,
         "createRpmPackage": true,
+        "dockerCrossCompile": false,
         "downloadFreshNode": true,
         "initialize": true,
         "isRelease": false,

src/dev/build/args.ts

@@ -22,6 +22,7 @@ export function readCliArgs(argv: string[]) {
       'skip-os-packages',
       'rpm',
       'deb',
+      'docker-cross-compile',
       'docker-images',
       'skip-docker-contexts',
       'skip-docker-ubi',
@@ -49,6 +50,7 @@ export function readCliArgs(argv: string[]) {
       rpm: null,
       deb: null,
       'docker-images': null,
+      'docker-cross-compile': false,
       'version-qualifier': '',
     },
     unknown: (flag) => {
@@ -94,6 +96,7 @@ export function readCliArgs(argv: string[]) {
   const buildOptions: BuildOptions = {
     isRelease: Boolean(flags.release),
     versionQualifier: flags['version-qualifier'],
+    dockerCrossCompile: Boolean(flags['docker-cross-compile']),
     initialize: !Boolean(flags['skip-initialize']),
     downloadFreshNode: !Boolean(flags['skip-node-download']),
     createGenericFolders: !Boolean(flags['skip-generic-folders']),

src/dev/build/build_distributables.ts

@@ -13,6 +13,7 @@ import * as Tasks from './tasks';
 
 export interface BuildOptions {
   isRelease: boolean;
+  dockerCrossCompile: boolean;
   downloadFreshNode: boolean;
   initialize: boolean;
   createGenericFolders: boolean;

src/dev/build/cli.ts

@@ -39,6 +39,7 @@ if (showHelp) {
         --rpm                   {dim Only build the rpm packages}
         --deb                   {dim Only build the deb packages}
         --docker-images         {dim Only build the Docker images}
+        --docker-cross-compile  {dim Produce arm64 and amd64 Docker images}
         --docker-contexts       {dim Only build the Docker build contexts}
         --skip-docker-ubi       {dim Don't build the docker ubi image}
         --skip-docker-ubuntu    {dim Don't build the docker ubuntu image}

src/dev/build/lib/build.test.ts

@@ -32,6 +32,7 @@ const config = new Config(
     buildSha: 'abcd1234',
     buildVersion: '8.0.0',
   },
+  false,
   true
 );
 

src/dev/build/lib/config.test.ts

@@ -29,6 +29,7 @@ const setup = async ({ targetAllPlatforms = true }: { targetAllPlatforms?: boole
   return await Config.create({
     isRelease: true,
     targetAllPlatforms,
+    dockerCrossCompile: false,
   });
 };
 

src/dev/build/lib/config.ts

@@ -17,6 +17,7 @@ interface Options {
   isRelease: boolean;
   targetAllPlatforms: boolean;
   versionQualifier?: string;
+  dockerCrossCompile: boolean;
 }
 
 interface Package {
@@ -29,7 +30,12 @@ interface Package {
 }
 
 export class Config {
-  static async create({ isRelease, targetAllPlatforms, versionQualifier }: Options) {
+  static async create({
+    isRelease,
+    targetAllPlatforms,
+    versionQualifier,
+    dockerCrossCompile,
+  }: Options) {
     const pkgPath = resolve(__dirname, '../../../../package.json');
     const pkg: Package = loadJsonFile.sync(pkgPath);
 
@@ -43,6 +49,7 @@ export class Config {
         versionQualifier,
         pkg,
       }),
+      dockerCrossCompile,
       isRelease
     );
   }
@@ -53,6 +60,7 @@ export class Config {
     private readonly nodeVersion: string,
     private readonly repoRoot: string,
     private readonly versionInfo: VersionInfo,
+    private readonly dockerCrossCompile: boolean,
     public readonly isRelease: boolean
   ) {}
 
@@ -70,6 +78,13 @@ export class Config {
     return this.nodeVersion;
   }
 
+  /**
+   * Get docker cross compile
+   */
+  getDockerCrossCompile() {
+    return this.dockerCrossCompile;
+  }
+
   /**
    * Convert an absolute path to a relative path, based from the repo
    */

src/dev/build/lib/runner.test.ts

@@ -50,6 +50,7 @@ const setup = async () => {
     isRelease: true,
     targetAllPlatforms: true,
     versionQualifier: '-SNAPSHOT',
+    dockerCrossCompile: false,
   });
 
   const run = createRunner({

src/dev/build/tasks/download_cloud_dependencies.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import del from 'del';
+import Axios from 'axios';
+import { Task, downloadToDisk, downloadToString } from '../lib';
+
+export const DownloadCloudDependencies: Task = {
+  description: 'Downloading cloud dependencies',
+
+  async run(config, log, build) {
+    const downloadBeat = async (beat: string, id: string) => {
+      const subdomain = config.isRelease ? 'artifacts' : 'snapshots';
+      const version = config.getBuildVersion();
+      const buildId = id.match(/[0-9]\.[0-9]\.[0-9]-[0-9a-z]{8}/);
+      const buildIdUrl = buildId ? `${buildId[0]}/` : '';
+
+      const localArchitecture = [process.arch === 'arm64' ? 'arm64' : 'x86_64'];
+      const allArchitectures = ['arm64', 'x86_64'];
+      const architectures = config.getDockerCrossCompile() ? allArchitectures : localArchitecture;
+      const downloads = architectures.map(async (arch) => {
+        const url = `https://${subdomain}-no-kpi.elastic.co/${buildIdUrl}downloads/beats/${beat}/${beat}-${version}-linux-${arch}.tar.gz`;
+        const checksum = await downloadToString({ log, url: url + '.sha512', expectStatus: 200 });
+        const destination = config.resolveFromRepo('.beats', Path.basename(url));
+        return downloadToDisk({
+          log,
+          url,
+          destination,
+          shaChecksum: checksum.split(' ')[0],
+          shaAlgorithm: 'sha512',
+          maxAttempts: 3,
+        });
+      });
+      return Promise.all(downloads);
+    };
+
+    let buildId = '';
+    if (!config.isRelease) {
+      const manifestUrl = `https://artifacts-api.elastic.co/v1/versions/${config.getBuildVersion()}/builds/latest`;
+      try {
+        const manifest = await Axios.get(manifestUrl);
+        buildId = manifest.data.build.build_id;
+      } catch (e) {
+        log.error(
+          `Unable to find Elastic artifacts for ${config.getBuildVersion()} at ${manifestUrl}.`
+        );
+        throw e;
+      }
+    }
+    await del([config.resolveFromRepo('.beats')]);
+
+    await downloadBeat('metricbeat', buildId);
+    await downloadBeat('filebeat', buildId);
+  },
+};

src/dev/build/tasks/nodejs/download_node_builds_task.test.ts

@@ -39,6 +39,7 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) {
   const config = await Config.create({
     isRelease: true,
     targetAllPlatforms: true,
+    dockerCrossCompile: false,
   });
 
   getNodeDownloadInfo.mockImplementation((_: Config, platform: Platform) => {

src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts

@@ -43,6 +43,7 @@ async function setup() {
   const config = await Config.create({
     isRelease: true,
     targetAllPlatforms: true,
+    dockerCrossCompile: false,
   });
 
   return { config };

src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts

@@ -48,6 +48,7 @@ async function setup(actualShaSums?: Record<string, string>) {
   const config = await Config.create({
     isRelease: true,
     targetAllPlatforms: true,
+    dockerCrossCompile: false,
   });
 
   getNodeShasums.mockReturnValue(

src/dev/build/tasks/os_packages/docker_generator/run.ts

@@ -73,6 +73,7 @@ export async function runDockerGenerator(
       : []),
   ];
 
+  const dockerCrossCompile = config.getDockerCrossCompile();
   const publicArtifactSubdomain = config.isRelease ? 'artifacts' : 'snapshots-no-kpi';
   const scope: TemplateContext = {
     artifactPrefix,
@@ -104,7 +105,7 @@ export async function runDockerGenerator(
     arm64: 'aarch64',
   };
   const buildArchitectureSupported = hostTarget[process.arch] === flags.architecture;
-  if (flags.architecture && !buildArchitectureSupported) {
+  if (flags.architecture && !buildArchitectureSupported && !dockerCrossCompile) {
     return;
   }
 

src/dev/build/tasks/os_packages/docker_generator/templates/build_docker_sh.template.ts

@@ -18,6 +18,7 @@ function generator({
   baseOSImage,
   architecture,
 }: TemplateContext) {
+  const dockerArchitecture = architecture === 'aarch64' ? 'linux/arm64' : 'linux/amd64';
   return dedent(`
   #!/usr/bin/env bash
   #
@@ -54,7 +55,7 @@ function generator({
   retry_docker_pull ${baseOSImage}
 
   echo "Building: kibana${imageFlavor}-docker"; \\
-  docker build -t ${imageTag}${imageFlavor}:${version} -f Dockerfile . || exit 1;
+  docker buildx build --platform ${dockerArchitecture} -t ${imageTag}${imageFlavor}:${version} -f Dockerfile . || exit 1;
 
   docker save ${imageTag}${imageFlavor}:${version} | gzip -c > ${dockerTargetFilename}