From be42eb8f17e2c331af01e75ab491cadd815b2d90 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 23 Mar 2022 16:06:40 -0500 Subject: [PATCH] [7.17] [build] Cross compile docker images (#128272) * [build] Cross compile docker images * typo * debug * Revert "[build] Cross compile docker images" This reverts commit 621780eb1d85076893e8a45b000b9886126c3153. * revert * support docker-cross-compile flag * fix types/tests * fix more tests * download cloud dependencies based on cross compile flag * fix array * fix more tests --- src/dev/build/args.test.ts | 7 +++ src/dev/build/args.ts | 3 + src/dev/build/build_distributables.ts | 1 + src/dev/build/cli.ts | 1 + src/dev/build/lib/build.test.ts | 1 + src/dev/build/lib/config.test.ts | 1 + src/dev/build/lib/config.ts | 17 +++++- src/dev/build/lib/runner.test.ts | 1 + .../tasks/download_cloud_dependencies.ts | 61 +++++++++++++++++++ .../nodejs/download_node_builds_task.test.ts | 1 + .../nodejs/extract_node_builds_task.test.ts | 1 + .../verify_existing_node_builds_task.test.ts | 1 + .../tasks/os_packages/docker_generator/run.ts | 3 +- .../templates/build_docker_sh.template.ts | 3 +- 14 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 src/dev/build/tasks/download_cloud_dependencies.ts diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index d475e5b22a0636..64f8158787c43d 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -36,6 +36,7 @@ it('build default and oss dist for current platform, without packages, by defaul "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerCrossCompile": false, "downloadFreshNode": true, "initialize": true, "isRelease": false, @@ -63,6 +64,7 @@ it('builds packages if --all-platforms is passed', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": true, + "dockerCrossCompile": false, "downloadFreshNode": true, "initialize": true, "isRelease": false, @@ -90,6 +92,7 @@ it('limits packages if --rpm passed with --all-platforms', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": true, + "dockerCrossCompile": false, "downloadFreshNode": true, "initialize": true, "isRelease": false, @@ -117,6 +120,7 @@ it('limits packages if --deb passed with --all-platforms', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerCrossCompile": false, "downloadFreshNode": true, "initialize": true, "isRelease": false, @@ -145,6 +149,7 @@ it('limits packages if --docker passed with --all-platforms', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerCrossCompile": false, "downloadFreshNode": true, "initialize": true, "isRelease": false, @@ -180,6 +185,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerCrossCompile": false, "downloadFreshNode": true, "initialize": true, "isRelease": false, @@ -208,6 +214,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () => "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": true, + "dockerCrossCompile": false, "downloadFreshNode": true, "initialize": true, "isRelease": false, diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index 5619ab34ac359a..721bd6e9e580ab 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -22,6 +22,7 @@ export function readCliArgs(argv: string[]) { 'skip-os-packages', 'rpm', 'deb', + 'docker-cross-compile', 'docker-images', 'skip-docker-contexts', 'skip-docker-ubi', @@ -49,6 +50,7 @@ export function readCliArgs(argv: string[]) { rpm: null, deb: null, 'docker-images': null, + 'docker-cross-compile': false, 'version-qualifier': '', }, unknown: (flag) => { @@ -94,6 +96,7 @@ export function readCliArgs(argv: string[]) { const buildOptions: BuildOptions = { isRelease: Boolean(flags.release), versionQualifier: flags['version-qualifier'], + dockerCrossCompile: Boolean(flags['docker-cross-compile']), initialize: !Boolean(flags['skip-initialize']), downloadFreshNode: !Boolean(flags['skip-node-download']), createGenericFolders: !Boolean(flags['skip-generic-folders']), diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 81ea67d1c52f66..c4b1458c8999db 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -13,6 +13,7 @@ import * as Tasks from './tasks'; export interface BuildOptions { isRelease: boolean; + dockerCrossCompile: boolean; downloadFreshNode: boolean; initialize: boolean; createGenericFolders: boolean; diff --git a/src/dev/build/cli.ts b/src/dev/build/cli.ts index 323a40d02de66b..792bdcec0d696f 100644 --- a/src/dev/build/cli.ts +++ b/src/dev/build/cli.ts @@ -39,6 +39,7 @@ if (showHelp) { --rpm {dim Only build the rpm packages} --deb {dim Only build the deb packages} --docker-images {dim Only build the Docker images} + --docker-cross-compile {dim Produce arm64 and amd64 Docker images} --docker-contexts {dim Only build the Docker build contexts} --skip-docker-ubi {dim Don't build the docker ubi image} --skip-docker-ubuntu {dim Don't build the docker ubuntu image} diff --git a/src/dev/build/lib/build.test.ts b/src/dev/build/lib/build.test.ts index 0c627d00a0dea8..f7eb75f0bf89f1 100644 --- a/src/dev/build/lib/build.test.ts +++ b/src/dev/build/lib/build.test.ts @@ -32,6 +32,7 @@ const config = new Config( buildSha: 'abcd1234', buildVersion: '8.0.0', }, + false, true ); diff --git a/src/dev/build/lib/config.test.ts b/src/dev/build/lib/config.test.ts index b2afe3337230df..a3006484232714 100644 --- a/src/dev/build/lib/config.test.ts +++ b/src/dev/build/lib/config.test.ts @@ -29,6 +29,7 @@ const setup = async ({ targetAllPlatforms = true }: { targetAllPlatforms?: boole return await Config.create({ isRelease: true, targetAllPlatforms, + dockerCrossCompile: false, }); }; diff --git a/src/dev/build/lib/config.ts b/src/dev/build/lib/config.ts index 33b98e1b94a042..fa6d3bfa4229d9 100644 --- a/src/dev/build/lib/config.ts +++ b/src/dev/build/lib/config.ts @@ -17,6 +17,7 @@ interface Options { isRelease: boolean; targetAllPlatforms: boolean; versionQualifier?: string; + dockerCrossCompile: boolean; } interface Package { @@ -29,7 +30,12 @@ interface Package { } export class Config { - static async create({ isRelease, targetAllPlatforms, versionQualifier }: Options) { + static async create({ + isRelease, + targetAllPlatforms, + versionQualifier, + dockerCrossCompile, + }: Options) { const pkgPath = resolve(__dirname, '../../../../package.json'); const pkg: Package = loadJsonFile.sync(pkgPath); @@ -43,6 +49,7 @@ export class Config { versionQualifier, pkg, }), + dockerCrossCompile, isRelease ); } @@ -53,6 +60,7 @@ export class Config { private readonly nodeVersion: string, private readonly repoRoot: string, private readonly versionInfo: VersionInfo, + private readonly dockerCrossCompile: boolean, public readonly isRelease: boolean ) {} @@ -70,6 +78,13 @@ export class Config { return this.nodeVersion; } + /** + * Get docker cross compile + */ + getDockerCrossCompile() { + return this.dockerCrossCompile; + } + /** * Convert an absolute path to a relative path, based from the repo */ diff --git a/src/dev/build/lib/runner.test.ts b/src/dev/build/lib/runner.test.ts index 2a08da2797a9de..3b606465fcecc6 100644 --- a/src/dev/build/lib/runner.test.ts +++ b/src/dev/build/lib/runner.test.ts @@ -50,6 +50,7 @@ const setup = async () => { isRelease: true, targetAllPlatforms: true, versionQualifier: '-SNAPSHOT', + dockerCrossCompile: false, }); const run = createRunner({ diff --git a/src/dev/build/tasks/download_cloud_dependencies.ts b/src/dev/build/tasks/download_cloud_dependencies.ts new file mode 100644 index 00000000000000..31873550f6b4ac --- /dev/null +++ b/src/dev/build/tasks/download_cloud_dependencies.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import del from 'del'; +import Axios from 'axios'; +import { Task, downloadToDisk, downloadToString } from '../lib'; + +export const DownloadCloudDependencies: Task = { + description: 'Downloading cloud dependencies', + + async run(config, log, build) { + const downloadBeat = async (beat: string, id: string) => { + const subdomain = config.isRelease ? 'artifacts' : 'snapshots'; + const version = config.getBuildVersion(); + const buildId = id.match(/[0-9]\.[0-9]\.[0-9]-[0-9a-z]{8}/); + const buildIdUrl = buildId ? `${buildId[0]}/` : ''; + + const localArchitecture = [process.arch === 'arm64' ? 'arm64' : 'x86_64']; + const allArchitectures = ['arm64', 'x86_64']; + const architectures = config.getDockerCrossCompile() ? allArchitectures : localArchitecture; + const downloads = architectures.map(async (arch) => { + const url = `https://${subdomain}-no-kpi.elastic.co/${buildIdUrl}downloads/beats/${beat}/${beat}-${version}-linux-${arch}.tar.gz`; + const checksum = await downloadToString({ log, url: url + '.sha512', expectStatus: 200 }); + const destination = config.resolveFromRepo('.beats', Path.basename(url)); + return downloadToDisk({ + log, + url, + destination, + shaChecksum: checksum.split(' ')[0], + shaAlgorithm: 'sha512', + maxAttempts: 3, + }); + }); + return Promise.all(downloads); + }; + + let buildId = ''; + if (!config.isRelease) { + const manifestUrl = `https://artifacts-api.elastic.co/v1/versions/${config.getBuildVersion()}/builds/latest`; + try { + const manifest = await Axios.get(manifestUrl); + buildId = manifest.data.build.build_id; + } catch (e) { + log.error( + `Unable to find Elastic artifacts for ${config.getBuildVersion()} at ${manifestUrl}.` + ); + throw e; + } + } + await del([config.resolveFromRepo('.beats')]); + + await downloadBeat('metricbeat', buildId); + await downloadBeat('filebeat', buildId); + }, +}; diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts index f5de91794ae1e1..80e9120a57b65b 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts @@ -39,6 +39,7 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + dockerCrossCompile: false, }); getNodeDownloadInfo.mockImplementation((_: Config, platform: Platform) => { diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts index 9f869b99c18aec..28974349ba4553 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts @@ -43,6 +43,7 @@ async function setup() { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + dockerCrossCompile: false, }); return { config }; diff --git a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts index c636db145694c0..e6787bed1c0567 100644 --- a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts @@ -48,6 +48,7 @@ async function setup(actualShaSums?: Record) { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + dockerCrossCompile: false, }); getNodeShasums.mockReturnValue( diff --git a/src/dev/build/tasks/os_packages/docker_generator/run.ts b/src/dev/build/tasks/os_packages/docker_generator/run.ts index 59ffb590a9a0fe..5f2bc3de1c4ac5 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/run.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/run.ts @@ -73,6 +73,7 @@ export async function runDockerGenerator( : []), ]; + const dockerCrossCompile = config.getDockerCrossCompile(); const publicArtifactSubdomain = config.isRelease ? 'artifacts' : 'snapshots-no-kpi'; const scope: TemplateContext = { artifactPrefix, @@ -104,7 +105,7 @@ export async function runDockerGenerator( arm64: 'aarch64', }; const buildArchitectureSupported = hostTarget[process.arch] === flags.architecture; - if (flags.architecture && !buildArchitectureSupported) { + if (flags.architecture && !buildArchitectureSupported && !dockerCrossCompile) { return; } diff --git a/src/dev/build/tasks/os_packages/docker_generator/templates/build_docker_sh.template.ts b/src/dev/build/tasks/os_packages/docker_generator/templates/build_docker_sh.template.ts index 05b9b4d100c535..934f1b3c020e0b 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/templates/build_docker_sh.template.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/templates/build_docker_sh.template.ts @@ -18,6 +18,7 @@ function generator({ baseOSImage, architecture, }: TemplateContext) { + const dockerArchitecture = architecture === 'aarch64' ? 'linux/arm64' : 'linux/amd64'; return dedent(` #!/usr/bin/env bash # @@ -54,7 +55,7 @@ function generator({ retry_docker_pull ${baseOSImage} echo "Building: kibana${imageFlavor}-docker"; \\ - docker build -t ${imageTag}${imageFlavor}:${version} -f Dockerfile . || exit 1; + docker buildx build --platform ${dockerArchitecture} -t ${imageTag}${imageFlavor}:${version} -f Dockerfile . || exit 1; docker save ${imageTag}${imageFlavor}:${version} | gzip -c > ${dockerTargetFilename}