From 8fb4f235cd6134dc06c872c15021b38bd2ae2836 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 18 Feb 2021 16:18:40 -0600 Subject: [PATCH 1/3] [build] Add task skips intended for partial builds Currently the build supports skips for leaf nodes: archive creation and OS packages. This adds skip options for folder cleanup and source build folders. This is intended to support two scenarios: 1) Debugging. Creating the build source folders and compiling assets takes up the bulk of build time. This can make testing package creation changes time consuming. 2) Build resuming. When creating docker images, we want the option to drop an archive in the target folder and run only the docker build step. Three flags have been added: ``` --skip-initialize Don't clean the target and build folders --skip-generic-folders Don't create the base source folders --skip-platform-folders Don't create the platform specific folders ``` The build syntax for this is verbose and probably due for refactoring. The long term goal is to migrate this to bazel, and the changes here are intended as stopgap until then. Testing - standalone Docker image ``` yarn kbn clean yarn kbn bootstrap mv ~/Downloads/kibana-8.0.0-SNAPSHOT-linux-x86_64.tar.gz target node scripts/build \ --no-oss \ --skip-initialize \ --skip-generic-folders \ --skip-platform-folders \ --skip-archives \ --docker-images \ --skip-docker-ubi \ --skip-docker-contexts ``` --- src/dev/build/args.ts | 6 ++ src/dev/build/build_distributables.ts | 68 +++++++++++-------- .../tasks/os_packages/docker_generator/run.ts | 27 ++++---- 3 files changed, 60 insertions(+), 41 deletions(-) diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index c594eacd08c014..bbfbd3e6f8813a 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -18,6 +18,9 @@ export function readCliArgs(argv: string[]) { 'oss', 'no-oss', 'skip-archives', + 'skip-initialize', + 'skip-generic-folders', + 'skip-platform-folders', 'skip-os-packages', 'rpm', 'deb', @@ -93,7 +96,10 @@ export function readCliArgs(argv: string[]) { versionQualifier: flags['version-qualifier'], buildOssDist: flags.oss !== false, buildDefaultDist: !flags.oss, + initialize: !Boolean(flags['skip-initialize']), downloadFreshNode: !Boolean(flags['skip-node-download']), + createGenericFolders: !Boolean(flags['skip-generic-folders']), + createPlatformFolders: !Boolean(flags['skip-platform-folders']), createArchives: !Boolean(flags['skip-archives']), createRpmPackage: isOsPackageDesired('rpm'), createDebPackage: isOsPackageDesired('deb'), diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 237fc71811a41c..92b453863060b9 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -16,6 +16,8 @@ export interface BuildOptions { buildOssDist: boolean; buildDefaultDist: boolean; downloadFreshNode: boolean; + createGenericFolders: boolean; + CreatePlatformFolders: boolean; createArchives: boolean; createRpmPackage: boolean; createDebPackage: boolean; @@ -41,45 +43,53 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions /** * verify, reset, and initialize the build environment */ - await run(Tasks.VerifyEnv); - await run(Tasks.Clean); - await run(options.downloadFreshNode ? Tasks.DownloadNodeBuilds : Tasks.VerifyExistingNodeBuilds); - await run(Tasks.ExtractNodeBuilds); + if (options.initialize) { + await run(Tasks.VerifyEnv); + await run(Tasks.Clean); + await run( + options.downloadFreshNode ? Tasks.DownloadNodeBuilds : Tasks.VerifyExistingNodeBuilds + ); + await run(Tasks.ExtractNodeBuilds); + } /** * run platform-generic build tasks */ - await run(Tasks.CopySource); - await run(Tasks.CopyBinScripts); - await run(Tasks.ReplaceFavicon); - await run(Tasks.CreateEmptyDirsAndFiles); - await run(Tasks.CreateReadme); - await run(Tasks.BuildBazelPackages); - await run(Tasks.BuildPackages); - await run(Tasks.BuildKibanaPlatformPlugins); - await run(Tasks.TranspileBabel); - await run(Tasks.CreatePackageJson); - await run(Tasks.InstallDependencies); - await run(Tasks.CleanPackages); - await run(Tasks.CreateNoticeFile); - await run(Tasks.UpdateLicenseFile); - await run(Tasks.RemovePackageJsonDeps); - await run(Tasks.CleanTypescript); - await run(Tasks.CleanExtraFilesFromModules); - await run(Tasks.CleanEmptyFolders); + if (options.createGenericFolders) { + await run(Tasks.CopySource); + await run(Tasks.CopyBinScripts); + await run(Tasks.ReplaceFavicon); + await run(Tasks.CreateEmptyDirsAndFiles); + await run(Tasks.CreateReadme); + await run(Tasks.BuildBazelPackages); + await run(Tasks.BuildPackages); + await run(Tasks.BuildKibanaPlatformPlugins); + await run(Tasks.TranspileBabel); + await run(Tasks.CreatePackageJson); + await run(Tasks.InstallDependencies); + await run(Tasks.CleanPackages); + await run(Tasks.CreateNoticeFile); + await run(Tasks.UpdateLicenseFile); + await run(Tasks.RemovePackageJsonDeps); + await run(Tasks.CleanTypescript); + await run(Tasks.CleanExtraFilesFromModules); + await run(Tasks.CleanEmptyFolders); + } /** * copy generic build outputs into platform-specific build * directories and perform platform/architecture-specific steps */ - await run(Tasks.CreateArchivesSources); - await run(Tasks.PatchNativeModules); - await run(Tasks.InstallChromium); - await run(Tasks.CleanExtraBinScripts); - await run(Tasks.CleanNodeBuilds); + if (options.createPlatformFolders) { + await run(Tasks.CreateArchivesSources); + await run(Tasks.PatchNativeModules); + await run(Tasks.InstallChromium); + await run(Tasks.CleanExtraBinScripts); + await run(Tasks.CleanNodeBuilds); - await run(Tasks.PathLength); - await run(Tasks.UuidVerification); + await run(Tasks.PathLength); + await run(Tasks.UuidVerification); + } /** * package platform-specific builds into archives diff --git a/src/dev/build/tasks/os_packages/docker_generator/run.ts b/src/dev/build/tasks/os_packages/docker_generator/run.ts index 21d2582f205f37..b8029328ac94a2 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/run.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/run.ts @@ -83,6 +83,16 @@ export async function runDockerGenerator( revision: config.getBuildSha(), }; + type HostArchitectureToDocker = Record; + const hostTarget: HostArchitectureToDocker = { + x64: 'x64', + arm64: 'aarch64', + }; + const buildArchitectureSupported = hostTarget[process.arch] === flags.architecture && flags.image; + if (!buildArchitectureSupported) { + return; + } + // Verify if we have the needed kibana target in order // to build the kibana docker image. // Also create the docker build target folder @@ -132,18 +142,11 @@ export async function runDockerGenerator( await chmodAsync(`${resolve(dockerBuildDir, 'build_docker.sh')}`, '755'); // Only build images on native targets - type HostArchitectureToDocker = Record; - const hostTarget: HostArchitectureToDocker = { - x64: 'x64', - arm64: 'aarch64', - }; - const buildImage = hostTarget[process.arch] === flags.architecture && flags.image; - if (buildImage) { - await exec(log, `./build_docker.sh`, [], { - cwd: dockerBuildDir, - level: 'info', - }); - } + + await exec(log, `./build_docker.sh`, [], { + cwd: dockerBuildDir, + level: 'info', + }); // Pack Dockerfiles and create a target for them if (flags.context) { From 9588194e7f6ab92a1cf6d8af02f2449270e0174f Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 19 Feb 2021 15:01:31 -0600 Subject: [PATCH 2/3] fix types --- src/dev/build/build_distributables.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 92b453863060b9..f0403fac1e26b3 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -16,8 +16,9 @@ export interface BuildOptions { buildOssDist: boolean; buildDefaultDist: boolean; downloadFreshNode: boolean; + initialize: boolean; createGenericFolders: boolean; - CreatePlatformFolders: boolean; + createPlatformFolders: boolean; createArchives: boolean; createRpmPackage: boolean; createDebPackage: boolean; From 1f77a5a9d0b5f076d3399d13832041c036011206 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 22 Feb 2021 09:38:32 -0600 Subject: [PATCH 3/3] update snapshots --- src/dev/build/args.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index 2397c18c04d07f..e749af73241cf1 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -33,8 +33,11 @@ it('build default and oss dist for current platform, without packages, by defaul "createDockerCentOS": false, "createDockerContexts": true, "createDockerUBI": false, + "createGenericFolders": true, + "createPlatformFolders": true, "createRpmPackage": false, "downloadFreshNode": true, + "initialize": true, "isRelease": false, "targetAllPlatforms": false, "versionQualifier": "", @@ -57,8 +60,11 @@ it('builds packages if --all-platforms is passed', () => { "createDockerCentOS": true, "createDockerContexts": true, "createDockerUBI": true, + "createGenericFolders": true, + "createPlatformFolders": true, "createRpmPackage": true, "downloadFreshNode": true, + "initialize": true, "isRelease": false, "targetAllPlatforms": true, "versionQualifier": "", @@ -81,8 +87,11 @@ it('limits packages if --rpm passed with --all-platforms', () => { "createDockerCentOS": false, "createDockerContexts": true, "createDockerUBI": false, + "createGenericFolders": true, + "createPlatformFolders": true, "createRpmPackage": true, "downloadFreshNode": true, + "initialize": true, "isRelease": false, "targetAllPlatforms": true, "versionQualifier": "", @@ -105,8 +114,11 @@ it('limits packages if --deb passed with --all-platforms', () => { "createDockerCentOS": false, "createDockerContexts": true, "createDockerUBI": false, + "createGenericFolders": true, + "createPlatformFolders": true, "createRpmPackage": false, "downloadFreshNode": true, + "initialize": true, "isRelease": false, "targetAllPlatforms": true, "versionQualifier": "", @@ -130,8 +142,11 @@ it('limits packages if --docker passed with --all-platforms', () => { "createDockerCentOS": true, "createDockerContexts": true, "createDockerUBI": true, + "createGenericFolders": true, + "createPlatformFolders": true, "createRpmPackage": false, "downloadFreshNode": true, + "initialize": true, "isRelease": false, "targetAllPlatforms": true, "versionQualifier": "", @@ -162,8 +177,11 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "createDockerCentOS": true, "createDockerContexts": true, "createDockerUBI": false, + "createGenericFolders": true, + "createPlatformFolders": true, "createRpmPackage": false, "downloadFreshNode": true, + "initialize": true, "isRelease": false, "targetAllPlatforms": true, "versionQualifier": "", @@ -187,8 +205,11 @@ it('limits packages if --all-platforms passed with --skip-docker-centos', () => "createDockerCentOS": false, "createDockerContexts": true, "createDockerUBI": true, + "createGenericFolders": true, + "createPlatformFolders": true, "createRpmPackage": true, "downloadFreshNode": true, + "initialize": true, "isRelease": false, "targetAllPlatforms": true, "versionQualifier": "",