From 5045cd34686de546095d9899366f2a31802e940d Mon Sep 17 00:00:00 2001 From: Jon Edvald Date: Wed, 13 Mar 2019 17:56:59 +0100 Subject: [PATCH] feat(maven-container): automatically fetch Maven and OpenJDK Along the way, improved the external tool helper to support libraries as well as executable binaries. --- .../reference/module-types/maven-container.md | 2 +- .../src/plugins/kubernetes/helm/helm-cli.ts | 6 +- .../maven-container/maven-container.ts | 64 +++---- .../src/plugins/maven-container/maven.ts | 27 +++ .../src/plugins/maven-container/openjdk.ts | 70 ++++++++ garden-service/src/util/ext-tools.ts | 157 +++++++++++------- 6 files changed, 221 insertions(+), 105 deletions(-) create mode 100644 garden-service/src/plugins/maven-container/maven.ts create mode 100644 garden-service/src/plugins/maven-container/openjdk.ts diff --git a/docs/reference/module-types/maven-container.md b/docs/reference/module-types/maven-container.md index 0a310f8e58..d3bd5d72ab 100644 --- a/docs/reference/module-types/maven-container.md +++ b/docs/reference/module-types/maven-container.md @@ -578,7 +578,7 @@ module: ### `module.jdkVersion` [module](#module) > jdkVersion -The Java version to run +The JDK version to use. | Type | Required | | ---- | -------- | diff --git a/garden-service/src/plugins/kubernetes/helm/helm-cli.ts b/garden-service/src/plugins/kubernetes/helm/helm-cli.ts index 830aa0243a..2c6ebf6b28 100644 --- a/garden-service/src/plugins/kubernetes/helm/helm-cli.ts +++ b/garden-service/src/plugins/kubernetes/helm/helm-cli.ts @@ -17,7 +17,7 @@ const helmCmd = new BinaryCmd({ sha256: "166318b2159613f87a7cb02af1614c96244b3d3c119f8e010429c1b4449681d5", extract: { format: "tar", - executablePath: ["darwin-amd64", "helm"], + targetPath: ["darwin-amd64", "helm"], }, }, linux: { @@ -25,7 +25,7 @@ const helmCmd = new BinaryCmd({ sha256: "15eca6ad225a8279de80c7ced42305e24bc5ac60bb7d96f2d2fa4af86e02c794", extract: { format: "tar", - executablePath: ["linux-amd64", "helm"], + targetPath: ["linux-amd64", "helm"], }, }, win32: { @@ -33,7 +33,7 @@ const helmCmd = new BinaryCmd({ sha256: "63fdb71ad6fac0572a21ad81da7508b1f0cae960ea944670f4d2f7fbaf23acb2", extract: { format: "zip", - executablePath: ["windows-amd64", "helm.exe"], + targetPath: ["windows-amd64", "helm.exe"], }, }, }, diff --git a/garden-service/src/plugins/maven-container/maven-container.ts b/garden-service/src/plugins/maven-container/maven-container.ts index 9859d9fcf9..3359b5a09c 100644 --- a/garden-service/src/plugins/maven-container/maven-container.ts +++ b/garden-service/src/plugins/maven-container/maven-container.ts @@ -6,7 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import * as execa from "execa" import * as Joi from "joi" import { omit, pick, get } from "lodash" import { copy, pathExists, readFile } from "fs-extra" @@ -16,6 +15,7 @@ import { ContainerServiceSpec, ContainerTestSpec, ContainerModuleConfig, + ContainerTaskSpec, } from "../container/config" import { validateWithPath } from "../../config/common" import { BuildModuleParams, ConfigureModuleParams, GetBuildStatusParams } from "../../types/plugin/params" @@ -29,6 +29,8 @@ import { STATIC_DIR } from "../../constants" import { xml2json } from "xml-js" import { containerModuleSpecSchema } from "../container/config" import { providerConfigBaseSchema } from "../../config/project" +import { openJdks } from "./openjdk" +import { maven } from "./maven" const defaultDockerfilePath = resolve(STATIC_DIR, "maven-container", "Dockerfile") @@ -41,10 +43,11 @@ interface MavenContainerModuleSpec extends ContainerModuleSpec { // type MavenContainerModuleConfig = ModuleConfig interface MavenContainerModule< - M extends ContainerModuleSpec = MavenContainerModuleSpec, + M extends MavenContainerModuleSpec = MavenContainerModuleSpec, S extends ContainerServiceSpec = ContainerServiceSpec, - T extends ContainerTestSpec = ContainerTestSpec - > extends Module { } + T extends ContainerTestSpec = ContainerTestSpec, + W extends ContainerTaskSpec = ContainerTaskSpec + > extends Module { } const mavenKeys = { jarPath: Joi.string() @@ -53,16 +56,15 @@ const mavenKeys = { .example("target/my-module.jar"), jdkVersion: Joi.number() .integer() - .min(8) + .allow(8, 11) .default(8) - .description("The Java version to run"), + .description("The JDK version to use."), } const mavenFieldsSchema = Joi.object() .keys(mavenKeys) export const mavenContainerModuleSpecSchema = containerModuleSpecSchema.keys(mavenKeys) - export const mavenContainerConfigSchema = providerConfigBaseSchema export const gardenPlugin = (): GardenPlugin => { @@ -95,9 +97,11 @@ async function configure(params: ConfigureModuleParams) { let containerConfig: ContainerModuleConfig = { ...moduleConfig } containerConfig.spec = omit(moduleConfig.spec, Object.keys(mavenKeys)) + const jdkVersion = mavenFields.jdkVersion! + containerConfig.spec.buildArgs = { JAR_PATH: mavenFields.jarPath!, - JDK_VERSION: mavenFields.jdkVersion!.toString(), + JDK_VERSION: jdkVersion.toString(), } const configured = await configureContainerModule({ ...params, moduleConfig: containerConfig }) @@ -127,7 +131,7 @@ async function getBuildStatus(params: GetBuildStatusParams async function build(params: BuildModuleParams) { // Run the maven build const { ctx, module, log } = params - let { jarPath } = module.spec + let { jarPath, jdkVersion } = module.spec const pom = await loadPom(module.path) const artifactId = get(pom, ["project", "artifactId", "_text"]) @@ -136,6 +140,11 @@ async function build(params: BuildModuleParams) { throw new ConfigurationError(`Could not read artifact ID from pom.xml in ${module.path}`, { path: module.path }) } + log.setState(`Creating jar artifact...`) + + const openJdk = openJdks[jdkVersion] + const openJdkPath = await openJdk.getPath(log) + const mvnArgs = [ "package", "--batch-mode", @@ -145,8 +154,14 @@ async function build(params: BuildModuleParams) { ] const mvnCmdStr = "mvn " + mvnArgs.join(" ") - log.setState(`Creating jar artifact...`) - await mvn(ctx.projectRoot, mvnArgs) + await maven.exec({ + args: mvnArgs, + cwd: ctx.projectRoot, + log, + env: { + JAVA_HOME: openJdkPath, + }, + }) // Copy the artifact to the module build directory const resolvedJarPath = resolve(module.path, jarPath) @@ -164,10 +179,6 @@ async function build(params: BuildModuleParams) { return buildContainerModule(params) } -async function mvn(cwd: string, args: string[]) { - return execa.stdout("mvn", args, { cwd, maxBuffer: 10 * 1024 * 1024 }) -} - async function loadPom(dir: string) { try { const pomPath = resolve(dir, "pom.xml") @@ -177,26 +188,3 @@ async function loadPom(dir: string) { throw new ConfigurationError(`Could not load pom.xml from directory ${dir}`, { dir }) } } - -// TODO: see if we could make this perform adequately, or perhaps use this only on Linux... -// -// async function mvn(jdkVersion: number, projectRoot: string, args: string[]) { -// const mvnImage = `maven:3.6.0-jdk-${jdkVersion}-slim` -// const m2Path = resolve(homedir(), ".m2") - -// const dockerArgs = [ -// "run", -// "--rm", -// "--interactive", -// "--volume", `${m2Path}:/root/.m2`, -// "--volume", `${projectRoot}:/project`, -// "--workdir", "/project", -// mvnImage, -// "--", -// ...args, -// ] - -// console.log(dockerArgs.join(" ")) - -// return execa.stdout("docker", dockerArgs, { maxBuffer: 10 * 1024 * 1024 }) -// } diff --git a/garden-service/src/plugins/maven-container/maven.ts b/garden-service/src/plugins/maven-container/maven.ts new file mode 100644 index 0000000000..462bd0d98d --- /dev/null +++ b/garden-service/src/plugins/maven-container/maven.ts @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { BinaryCmd, LibraryPlatformSpec } from "../../util/ext-tools" + +const spec: LibraryPlatformSpec = { + url: "http://mirror.23media.de/apache/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz", + sha256: "6a1b346af36a1f1a491c1c1a141667c5de69b42e6611d3687df26868bc0f4637", + extract: { + format: "tar", + targetPath: ["apache-maven-3.6.0", "bin", "mvn"], + }, +} + +export const maven = new BinaryCmd({ + name: "maven", + specs: { + darwin: spec, + linux: spec, + win32: spec, + }, +}) diff --git a/garden-service/src/plugins/maven-container/openjdk.ts b/garden-service/src/plugins/maven-container/openjdk.ts new file mode 100644 index 0000000000..293004fc25 --- /dev/null +++ b/garden-service/src/plugins/maven-container/openjdk.ts @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2018 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { Library, LibraryPlatformSpec } from "../../util/ext-tools" + +const jdk8Version = "jdk8u202-b08" + +function jdk8Spec(filename: string, sha256: string): LibraryPlatformSpec { + return { + url: `https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/${jdk8Version}/${filename}`, + sha256, + extract: { + format: "tar", + targetPath: [jdk8Version, "Contents", "Home"], + }, + } +} + +function jdk11Spec(filename: string, sha256: string): LibraryPlatformSpec { + return { + url: `https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.2%2B9/${filename}`, + sha256, + extract: { + format: "tar", + targetPath: ["jdk-11.0.2+9", "Contents", "Home"], + }, + } +} + +export const openJdks: { [version: number]: Library } = { + 8: new Library({ + name: "openjdk-8", + specs: { + darwin: jdk8Spec( + "OpenJDK8U-jdk_x64_mac_hotspot_8u202b08.tar.gz", + "059f7c18faa6722aa636bbd79bcdff3aee6a6da5b34940b072ea6e3af85bbe1d", + ), + linux: jdk8Spec( + "OpenJDK8U-jdk_x64_linux_hotspot_8u202b08.tar.gz", + "f5a1c9836beb3ca933ec3b1d39568ecbb68bd7e7ca6a9989a21ff16a74d910ab", + ), + win32: jdk8Spec( + "OpenJDK8U-jdk_x64_windows_hotspot_8u202b08.zip", + "2637dab3bc81274e19991eebc27684276b482dd71d0f84fedf703d4fba3576e5", + ), + }, + }), + 11: new Library({ + name: "openjdk-11", + specs: { + darwin: jdk11Spec( + "OpenJDK11U-jdk_x64_mac_hotspot_11.0.2_9.tar.gz", + "fffd4ed283e5cd443760a8ec8af215c8ca4d33ec5050c24c1277ba64b5b5e81a", + ), + linux: jdk11Spec( + "OpenJDK11U-jdk_x64_linux_hotspot_11.0.2_9.tar.gz", + "d02089d834f7702ac1a9776d8d0d13ee174d0656cf036c6b68b9ffb71a6f610e", + ), + win32: jdk11Spec( + "OpenJDK11U-jdk_x64_windows_hotspot_11.0.2_9.zip", + "bde1648333abaf49c7175c9ee8ba9115a55fc160838ff5091f07d10c4bb50b3a", + ), + }, + }), +} diff --git a/garden-service/src/util/ext-tools.ts b/garden-service/src/util/ext-tools.ts index 6a986eb1fb..4b616ec284 100644 --- a/garden-service/src/util/ext-tools.ts +++ b/garden-service/src/util/ext-tools.ts @@ -9,7 +9,7 @@ import { platform, homedir } from "os" import { pathExists, createWriteStream, ensureDir, chmod, remove, move } from "fs-extra" import { ConfigurationError, ParameterError, GardenBaseError } from "../exceptions" -import { join, dirname, basename } from "path" +import { join, dirname, basename, sep } from "path" import { hashString } from "./util" import Axios from "axios" import * as execa from "execa" @@ -25,32 +25,27 @@ const AsyncLock = require("async-lock") const globalGardenPath = join(homedir(), ".garden") const toolsPath = join(globalGardenPath, "tools") -interface ExecParams { - cwd?: string - log: LogEntry - args?: string[] - timeout?: number -} - -abstract class Cmd { - abstract async exec(params: ExecParams): Promise - abstract async stdout(params: ExecParams): Promise +export interface LibraryExtractSpec { + // Archive format. Note: the "tar" format also implicitly supports gzip and bz2 compression. + format: "tar" | "zip" + // Path to the target file or directory, relative to the download directory, after downloading and + // extracting the archive. For BinaryCmds, this should point to the executable in the archive. + targetPath: string[] } -interface BinarySpec { +export interface LibraryPlatformSpec { url: string - sha256?: string // optionally specify sha256 checksum for validation - extract?: { - format: "tar" | "zip", // note: the "tar" format also supports gzip compression - executablePath: string[], // the path of the executable in the archive - } + // Optionally specify sha256 checksum for validation. + sha256?: string + // If the URL contains an archive, provide extraction instructions. + extract?: LibraryExtractSpec, } // TODO: support different architectures? (the Garden class currently errors on non-x64 archs, and many tools may // only be available in x64). -interface BinaryCmdSpec { +interface LibrarySpec { name: string - specs: { [key in SupportedPlatform]: BinarySpec } + specs: { [key in SupportedPlatform]: LibraryPlatformSpec } } export class DownloadError extends GardenBaseError { @@ -58,28 +53,24 @@ export class DownloadError extends GardenBaseError { } /** - * This helper class allows you to declare a tool dependency by providing a URL to a single-file binary, - * or an archive containing an executable, for each of our supported platforms. When executing the tool, - * the appropriate URL for the current platform will be downloaded and cached in the user's home directory + * This helper class allows you to declare a library dependency by providing a URL to a file or an archive, + * for each of our supported platforms. When requesting the path to the library, the appropriate URL for the + * current platform will be downloaded, extracted (if applicable) and cached in the user's home directory * (under .garden/tools//). * - * Note: The binary or archive currently needs to be self-contained and work without further installation steps. + * Note: The file or archive currently needs to be self-contained and work without further installation steps. */ -export class BinaryCmd extends Cmd { +export class Library { name: string - spec: BinarySpec + spec: LibraryPlatformSpec private lock: any private toolPath: string private versionDirname: string - private versionPath: string - private executablePath: string - private executableSubpath: string[] - private defaultCwd: string - - constructor(spec: BinaryCmdSpec) { - super() + protected versionPath: string + protected targetSubpath: string[] + constructor(spec: LibrarySpec) { const currentPlatform = platform() const platformSpec = spec.specs[currentPlatform] @@ -98,23 +89,26 @@ export class BinaryCmd extends Cmd { this.versionDirname = hashString(this.spec.url, 16) this.versionPath = join(this.toolPath, this.versionDirname) - this.executableSubpath = this.spec.extract - ? this.spec.extract.executablePath + this.targetSubpath = this.spec.extract + ? this.spec.extract.targetPath : [basename(this.spec.url)] - this.executablePath = join(this.versionPath, ...this.executableSubpath) - this.defaultCwd = dirname(this.executablePath) } - private async download(log: LogEntry) { + async getPath(log: LogEntry) { + await this.download(log) + return join(this.versionPath, ...this.targetSubpath) + } + + protected async download(log: LogEntry) { return this.lock.acquire("download", async () => { - if (await pathExists(this.executablePath)) { + if (await pathExists(this.versionPath)) { return } const tmpPath = join(this.toolPath, this.versionDirname + "." + uuid.v4().substr(0, 8)) - const tmpExecutable = join(tmpPath, ...this.executableSubpath) + const targetAbsPath = join(tmpPath, ...this.targetSubpath) - const logEntry = log.verbose(`Fetching ${this.name}...`) + const logEntry = log.info({ symbol: "info", msg: `Fetching ${this.name}...` }) const debug = logEntry.debug(`Downloading ${this.spec.url}...`) await ensureDir(tmpPath) @@ -122,14 +116,13 @@ export class BinaryCmd extends Cmd { try { await this.fetch(tmpPath, log) - if (!(await pathExists(tmpExecutable))) { + if (this.spec.extract && !(await pathExists(targetAbsPath))) { throw new ConfigurationError( - `Archive ${this.spec.url} does not contain a file at ${join(...this.spec.extract!.executablePath)}`, + `Archive ${this.spec.url} does not contain a file or directory at ${this.targetSubpath.join(sep)}`, { name: this.name, spec: this.spec }, ) } - await chmod(tmpExecutable, 0o755) await move(tmpPath, this.versionPath, { overwrite: true }) } finally { @@ -144,22 +137,7 @@ export class BinaryCmd extends Cmd { }) } - async exec({ cwd, args, log, timeout }: ExecParams) { - await this.download(log) - return execa(this.executablePath, args || [], { cwd: cwd || this.defaultCwd, timeout }) - } - - async stdout(params: ExecParams) { - const res = await this.exec(params) - return res.stdout - } - - async spawn({ cwd, args, log }: ExecParams) { - await this.download(log) - return spawn(this.executablePath, args || [], { cwd }) - } - - private async fetch(targetPath: string, log: LogEntry) { + protected async fetch(tmpPath: string, log: LogEntry) { const response = await Axios({ method: "GET", url: this.spec.url, @@ -195,22 +173,22 @@ export class BinaryCmd extends Cmd { }) if (!this.spec.extract) { - const targetExecutable = join(targetPath, ...this.executableSubpath) + const targetExecutable = join(tmpPath, ...this.targetSubpath) response.data.pipe(createWriteStream(targetExecutable)) response.data.on("end", () => resolve()) } else { const format = this.spec.extract.format - let extractor + let extractor: any if (format === "tar") { extractor = tar.x({ - C: targetPath, + C: tmpPath, strict: true, onwarn: entry => console.log(entry), }) extractor.on("end", () => resolve()) } else if (format === "zip") { - extractor = Extract({ path: targetPath }) + extractor = Extract({ path: tmpPath }) extractor.on("close", () => resolve()) } else { reject(new ParameterError(`Invalid archive format: ${format}`, { name: this.name, spec: this.spec })) @@ -227,3 +205,56 @@ export class BinaryCmd extends Cmd { }) } } + +interface ExecParams { + args?: string[] + cwd?: string + env?: { [key: string]: string } + log: LogEntry + timeout?: number +} + +/** + * This helper class allows you to declare a tool dependency by providing a URL to a single-file binary, + * or an archive containing an executable, for each of our supported platforms. When executing the tool, + * the appropriate URL for the current platform will be downloaded and cached in the user's home directory + * (under .garden/tools//). + * + * Note: The binary or archive currently needs to be self-contained and work without further installation steps. + */ +export class BinaryCmd extends Library { + name: string + spec: LibraryPlatformSpec + + private chmodDone: boolean + + constructor(spec: LibrarySpec) { + super(spec) + this.chmodDone = false + } + + async getPath(log: LogEntry) { + const path = await super.getPath(log) + // Make sure the target path is executable + if (!this.chmodDone) { + await chmod(path, 0o755) + this.chmodDone = true + } + return path + } + + async exec({ args, cwd, env, log, timeout }: ExecParams) { + const path = await this.getPath(log) + return execa(path, args || [], { cwd: cwd || dirname(path), timeout, env }) + } + + async stdout(params: ExecParams) { + const res = await this.exec(params) + return res.stdout + } + + async spawn({ args, cwd, env, log }: ExecParams) { + const path = await this.getPath(log) + return spawn(path, args || [], { cwd: cwd || dirname(path), env }) + } +}