diff --git a/ts/build.ts b/ts/build.ts index 3bbf373..4357539 100644 --- a/ts/build.ts +++ b/ts/build.ts @@ -9,6 +9,9 @@ export type BuildOptions = { optimize?: Optimize; zigCwd: string; step: string; + prefix?: string; + cacheDir?: string; + globalCacheDir?: string; /** Skip logging (useful when called from buildArtifacts which has its own logging) */ quiet?: boolean; }; @@ -23,6 +26,18 @@ export async function build(opts: BuildOptions): Promise { args.push(`-Doptimize=${opts.optimize}`); } + if (opts.prefix) { + args.push("--prefix", opts.prefix); + } + + if (opts.cacheDir) { + args.push("--cache-dir", opts.cacheDir); + } + + if (opts.globalCacheDir) { + args.push("--global-cache-dir", opts.globalCacheDir); + } + if (!opts.quiet) { logInfo(`Building for ${opts.target}...`); } diff --git a/ts/buildArtifacts.ts b/ts/buildArtifacts.ts index 7e3ecbb..ab2187d 100644 --- a/ts/buildArtifacts.ts +++ b/ts/buildArtifacts.ts @@ -1,16 +1,23 @@ import {promises as fs} from "node:fs"; +import {availableParallelism} from "node:os"; import {join} from "node:path"; import {type ParseArgsOptionsConfig, parseArgs} from "node:util"; import {build} from "./build.js"; import {loadConfig} from "./config.js"; -import {type Target, validateOptimize} from "./lib.js"; +import {type Target, parsePositiveIntOption, runWithConcurrency, validateOptimize} from "./lib.js"; import {logDetail, logInfo, logStep, logSuccess} from "./log.js"; +const defaultConcurrency = String(Math.max(1, Math.min(availableParallelism(), 4))); + const buildArtifactsCliOptions = { "artifacts-dir": { default: "artifacts", type: "string", }, + concurrency: { + default: defaultConcurrency, + type: "string", + }, optimize: { type: "string", }, @@ -28,15 +35,16 @@ type MoveArtifactOpts = { artifactsDir: string; target: Target; binaryName: string; + buildPrefix?: string; }; export async function moveArtifact(opts: MoveArtifactOpts): Promise { const destDir = join(opts.artifactsDir, opts.target); await fs.mkdir(destDir, {recursive: true}); - await fs.rename( - join(opts.zigCwd, "zig-out", "lib", `${opts.binaryName}.node`), - join(destDir, `${opts.binaryName}.node`) - ); + + const outputLibDir = opts.buildPrefix ? join(opts.buildPrefix, "lib") : join(opts.zigCwd, "zig-out", "lib"); + + await fs.rename(join(outputLibDir, `${opts.binaryName}.node`), join(destDir, `${opts.binaryName}.node`)); } export async function buildArtifactsCli(): Promise { @@ -52,30 +60,46 @@ export async function buildArtifactsCli(): Promise { if (!step) { throw new Error("--step is required (or set zapi.step in package.json)"); } + + const concurrency = parsePositiveIntOption( + "concurrency", + values.concurrency as string | undefined, + Number(defaultConcurrency) + ); const total = config.targets.length; + const artifactsDir = values["artifacts-dir"]; + const buildRoot = join(artifactsDir, ".zapi-build"); + + logInfo(`Building ${config.binaryName} for ${total} target(s) with concurrency ${concurrency}...`); - logInfo(`Building ${config.binaryName} for ${total} target(s)...`); + await runWithConcurrency(config.targets, concurrency, async (target, index) => { + const targetBuildDir = join(buildRoot, target); + const prefix = join(targetBuildDir, "prefix"); + const cacheDir = join(targetBuildDir, "cache"); + const globalCacheDir = join(targetBuildDir, "global-cache"); - for (let i = 0; i < config.targets.length; i++) { - const target = config.targets[i]; - logStep(i + 1, total, `Building for ${target}...`); + logStep(index + 1, total, `Building for ${target}...`); await build({ + cacheDir, + globalCacheDir, optimize, + prefix, quiet: true, step, target, zigCwd: values["zig-cwd"], }); - logDetail(`Moving artifact to ${join(values["artifacts-dir"], target)}`); + logDetail(`Moving artifact to ${join(artifactsDir, target)}`); await moveArtifact({ - artifactsDir: values["artifacts-dir"], + artifactsDir, binaryName: config.binaryName, + buildPrefix: prefix, target, zigCwd: values["zig-cwd"], }); - } + }); - logSuccess(`Built ${total} artifact(s) to ${values["artifacts-dir"]}/`); + logSuccess(`Built ${total} artifact(s) to ${artifactsDir}/`); } diff --git a/ts/cli.ts b/ts/cli.ts index b28e102..975ebbb 100644 --- a/ts/cli.ts +++ b/ts/cli.ts @@ -24,18 +24,21 @@ Commands: --zig-cwd Working directory for zig build (default: .) build-artifacts Build for all configured targets - --optimize Optimization level - --step Zig build step (required) - --zig-cwd Working directory for zig build (default: .) + --optimize Optimization level + --step Zig build step (required) + --zig-cwd Working directory for zig build (default: .) --artifacts-dir Output directory for artifacts (default: artifacts) + --concurrency Parallel target builds (default: min(cpu, 4)) prepublish Prepare npm packages for publishing --artifacts-dir Directory containing built artifacts (default: artifacts) --npm-dir Directory for npm packages (default: npm) + --concurrency Parallel filesystem prep jobs (default: min(cpu, 4)) publish Publish all packages to npm --npm-dir Directory containing npm packages (default: npm) --dry-run Preview what would be published without publishing + --concurrency Parallel target publishes (default: 1) [-- ] Additional arguments passed to npm publish Options: diff --git a/ts/config.ts b/ts/config.ts index 2b8c8b7..11c1240 100644 --- a/ts/config.ts +++ b/ts/config.ts @@ -1,7 +1,9 @@ import {promises as fs} from "node:fs"; -import {join} from "node:path"; +import {basename, join} from "node:path"; import {TARGETS, type Target} from "./lib.js"; +const BINARY_NAME_RE = /^[A-Za-z0-9._-]+$/; + export type Config = { binaryName: string; targets: Target[]; @@ -37,6 +39,18 @@ export async function loadConfig(cwd = "."): Promise { return {config, pkgJson}; } +function validateBinaryName(binaryName: string): void { + if (!BINARY_NAME_RE.test(binaryName)) { + throw new Error("zapi.binaryName must match /^[A-Za-z0-9._-]+$/"); + } + if (binaryName !== basename(binaryName)) { + throw new Error("zapi.binaryName must not include path segments"); + } + if (binaryName.includes("/") || binaryName.includes("\\") || binaryName.includes("..")) { + throw new Error("zapi.binaryName must not contain path separators or '..'"); + } +} + export function parsePkgJson(pkgJson: PkgJson): Config { const napi = pkgJson.zapi; if (typeof napi !== "object" || napi === null) { @@ -46,6 +60,8 @@ export function parsePkgJson(pkgJson: PkgJson): Config { if (typeof binaryName !== "string") { throw new Error("zapi.binaryName must be a string"); } + validateBinaryName(binaryName); + const targets = napi.targets; if (!Array.isArray(targets) || targets.length === 0) { throw new Error("zapi.targets must be a non-empty array"); diff --git a/ts/lib.ts b/ts/lib.ts index 6dd3f7a..940845e 100644 --- a/ts/lib.ts +++ b/ts/lib.ts @@ -69,6 +69,50 @@ export function requireOption(name: string, value: string | undefined): string { return value; } +/** + * Parse a positive integer CLI option. + */ +export function parsePositiveIntOption(name: string, value: string | undefined, defaultValue = 1): number { + if (value == null || value === "") { + return defaultValue; + } + + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`--${name} must be a positive integer`); + } + return parsed; +} + +/** + * Run async jobs with bounded concurrency. + */ +export async function runWithConcurrency( + items: readonly T[], + concurrency: number, + worker: (item: T, index: number) => Promise +): Promise { + if (items.length === 0) { + return; + } + + const limit = Math.max(1, Math.min(concurrency, items.length)); + let nextIndex = 0; + + await Promise.all( + Array.from({length: limit}, async () => { + while (true) { + const index = nextIndex; + nextIndex += 1; + if (index >= items.length) { + return; + } + await worker(items[index], index); + } + }) + ); +} + export function getZigTriple(target: Target): string { switch (target) { case "x86_64-unknown-linux-gnu": @@ -87,18 +131,24 @@ export function getZigTriple(target: Target): string { } // from napi-rs +let isMuslCache: boolean | null = null; + const isMusl = () => { - let musl: boolean | null = false; - if (process.platform === "linux") { - musl = isMuslFromFilesystem(); - if (musl === null) { - musl = isMuslFromReport(); - } - if (musl === null) { - musl = isMuslFromChildProcess(); - } + if (process.platform !== "linux") { + return false; + } + + if (isMuslCache !== null) { + return isMuslCache; + } + + let musl = isMuslFromFilesystem(); + if (musl === null) { + musl = isMuslFromReport(); } - return musl; + + isMuslCache = musl ?? false; + return isMuslCache; }; const isFileMusl = (f: string) => f.includes("libc.musl-") || f.includes("ld-musl-"); @@ -129,15 +179,6 @@ const isMuslFromReport = () => { return false; }; -const isMuslFromChildProcess = () => { - try { - return require("node:child_process").execSync("ldd --version", {encoding: "utf8"}).includes("musl"); - } catch (_e) { - // If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false - return false; - } -}; - export function getTarget(platform: NodeJS.Platform, arch: NodeJS.Architecture): Target { if (platform === "darwin") { if (arch === "arm64") { diff --git a/ts/prepublish.ts b/ts/prepublish.ts index 2dc3c4a..ae60fae 100644 --- a/ts/prepublish.ts +++ b/ts/prepublish.ts @@ -1,22 +1,27 @@ import {promises as fs} from "node:fs"; +import {availableParallelism} from "node:os"; import {join} from "node:path"; import {type ParseArgsOptionsConfig, parseArgs} from "node:util"; import {type Config, type PkgJson, getTargetParts, loadConfig} from "./config.js"; +import {parsePositiveIntOption, runWithConcurrency} from "./lib.js"; import {logDetail, logInfo, logSuccess, logWarn} from "./log.js"; +const defaultConcurrency = String(Math.max(1, Math.min(availableParallelism(), 4))); + export type CreateNpmDirsOpts = { "npm-dir": string; + concurrency: number; }; export async function createNpmDirs(config: Config, opts: CreateNpmDirsOpts): Promise { - for (const target of config.targets) { + await runWithConcurrency(config.targets, opts.concurrency, async (target) => { await fs.mkdir(join(opts["npm-dir"], target), {recursive: true}); - } + }); } export async function moveArtifacts(_pkgJson: PkgJson, config: Config, opts: PrepublishOpts): Promise { logInfo("Moving artifacts to npm packages..."); - for (const target of config.targets) { + await runWithConcurrency(config.targets, opts.concurrency, async (target) => { const artifactPath = join(opts["artifacts-dir"], target, `${config.binaryName}.node`); const destPath = join(opts["npm-dir"], target, `${config.binaryName}.node`); @@ -28,12 +33,12 @@ export async function moveArtifacts(_pkgJson: PkgJson, config: Config, opts: Pre throw err; }); logDetail(`${target} → ${destPath}`); - } + }); } export async function updateTargetPkgJsons(pkgJson: PkgJson, config: Config, opts: PrepublishOpts): Promise { logInfo("Generating target package.json files..."); - for (const target of config.targets) { + await runWithConcurrency(config.targets, opts.concurrency, async (target) => { const {platform, arch, abi} = getTargetParts(target); const libc = platform !== "linux" ? undefined : abi === "gnu" ? "glibc" : abi === "musl" ? "musl" : undefined; @@ -59,7 +64,7 @@ This is the ${target} target package for ${pkgJson.name}. ` ); logDetail(`Created ${join(targetDir, "package.json")}`); - } + }); } export function updateOptionalDependencies(pkgJson: PkgJson, config: Config): PkgJson { @@ -74,6 +79,7 @@ export function updateOptionalDependencies(pkgJson: PkgJson, config: Config): Pk type PrepublishOpts = { "artifacts-dir": string; "npm-dir": string; + concurrency: number; }; const prepublishOptions = { @@ -81,6 +87,10 @@ const prepublishOptions = { default: "artifacts", type: "string", }, + concurrency: { + default: defaultConcurrency, + type: "string", + }, "npm-dir": { default: "npm", type: "string", @@ -94,12 +104,23 @@ export async function prepublishCli(): Promise { }); const {pkgJson, config} = await loadConfig(); - - logInfo(`Preparing ${pkgJson.name}@${pkgJson.version} for publishing...`); - - await createNpmDirs(config, values); - await moveArtifacts(pkgJson, config, values); - await updateTargetPkgJsons(pkgJson, config, values); + const concurrency = parsePositiveIntOption( + "concurrency", + values.concurrency as string | undefined, + Number(defaultConcurrency) + ); + + logInfo(`Preparing ${pkgJson.name}@${pkgJson.version} for publishing with concurrency ${concurrency}...`); + + const opts: PrepublishOpts = { + "artifacts-dir": values["artifacts-dir"], + concurrency, + "npm-dir": values["npm-dir"], + }; + + await createNpmDirs(config, opts); + await moveArtifacts(pkgJson, config, opts); + await updateTargetPkgJsons(pkgJson, config, opts); logInfo("Updating package.json with optionalDependencies..."); const updatedPkgJson = await updateOptionalDependencies(pkgJson, config); diff --git a/ts/publish.ts b/ts/publish.ts index 7ab4fb1..120e81c 100644 --- a/ts/publish.ts +++ b/ts/publish.ts @@ -2,9 +2,14 @@ import {spawn} from "node:child_process"; import {join} from "node:path"; import {type ParseArgsOptionsConfig, parseArgs} from "node:util"; import {loadConfig} from "./config.js"; +import {parsePositiveIntOption, runWithConcurrency} from "./lib.js"; import {logDetail, logInfo, logStep, logSuccess} from "./log.js"; const publishOptions = { + concurrency: { + default: "1", + type: "string", + }, "dry-run": { default: false, type: "boolean", @@ -18,6 +23,7 @@ const publishOptions = { export type PublishOpts = { "npm-dir": string; "dry-run": boolean; + concurrency: number; }; function extraPublishArgs(): string[] { @@ -60,7 +66,9 @@ export async function publish(opts: PublishOpts): Promise { const total = config.targets.length + 1; // +1 for main package if (opts["dry-run"]) { - logInfo(`[DRY RUN] Would publish ${config.targets.length} target package(s) + main package`); + logInfo( + `[DRY RUN] Would publish ${config.targets.length} target package(s) + main package with concurrency ${opts.concurrency}` + ); logDetail(`Extra npm args: ${extraArgs.length > 0 ? extraArgs.join(" ") : "(none)"}`); for (let i = 0; i < config.targets.length; i++) { @@ -77,14 +85,16 @@ export async function publish(opts: PublishOpts): Promise { return; } - logInfo(`Publishing ${config.targets.length} target package(s) + main package...`); - - for (let i = 0; i < config.targets.length; i++) { - const target = config.targets[i]; - logStep(i + 1, total, `Publishing ${target}...`); + logInfo( + `Publishing ${config.targets.length} target package(s) + main package with concurrency ${opts.concurrency}...` + ); + let started = 0; + await runWithConcurrency(config.targets, opts.concurrency, async (target) => { + started += 1; + logStep(started, total, `Publishing ${target}...`); await runNpm(publishArgv, join(process.cwd(), opts["npm-dir"], target)); - } + }); logStep(total, total, "Publishing main package..."); await runNpm(publishArgv, process.cwd()); @@ -100,6 +110,7 @@ export async function publishCli(): Promise { }); await publish({ + concurrency: parsePositiveIntOption("concurrency", values["concurrency"] as string | undefined, 1), "dry-run": values["dry-run"] as boolean, "npm-dir": values["npm-dir"] as string, });