Skip to content

Commit

Permalink
feat: yarn 3 support for native modules via new electron/rebuild comp…
Browse files Browse the repository at this point in the history
…ilation (#8112)
  • Loading branch information
mmaietta committed Mar 9, 2024
1 parent 3d4cc7a commit 9edfee6
Show file tree
Hide file tree
Showing 14 changed files with 743 additions and 30 deletions.
5 changes: 5 additions & 0 deletions .changeset/twelve-points-whisper.md
@@ -0,0 +1,5 @@
---
"app-builder-lib": minor
---

feat: implementing electron/rebuild with config option `useLegacyRebuilder` default: `true` to support Yarn 3
5 changes: 5 additions & 0 deletions .github/actions/pretest/action.yml
Expand Up @@ -20,6 +20,11 @@ runs:
with:
version: ${{ inputs.version }}

- name: Setup python
uses: actions/setup-python@v5
with:
python-version: '3.11'

- name: Setup node
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
with:
Expand Down
3 changes: 3 additions & 0 deletions docs/configuration/configuration.md
Expand Up @@ -93,6 +93,9 @@ Env file `electron-builder.env` in the current dir ([example](https://github.com
<p><code id="Configuration-npmRebuild">npmRebuild</code> = <code>true</code> Boolean - Whether to <a href="https://docs.npmjs.com/cli/rebuild">rebuild</a> native dependencies before starting to package the app.</p>
</li>
<li>
<p><code id="Configuration-nativeRebuilder">nativeRebuilder</code> = <code>legacy</code> “legacy” | “sequential” | “parallel” | “undefined” - Use <code>legacy</code> app-builder binary for installing native dependencies, or @electron/rebuild in <code>sequential</code> or <code>parallel</code> compilation modes.</p>
</li>
<li>
<p><code id="Configuration-buildNumber">buildNumber</code> String | “undefined” - The build number. Maps to the <code>--iteration</code> flag for builds using FPM on Linux. If not defined, then it will fallback to <code>BUILD_NUMBER</code> or <code>TRAVIS_BUILD_NUMBER</code> or <code>APPVEYOR_BUILD_NUMBER</code> or <code>CIRCLE_BUILD_NUM</code> or <code>BUILD_BUILDNUMBER</code> or <code>CI_PIPELINE_IID</code> env.</p>
</li>
</ul>
Expand Down
1 change: 1 addition & 0 deletions packages/app-builder-lib/package.json
Expand Up @@ -49,6 +49,7 @@
"@develar/schema-utils": "~2.6.5",
"@electron/notarize": "2.2.1",
"@electron/osx-sign": "1.0.5",
"@electron/rebuild": "3.6.0",
"@electron/universal": "1.5.1",
"@malept/flatpak-bundler": "^0.4.0",
"@types/fs-extra": "9.0.13",
Expand Down
17 changes: 17 additions & 0 deletions packages/app-builder-lib/scheme.json
Expand Up @@ -7463,6 +7463,23 @@
}
]
},
"nativeRebuilder": {
"anyOf": [
{
"enum": [
"legacy",
"parallel",
"sequential"
],
"type": "string"
},
{
"type": "null"
}
],
"default": "legacy",
"description": "Use `legacy` app-builder binary for installing native dependencies, or @electron/rebuild in `sequential` or `parallel` compilation modes."
},
"nodeGypRebuild": {
"default": false,
"description": "Whether to execute `node-gyp rebuild` before starting to package the app.\n\nDon't [use](https://github.com/electron-userland/electron-builder/issues/683#issuecomment-241214075) [npm](http://electron.atom.io/docs/tutorial/using-native-node-modules/#using-npm) (neither `.npmrc`) for configuring electron headers. Use `electron-builder node-gyp-rebuild` instead.",
Expand Down
5 changes: 5 additions & 0 deletions packages/app-builder-lib/src/configuration.ts
Expand Up @@ -131,6 +131,11 @@ export interface Configuration extends PlatformSpecificBuildOptions {
* @default true
*/
readonly npmRebuild?: boolean
/**
* Use `legacy` app-builder binary for installing native dependencies, or @electron/rebuild in `sequential` or `parallel` compilation modes.
* @default legacy
*/
readonly nativeRebuilder?: "legacy" | "sequential" | "parallel" | null

/**
* The build number. Maps to the `--iteration` flag for builds using FPM on Linux.
Expand Down
13 changes: 11 additions & 2 deletions packages/app-builder-lib/src/electron/electronVersion.ts
@@ -1,3 +1,5 @@
import { getProjectRootPath } from "@electron/rebuild/lib/search-module"

import { InvalidConfigurationError, log } from "builder-util"
import { parseXml } from "builder-util-runtime"
import { httpExecutor } from "builder-util/out/nodeHttpExecutor"
Expand Down Expand Up @@ -56,8 +58,15 @@ export async function computeElectronVersion(projectDir: string): Promise<string
return result
}

const metadata = await orNullIfFileNotExist(readJson(path.join(projectDir, "package.json")))
const dependency = metadata ? findFromPackageMetadata(metadata) : null
const potentialRootDirs = [projectDir, await getProjectRootPath(projectDir)]
let dependency: NameAndVersion | null = null
for await (const dir of potentialRootDirs) {
const metadata = await orNullIfFileNotExist(readJson(path.join(dir, "package.json")))
dependency = metadata ? findFromPackageMetadata(metadata) : null
if (dependency) {
break
}
}
if (dependency?.name === "electron-nightly") {
log.info("You are using a nightly version of electron, be warned that those builds are highly unstable.")
const feedXml = await httpExecutor.request({
Expand Down
2 changes: 1 addition & 1 deletion packages/app-builder-lib/src/util/packageMetadata.ts
Expand Up @@ -70,7 +70,7 @@ export function checkMetadata(metadata: Metadata, devMetadata: any | null, appPa
const devDependencies = (metadata as any).devDependencies
if (devDependencies != null && ("electron-rebuild" in devDependencies || "@electron/rebuild" in devDependencies)) {
log.info(
'@electron/rebuild not required if you use electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
'@electron/rebuild already used by electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
)
}

Expand Down
59 changes: 59 additions & 0 deletions packages/app-builder-lib/src/util/rebuild/rebuild.ts
@@ -0,0 +1,59 @@
import * as cp from "child_process"
import * as path from "path"
import { RebuildOptions } from "@electron/rebuild"
import { log } from "builder-util"

export const rebuild = async (options: RebuildOptions): Promise<void> => {
const { arch } = options
log.info({ arch }, `installing native dependencies`)

const child = cp.fork(path.resolve(__dirname, "remote-rebuild.js"), [JSON.stringify(options)], {
stdio: ["pipe", "pipe", "pipe", "ipc"],
})

let pendingError: Error

child.stdout?.on("data", chunk => {
log.info(chunk.toString())
})
child.stderr?.on("data", chunk => {
log.error(chunk.toString())
})

child.on("message", (message: { msg: string; moduleName: string; err: { message: string; stack: string } }) => {
const { moduleName, msg } = message
switch (msg) {
case "module-found": {
log.info({ moduleName, arch }, "preparing")
break
}
case "module-done": {
log.info({ moduleName, arch }, "finished")
break
}
case "module-skip": {
log.debug?.({ moduleName, arch }, "skipped. set ENV=electron-rebuild to determine why")
break
}
case "rebuild-error": {
pendingError = new Error(message.err.message)
pendingError.stack = message.err.stack
break
}
case "rebuild-done": {
log.info("completed installing native dependencies")
break
}
}
})

await new Promise<void>((resolve, reject) => {
child.on("exit", code => {
if (code === 0 && !pendingError) {
resolve()
} else {
reject(pendingError || new Error(`Rebuilder failed with exit code: ${code}`))
}
})
})
}
30 changes: 30 additions & 0 deletions packages/app-builder-lib/src/util/rebuild/remote-rebuild.ts
@@ -0,0 +1,30 @@
import { rebuild, RebuildOptions } from "@electron/rebuild"

if (!process.send) {
console.error("The remote rebuilder expects to be spawned with an IPC channel")
process.exit(1)
}

const options: RebuildOptions = JSON.parse(process.argv[2])

const rebuilder = rebuild(options)

rebuilder.lifecycle.on("module-found", (moduleName: string) => process.send?.({ msg: "module-found", moduleName }))
rebuilder.lifecycle.on("module-done", (moduleName: string) => process.send?.({ msg: "module-done", moduleName }))
rebuilder.lifecycle.on("module-skip", (moduleName: string) => process.send?.({ msg: "module-skip", moduleName }))

rebuilder
.then(() => {
process.send?.({ msg: "rebuild-done" })
return process.exit(0)
})
.catch(err => {
process.send?.({
msg: "rebuild-error",
err: {
message: err.message,
stack: err.stack,
},
})
process.exit(0)
})
54 changes: 44 additions & 10 deletions packages/app-builder-lib/src/util/yarn.ts
Expand Up @@ -4,11 +4,15 @@ import { Lazy } from "lazy-val"
import { homedir } from "os"
import * as path from "path"
import { Configuration } from "../configuration"
import { executeAppBuilderAndWriteJson } from "./appBuilder"
import { NodeModuleDirInfo } from "./packageDependencies"
import * as electronRebuild from "@electron/rebuild"
import { getProjectRootPath } from "@electron/rebuild/lib/search-module"
import { rebuild as remoteRebuild } from "./rebuild/rebuild"
import { executeAppBuilderAndWriteJson } from "./appBuilder"
import { RebuildMode } from "@electron/rebuild/lib/types"

export async function installOrRebuild(config: Configuration, appDir: string, options: RebuildOptions, forceInstall = false) {
const effectiveOptions = {
const effectiveOptions: RebuildOptions = {
buildFromSource: config.buildDependenciesFromSource === true,
additionalArgs: asArray(config.npmArgs),
...options,
Expand All @@ -24,9 +28,9 @@ export async function installOrRebuild(config: Configuration, appDir: string, op
}

if (forceInstall || !isDependenciesInstalled) {
await installDependencies(appDir, effectiveOptions)
await installDependencies(config, appDir, effectiveOptions)
} else {
await rebuild(appDir, effectiveOptions)
await rebuild(config, appDir, effectiveOptions)
}
}

Expand Down Expand Up @@ -83,7 +87,7 @@ function checkYarnBerry() {
return yarnMajorVersion >= 2
}

function installDependencies(appDir: string, options: RebuildOptions): Promise<any> {
async function installDependencies(config: Configuration, appDir: string, options: RebuildOptions): Promise<any> {
const platform = options.platform || process.platform
const arch = options.arch || process.arch
const additionalArgs = options.additionalArgs
Expand Down Expand Up @@ -113,10 +117,14 @@ function installDependencies(appDir: string, options: RebuildOptions): Promise<a
if (additionalArgs != null) {
execArgs.push(...additionalArgs)
}
return spawn(execPath, execArgs, {
await spawn(execPath, execArgs, {
cwd: appDir,
env: getGypEnv(options.frameworkInfo, platform, arch, options.buildFromSource === true),
})

// Some native dependencies no longer use `install` hook for building their native module, (yarn 3+ removed implicit link of `install` and `rebuild` steps)
// https://github.com/electron-userland/electron-builder/issues/8024
return rebuild(config, appDir, options)
}

export async function nodeGypRebuild(platform: NodeJS.Platform, arch: string, frameworkInfo: DesktopFrameworkInfo) {
Expand Down Expand Up @@ -164,8 +172,8 @@ export interface RebuildOptions {
}

/** @internal */
export async function rebuild(appDir: string, options: RebuildOptions) {
const configuration: any = {
export async function rebuild(config: Configuration, appDir: string, options: RebuildOptions) {
const configuration = {
dependencies: await options.productionDeps.value,
nodeExecPath: process.execPath,
platform: options.platform || process.platform,
Expand All @@ -174,7 +182,33 @@ export async function rebuild(appDir: string, options: RebuildOptions) {
execPath: process.env.npm_execpath || process.env.NPM_CLI_JS,
buildFromSource: options.buildFromSource === true,
}
if ([undefined, null, "legacy"].includes(config.nativeRebuilder)) {
const env = getGypEnv(options.frameworkInfo, configuration.platform, configuration.arch, options.buildFromSource === true)
return executeAppBuilderAndWriteJson(["rebuild-node-modules"], configuration, { env, cwd: appDir })
}

const env = getGypEnv(options.frameworkInfo, configuration.platform, configuration.arch, options.buildFromSource === true)
await executeAppBuilderAndWriteJson(["rebuild-node-modules"], configuration, { env, cwd: appDir })
const {
frameworkInfo: { version: electronVersion },
} = options
const { arch, buildFromSource } = configuration
const logInfo = {
electronVersion,
arch,
buildFromSource,
appDir: log.filePath(appDir) || "./",
}
log.info(logInfo, "executing @electron/rebuild")

const rebuildOptions: electronRebuild.RebuildOptions = {
buildPath: appDir,
electronVersion,
arch,
debug: log.isDebugEnabled,
projectRootPath: await getProjectRootPath(appDir),
mode: config.nativeRebuilder as RebuildMode,
}
if (buildFromSource) {
rebuildOptions.prebuildTagPrefix = "totally-not-a-real-prefix-to-force-rebuild"
}
return remoteRebuild(rebuildOptions)
}

0 comments on commit 9edfee6

Please sign in to comment.