Add event for swc load failure and attempt patching lockfile (#36527)

packages/next/build/swc/index.js

@@ -1,7 +1,10 @@
 import { platform, arch } from 'os'
 import { platformArchTriples } from 'next/dist/compiled/@napi-rs/triples'
+import { version as nextVersion, optionalDependencies } from 'next/package.json'
 import * as Log from '../output/log'
 import { getParserOptions } from './options'
+import { eventSwcLoadFailure } from '../../telemetry/events/swc-load-failure'
+import { patchIncorrectLockfile } from '../../lib/patch-incorrect-lockfile'
 
 const ArchName = arch()
 const PlatformName = platform()
@@ -18,6 +21,9 @@ async function loadBindings() {
     attempts = attempts.concat(a)
   }
 
+  // TODO: fetch wasm and fallback when loading native fails
+  // so that users aren't blocked on this, we still want to
+  // report the native load failure so we can patch though
   try {
     let bindings = await loadWasm()
     return bindings
@@ -41,13 +47,50 @@ function loadBindingsSync() {
 
 function logLoadFailure(attempts) {
   for (let attempt of attempts) {
-    Log.info(attempt)
+    Log.warn(attempt)
   }
+  let glibcVersion
+  let installedSwcPackages
 
-  Log.error(
-    `Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
-  )
-  process.exit(1)
+  try {
+    glibcVersion = process.report?.getReport().header.glibcVersionRuntime
+  } catch (_) {}
+
+  try {
+    const pkgNames = Object.keys(optionalDependencies || {}).filter((pkg) =>
+      pkg.startsWith('@next/swc')
+    )
+    const installedPkgs = []
+
+    for (const pkg of pkgNames) {
+      try {
+        const { version } = require(`${pkg}/package.json`)
+        installedPkgs.push(`${pkg}@${version}`)
+      } catch (_) {}
+    }
+
+    if (installedPkgs.length > 0) {
+      installedSwcPackages = installedPkgs.sort().join(',')
+    }
+  } catch (_) {}
+
+  patchIncorrectLockfile(process.cwd())
+    .then(() => {
+      return eventSwcLoadFailure({
+        nextVersion,
+        glibcVersion,
+        installedSwcPackages,
+        arch: process.arch,
+        platform: process.platform,
+        nodeVersion: process.versions.node,
+      })
+    })
+    .finally(() => {
+      Log.error(
+        `Failed to load SWC binary for ${PlatformName}/${ArchName}, see more info here: https://nextjs.org/docs/messages/failed-loading-swc`
+      )
+      process.exit(1)
+    })
 }
 
 async function loadWasm() {

packages/next/lib/patch-incorrect-lockfile.ts

@@ -0,0 +1,94 @@
+import { promises } from 'fs'
+import '../server/node-polyfill-fetch'
+import * as Log from '../build/output/log'
+import findUp from 'next/dist/compiled/find-up'
+
+/**
+ * Attempts to patch npm package-lock.json when it
+ * fails to include optionalDependencies for other platforms
+ * this can occur when the package-lock is rebuilt from a current
+ * node_modules install instead of pulling fresh package data
+ */
+export async function patchIncorrectLockfile(dir: string) {
+  const lockfilePath = await findUp('package-lock.json', { cwd: dir })
+
+  if (!lockfilePath) {
+    // if no lockfile present there is no action to take
+    return
+  }
+  const content = await promises.readFile(lockfilePath, 'utf8')
+  const lockfileParsed = JSON.parse(content)
+
+  const packageKeys = Object.keys(lockfileParsed.dependencies)
+  const foundSwcPkgs = new Set()
+  const nextPkg = lockfileParsed.packages['node_modules/next']
+
+  if (!nextPkg) {
+    return console.error('Failed to locate next in', lockfilePath)
+  }
+  const nextVersion = nextPkg.version
+
+  const expectedSwcPkgs = Object.keys(nextPkg?.optionalDependencies).filter(
+    (pkg) => pkg.startsWith('@next/swc-')
+  )
+
+  packageKeys.forEach((pkgKey) => {
+    const swcIndex = pkgKey.indexOf('@next/swc-')
+    if (swcIndex > -1) {
+      foundSwcPkgs.add(pkgKey.substring(swcIndex))
+    }
+  })
+
+  // if swc package keys are missing manually populate them
+  // so installs on different platforms can succeed
+  // user will need to run npm i after to ensure it's corrected
+  if (foundSwcPkgs.size !== expectedSwcPkgs.length) {
+    Log.warn(`Found lockfile missing swc dependencies, patching..`)
+
+    try {
+      // populate fields for each missing swc pkg
+      for (const pkg of expectedSwcPkgs) {
+        if (!foundSwcPkgs.has(pkg)) {
+          console.log('fetching', pkg)
+          const res = await fetch(`https://registry.npmjs.org/${pkg}`)
+
+          if (!res.ok) {
+            throw new Error(
+              `Failed to fetch registry info for ${pkg}, got status ${res.status}`
+            )
+          }
+          const data = await res.json()
+          const version = data.versions[nextVersion]
+
+          if (!version) {
+            throw new Error(
+              `Failed to find matching version for ${pkg} at ${nextVersion}`
+            )
+          }
+          lockfileParsed.packages[`node_modules/${pkg}`] = {
+            version: nextVersion,
+            resolved: version.dist.tarball,
+            integrity: version.dist.integrity,
+            cpu: version.cpu,
+            optional: true,
+            os: version.os,
+            engines: version.engines,
+          }
+        }
+      }
+
+      await promises.writeFile(
+        lockfilePath,
+        JSON.stringify(lockfileParsed, null, 2)
+      )
+      Log.warn(
+        'Lockfile was successfully patched, please run "npm install" to ensure @next/swc dependencies are downloaded'
+      )
+    } catch (err) {
+      Log.error(
+        `Failed to patch lockfile, please try uninstalling and reinstalling next in this workspace`
+      )
+      console.error(err)
+    }
+  }
+}

packages/next/telemetry/events/swc-load-failure.ts

@@ -0,0 +1,31 @@
+import { traceGlobals } from '../../trace/shared'
+import { Telemetry } from '../storage'
+
+const EVENT_PLUGIN_PRESENT = 'NEXT_SWC_LOAD_FAILURE'
+export type EventSwcLoadFailure = {
+  eventName: string
+  payload: {
+    platform: string
+    arch: string
+    nodeVersion: string
+    nextVersion: string
+    wasm?: string
+    glibcVersion?: string
+    installedSwcPackages?: string
+  }
+}
+
+export async function eventSwcLoadFailure(
+  event: EventSwcLoadFailure['payload']
+): Promise<void> {
+  const telemetry: Telemetry = traceGlobals.get('telemetry')
+  // can't continue if telemetry isn't set
+  if (!telemetry) return
+
+  telemetry.record({
+    eventName: EVENT_PLUGIN_PRESENT,
+    payload: event,
+  })
+  // ensure this event is flushed before process exits
+  await telemetry.flush()
+}

test/integration/telemetry/test/index.test.js

@@ -98,6 +98,25 @@ describe('Telemetry CLI', () => {
     expect(stderr2).toMatch(/isSrcDir.*?true/)
   })
 
+  it('emits event when swc fails to load', async () => {
+    await fs.remove(path.join(appDir, '.next'))
+    const { stderr } = await runNextCommand(['build', appDir], {
+      stderr: true,
+      env: {
+        // block swc from loading
+        NODE_OPTIONS: '--no-addons',
+        NEXT_TELEMETRY_DEBUG: 1,
+      },
+    })
+    expect(stderr).toMatch(/NEXT_SWC_LOAD_FAILURE/)
+    expect(stderr).toContain(
+      `"nextVersion": "${require('next/package.json').version}"`
+    )
+    expect(stderr).toContain(`"arch": "${process.arch}"`)
+    expect(stderr).toContain(`"platform": "${process.platform}"`)
+    expect(stderr).toContain(`"nodeVersion": "${process.versions.node}"`)
+  })
+
   it('logs completed `next build` with warnings', async () => {
     await fs.rename(
       path.join(appDir, 'pages', 'warning.skip'),

test/lib/create-next-install.js

@@ -9,7 +9,8 @@ const { linkPackages } =
 async function createNextInstall(
   dependencies,
   installCommand,
-  packageJson = {}
+  packageJson = {},
+  packageLockPath = ''
 ) {
   const tmpDir = await fs.realpath(process.env.NEXT_TEST_DIR || os.tmpdir())
   const origRepoDir = path.join(__dirname, '../../')
@@ -49,14 +50,18 @@ async function createNextInstall(
     })
   }
 
-  const pkgPaths = await linkPackages(tmpRepoDir)
-  const combinedDependencies = {
-    ...Object.keys(dependencies).reduce((prev, pkg) => {
-      const pkgPath = pkgPaths.get(pkg)
-      prev[pkg] = pkgPath || dependencies[pkg]
-      return prev
-    }, {}),
-    next: pkgPaths.get('next'),
+  let combinedDependencies = dependencies
+
+  if (!(packageJson && packageJson.nextPrivateSkipLocalDeps)) {
+    const pkgPaths = await linkPackages(tmpRepoDir)
+    combinedDependencies = {
+      ...Object.keys(dependencies).reduce((prev, pkg) => {
+        const pkgPath = pkgPaths.get(pkg)
+        prev[pkg] = pkgPath || dependencies[pkg]
+        return prev
+      }, {}),
+      next: pkgPaths.get('next'),
+    }
   }
 
   await fs.ensureDir(installDir)
@@ -73,6 +78,13 @@ async function createNextInstall(
     )
   )
 
+  if (packageLockPath) {
+    await fs.copy(
+      packageLockPath,
+      path.join(installDir, path.basename(packageLockPath))
+    )
+  }
+
   if (installCommand) {
     const installString =
       typeof installCommand === 'function'

test/lib/e2e-utils.ts

@@ -119,6 +119,7 @@ export async function createNext(opts: {
   buildCommand?: string
   packageJson?: PackageJson
   startCommand?: string
+  packageLockPath?: string
 }): Promise<NextInstance> {
   try {
     if (nextInstance) {
@@ -149,7 +150,7 @@ export async function createNext(opts: {
   } catch (err) {
     require('console').error('Failed to create next instance', err)
     try {
-      await nextInstance.destroy()
+      nextInstance.destroy()
     } catch (_) {}
     process.exit(1)
   }

test/lib/next-modes/base.ts

@@ -32,6 +32,7 @@ export class NextInstance {
   protected _url: string
   protected _parsedUrl: URL
   protected packageJson: PackageJson
+  protected packageLockPath?: string
   protected basePath?: string
 
   constructor({
@@ -42,6 +43,7 @@ export class NextInstance {
     buildCommand,
     startCommand,
     packageJson = {},
+    packageLockPath,
   }: {
     files: {
       [filename: string]: string | FileRef
@@ -50,6 +52,7 @@ export class NextInstance {
       [name: string]: string
     }
     packageJson?: PackageJson
+    packageLockPath?: string
     nextConfig?: NextConfig
     installCommand?: InstallCommand
     buildCommand?: string
@@ -62,6 +65,7 @@ export class NextInstance {
     this.buildCommand = buildCommand
     this.startCommand = startCommand
     this.packageJson = packageJson
+    this.packageLockPath = packageLockPath
     this.events = {}
     this.isDestroyed = false
     this.isStopping = false
@@ -131,7 +135,8 @@ export class NextInstance {
         this.testDir = await createNextInstall(
           finalDependencies,
           this.installCommand,
-          this.packageJson
+          this.packageJson,
+          this.packageLockPath
         )
       }
       console.log('created next.js install, writing test files')