feat(next-swc): try to fallback native bindings with MODULE_NOT_FOUND

packages/next/src/build/swc/index.ts

@@ -7,7 +7,7 @@ import * as Log from '../output/log'
 import { getParserOptions } from './options'
 import { eventSwcLoadFailure } from '../../telemetry/events/swc-load-failure'
 import { patchIncorrectLockfile } from '../../lib/patch-incorrect-lockfile'
-import { downloadWasmSwc } from '../../lib/download-wasm-swc'
+import { downloadWasmSwc, downloadNativeNextSwc } from '../../lib/download-swc'
 import { spawn } from 'child_process'
 import { NextConfigComplete, TurboLoaderItem } from '../../server/config-shared'
 import { isDeepStrictEqual } from 'util'
@@ -88,6 +88,8 @@ let pendingBindings: any
 let swcTraceFlushGuard: any
 let swcHeapProfilerFlushGuard: any
 let swcCrashReporterFlushGuard: any
+let downloadNativeBindingsPromise: Promise<void> | undefined = undefined
+
 export const lockfilePatchPromise: { cur?: Promise<void> } = {}
 
 export interface Binding {
@@ -155,9 +157,31 @@ export async function loadBindings(): Promise<Binding> {
       }
     }
 
+    // Trickle down loading `fallback` bindings:
+    //
+    // - First, try to load native bindings installed in node_modules.
+    // - If that fails with `ERR_MODULE_NOT_FOUND`, treat it as case of https://github.com/npm/cli/issues/4828
+    // that host system where generated package lock is not matching to the guest system running on, try to manually
+    // download corresponding target triple and load it. This won't be triggered if native bindings are failed to load
+    // with other reasons than `ERR_MODULE_NOT_FOUND`.
+    // - Lastly, falls back to wasm binding where possible.
     try {
       return resolve(loadNative(isCustomTurbopack))
     } catch (a) {
+      if (
+        Array.isArray(a) &&
+        a.every((m) => m.includes('it was not installed'))
+      ) {
+        let fallbackBindings = await tryLoadNativeWithFallback(
+          attempts,
+          isCustomTurbopack
+        )
+
+        if (fallbackBindings) {
+          return resolve(fallbackBindings)
+        }
+      }
+
       attempts = attempts.concat(a)
     }
 
@@ -177,6 +201,33 @@ export async function loadBindings(): Promise<Binding> {
   return pendingBindings
 }
 
+async function tryLoadNativeWithFallback(
+  attempts: Array<string>,
+  isCustomTurbopack: boolean
+) {
+  const nativeBindingsDirectory = path.join(
+    path.dirname(require.resolve('next/package.json')),
+    'next-swc-fallback'
+  )
+
+  if (!downloadNativeBindingsPromise) {
+    downloadNativeBindingsPromise = downloadNativeNextSwc(
+      nextVersion,
+      nativeBindingsDirectory,
+      triples.map((triple: any) => triple.platformArchABI)
+    )
+  }
+  await downloadNativeBindingsPromise
+
+  try {
+    let bindings = loadNative(isCustomTurbopack, nativeBindingsDirectory)
+    return bindings
+  } catch (a: any) {
+    attempts.concat(a)
+  }
+  return undefined
+}
+
 async function tryLoadWasmWithFallback(
   attempts: any,
   isCustomTurbopack: boolean
@@ -776,7 +827,7 @@ async function loadWasm(importPath = '', isCustomTurbopack: boolean) {
   throw attempts
 }
 
-function loadNative(isCustomTurbopack = false) {
+function loadNative(isCustomTurbopack = false, importPath?: string) {
   if (nativeBindings) {
     return nativeBindings
   }
@@ -794,10 +845,18 @@ function loadNative(isCustomTurbopack = false) {
 
   if (!bindings) {
     for (const triple of triples) {
-      let pkg = `@next/swc-${triple.platformArchABI}`
+      let pkg = importPath
+        ? path.join(
+            importPath,
+            `@next/swc-${triple.platformArchABI}`,
+            `next-swc.${triple.platformArchABI}.node`
+          )
+        : `@next/swc-${triple.platformArchABI}`
       try {
         bindings = require(pkg)
-        checkVersionMismatch(require(`${pkg}/package.json`))
+        if (!importPath) {
+          checkVersionMismatch(require(`${pkg}/package.json`))
+        }
         break
       } catch (e: any) {
         if (e?.code === 'MODULE_NOT_FOUND') {

packages/next/src/lib/download-swc.ts

@@ -0,0 +1,172 @@
+import os from 'os'
+import fs from 'fs'
+import path from 'path'
+import * as Log from '../build/output/log'
+import tar from 'next/dist/compiled/tar'
+const { fetch } = require('next/dist/compiled/undici') as {
+  fetch: typeof global.fetch
+}
+const { WritableStream } = require('node:stream/web') as {
+  WritableStream: typeof global.WritableStream
+}
+import { fileExists } from './file-exists'
+import { getRegistry } from './helpers/get-registry'
+
+const MAX_VERSIONS_TO_CACHE = 8
+
+// get platform specific cache directory adapted from playwright's handling
+// https://github.com/microsoft/playwright/blob/7d924470d397975a74a19184c136b3573a974e13/packages/playwright-core/src/utils/registry.ts#L141
+async function getCacheDirectory() {
+  let result
+  const envDefined = process.env['NEXT_SWC_PATH']
+
+  if (envDefined) {
+    result = envDefined
+  } else {
+    let systemCacheDirectory
+    if (process.platform === 'linux') {
+      systemCacheDirectory =
+        process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache')
+    } else if (process.platform === 'darwin') {
+      systemCacheDirectory = path.join(os.homedir(), 'Library', 'Caches')
+    } else if (process.platform === 'win32') {
+      systemCacheDirectory =
+        process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local')
+    } else {
+      /// Attempt to use generic tmp location for un-handled platform
+      if (!systemCacheDirectory) {
+        for (const dir of [
+          path.join(os.homedir(), '.cache'),
+          path.join(os.tmpdir()),
+        ]) {
+          if (await fileExists(dir)) {
+            systemCacheDirectory = dir
+            break
+          }
+        }
+      }
+
+      if (!systemCacheDirectory) {
+        console.error(new Error('Unsupported platform: ' + process.platform))
+        process.exit(0)
+      }
+    }
+    result = path.join(systemCacheDirectory, 'next-swc')
+  }
+
+  if (!path.isAbsolute(result)) {
+    // It is important to resolve to the absolute path:
+    //   - for unzipping to work correctly;
+    //   - so that registry directory matches between installation and execution.
+    // INIT_CWD points to the root of `npm/yarn install` and is probably what
+    // the user meant when typing the relative path.
+    result = path.resolve(process.env['INIT_CWD'] || process.cwd(), result)
+  }
+  return result
+}
+
+async function extractBinary(
+  outputDirectory: string,
+  pkgName: string,
+  tarFileName: string
+) {
+  const cacheDirectory = await getCacheDirectory()
+
+  const extractFromTar = async () => {
+    await tar.x({
+      file: path.join(cacheDirectory, tarFileName),
+      cwd: outputDirectory,
+      strip: 1,
+    })
+  }
+
+  if (!(await fileExists(path.join(cacheDirectory, tarFileName)))) {
+    Log.info(`Downloading swc package ${pkgName}...`)
+    await fs.promises.mkdir(cacheDirectory, { recursive: true })
+    const tempFile = path.join(
+      cacheDirectory,
+      `${tarFileName}.temp-${Date.now()}`
+    )
+
+    const registry = getRegistry()
+
+    await fetch(`${registry}${pkgName}/-/${tarFileName}`).then((res) => {
+      const { ok, body } = res
+      if (!ok) {
+        throw new Error(`request failed with status ${res.status}`)
+      }
+      if (!body) {
+        throw new Error('request failed with empty body')
+      }
+      const cacheWriteStream = fs.createWriteStream(tempFile)
+      return body.pipeTo(
+        new WritableStream({
+          write(chunk) {
+            cacheWriteStream.write(chunk)
+          },
+          close() {
+            cacheWriteStream.close()
+          },
+        })
+      )
+    })
+    await fs.promises.rename(tempFile, path.join(cacheDirectory, tarFileName))
+  }
+  await extractFromTar()
+
+  const cacheFiles = await fs.promises.readdir(cacheDirectory)
+
+  if (cacheFiles.length > MAX_VERSIONS_TO_CACHE) {
+    cacheFiles.sort((a, b) => {
+      if (a.length < b.length) return -1
+      return a.localeCompare(b)
+    })
+
+    // prune oldest versions in cache
+    for (let i = 0; i++; i < cacheFiles.length - MAX_VERSIONS_TO_CACHE) {
+      await fs.promises
+        .unlink(path.join(cacheDirectory, cacheFiles[i]))
+        .catch(() => {})
+    }
+  }
+}
+
+export async function downloadNativeNextSwc(
+  version: string,
+  bindingsDirectory: string,
+  triplesABI: Array<string>
+) {
+  for (const triple of triplesABI) {
+    const pkgName = `@next/swc-${triple}`
+    const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
+    const outputDirectory = path.join(bindingsDirectory, pkgName)
+
+    if (await fileExists(outputDirectory)) {
+      // if the package is already downloaded a different
+      // failure occurred than not being present
+      return
+    }
+
+    await fs.promises.mkdir(outputDirectory, { recursive: true })
+    await extractBinary(outputDirectory, pkgName, tarFileName)
+  }
+}
+
+export async function downloadWasmSwc(
+  version: string,
+  wasmDirectory: string,
+  variant: 'nodejs' | 'web' = 'nodejs'
+) {
+  const pkgName = `@next/swc-wasm-${variant}`
+  const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
+  const outputDirectory = path.join(wasmDirectory, pkgName)
+
+  if (await fileExists(outputDirectory)) {
+    // if the package is already downloaded a different
+    // failure occurred than not being present
+    return
+  }
+
+  await fs.promises.mkdir(outputDirectory, { recursive: true })
+  await extractBinary(outputDirectory, pkgName, tarFileName)
+}