feat(next): turbopack build support, fix incorrect handling of external packages (#14475)

Fixes #14338 and
vercel/next.js#85110 (comment).

This PR should fix support for using turbopack to build your Next.js
project (which is now the default in Next.js 16, unless you pass `next
build --webpack`).

Previously, we were directly externalizing dependencies like `libsql` as
they cause an error when being bundled. However, as those dependencies
were not installed at the top-level by the user, the resulting
`require('libsql')` calls could fail in production.

Turbopack threw the following warning:

```bash
Packages that should be external need to be installed in the project directory, so they can be resolved from the output files.\nTry to install it into the project directory by running
````

However, we ignored this warning, as we cannot force users to manually
install dependencies of our dependencies.

In Next.js 16, when using Turbopack build, this is no longer a warning
but an error, which is why attempting to use turbopack build right now
will not work.

This PR properly fixes this externalization issue, by externalizing the
`entry points` of those problematic packages (our db adapters). This
means we no longer need to hide the warning.

Author: AlessioGr

packages/next/src/withPayload.js

@@ -15,24 +15,6 @@ export const withPayload = (nextConfig = {}, options = {}) => {
     env.NEXT_PUBLIC_ENABLE_ROUTER_CACHE_REFRESH = 'true'
   }
 
-  if (process.env.PAYLOAD_PATCH_TURBOPACK_WARNINGS !== 'false') {
-    const turbopackWarningText =
-      'Packages that should be external need to be installed in the project directory, so they can be resolved from the output files.\nTry to install it into the project directory by running'
-
-    const consoleWarn = console.warn
-    console.warn = (...args) => {
-      // Force to disable serverExternalPackages warnings: https://github.com/vercel/next.js/issues/68805
-      if (
-        (typeof args[1] === 'string' && args[1].includes(turbopackWarningText)) ||
-        (typeof args[0] === 'string' && args[0].includes(turbopackWarningText))
-      ) {
-        return
-      }
-
-      consoleWarn(...args)
-    }
-  }
-
   const poweredByHeader = {
     key: 'X-Powered-By',
     value: 'Next.js, Payload',
@@ -88,25 +70,32 @@ export const withPayload = (nextConfig = {}, options = {}) => {
     },
     serverExternalPackages: [
       ...(nextConfig.serverExternalPackages || []),
-      'drizzle-kit',
-      'drizzle-kit/api',
-      'pino',
-      'libsql',
-      'pino-pretty',
-      'graphql',
-      // Do not bundle server-only packages during dev to improve compile speed
+      // These packages always need to be external, both during dev and production. This is because they install dependencies
+      // that will error when trying to bundle them (e.g. drizzle-kit, libsql, esbuild etc.).
+      // We cannot externalize those problem-packages directly. We can only externalize packages that are manually installed
+      // by the end user. Otherwise, the require('externalPackage') calls generated by the bundler would fail during runtime,
+      // as you cannot import dependencies of dependencies in a lot of package managers like pnpm. We'd have to force users
+      // to install the dependencies directly.
+      // Thus, we externalize the "entry-point" = the package that is installed by the end user, which would be our db adapters.
+      //
+      //
+      // External because it installs mongoose (part of default serverExternalPackages: https://github.com/vercel/next.js/blob/canary/packages/next/src/lib/server-external-packages.json => would throw warning if we don't exclude the entry-point package):
+      '@payloadcms/db-mongodb',
+      // External because they install dependencies like drizzle, libsql, esbuild etc.:
+      '@payloadcms/db-postgres',
+      '@payloadcms/db-sqlite',
+      '@payloadcms/db-vercel-postgres',
+      '@payloadcms/drizzle',
+      // External because they install @aws-sdk/client-s3:
+      '@payloadcms/payload-cloud',
+      // TODO: We need to externalize @payloadcms/storage-s3 as well, once Next.js has the ability to exclude @payloadcms/storage-s3/client from being externalized.
+      // Do not bundle additional server-only packages during dev to improve compilation speed
       ...(process.env.NODE_ENV === 'development' && options.devBundleServerPackages === false
         ? [
             'payload',
-            '@payloadcms/db-mongodb',
-            '@payloadcms/db-postgres',
-            '@payloadcms/db-sqlite',
-            '@payloadcms/db-vercel-postgres',
-            '@payloadcms/drizzle',
             '@payloadcms/email-nodemailer',
             '@payloadcms/email-resend',
             '@payloadcms/graphql',
-            '@payloadcms/payload-cloud',
             '@payloadcms/plugin-redirects',
             // TODO: Add the following packages, excluding their /client subpath exports, once Next.js supports it
             //'@payloadcms/plugin-cloud-storage',
@@ -129,45 +118,14 @@ export const withPayload = (nextConfig = {}, options = {}) => {
 
       return {
         ...incomingWebpackConfig,
-        externals: [
-          ...(incomingWebpackConfig?.externals || []),
-          'drizzle-kit',
-          'drizzle-kit/api',
-          'sharp',
-          'libsql',
-          'require-in-the-middle',
-        ],
-        ignoreWarnings: [
-          ...(incomingWebpackConfig?.ignoreWarnings || []),
-          { module: /node_modules\/mongodb\/lib\/utils\.js/ },
-          { file: /node_modules\/mongodb\/lib\/utils\.js/ },
-          { module: /node_modules\/mongodb\/lib\/bson\.js/ },
-          { file: /node_modules\/mongodb\/lib\/bson\.js/ },
-        ],
+        externals: [...(incomingWebpackConfig?.externals || []), 'require-in-the-middle'],
         plugins: [
           ...(incomingWebpackConfig?.plugins || []),
           // Fix cloudflare:sockets error: https://github.com/vercel/next.js/discussions/50177
           new webpackOptions.webpack.IgnorePlugin({
             resourceRegExp: /^pg-native$|^cloudflare:sockets$/,
           }),
         ],
-        resolve: {
-          ...(incomingWebpackConfig?.resolve || {}),
-          alias: {
-            ...(incomingWebpackConfig?.resolve?.alias || {}),
-          },
-          fallback: {
-            ...(incomingWebpackConfig?.resolve?.fallback || {}),
-            '@aws-sdk/credential-providers': false,
-            '@mongodb-js/zstd': false,
-            aws4: false,
-            kerberos: false,
-            'mongodb-client-encryption': false,
-            snappy: false,
-            'supports-color': false,
-            'yocto-queue': false,
-          },
-        },
       }
     },
   }

test/plugin-multi-tenant/e2e.spec.ts

@@ -577,17 +577,22 @@ test.describe('Multi Tenant', () => {
         page,
         serverURL,
       })
+      await wait(1000)
 
       await goToListDoc({
         cellClass: '.cell-name',
         page,
         textToMatch: 'Blue Dog',
         urlUtil: tenantsURL,
       })
+      await wait(1000)
 
       await expect(page.locator('#field-name')).toBeVisible()
       await page.locator('#field-name').fill('Red Dog')
+      await wait(1000)
+
       await saveDocAndAssert(page)
+      await wait(1000)
 
       await page.goto(tenantsURL.list)
       // Wait for backend tenant cache to update after save operation
@@ -599,17 +604,22 @@ test.describe('Multi Tenant', () => {
           return (await getTenantOptions({ page })).sort()
         })
         .toEqual(['Red Dog', 'Steel Cat', 'Public Tenant', 'Anchor Bar'].sort())
+      await wait(1000)
 
       await goToListDoc({
         cellClass: '.cell-name',
         page,
         textToMatch: 'Red Dog',
         urlUtil: tenantsURL,
       })
+      await wait(1000)
 
       // Change the tenant back to the original name
       await page.locator('#field-name').fill('Blue Dog')
+      await wait(1000)
+
       await saveDocAndAssert(page)
+      await wait(1000)
 
       await page.goto(tenantsURL.list)
       // Wait for backend tenant cache to update after save operation