From 1c914279762d3fdfedd3d00d4502f4ea6a418aeb Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 29 Aug 2023 16:35:23 -0700 Subject: [PATCH 01/54] Remove render workers in favor of esm loader --- packages/next/src/bin/next.ts | 71 ++++++--- packages/next/src/build/index.ts | 5 +- packages/next/src/cli/next-build-args.ts | 17 ++ packages/next/src/cli/next-build.ts | 22 +-- packages/next/src/cli/next-dev-args.ts | 25 +++ packages/next/src/cli/next-dev.ts | 147 +----------------- packages/next/src/cli/next-export-args.ts | 14 ++ packages/next/src/cli/next-export.ts | 17 +- packages/next/src/cli/next-info-args.ts | 13 ++ packages/next/src/cli/next-info.ts | 18 +-- packages/next/src/cli/next-lint-args.ts | 44 ++++++ packages/next/src/cli/next-lint.ts | 49 +----- packages/next/src/cli/next-start-args.ts | 15 ++ packages/next/src/cli/next-start.ts | 18 +-- packages/next/src/cli/next-telemetry-args.ts | 10 ++ packages/next/src/cli/next-telemetry.ts | 14 +- packages/next/src/lib/command-args.ts | 17 ++ packages/next/src/lib/commands.ts | 4 +- packages/next/src/lib/get-project-dir.ts | 5 +- packages/next/src/server/base-server.ts | 6 + packages/next/src/server/config-utils.ts | 2 +- packages/next/src/server/esm-loader.mts | 27 ++++ .../import-overrides.ts} | 56 ++----- packages/next/src/server/lib/mock-request.ts | 62 +++++++- packages/next/src/server/lib/render-server.ts | 34 ++-- .../next/src/server/lib/route-resolver.ts | 18 ++- packages/next/src/server/lib/router-server.ts | 116 +++----------- .../server/lib/router-utils/resolve-routes.ts | 67 ++++---- packages/next/src/server/lib/start-server.ts | 5 + packages/next/src/server/next-server.ts | 8 +- packages/next/src/server/next.ts | 32 ++++ packages/next/src/server/require-hook.js | 31 ++++ packages/next/src/shared/lib/constants.ts | 8 + packages/next/taskfile-swc.js | 5 +- packages/next/taskfile.js | 2 +- test/e2e/app-dir/app/index.test.ts | 4 +- 36 files changed, 515 insertions(+), 493 deletions(-) create mode 100755 packages/next/src/cli/next-build-args.ts create mode 100644 packages/next/src/cli/next-dev-args.ts create mode 100755 packages/next/src/cli/next-export-args.ts create mode 100755 packages/next/src/cli/next-info-args.ts create mode 100755 packages/next/src/cli/next-lint-args.ts create mode 100755 packages/next/src/cli/next-start-args.ts create mode 100755 packages/next/src/cli/next-telemetry-args.ts create mode 100644 packages/next/src/lib/command-args.ts create mode 100644 packages/next/src/server/esm-loader.mts rename packages/next/src/server/{require-hook.ts => lib/import-overrides.ts} (75%) create mode 100644 packages/next/src/server/require-hook.js diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 91b15669d179a..705d1ad287f5b 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -3,16 +3,15 @@ import * as log from '../build/output/log' import arg from 'next/dist/compiled/arg/index.js' import { NON_STANDARD_NODE_ENV } from '../lib/constants' import { commands } from '../lib/commands' -;['react', 'react-dom'].forEach((dependency) => { - try { - // When 'npm link' is used it checks the clone location. Not the project. - require.resolve(dependency) - } catch (err) { - console.warn( - `The module '${dependency}' was not found. Next.js requires that you include it in 'dependencies' of your 'package.json'. To add it, run 'npm install ${dependency}'` - ) - } -}) +import { commandArgs } from '../lib/command-args' +import loadConfig from '../server/config' +import { + PHASE_DEVELOPMENT_SERVER, + PHASE_PRODUCTION_BUILD, +} from '../shared/lib/constants' +import { getProjectDir } from '../lib/get-project-dir' +import { getValidatedArgs } from '../lib/get-validated-args' +import { findPagesDir } from '../lib/find-pages-dir' const defaultCommand = 'dev' const args = arg( @@ -122,13 +121,49 @@ if (!process.env.NEXT_MANUAL_SIG_HANDLE && command !== 'dev') { process.on('SIGTERM', () => process.exit(0)) process.on('SIGINT', () => process.exit(0)) } +async function main() { + const currentArgsSpec = commandArgs[command]() + const validatedArgs = getValidatedArgs(currentArgsSpec, forwardedArgs) + const dir = getProjectDir( + process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0] + ) + const config = await loadConfig( + command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_BUILD, + dir + ) + // TODO: set config to env variable to be re-used so we don't reload + // un-necessarily + const dirsResult = findPagesDir(dir, true) + + if (dirsResult.appDir) { + process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental + .serverActions + ? 'experimental' + : 'next' + } -commands[command]() - .then((exec) => exec(forwardedArgs)) - .then(() => { - if (command === 'build' || command === 'experimental-compile') { - // ensure process exits after build completes so open handles/connections - // don't cause process to hang - process.exit(0) + require('../server/require-hook') + + for (const dependency of ['react', 'react-dom']) { + try { + // When 'npm link' is used it checks the clone location. Not the project. + require.resolve(dependency) + } catch (err) { + console.warn( + `The module '${dependency}' was not found. Next.js requires that you include it in 'dependencies' of your 'package.json'. To add it, run 'npm install ${dependency}'` + ) } - }) + } + + await commands[command]() + .then((exec) => exec(validatedArgs)) + .then(() => { + if (command === 'build' || command === 'experimental-compile') { + // ensure process exits after build completes so open handles/connections + // don't cause process to hang + process.exit(0) + } + }) +} + +main() diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 3bc1c9eaef520..94f5ebc8178b0 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -144,7 +144,10 @@ import { createValidFileMatcher } from '../server/lib/find-page-file' import { startTypeChecking } from './type-check' import { generateInterceptionRoutesRewrites } from '../lib/generate-interception-routes-rewrites' import { buildDataRoute } from '../server/lib/router-utils/build-data-route' -import { baseOverrides, experimentalOverrides } from '../server/require-hook' +import { + baseOverrides, + experimentalOverrides, +} from '../server/lib/import-overrides' import { initialize } from '../server/lib/incremental-cache-server' import { nodeFs } from '../server/lib/node-fs-methods' diff --git a/packages/next/src/cli/next-build-args.ts b/packages/next/src/cli/next-build-args.ts new file mode 100755 index 0000000000000..814ac16df8396 --- /dev/null +++ b/packages/next/src/cli/next-build-args.ts @@ -0,0 +1,17 @@ +import arg from 'next/dist/compiled/arg/index.js' + +export const validArgs: arg.Spec = { + // Types + '--help': Boolean, + '--profile': Boolean, + '--debug': Boolean, + '--no-lint': Boolean, + '--no-mangling': Boolean, + '--experimental-app-only': Boolean, + '--experimental-turbo': Boolean, + '--experimental-turbo-root': String, + '--build-mode': String, + // Aliases + '-h': '--help', + '-d': '--debug', +} diff --git a/packages/next/src/cli/next-build.ts b/packages/next/src/cli/next-build.ts index c20762c828908..f26b31d5e2768 100755 --- a/packages/next/src/cli/next-build.ts +++ b/packages/next/src/cli/next-build.ts @@ -1,33 +1,13 @@ #!/usr/bin/env node import { existsSync } from 'fs' -import arg from 'next/dist/compiled/arg/index.js' import * as Log from '../build/output/log' import { CliCommand } from '../lib/commands' import build from '../build' import { printAndExit } from '../server/lib/utils' import isError from '../lib/is-error' import { getProjectDir } from '../lib/get-project-dir' -import { getValidatedArgs } from '../lib/get-validated-args' - -const nextBuild: CliCommand = (argv) => { - const validArgs: arg.Spec = { - // Types - '--help': Boolean, - '--profile': Boolean, - '--debug': Boolean, - '--no-lint': Boolean, - '--no-mangling': Boolean, - '--experimental-app-only': Boolean, - '--experimental-turbo': Boolean, - '--experimental-turbo-root': String, - '--build-mode': String, - // Aliases - '-h': '--help', - '-d': '--debug', - } - - const args = getValidatedArgs(validArgs, argv) +const nextBuild: CliCommand = (args) => { if (args['--help']) { printAndExit( ` diff --git a/packages/next/src/cli/next-dev-args.ts b/packages/next/src/cli/next-dev-args.ts new file mode 100644 index 0000000000000..3ab378660c386 --- /dev/null +++ b/packages/next/src/cli/next-dev-args.ts @@ -0,0 +1,25 @@ +import arg from 'next/dist/compiled/arg/index.js' + +export const validArgs: arg.Spec = { + // Types + '--help': Boolean, + '--port': Number, + '--hostname': String, + '--turbo': Boolean, + '--experimental-turbo': Boolean, + '--experimental-https': Boolean, + '--experimental-https-key': String, + '--experimental-https-cert': String, + '--experimental-test-proxy': Boolean, + '--experimental-upload-trace': String, + + // To align current messages with native binary. + // Will need to adjust subcommand later. + '--show-all': Boolean, + '--root': String, + + // Aliases + '-h': '--help', + '-p': '--port', + '-H': '--hostname', +} diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index 969712a27492d..7badb769eecfe 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -1,10 +1,5 @@ #!/usr/bin/env node -import arg from 'next/dist/compiled/arg/index.js' import type { StartServerOptions } from '../server/lib/start-server' -import { - genRouterWorkerExecArgv, - getNodeOptionsWithoutInspect, -} from '../server/lib/utils' import { getPort, printAndExit } from '../server/lib/utils' import * as Log from '../build/output/log' import { CliCommand } from '../lib/commands' @@ -20,13 +15,10 @@ import { findRootDir } from '../lib/find-root' import { fileExists, FileType } from '../lib/file-exists' import { getNpxCommand } from '../lib/helpers/get-npx-command' import Watchpack from 'watchpack' -import { resetEnv, initialEnv } from '@next/env' -import { getValidatedArgs } from '../lib/get-validated-args' -import { Worker } from 'next/dist/compiled/jest-worker' -import type { ChildProcess } from 'child_process' -import { checkIsNodeDebugging } from '../server/lib/is-node-debugging' +import { resetEnv } from '@next/env' import { createSelfSignedCertificate } from '../lib/mkcert' import uploadTrace from '../trace/upload-trace' +import { startServer } from '../server/lib/start-server' let dir: string let config: NextConfigComplete @@ -124,116 +116,7 @@ function watchConfigFiles( wp.on('change', onChange) } -type StartServerWorker = Worker & - Pick - -async function createRouterWorker(fullConfig: NextConfigComplete): Promise<{ - worker: StartServerWorker - cleanup: () => Promise -}> { - const isNodeDebugging = checkIsNodeDebugging() - const worker = new Worker(require.resolve('../server/lib/start-server'), { - numWorkers: 1, - // TODO: do we want to allow more than 8 OOM restarts? - maxRetries: 8, - forkOptions: { - execArgv: await genRouterWorkerExecArgv( - isNodeDebugging === undefined ? false : isNodeDebugging - ), - env: { - FORCE_COLOR: '1', - ...(initialEnv as any), - NODE_OPTIONS: getNodeOptionsWithoutInspect(), - ...(process.env.NEXT_CPU_PROF - ? { __NEXT_PRIVATE_CPU_PROFILE: `CPU.router` } - : {}), - WATCHPACK_WATCHER_LIMIT: '20', - EXPERIMENTAL_TURBOPACK: process.env.EXPERIMENTAL_TURBOPACK, - __NEXT_PRIVATE_PREBUNDLED_REACT: !!fullConfig.experimental.serverActions - ? 'experimental' - : 'next', - }, - }, - exposedMethods: ['startServer'], - }) as Worker & - Pick - - const cleanup = () => { - for (const curWorker of ((worker as any)._workerPool?._workers || []) as { - _child?: ChildProcess - }[]) { - curWorker._child?.kill('SIGINT') - } - process.exit(0) - } - - // If the child routing worker exits we need to exit the entire process - for (const curWorker of ((worker as any)._workerPool?._workers || []) as { - _child?: ChildProcess - }[]) { - curWorker._child?.on('exit', cleanup) - } - - process.on('exit', cleanup) - process.on('SIGINT', cleanup) - process.on('SIGTERM', cleanup) - process.on('uncaughtException', cleanup) - process.on('unhandledRejection', cleanup) - - const workerStdout = worker.getStdout() - const workerStderr = worker.getStderr() - - workerStdout.on('data', (data) => { - process.stdout.write(data) - }) - workerStderr.on('data', (data) => { - process.stderr.write(data) - }) - - return { - worker, - cleanup: async () => { - // Remove process listeners for childprocess too. - for (const curWorker of ((worker as any)._workerPool?._workers || []) as { - _child?: ChildProcess - }[]) { - curWorker._child?.off('exit', cleanup) - } - process.off('exit', cleanup) - process.off('SIGINT', cleanup) - process.off('SIGTERM', cleanup) - process.off('uncaughtException', cleanup) - process.off('unhandledRejection', cleanup) - await worker.end() - }, - } -} - -const nextDev: CliCommand = async (argv) => { - const validArgs: arg.Spec = { - // Types - '--help': Boolean, - '--port': Number, - '--hostname': String, - '--turbo': Boolean, - '--experimental-turbo': Boolean, - '--experimental-https': Boolean, - '--experimental-https-key': String, - '--experimental-https-cert': String, - '--experimental-test-proxy': Boolean, - '--experimental-upload-trace': String, - - // To align current messages with native binary. - // Will need to adjust subcommand later. - '--show-all': Boolean, - '--root': String, - - // Aliases - '-h': '--help', - '-p': '--port', - '-H': '--hostname', - } - const args = getValidatedArgs(validArgs, argv) +const nextDev: CliCommand = async (args) => { if (args['--help']) { console.log(` Description @@ -308,6 +191,7 @@ const nextDev: CliCommand = async (argv) => { // some set-ups that rely on listening on other interfaces const host = args['--hostname'] config = await loadConfig(PHASE_DEVELOPMENT_SERVER, dir) + const isExperimentalTestProxy = args['--experimental-test-proxy'] if (args['--experimental-upload-trace']) { @@ -404,7 +288,6 @@ const nextDev: CliCommand = async (argv) => { } else { const runDevServer = async (reboot: boolean) => { try { - const workerInit = await createRouterWorker(config) if (!!args['--experimental-https']) { Log.warn( 'Self-signed certificates are currently an experimental feature, use at your own risk.' @@ -424,26 +307,22 @@ const nextDev: CliCommand = async (argv) => { certificate = await createSelfSignedCertificate(host) } - await workerInit.worker.startServer({ + await startServer({ ...devServerOptions, selfSignedCertificate: certificate, }) } else { - await workerInit.worker.startServer(devServerOptions) + await startServer(devServerOptions) } await preflight(reboot) - return { - cleanup: workerInit.cleanup, - } } catch (err) { console.error(err) process.exit(1) } } - let runningServer: Awaited> | undefined - + // TODO: move config file watching to inner process watchConfigFiles(devServerOptions.dir, async (filename) => { if (process.env.__NEXT_DISABLE_MEMORY_WATCHER) { Log.info( @@ -456,19 +335,9 @@ const nextDev: CliCommand = async (argv) => { filename )}. Restarting the server to apply the changes...` ) - - try { - if (runningServer) { - await runningServer.cleanup() - } - runningServer = await runDevServer(true) - } catch (err) { - console.error(err) - process.exit(1) - } }) - runningServer = await runDevServer(false) + await runDevServer(false) } } diff --git a/packages/next/src/cli/next-export-args.ts b/packages/next/src/cli/next-export-args.ts new file mode 100755 index 0000000000000..e6c66da77dd30 --- /dev/null +++ b/packages/next/src/cli/next-export-args.ts @@ -0,0 +1,14 @@ +import arg from 'next/dist/compiled/arg/index.js' + +export const validArgs: arg.Spec = { + // Types + '--help': Boolean, + '--silent': Boolean, + '--outdir': String, + '--threads': Number, + + // Aliases + '-h': '--help', + '-o': '--outdir', + '-s': '--silent', +} diff --git a/packages/next/src/cli/next-export.ts b/packages/next/src/cli/next-export.ts index b3fe1d21b1dc7..bf94fdf3f2e2b 100755 --- a/packages/next/src/cli/next-export.ts +++ b/packages/next/src/cli/next-export.ts @@ -1,7 +1,6 @@ #!/usr/bin/env node import { resolve, join } from 'path' import { existsSync } from 'fs' -import arg from 'next/dist/compiled/arg/index.js' import chalk from 'next/dist/compiled/chalk' import exportApp, { ExportError, ExportOptions } from '../export' import * as Log from '../build/output/log' @@ -9,23 +8,9 @@ import { printAndExit } from '../server/lib/utils' import { CliCommand } from '../lib/commands' import { trace } from '../trace' import { getProjectDir } from '../lib/get-project-dir' -import { getValidatedArgs } from '../lib/get-validated-args' -const nextExport: CliCommand = (argv) => { +const nextExport: CliCommand = (args) => { const nextExportCliSpan = trace('next-export-cli') - const validArgs: arg.Spec = { - // Types - '--help': Boolean, - '--silent': Boolean, - '--outdir': String, - '--threads': Number, - - // Aliases - '-h': '--help', - '-o': '--outdir', - '-s': '--silent', - } - const args = getValidatedArgs(validArgs, argv) if (args['--help']) { console.log(` Description diff --git a/packages/next/src/cli/next-info-args.ts b/packages/next/src/cli/next-info-args.ts new file mode 100755 index 0000000000000..dafea92660d84 --- /dev/null +++ b/packages/next/src/cli/next-info-args.ts @@ -0,0 +1,13 @@ +import arg from 'next/dist/compiled/arg/index.js' + +/** + * Supported CLI arguments. + */ +export const validArgs: arg.Spec = { + // Types + '--help': Boolean, + // Aliases + '-h': '--help', + // Detailed diagnostics + '--verbose': Boolean, +} diff --git a/packages/next/src/cli/next-info.ts b/packages/next/src/cli/next-info.ts index 54bf0b58fd52a..b8095852c9484 100755 --- a/packages/next/src/cli/next-info.ts +++ b/packages/next/src/cli/next-info.ts @@ -4,14 +4,12 @@ import os from 'os' import childProcess from 'child_process' import chalk from 'next/dist/compiled/chalk' -import arg from 'next/dist/compiled/arg/index.js' const { fetch } = require('next/dist/compiled/undici') as { fetch: typeof global.fetch } import { CliCommand } from '../lib/commands' import { PHASE_INFO } from '../shared/lib/constants' import loadConfig from '../server/config' -import { getValidatedArgs } from '../lib/get-validated-args' const dir = process.cwd() @@ -51,18 +49,6 @@ type PlatformTaskScript = darwin?: TaskScript } -/** - * Supported CLI arguments. - */ -const validArgs: arg.Spec = { - // Types - '--help': Boolean, - // Aliases - '-h': '--help', - // Detailed diagnostics - '--verbose': Boolean, -} - function getPackageVersion(packageName: string) { try { return require(`${packageName}/package.json`).version @@ -590,9 +576,7 @@ async function printVerbose() { * There are 2 modes, by default it collects basic next.js installation with runtime information. If * `--verbose` mode is enabled it'll try to collect, verify more data for next-swc installation and others. */ -const nextInfo: CliCommand = async (argv) => { - const args = getValidatedArgs(validArgs, argv) - +const nextInfo: CliCommand = async (args) => { if (args['--help']) { printHelp() return diff --git a/packages/next/src/cli/next-lint-args.ts b/packages/next/src/cli/next-lint-args.ts new file mode 100755 index 0000000000000..854af64be303f --- /dev/null +++ b/packages/next/src/cli/next-lint-args.ts @@ -0,0 +1,44 @@ +import arg from 'next/dist/compiled/arg/index.js' + +const validEslintArgs: arg.Spec = { + // Types + '--config': String, + '--ext': [String], + '--resolve-plugins-relative-to': String, + '--rulesdir': [String], + '--fix': Boolean, + '--fix-type': [String], + '--ignore-path': String, + '--no-ignore': Boolean, + '--quiet': Boolean, + '--max-warnings': Number, + '--no-inline-config': Boolean, + '--report-unused-disable-directives': String, + '--cache': Boolean, // Although cache is enabled by default, this dummy flag still exists to not cause any breaking changes + '--no-cache': Boolean, + '--cache-location': String, + '--cache-strategy': String, + '--error-on-unmatched-pattern': Boolean, + '--format': String, + '--output-file': String, + + // Aliases + '-c': '--config', + '-f': '--format', + '-o': '--output-file', +} + +export const validArgs: arg.Spec = { + // Types + '--help': Boolean, + '--base-dir': String, + '--dir': [String], + '--file': [String], + '--strict': Boolean, + + // Aliases + '-h': '--help', + '-b': '--base-dir', + '-d': '--dir', + ...validEslintArgs, +} diff --git a/packages/next/src/cli/next-lint.ts b/packages/next/src/cli/next-lint.ts index 3370cfc53397e..b1e00c5869ea6 100755 --- a/packages/next/src/cli/next-lint.ts +++ b/packages/next/src/cli/next-lint.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node +import type arg from 'next/dist/compiled/arg/index.js' import { existsSync } from 'fs' -import arg from 'next/dist/compiled/arg/index.js' import { join } from 'path' import chalk from 'next/dist/compiled/chalk' @@ -16,7 +16,6 @@ import { CompileError } from '../lib/compile-error' import { getProjectDir } from '../lib/get-project-dir' import { findPagesDir } from '../lib/find-pages-dir' import { verifyTypeScriptSetup } from '../lib/verifyTypeScriptSetup' -import { getValidatedArgs } from '../lib/get-validated-args' const eslintOptions = (args: arg.Spec, defaultCacheLocation: string) => ({ overrideConfigFile: args['--config'] || null, @@ -47,51 +46,7 @@ const eslintOptions = (args: arg.Spec, defaultCacheLocation: string) => ({ : false, }) -const nextLint: CliCommand = async (argv) => { - const validArgs: arg.Spec = { - // Types - '--help': Boolean, - '--base-dir': String, - '--dir': [String], - '--file': [String], - '--strict': Boolean, - - // Aliases - '-h': '--help', - '-b': '--base-dir', - '-d': '--dir', - } - - const validEslintArgs: arg.Spec = { - // Types - '--config': String, - '--ext': [String], - '--resolve-plugins-relative-to': String, - '--rulesdir': [String], - '--fix': Boolean, - '--fix-type': [String], - '--ignore-path': String, - '--no-ignore': Boolean, - '--quiet': Boolean, - '--max-warnings': Number, - '--no-inline-config': Boolean, - '--report-unused-disable-directives': String, - '--cache': Boolean, // Although cache is enabled by default, this dummy flag still exists to not cause any breaking changes - '--no-cache': Boolean, - '--cache-location': String, - '--cache-strategy': String, - '--error-on-unmatched-pattern': Boolean, - '--format': String, - '--output-file': String, - - // Aliases - '-c': '--config', - '-f': '--format', - '-o': '--output-file', - } - - const args = getValidatedArgs({ ...validArgs, ...validEslintArgs }, argv) - +const nextLint: CliCommand = async (args) => { if (args['--help']) { printAndExit( ` diff --git a/packages/next/src/cli/next-start-args.ts b/packages/next/src/cli/next-start-args.ts new file mode 100755 index 0000000000000..1c95bafd0bc71 --- /dev/null +++ b/packages/next/src/cli/next-start-args.ts @@ -0,0 +1,15 @@ +import arg from 'next/dist/compiled/arg/index.js' + +export const validArgs: arg.Spec = { + // Types + '--help': Boolean, + '--port': Number, + '--hostname': String, + '--keepAliveTimeout': Number, + '--experimental-test-proxy': Boolean, + + // Aliases + '-h': '--help', + '-p': '--port', + '-H': '--hostname', +} diff --git a/packages/next/src/cli/next-start.ts b/packages/next/src/cli/next-start.ts index ef54c2723de47..6af517729492d 100755 --- a/packages/next/src/cli/next-start.ts +++ b/packages/next/src/cli/next-start.ts @@ -1,27 +1,11 @@ #!/usr/bin/env node -import arg from 'next/dist/compiled/arg/index.js' import { startServer } from '../server/lib/start-server' import { getPort, printAndExit } from '../server/lib/utils' import { getProjectDir } from '../lib/get-project-dir' import { CliCommand } from '../lib/commands' -import { getValidatedArgs } from '../lib/get-validated-args' -const nextStart: CliCommand = async (argv) => { - const validArgs: arg.Spec = { - // Types - '--help': Boolean, - '--port': Number, - '--hostname': String, - '--keepAliveTimeout': Number, - '--experimental-test-proxy': Boolean, - - // Aliases - '-h': '--help', - '-p': '--port', - '-H': '--hostname', - } - const args = getValidatedArgs(validArgs, argv) +const nextStart: CliCommand = async (args) => { if (args['--help']) { console.log(` Description diff --git a/packages/next/src/cli/next-telemetry-args.ts b/packages/next/src/cli/next-telemetry-args.ts new file mode 100755 index 0000000000000..91b07af36d4fe --- /dev/null +++ b/packages/next/src/cli/next-telemetry-args.ts @@ -0,0 +1,10 @@ +import arg from 'next/dist/compiled/arg/index.js' + +export const validArgs: arg.Spec = { + // Types + '--enable': Boolean, + '--disable': Boolean, + '--help': Boolean, + // Aliases + '-h': '--help', +} diff --git a/packages/next/src/cli/next-telemetry.ts b/packages/next/src/cli/next-telemetry.ts index 7b6f458b17b55..e8fae460cf1ea 100755 --- a/packages/next/src/cli/next-telemetry.ts +++ b/packages/next/src/cli/next-telemetry.ts @@ -1,21 +1,9 @@ #!/usr/bin/env node import chalk from 'next/dist/compiled/chalk' -import arg from 'next/dist/compiled/arg/index.js' import { CliCommand } from '../lib/commands' import { Telemetry } from '../telemetry/storage' -import { getValidatedArgs } from '../lib/get-validated-args' - -const nextTelemetry: CliCommand = (argv) => { - const validArgs: arg.Spec = { - // Types - '--enable': Boolean, - '--disable': Boolean, - '--help': Boolean, - // Aliases - '-h': '--help', - } - const args = getValidatedArgs(validArgs, argv) +const nextTelemetry: CliCommand = (args) => { if (args['--help']) { console.log( ` diff --git a/packages/next/src/lib/command-args.ts b/packages/next/src/lib/command-args.ts new file mode 100644 index 0000000000000..00d1e4164e020 --- /dev/null +++ b/packages/next/src/lib/command-args.ts @@ -0,0 +1,17 @@ +import { getValidatedArgs } from './get-validated-args' + +export type CliCommand = (args: ReturnType) => void + +export const commandArgs: { + [command: string]: () => Parameters[0] +} = { + build: () => require('../cli/next-build-args').validArgs, + start: () => require('../cli/next-start-args').validArgs, + export: () => require('../cli/next-export-args').validArgs, + dev: () => require('../cli/next-dev-args').validArgs, + lint: () => require('../cli/next-lint-args').validArgs, + telemetry: () => require('../cli/next-telemetry-args').validArgs, + info: () => require('../cli/next-info-args').validArgs, + 'experimental-compile': () => require('../cli/next-build-args').validArgs, + 'experimental-generate': () => require('../cli/next-build-args').validArgs, +} diff --git a/packages/next/src/lib/commands.ts b/packages/next/src/lib/commands.ts index fdc6c0ff51976..d921a8583cb56 100644 --- a/packages/next/src/lib/commands.ts +++ b/packages/next/src/lib/commands.ts @@ -1,4 +1,6 @@ -export type CliCommand = (argv?: string[]) => void +import { getValidatedArgs } from './get-validated-args' + +export type CliCommand = (args: ReturnType) => void export const commands: { [command: string]: () => Promise } = { build: () => Promise.resolve(require('../cli/next-build').nextBuild), diff --git a/packages/next/src/lib/get-project-dir.ts b/packages/next/src/lib/get-project-dir.ts index c8530357c8823..ce47c791070a2 100644 --- a/packages/next/src/lib/get-project-dir.ts +++ b/packages/next/src/lib/get-project-dir.ts @@ -35,7 +35,10 @@ export function getProjectDir(dir?: string) { Log.error( `Invalid project directory provided, no such directory: ${path.resolve( dir || '.' - )}` + )}`, + new Error().stack, + process.env.NEXT_PRIVATE_WORKER, + process.argv ) process.exit(1) } diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 2229e95293b94..43c56d84557e9 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -44,6 +44,7 @@ import { getRedirectStatus } from '../lib/redirect-status' import { isEdgeRuntime } from '../lib/is-edge-runtime' import { APP_PATHS_MANIFEST, + INTERNAL_HEADERS, NEXT_BUILTIN_DOCUMENT, PAGES_MANIFEST, STATIC_STATUS_PAGES, @@ -1543,6 +1544,11 @@ export default abstract class Server { // For edge runtime 404 page, /_not-found needs to be treated as 404 page (process.env.NEXT_RUNTIME === 'edge' && pathname === '/_not-found') || pathname === '/404' + + for (const key of INTERNAL_HEADERS) { + delete req.headers[key] + } + const is500Page = pathname === '/500' const isAppPath = components.isAppPath const hasServerProps = !!components.getServerSideProps diff --git a/packages/next/src/server/config-utils.ts b/packages/next/src/server/config-utils.ts index 9270cf3c46f22..7014f439df03c 100644 --- a/packages/next/src/server/config-utils.ts +++ b/packages/next/src/server/config-utils.ts @@ -10,7 +10,7 @@ export function loadWebpackHook() { // hook the Node.js require so that webpack requires are // routed to the bundled and now initialized webpack version - require('../server/require-hook').addHookAliases( + require('../server/lib/import-overrides').addHookAliases( [ ['webpack', 'next/dist/compiled/webpack/webpack-lib'], ['webpack/package', 'next/dist/compiled/webpack/package'], diff --git a/packages/next/src/server/esm-loader.mts b/packages/next/src/server/esm-loader.mts new file mode 100644 index 0000000000000..f7c7b90582b14 --- /dev/null +++ b/packages/next/src/server/esm-loader.mts @@ -0,0 +1,27 @@ +import module from 'module' + +const require = module.createRequire(import.meta.url) +const { + overrideReact, + hookPropertyMap, +} = require('next/dist/server/lib/import-overrides') + +export function resolve(specifier: string, context: any, nextResolve: any) { + // In case the environment variable is set after the module is loaded. + overrideReact() + + const hookResolved = hookPropertyMap.get(specifier) + if (hookResolved) { + specifier = hookResolved + } + + if (specifier.endsWith('next/dist/bin/next')) { + return { + url: specifier, + shortCircuit: true, + format: 'commonjs', + } + } + + return nextResolve(specifier, context) +} diff --git a/packages/next/src/server/require-hook.ts b/packages/next/src/server/lib/import-overrides.ts similarity index 75% rename from packages/next/src/server/require-hook.ts rename to packages/next/src/server/lib/import-overrides.ts index 1f53b3b479109..5d786fa8f36e0 100644 --- a/packages/next/src/server/require-hook.ts +++ b/packages/next/src/server/lib/import-overrides.ts @@ -1,23 +1,6 @@ -// Synchronously inject a require hook for webpack and webpack/. It's required to use the internal ncc webpack version. -// This is needed for userland plugins to attach to the same webpack instance as Next.js'. -// Individually compiled modules are as defined for the compilation in bundles/webpack/packages/*. +import { dirname } from 'path' -// This module will only be loaded once per process. - -const { dirname } = require('path') -const mod = require('module') -const resolveFilename = mod._resolveFilename -const hookPropertyMap = new Map() - -let aliasedPrebundledReact = false - -const resolve = process.env.NEXT_MINIMAL - ? // @ts-ignore - __non_webpack_require__.resolve - : require.resolve - -const toResolveMap = (map: Record): [string, string][] => - Object.entries(map).map(([key, value]) => [key, resolve(value)]) +export const hookPropertyMap = new Map() // these must use require.resolve to be statically analyzable export const defaultOverrides = { @@ -72,6 +55,16 @@ export const experimentalOverrides = { 'next/dist/compiled/react-server-dom-webpack-experimental/server.node', } +let aliasedPrebundledReact = false + +const resolve = process.env.NEXT_MINIMAL + ? // @ts-ignore + __non_webpack_require__.resolve + : require.resolve + +const toResolveMap = (map: Record): [string, string][] => + Object.entries(map).map(([key, value]) => [key, resolve(value)]) + export function addHookAliases(aliases: [string, string][] = []) { for (const [key, value] of aliases) { hookPropertyMap.set(key, value) @@ -82,9 +75,10 @@ export function addHookAliases(aliases: [string, string][] = []) { addHookAliases(toResolveMap(defaultOverrides)) // Override built-in React packages if necessary -function overrideReact() { - if (process.env.__NEXT_PRIVATE_PREBUNDLED_REACT) { +export function overrideReact() { + if (process.env.__NEXT_PRIVATE_PREBUNDLED_REACT && !aliasedPrebundledReact) { aliasedPrebundledReact = true + console.log('aliased react to', process.env.__NEXT_PRIVATE_PREBUNDLED_REACT) // Require these modules with static paths to make sure they are tracked by // NFT when building the app in standalone mode, as we are now conditionally @@ -97,23 +91,3 @@ function overrideReact() { } } overrideReact() - -mod._resolveFilename = function ( - originalResolveFilename: typeof resolveFilename, - requestMap: Map, - request: string, - parent: any, - isMain: boolean, - options: any -) { - if (process.env.__NEXT_PRIVATE_PREBUNDLED_REACT && !aliasedPrebundledReact) { - // In case the environment variable is set after the module is loaded. - overrideReact() - } - - const hookResolved = requestMap.get(request) - if (hookResolved) request = hookResolved - return originalResolveFilename.call(mod, request, parent, isMain, options) - - // We use `bind` here to avoid referencing outside variables to create potential memory leaks. -}.bind(null, resolveFilename, hookPropertyMap) diff --git a/packages/next/src/server/lib/mock-request.ts b/packages/next/src/server/lib/mock-request.ts index 2038751122596..563583db9bf0c 100644 --- a/packages/next/src/server/lib/mock-request.ts +++ b/packages/next/src/server/lib/mock-request.ts @@ -18,6 +18,7 @@ interface MockedRequestOptions { url: string headers: IncomingHttpHeaders method: string + readable?: Stream.Readable socket?: Socket | null } @@ -33,6 +34,8 @@ export class MockedRequest extends Stream.Readable implements IncomingMessage { public readonly httpVersionMajor = 1 public readonly httpVersionMinor = 0 + private bodyReadable?: Stream.Readable + // If we don't actually have a socket, we'll just use a mock one that // always returns false for the `encrypted` property. public socket: Socket = new Proxy({} as TLSSocket, { @@ -47,13 +50,25 @@ export class MockedRequest extends Stream.Readable implements IncomingMessage { }, }) - constructor({ url, headers, method, socket = null }: MockedRequestOptions) { + constructor({ + url, + headers, + method, + socket = null, + readable, + }: MockedRequestOptions) { super() this.url = url this.headers = headers this.method = method + if (readable) { + this.bodyReadable = readable + this.bodyReadable.on('end', () => this.emit('end')) + this.bodyReadable.on('close', () => this.emit('close')) + } + if (socket) { this.socket = socket } @@ -70,9 +85,13 @@ export class MockedRequest extends Stream.Readable implements IncomingMessage { return headers } - public _read(): void { - this.emit('end') - this.emit('close') + public _read(size: number): void { + if (this.bodyReadable) { + return this.bodyReadable._read(size) + } else { + this.emit('end') + this.emit('close') + } } /** @@ -120,6 +139,7 @@ export interface MockedResponseOptions { statusCode?: number socket?: Socket | null headers?: OutgoingHttpHeaders + resWriter?: (chunk: Buffer | string) => boolean } export class MockedResponse extends Stream.Writable implements ServerResponse { @@ -151,6 +171,11 @@ export class MockedResponse extends Stream.Writable implements ServerResponse { */ public readonly headers: Headers + private resWriter: MockedResponseOptions['resWriter'] + + public readonly headPromise: Promise + private headPromiseResolve?: () => void + constructor(res: MockedResponseOptions = {}) { super() @@ -166,7 +191,18 @@ export class MockedResponse extends Stream.Writable implements ServerResponse { this.on('finish', () => resolve(true)) this.on('end', () => resolve(true)) this.on('error', (err) => reject(err)) + }).then((val) => { + this.headPromiseResolve?.() + return val + }) + + this.headPromise = new Promise((resolve) => { + this.headPromiseResolve = resolve }) + + if (res.resWriter) { + this.resWriter = res.resWriter + } } public appendHeader(name: string, value: string | string[]): this { @@ -197,6 +233,9 @@ export class MockedResponse extends Stream.Writable implements ServerResponse { } public write(chunk: Buffer | string) { + if (this.resWriter) { + return this.resWriter(chunk) + } this.buffers.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) return true @@ -285,6 +324,7 @@ export class MockedResponse extends Stream.Writable implements ServerResponse { this.statusCode = statusCode this.headersSent = true + this.headPromiseResolve?.() return this } @@ -395,6 +435,8 @@ interface RequestResponseMockerOptions { url: string headers?: IncomingHttpHeaders method?: string + bodyReadable?: Stream.Readable + resWriter?: (chunk: Buffer | string) => boolean socket?: Socket | null } @@ -402,10 +444,18 @@ export function createRequestResponseMocks({ url, headers = {}, method = 'GET', + bodyReadable, + resWriter, socket = null, }: RequestResponseMockerOptions) { return { - req: new MockedRequest({ url, headers, method, socket }), - res: new MockedResponse({ socket }), + req: new MockedRequest({ + url, + headers, + method, + socket, + readable: bodyReadable, + }), + res: new MockedResponse({ socket, resWriter }), } } diff --git a/packages/next/src/server/lib/render-server.ts b/packages/next/src/server/lib/render-server.ts index 9c2eee1fd2086..2512353f1fe92 100644 --- a/packages/next/src/server/lib/render-server.ts +++ b/packages/next/src/server/lib/render-server.ts @@ -1,8 +1,5 @@ -import type { RequestHandler } from '../next' +import type { NextServer, RequestHandler } from '../next' -// this must come first as it includes require hooks -import { initializeServerWorker } from './setup-server-worker' -import { formatHostname } from './format-hostname' import next from '../next' import { PropagateToWorkersField } from './router-utils/types' @@ -11,8 +8,12 @@ export const WORKER_SELF_EXIT_CODE = 77 let result: | undefined | { - port: number - hostname: string + requestHandler: ReturnType< + InstanceType['getRequestHandler'] + > + upgradeHandler: ReturnType< + InstanceType['getUpgradeHandler'] + > } let app: ReturnType | undefined @@ -72,8 +73,9 @@ export async function initialize(opts: { isNodeDebugging: boolean keepAliveTimeout?: number serverFields?: any + server?: any experimentalTestProxy: boolean -}): Promise> { +}) { // if we already setup the server return as we only need to do // this on first worker boot if (result) { @@ -88,23 +90,13 @@ export async function initialize(opts: { let requestHandler: RequestHandler let upgradeHandler: any - const { port, server, hostname } = await initializeServerWorker( - (...args) => { - return requestHandler(...args) - }, - (...args) => { - return upgradeHandler(...args) - }, - opts - ) - app = next({ ...opts, _routerWorker: opts.workerType === 'router', _renderWorker: opts.workerType === 'render', - hostname, + hostname: opts.hostname || 'localhost', customServer: false, - httpServer: server, + httpServer: opts.server, port: opts.port, isNodeDebugging: opts.isNodeDebugging, }) @@ -114,8 +106,8 @@ export async function initialize(opts: { await app.prepare(opts.serverFields) result = { - port, - hostname: formatHostname(hostname), + requestHandler, + upgradeHandler, } return result diff --git a/packages/next/src/server/lib/route-resolver.ts b/packages/next/src/server/lib/route-resolver.ts index 83e122f2cc70a..332d0138a916e 100644 --- a/packages/next/src/server/lib/route-resolver.ts +++ b/packages/next/src/server/lib/route-resolver.ts @@ -22,6 +22,7 @@ import { signalFromNodeResponse } from '../web/spec-extension/adapters/next-requ import { getMiddlewareRouteMatcher } from '../../shared/lib/router/utils/middleware-route-matcher' import type { RenderWorker } from './router-server' import { pipeReadable } from '../pipe-readable' +import { Stream } from 'stream' type RouteResult = | { @@ -210,8 +211,21 @@ export async function makeResolver( pages: { async initialize() { return { - port: middlewareServerAddr.port, - hostname: formatHostname(middlewareServerAddr.hostname), + async requestHandler(req, res) { + fetch( + `http://localhost:${middlewareServerAddr.port}${req.url || ''}` + ).then((fetchRes) => { + for (const [key, value] of fetchRes.headers) { + res.setHeader(key, value) + } + if (fetchRes.body) { + Stream.Readable.fromWeb(fetchRes.body as any).pipe(res) + } else { + res.end() + } + }) + }, + async upgradeHandler() {}, } }, async deleteCache() {}, diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 6ba6beea2a3a0..c94eb4268dcc0 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -11,14 +11,11 @@ import path from 'path' import loadConfig from '../config' import { serveStatic } from '../serve-static' import setupDebug from 'next/dist/compiled/debug' -import { splitCookiesString, toNodeOutgoingHttpHeaders } from '../web/utils' import { Telemetry } from '../../telemetry/storage' import { DecodeError } from '../../shared/lib/utils' -import { filterReqHeaders, ipcForbiddenHeaders } from './server-ipc/utils' import { findPagesDir } from '../../lib/find-pages-dir' import { setupFsCheck } from './router-utils/filesystem' import { proxyRequest } from './router-utils/proxy-request' -import { invokeRequest } from './server-ipc/invoke-request' import { isAbortError, pipeReadable } from '../pipe-readable' import { createRequestResponseMocks } from './mock-request' import { createIpcServer, createWorker } from './server-ipc' @@ -28,13 +25,13 @@ import { NextUrlWithParsedQuery, getRequestMeta } from '../request-meta' import { pathHasPrefix } from '../../shared/lib/router/utils/path-has-prefix' import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-prefix' import setupCompression from 'next/dist/compiled/compression' +import { signalFromNodeResponse } from '../web/spec-extension/adapters/next-request' import { PHASE_PRODUCTION_SERVER, PHASE_DEVELOPMENT_SERVER, PERMANENT_REDIRECT_STATUS, } from '../../shared/lib/constants' -import { signalFromNodeResponse } from '../web/spec-extension/adapters/next-request' const debug = setupDebug('next:router-server:main') @@ -56,6 +53,7 @@ export async function initialize(opts: { dir: string port: number dev: boolean + server?: import('http').Server minimalMode?: boolean hostname?: string workerType: 'router' | 'render' @@ -186,23 +184,11 @@ export async function initialize(opts: { // Set global environment variables for the app render server to use. process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT = ipcPort + '' process.env.__NEXT_PRIVATE_ROUTER_IPC_KEY = ipcValidationKey - process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental - .serverActions - ? 'experimental' - : 'next' renderWorkers.app = require('./render-server') as typeof import('./render-server') - const { initialEnv } = require('@next/env') as typeof import('@next/env') - renderWorkers.pages = await createWorker( - ipcPort, - ipcValidationKey, - opts.isNodeDebugging, - 'pages', - config, - initialEnv - ) + renderWorkers.pages = renderWorkers.app const renderWorkerOpts: Parameters[0] = { port: opts.port, @@ -211,6 +197,7 @@ export async function initialize(opts: { hostname: opts.hostname, minimalMode: opts.minimalMode, dev: !!opts.dev, + server: opts.server, isNodeDebugging: !!opts.isNodeDebugging, serverFields: devInstance?.serverFields || {}, experimentalTestProxy: !!opts.experimentalTestProxy, @@ -325,7 +312,6 @@ export async function initialize(opts: { async function invokeRender( parsedUrl: NextUrlWithParsedQuery, type: keyof typeof renderWorkers, - handleIndex: number, invokePath: string, additionalInvokeHeaders: Record = {} ) { @@ -360,29 +346,22 @@ export async function initialize(opts: { throw new Error(`Failed to initialize render worker ${type}`) } - const renderUrl = `http://${workerResult.hostname}:${workerResult.port}${req.url}` - const invokeHeaders: typeof req.headers = { - ...req.headers, 'x-middleware-invoke': '', 'x-invoke-path': invokePath, 'x-invoke-query': encodeURIComponent(JSON.stringify(parsedUrl.query)), ...(additionalInvokeHeaders || {}), } + Object.assign(req.headers, invokeHeaders) - debug('invokeRender', renderUrl, invokeHeaders) + debug('invokeRender', req.url, invokeHeaders) - let invokeRes try { - invokeRes = await invokeRequest( - renderUrl, - { - headers: invokeHeaders, - method: req.method, - signal: signalFromNodeResponse(res), - }, - getRequestMeta(req, '__NEXT_CLONABLE_BODY')?.cloneBodyStream() + const initResult = await renderWorkers.pages?.initialize( + renderWorkerOpts ) + await initResult?.requestHandler(req, res) + return } catch (e) { // If the client aborts before we can receive a response object (when // the headers are flushed), then we can early exit without further @@ -392,54 +371,6 @@ export async function initialize(opts: { } throw e } - - debug('invokeRender res', invokeRes.status, invokeRes.headers) - - // when we receive x-no-fallback we restart - if (invokeRes.headers.get('x-no-fallback')) { - // eslint-disable-next-line - await handleRequest(handleIndex + 1) - return - } - - for (const [key, value] of Object.entries( - filterReqHeaders( - toNodeOutgoingHttpHeaders(invokeRes.headers), - ipcForbiddenHeaders - ) - )) { - if (value !== undefined) { - if (key === 'set-cookie') { - const curValue = res.getHeader(key) as string - const newValue: string[] = [] as string[] - - for (const cookie of Array.isArray(curValue) - ? curValue - : splitCookiesString(curValue || '')) { - newValue.push(cookie) - } - for (const val of (Array.isArray(value) - ? value - : value - ? [value] - : []) as string[]) { - newValue.push(val) - } - res.setHeader(key, newValue) - } else { - res.setHeader(key, value as string) - } - } - } - res.statusCode = invokeRes.status || 200 - res.statusMessage = invokeRes.statusText || '' - - if (invokeRes.body) { - await pipeReadable(invokeRes.body, res) - } else { - res.end() - } - return } const handleRequest = async (handleIndex: number) => { @@ -555,7 +486,8 @@ export async function initialize(opts: { (fsChecker.appFiles.has(matchedOutput.itemPath) || fsChecker.pageFiles.has(matchedOutput.itemPath)) ) { - await invokeRender(parsedUrl, 'pages', handleIndex, '/_error', { + res.statusCode = 500 + await invokeRender(parsedUrl, 'pages', '/_error', { 'x-invoke-status': '500', 'x-invoke-error': JSON.stringify({ message: `A conflicting public file and page file was found for path ${matchedOutput.itemPath} https://nextjs.org/docs/messages/conflicting-public-file-page`, @@ -579,15 +511,10 @@ export async function initialize(opts: { } if (!(req.method === 'GET' || req.method === 'HEAD')) { res.setHeader('Allow', ['GET', 'HEAD']) - return await invokeRender( - url.parse('/405', true), - 'pages', - handleIndex, - '/405', - { - 'x-invoke-status': '405', - } - ) + res.statusCode = 405 + return await invokeRender(url.parse('/405', true), 'pages', '/405', { + 'x-invoke-status': '405', + }) } try { @@ -642,10 +569,10 @@ export async function initialize(opts: { if (typeof err.statusCode === 'number') { const invokePath = `/${err.statusCode}` const invokeStatus = `${err.statusCode}` + res.statusCode = err.statusCode return await invokeRender( url.parse(invokePath, true), 'pages', - handleIndex, invokePath, { 'x-invoke-status': invokeStatus, @@ -662,7 +589,6 @@ export async function initialize(opts: { return await invokeRender( parsedUrl, matchedOutput.type === 'appFile' ? 'app' : 'pages', - handleIndex, parsedUrl.pathname || '/', { 'x-invoke-output': matchedOutput.itemPath, @@ -687,18 +613,20 @@ export async function initialize(opts: { ? devInstance?.serverFields.hasAppNotFound : await fsChecker.getItem('/_not-found') + res.statusCode = 404 + if (appNotFound) { return await invokeRender( parsedUrl, 'app', - handleIndex, opts.dev ? '/not-found' : '/_not-found', { 'x-invoke-status': '404', } ) } - await invokeRender(parsedUrl, 'pages', handleIndex, '/404', { + + await invokeRender(parsedUrl, 'pages', '/404', { 'x-invoke-status': '404', }) } @@ -716,10 +644,10 @@ export async function initialize(opts: { } else { console.error(err) } + res.statusCode = Number(invokeStatus) return await invokeRender( url.parse(invokePath, true), 'pages', - 0, invokePath, { 'x-invoke-status': invokeStatus, diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index 3ae8d5995036d..313e0adaf2369 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -14,7 +14,6 @@ import { Header } from '../../../lib/load-custom-routes' import { stringifyQuery } from '../../server-route-utils' import { formatHostname } from '../format-hostname' import { toNodeOutgoingHttpHeaders } from '../../web/utils' -import { invokeRequest } from '../server-ipc/invoke-request' import { isAbortError } from '../../pipe-readable' import { getCookieParser, setLazyProp } from '../../api-utils' import { getHostname } from '../../../shared/lib/get-hostname' @@ -38,6 +37,7 @@ import { matchHas, prepareDestination, } from '../../../shared/lib/router/utils/prepare-destination' +import { createRequestResponseMocks } from '../mock-request' const debug = setupDebug('next:router-server:resolve-routes') @@ -99,7 +99,6 @@ export function getResolveRoutes( async function resolveRoutes({ req, isUpgradeReq, - signal, invokedOutputs, }: { req: IncomingMessage @@ -445,22 +444,6 @@ export function getResolveRoutes( if (!workerResult) { throw new Error(`Failed to initialize render worker "middleware"`) } - const stringifiedQuery = stringifyQuery( - req as any, - getRequestMeta(req, '__NEXT_INIT_QUERY') || {} - ) - const parsedInitUrl = new URL( - getRequestMeta(req, '__NEXT_INIT_URL') || '/', - 'http://n' - ) - - const curUrl = config.skipMiddlewareUrlNormalize - ? `${parsedInitUrl.pathname}${parsedInitUrl.search}` - : `${parsedUrl.pathname}${stringifiedQuery ? '?' : ''}${ - stringifiedQuery || '' - }` - - const renderUrl = `http://${workerResult.hostname}:${workerResult.port}${curUrl}` const invokeHeaders: typeof req.headers = { ...req.headers, @@ -470,19 +453,43 @@ export function getResolveRoutes( 'x-middleware-invoke': '1', } - debug('invoking middleware', renderUrl, invokeHeaders) + debug('invoking middleware', req.url, invokeHeaders) let middlewareRes + let bodyStream: ReadableStream | undefined = undefined try { - middlewareRes = await invokeRequest( - renderUrl, - { - headers: invokeHeaders, - method: req.method, - signal, + let readableController: ReadableStreamController + const { req: mockedReq, res: mockedRes } = + await createRequestResponseMocks({ + url: req.url || '/', + method: req.method || 'GET', + headers: filterReqHeaders(invokeHeaders, ipcForbiddenHeaders), + bodyReadable: getRequestMeta( + req, + '__NEXT_CLONABLE_BODY' + )?.cloneBodyStream(), + resWriter(chunk) { + readableController.enqueue(Buffer.from(chunk)) + return true + }, + }) + + bodyStream = new ReadableStream({ + start(controller) { + readableController = controller }, - getRequestMeta(req, '__NEXT_CLONABLE_BODY')?.cloneBodyStream() + }) + + const initResult = await renderWorkers.pages?.initialize( + renderWorkerOpts ) + + mockedRes.on('close', () => { + readableController.close() + }) + initResult?.requestHandler(mockedReq, mockedRes) + await mockedRes.headPromise + middlewareRes = mockedRes } catch (e) { // If the client aborts before we can receive a response object // (when the headers are flushed), then we can early exit without @@ -501,7 +508,7 @@ export function getResolveRoutes( middlewareRes.headers ) as Record - debug('middleware res', middlewareRes.status, middlewareHeaders) + debug('middleware res', middlewareRes.statusCode, middlewareHeaders) if (middlewareHeaders['x-middleware-override-headers']) { const overriddenHeaders: Set = new Set() @@ -613,7 +620,7 @@ export function getResolveRoutes( parsedUrl, resHeaders, finished: true, - statusCode: middlewareRes.status, + statusCode: middlewareRes.statusCode, } } @@ -622,8 +629,8 @@ export function getResolveRoutes( parsedUrl, resHeaders, finished: true, - bodyStream: middlewareRes.body, - statusCode: middlewareRes.status, + bodyStream, + statusCode: middlewareRes.statusCode, } } } diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 661bcd8701842..2641d9afadb0f 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -1,3 +1,4 @@ +import '../next' import '../node-polyfill-fetch' import '../require-hook' @@ -45,6 +46,7 @@ export async function getRequestHandlers({ dir, port, isDev, + server, hostname, minimalMode, isNodeDebugging, @@ -54,6 +56,7 @@ export async function getRequestHandlers({ dir: string port: number isDev: boolean + server?: import('http').Server hostname: string minimalMode?: boolean isNodeDebugging?: boolean @@ -66,6 +69,7 @@ export async function getRequestHandlers({ hostname, dev: isDev, minimalMode, + server, workerType: 'router', isNodeDebugging: isNodeDebugging || false, keepAliveTimeout, @@ -240,6 +244,7 @@ export async function startServer({ dir, port, isDev, + server, hostname, minimalMode, isNodeDebugging: Boolean(isNodeDebugging), diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index eef8514491d82..7c23358621868 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -976,13 +976,17 @@ export default class NextNodeServer extends BaseServer { private normalizeReq( req: BaseNextRequest | IncomingMessage ): BaseNextRequest { - return req instanceof IncomingMessage ? new NodeNextRequest(req) : req + return !(req instanceof NodeNextRequest) + ? new NodeNextRequest(req as IncomingMessage) + : req } private normalizeRes( res: BaseNextResponse | ServerResponse ): BaseNextResponse { - return res instanceof ServerResponse ? new NodeNextResponse(res) : res + return !(res instanceof NodeNextResponse) + ? new NodeNextResponse(res as ServerResponse) + : res } public getRequestHandler(): NodeRequestHandler { diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index 6cdd4282007b9..3a6b1959d3aad 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -5,6 +5,38 @@ import type { NextConfigComplete } from './config-shared' import type { IncomingMessage, ServerResponse } from 'http' import type { NextUrlWithParsedQuery } from './request-meta' +import { spawnSync } from 'child_process' + +// if we are not inside of the esm loader enabled +// worker we need to re-spawn with correct args +// we can't do this if imported in jest test file otherwise +// it duplicates tests +if ( + !process.env.NEXT_PRIVATE_WORKER && + typeof jest === 'undefined' && + process.env.__NEXT_PRIVATE_PREBUNDLED_REACT +) { + for (let i = 0; i < 1; i++) { + try { + const newArgs = [ + '--experimental-loader', + 'next/dist/esm/server/esm-loader.mjs', + '--no-warnings', + ...process.argv.splice(1), + ] + spawnSync(process.argv[0], newArgs, { + stdio: 'inherit', + env: { + ...process.env, + NEXT_PRIVATE_WORKER: '1', + }, + }) + } catch (err) { + console.error(`spawn failed retrying`, err) + } + } +} + import './require-hook' import './node-polyfill-fetch' import './node-polyfill-crypto' diff --git a/packages/next/src/server/require-hook.js b/packages/next/src/server/require-hook.js new file mode 100644 index 0000000000000..67f773de0d41a --- /dev/null +++ b/packages/next/src/server/require-hook.js @@ -0,0 +1,31 @@ +// Synchronously inject a require hook for webpack and webpack/. It's required to use the internal ncc webpack version. +// This is needed for userland plugins to attach to the same webpack instance as Next.js'. +// Individually compiled modules are as defined for the compilation in bundles/webpack/packages/*. + +// This module will only be loaded once per process. +const mod = require('module') +const resolveFilename = mod._resolveFilename + +const { overrideReact, hookPropertyMap } = require('./lib/import-overrides') + +mod._resolveFilename = function ( + originalResolveFilename, + requestMap, + request, + parent, + isMain, + options +) { + if (request === 'from-require' || request === 'from-import') { + console.log('require-hook', request) + } + // In case the environment variable is set after the module is loaded. + overrideReact() + + const hookResolved = requestMap.get(request) + if (hookResolved) request = hookResolved + + return originalResolveFilename.call(mod, request, parent, isMain, options) + + // We use `bind` here to avoid referencing outside variables to create potential memory leaks. +}.bind(null, resolveFilename, hookPropertyMap) diff --git a/packages/next/src/shared/lib/constants.ts b/packages/next/src/shared/lib/constants.ts index 3d0f78e7bd018..86a1c2059b585 100644 --- a/packages/next/src/shared/lib/constants.ts +++ b/packages/next/src/shared/lib/constants.ts @@ -10,6 +10,14 @@ export const COMPILER_NAMES = { edgeServer: 'edge-server', } as const +export const INTERNAL_HEADERS = [ + 'x-invoke-path', + 'x-invoke-status', + 'x-invoke-error', + 'x-invoke-query', + 'x-middleware-invoke', +] as const + export type CompilerNameValues = ValueOf export const COMPILER_INDEXES: { diff --git a/packages/next/taskfile-swc.js b/packages/next/taskfile-swc.js index 8ff3427c60e36..0d28ca9d0b5c0 100644 --- a/packages/next/taskfile-swc.js +++ b/packages/next/taskfile-swc.js @@ -140,7 +140,10 @@ module.exports = function (task) { if (ext) { const extRegex = new RegExp(ext.replace('.', '\\.') + '$', 'i') // Remove the extension if stripExtension is enabled or replace it with `.js` - file.base = file.base.replace(extRegex, stripExtension ? '' : '.js') + file.base = file.base.replace( + extRegex, + stripExtension ? '' : `.${ext === '.mts' ? 'm' : ''}js` + ) } if (output.map) { diff --git a/packages/next/taskfile.js b/packages/next/taskfile.js index 495da5ef09467..37452e54d08cd 100644 --- a/packages/next/taskfile.js +++ b/packages/next/taskfile.js @@ -2449,7 +2449,7 @@ export async function server(task, opts) { export async function server_esm(task, opts) { await task - .source('src/server/**/!(*.test).+(js|ts|tsx)') + .source('src/server/**/!(*.test).+(js|mts|ts|tsx)') .swc('server', { dev: opts.dev, esm: true }) .target('dist/esm/server') } diff --git a/test/e2e/app-dir/app/index.test.ts b/test/e2e/app-dir/app/index.test.ts index 9b4404f888462..54cf83cec0c7f 100644 --- a/test/e2e/app-dir/app/index.test.ts +++ b/test/e2e/app-dir/app/index.test.ts @@ -310,9 +310,7 @@ createNextDescribe( const res = await next.fetch('/dashboard') expect(res.headers.get('x-edge-runtime')).toBe('1') expect(res.headers.get('vary')).toBe( - isNextDeploy - ? 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url' - : 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url, Accept-Encoding' + 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url' ) }) From 7fd58541e564194d31e63cbb0397d4032075cf24 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 30 Aug 2023 22:24:22 -0700 Subject: [PATCH 02/54] test cases --- packages/next/src/bin/next.ts | 23 ++++++--- .../nextjs-require-cache-hot-reloader.ts | 24 ++------- .../next/src/server/lib/import-overrides.ts | 1 - packages/next/src/server/lib/router-server.ts | 50 ------------------- test/development/basic/node-builtins.test.ts | 4 +- .../api-body-parser/test/index.test.js | 4 +- 6 files changed, 24 insertions(+), 82 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 705d1ad287f5b..92d678333fa2e 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import '../server/require-hook' import * as log from '../build/output/log' import arg from 'next/dist/compiled/arg/index.js' import { NON_STANDARD_NODE_ENV } from '../lib/constants' @@ -127,23 +128,33 @@ async function main() { const dir = getProjectDir( process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0] ) + const origEnv = Object.assign({}, process.env) + + // TODO: set config to env variable to be re-used so we don't reload + // un-necessarily const config = await loadConfig( command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_BUILD, dir ) - // TODO: set config to env variable to be re-used so we don't reload - // un-necessarily - const dirsResult = findPagesDir(dir, true) + let dirsResult: ReturnType | undefined = undefined + + try { + dirsResult = findPagesDir(dir, true) + } catch (_) { + // handle this error further down + } - if (dirsResult.appDir) { + if (dirsResult?.appDir) { + // we need to reset env if we are going to create + // the worker process with the esm loader so that the + // initial env state is correct + process.env = origEnv process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental .serverActions ? 'experimental' : 'next' } - require('../server/require-hook') - for (const dependency of ['react', 'react-dom']) { try { // When 'npm link' is used it checks the clone location. Not the project. diff --git a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts index 107e2e03f07bb..b1458dd5af06a 100644 --- a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts +++ b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts @@ -20,16 +20,7 @@ const originModules = [ const RUNTIME_NAMES = ['webpack-runtime', 'webpack-api-runtime'] -const nextDeleteCacheRpc = async (filePaths: string[]) => { - if ((global as any)._nextDeleteCache) { - return (global as any)._nextDeleteCache(filePaths) - } -} - export function deleteAppClientCache() { - if ((global as any)._nextDeleteAppClientCache) { - return (global as any)._nextDeleteAppClientCache() - } // ensure we reset the cache for rsc components // loaded via react-server-dom-webpack const reactServerDomModId = require.resolve( @@ -88,25 +79,18 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance { apply(compiler: Compiler) { compiler.hooks.assetEmitted.tap(PLUGIN_NAME, (_file, { targetPath }) => { - nextDeleteCacheRpc([targetPath]) - - // Clear module context in other processes - if ((global as any)._nextClearModuleContext) { - ;(global as any)._nextClearModuleContext(targetPath) - } // Clear module context in this process clearModuleContext(targetPath) + deleteCache(targetPath) }) compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async (compilation) => { - const cacheEntriesToDelete = [] - for (const name of RUNTIME_NAMES) { const runtimeChunkPath = path.join( compilation.outputOptions.path!, `${name}.js` ) - cacheEntriesToDelete.push(runtimeChunkPath) + deleteCache(runtimeChunkPath) } // we need to make sure to clear all server entries from cache @@ -116,15 +100,15 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance { const isAppPath = entry.toString().startsWith('app/') return entry.toString().startsWith('pages/') || isAppPath }) + deleteAppClientCache() for (const page of entries) { const outputPath = path.join( compilation.outputOptions.path!, page + '.js' ) - cacheEntriesToDelete.push(outputPath) + await deleteCache(outputPath) } - await nextDeleteCacheRpc(cacheEntriesToDelete) }) } } diff --git a/packages/next/src/server/lib/import-overrides.ts b/packages/next/src/server/lib/import-overrides.ts index 5d786fa8f36e0..12f0421ffa853 100644 --- a/packages/next/src/server/lib/import-overrides.ts +++ b/packages/next/src/server/lib/import-overrides.ts @@ -78,7 +78,6 @@ addHookAliases(toResolveMap(defaultOverrides)) export function overrideReact() { if (process.env.__NEXT_PRIVATE_PREBUNDLED_REACT && !aliasedPrebundledReact) { aliasedPrebundledReact = true - console.log('aliased react to', process.env.__NEXT_PRIVATE_PREBUNDLED_REACT) // Require these modules with static paths to make sure they are tracked by // NFT when building the app in standalone mode, as we are now conditionally diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index c94eb4268dcc0..39f17e1e3d8b7 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -209,56 +209,6 @@ export async function initialize(opts: { pages: await renderWorkers.pages?.initialize(renderWorkerOpts), } - if (devInstance) { - const originalNextDeleteCache = (global as any)._nextDeleteCache - ;(global as any)._nextDeleteCache = async (filePaths: string[]) => { - // Multiple instances of Next.js can be instantiated, since this is a global we have to call the original if it exists. - if (originalNextDeleteCache) { - await originalNextDeleteCache(filePaths) - } - try { - await Promise.all([ - renderWorkers.pages?.deleteCache(filePaths), - renderWorkers.app?.deleteCache(filePaths), - ]) - } catch (err) { - console.error(err) - } - } - const originalNextDeleteAppClientCache = (global as any) - ._nextDeleteAppClientCache - ;(global as any)._nextDeleteAppClientCache = async () => { - // Multiple instances of Next.js can be instantiated, since this is a global we have to call the original if it exists. - if (originalNextDeleteAppClientCache) { - await originalNextDeleteAppClientCache() - } - try { - await Promise.all([ - renderWorkers.pages?.deleteAppClientCache(), - renderWorkers.app?.deleteAppClientCache(), - ]) - } catch (err) { - console.error(err) - } - } - const originalNextClearModuleContext = (global as any) - ._nextClearModuleContext - ;(global as any)._nextClearModuleContext = async (targetPath: string) => { - // Multiple instances of Next.js can be instantiated, since this is a global we have to call the original if it exists. - if (originalNextClearModuleContext) { - await originalNextClearModuleContext() - } - try { - await Promise.all([ - renderWorkers.pages?.clearModuleContext(targetPath), - renderWorkers.app?.clearModuleContext(targetPath), - ]) - } catch (err) { - console.error(err) - } - } - } - const logError = async ( type: 'uncaughtException' | 'unhandledRejection', err: Error | undefined diff --git a/test/development/basic/node-builtins.test.ts b/test/development/basic/node-builtins.test.ts index 27797f6c564d0..3c5c6a6a335f8 100644 --- a/test/development/basic/node-builtins.test.ts +++ b/test/development/basic/node-builtins.test.ts @@ -86,7 +86,7 @@ createNextDescribe( expect(parsedData.https).toBe(true) expect(parsedData.os).toBe('\n') expect(parsedData.path).toBe('/hello/world/test.txt') - expect(parsedData.process).toInclude('next-render-worker-pages') + expect(parsedData.process).toInclude('next-router-worker') expect(parsedData.querystring).toBe('a=b') expect(parsedData.stringDecoder).toBe(true) expect(parsedData.sys).toBe(true) @@ -112,7 +112,7 @@ createNextDescribe( expect(parsedData.https).toBe(true) expect(parsedData.os).toBe('\n') expect(parsedData.path).toBe('/hello/world/test.txt') - expect(parsedData.process).toInclude('next-render-worker-pages') + expect(parsedData.process).toInclude('next-router-worker') expect(parsedData.querystring).toBe('a=b') expect(parsedData.stringDecoder).toBe(true) expect(parsedData.sys).toBe(true) diff --git a/test/integration/api-body-parser/test/index.test.js b/test/integration/api-body-parser/test/index.test.js index 83501cfa63727..ddca1496237bd 100644 --- a/test/integration/api-body-parser/test/index.test.js +++ b/test/integration/api-body-parser/test/index.test.js @@ -27,9 +27,7 @@ function runTests() { killApp(app) }) - // TODO: we can't allow req fields with the proxying required for separate - // workers - it.skip('should not throw if request body is already parsed in custom middleware', async () => { + it('should not throw if request body is already parsed in custom middleware', async () => { await startServer() const data = await makeRequest() expect(data).toEqual([{ title: 'Nextjs' }]) From 1723f9318cba2e39fa9580b7e6e616029624b256 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 30 Aug 2023 22:26:51 -0700 Subject: [PATCH 03/54] telemetry case --- packages/next/src/bin/next.ts | 53 ++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 92d678333fa2e..4108647bad7d8 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -125,34 +125,37 @@ if (!process.env.NEXT_MANUAL_SIG_HANDLE && command !== 'dev') { async function main() { const currentArgsSpec = commandArgs[command]() const validatedArgs = getValidatedArgs(currentArgsSpec, forwardedArgs) - const dir = getProjectDir( - process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0] - ) - const origEnv = Object.assign({}, process.env) - // TODO: set config to env variable to be re-used so we don't reload - // un-necessarily - const config = await loadConfig( - command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_BUILD, - dir - ) - let dirsResult: ReturnType | undefined = undefined + if (command !== 'telemetry') { + const dir = getProjectDir( + process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0] + ) + const origEnv = Object.assign({}, process.env) - try { - dirsResult = findPagesDir(dir, true) - } catch (_) { - // handle this error further down - } + // TODO: set config to env variable to be re-used so we don't reload + // un-necessarily + const config = await loadConfig( + command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_BUILD, + dir + ) + let dirsResult: ReturnType | undefined = undefined - if (dirsResult?.appDir) { - // we need to reset env if we are going to create - // the worker process with the esm loader so that the - // initial env state is correct - process.env = origEnv - process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental - .serverActions - ? 'experimental' - : 'next' + try { + dirsResult = findPagesDir(dir, true) + } catch (_) { + // handle this error further down + } + + if (dirsResult?.appDir) { + // we need to reset env if we are going to create + // the worker process with the esm loader so that the + // initial env state is correct + process.env = origEnv + process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental + .serverActions + ? 'experimental' + : 'next' + } } for (const dependency of ['react', 'react-dom']) { From 1875a420e1ebbbe9e381361c807bf170e91ef671 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 30 Aug 2023 22:29:11 -0700 Subject: [PATCH 04/54] fix type --- packages/next/src/server/lib/route-resolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/server/lib/route-resolver.ts b/packages/next/src/server/lib/route-resolver.ts index 332d0138a916e..d9f0eee348247 100644 --- a/packages/next/src/server/lib/route-resolver.ts +++ b/packages/next/src/server/lib/route-resolver.ts @@ -228,9 +228,9 @@ export async function makeResolver( async upgradeHandler() {}, } }, + deleteAppClientCache() {}, async deleteCache() {}, async clearModuleContext() {}, - async deleteAppClientCache() {}, async propagateServerField() {}, } as Partial as any, }, From d7d7fbbbcf054a7d80f75c758fc64ad45bed6c37 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 30 Aug 2023 22:56:04 -0700 Subject: [PATCH 05/54] fix require cache case --- packages/next/src/bin/next.ts | 6 +++--- .../webpack/plugins/nextjs-require-cache-hot-reloader.ts | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 4108647bad7d8..a9b06e5f466b9 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -7,8 +7,8 @@ import { commands } from '../lib/commands' import { commandArgs } from '../lib/command-args' import loadConfig from '../server/config' import { + PHASE_PRODUCTION_SERVER, PHASE_DEVELOPMENT_SERVER, - PHASE_PRODUCTION_BUILD, } from '../shared/lib/constants' import { getProjectDir } from '../lib/get-project-dir' import { getValidatedArgs } from '../lib/get-validated-args' @@ -126,7 +126,7 @@ async function main() { const currentArgsSpec = commandArgs[command]() const validatedArgs = getValidatedArgs(currentArgsSpec, forwardedArgs) - if (command !== 'telemetry') { + if (command === 'start' || command === 'dev') { const dir = getProjectDir( process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0] ) @@ -135,7 +135,7 @@ async function main() { // TODO: set config to env variable to be re-used so we don't reload // un-necessarily const config = await loadConfig( - command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_BUILD, + command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, dir ) let dirsResult: ReturnType | undefined = undefined diff --git a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts index b1458dd5af06a..93e2ad3ede3f2 100644 --- a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts +++ b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts @@ -96,11 +96,16 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance { // we need to make sure to clear all server entries from cache // since they can have a stale webpack-runtime cache // which needs to always be in-sync + let hasAppEntry = false const entries = [...compilation.entries.keys()].filter((entry) => { const isAppPath = entry.toString().startsWith('app/') + if (isAppPath) hasAppEntry = true return entry.toString().startsWith('pages/') || isAppPath }) - deleteAppClientCache() + + if (hasAppEntry) { + deleteAppClientCache() + } for (const page of entries) { const outputPath = path.join( From 6a91245d0752a3e26f990e33a7fa74fba67430a3 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 30 Aug 2023 23:15:22 -0700 Subject: [PATCH 06/54] tweak request handler --- packages/next/src/server/lib/router-server.ts | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 39f17e1e3d8b7..432ddd5944167 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -26,6 +26,7 @@ import { pathHasPrefix } from '../../shared/lib/router/utils/path-has-prefix' import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-prefix' import setupCompression from 'next/dist/compiled/compression' import { signalFromNodeResponse } from '../web/spec-extension/adapters/next-request' +import { filterReqHeaders, ipcForbiddenHeaders } from './server-ipc/utils' import { PHASE_PRODUCTION_SERVER, @@ -263,6 +264,7 @@ export async function initialize(opts: { parsedUrl: NextUrlWithParsedQuery, type: keyof typeof renderWorkers, invokePath: string, + handleIndex: number, additionalInvokeHeaders: Record = {} ) { // invokeRender expects /api routes to not be locale prefixed @@ -310,7 +312,49 @@ export async function initialize(opts: { const initResult = await renderWorkers.pages?.initialize( renderWorkerOpts ) - await initResult?.requestHandler(req, res) + + let bodyStream: ReadableStream | undefined = undefined + let readableController: ReadableStreamController + const { req: mockedReq, res: mockedRes } = + await createRequestResponseMocks({ + url: req.url || '/', + method: req.method || 'GET', + headers: filterReqHeaders(invokeHeaders, ipcForbiddenHeaders), + bodyReadable: getRequestMeta( + req, + '__NEXT_CLONABLE_BODY' + )?.cloneBodyStream(), + resWriter(chunk) { + readableController.enqueue(Buffer.from(chunk)) + return true + }, + }) + + bodyStream = new ReadableStream({ + start(controller) { + readableController = controller + }, + }) + + mockedRes.on('close', () => { + readableController.close() + }) + initResult?.requestHandler(mockedReq, mockedRes) + await mockedRes.headPromise + + // when we receive x-no-fallback we restart + if (mockedRes.getHeader('x-no-fallback')) { + // eslint-disable-next-line + await handleRequest(handleIndex + 1) + return + } + + for (const key in Object.entries(mockedRes.headers)) { + res.setHeader(key, mockedRes.getHeader(key) as string) + } + res.statusCode = mockedRes.statusCode + res.statusMessage = mockedRes.statusMessage + await pipeReadable(bodyStream, res) return } catch (e) { // If the client aborts before we can receive a response object (when @@ -437,7 +481,7 @@ export async function initialize(opts: { fsChecker.pageFiles.has(matchedOutput.itemPath)) ) { res.statusCode = 500 - await invokeRender(parsedUrl, 'pages', '/_error', { + await invokeRender(parsedUrl, 'pages', '/_error', handleIndex, { 'x-invoke-status': '500', 'x-invoke-error': JSON.stringify({ message: `A conflicting public file and page file was found for path ${matchedOutput.itemPath} https://nextjs.org/docs/messages/conflicting-public-file-page`, @@ -462,9 +506,15 @@ export async function initialize(opts: { if (!(req.method === 'GET' || req.method === 'HEAD')) { res.setHeader('Allow', ['GET', 'HEAD']) res.statusCode = 405 - return await invokeRender(url.parse('/405', true), 'pages', '/405', { - 'x-invoke-status': '405', - }) + return await invokeRender( + url.parse('/405', true), + 'pages', + '/405', + handleIndex, + { + 'x-invoke-status': '405', + } + ) } try { @@ -524,6 +574,7 @@ export async function initialize(opts: { url.parse(invokePath, true), 'pages', invokePath, + handleIndex, { 'x-invoke-status': invokeStatus, } @@ -540,6 +591,7 @@ export async function initialize(opts: { parsedUrl, matchedOutput.type === 'appFile' ? 'app' : 'pages', parsedUrl.pathname || '/', + handleIndex, { 'x-invoke-output': matchedOutput.itemPath, } @@ -570,13 +622,14 @@ export async function initialize(opts: { parsedUrl, 'app', opts.dev ? '/not-found' : '/_not-found', + handleIndex, { 'x-invoke-status': '404', } ) } - await invokeRender(parsedUrl, 'pages', '/404', { + await invokeRender(parsedUrl, 'pages', '/404', handleIndex, { 'x-invoke-status': '404', }) } @@ -599,6 +652,7 @@ export async function initialize(opts: { url.parse(invokePath, true), 'pages', invokePath, + 0, { 'x-invoke-status': invokeStatus, } From 0f286f8b706a4c09af3c0b6edf078cbf9ab9413e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 30 Aug 2023 23:23:22 -0700 Subject: [PATCH 07/54] fix req headers --- packages/next/src/server/lib/router-server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 432ddd5944167..86df8f6a280ad 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -299,6 +299,7 @@ export async function initialize(opts: { } const invokeHeaders: typeof req.headers = { + ...req.headers, 'x-middleware-invoke': '', 'x-invoke-path': invokePath, 'x-invoke-query': encodeURIComponent(JSON.stringify(parsedUrl.query)), From 809620a851534c5532063b848273bf840325cec2 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 30 Aug 2023 23:32:21 -0700 Subject: [PATCH 08/54] fix response headers --- packages/next/src/server/lib/router-server.ts | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 86df8f6a280ad..e1a8485476653 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -33,6 +33,7 @@ import { PHASE_DEVELOPMENT_SERVER, PERMANENT_REDIRECT_STATUS, } from '../../shared/lib/constants' +import { toNodeOutgoingHttpHeaders } from '../web/utils' const debug = setupDebug('next:router-server:main') @@ -350,8 +351,28 @@ export async function initialize(opts: { return } - for (const key in Object.entries(mockedRes.headers)) { - res.setHeader(key, mockedRes.getHeader(key) as string) + for (const [key, value] of Object.entries({ + ...filterReqHeaders( + toNodeOutgoingHttpHeaders(mockedRes.headers), + ipcForbiddenHeaders + ), + })) { + if ( + [ + 'content-length', + 'x-middleware-rewrite', + 'x-middleware-redirect', + 'x-middleware-refresh', + 'x-middleware-invoke', + 'x-invoke-path', + 'x-invoke-query', + ].includes(key) + ) { + continue + } + if (value) { + res.setHeader(key, value) + } } res.statusCode = mockedRes.statusCode res.statusMessage = mockedRes.statusMessage From 733b9cbd5fb72721d855decd0d6d0e87f611189d Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 08:46:27 -0700 Subject: [PATCH 09/54] avoid using mocked req/res for request handler --- packages/next/src/server/base-server.ts | 4 ++ packages/next/src/server/lib/router-server.ts | 71 +++---------------- packages/next/src/server/next-server.ts | 6 +- packages/next/src/server/next.ts | 2 +- test/e2e/app-dir/app/index.test.ts | 4 +- .../api-body-parser/pages/api/index.js | 13 +++- test/integration/api-body-parser/server.js | 1 + .../api-body-parser/test/index.test.js | 6 +- 8 files changed, 34 insertions(+), 73 deletions(-) diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 43c56d84557e9..b383265a0c3a3 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -1283,6 +1283,10 @@ export default abstract class Server { res.statusCode = 200 return await this.run(req, res, parsedUrl) } catch (err: any) { + if (err instanceof NoFallbackError) { + throw err + } + if ( (err && typeof err === 'object' && err.code === 'ERR_INVALID_URL') || err instanceof DecodeError || diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index e1a8485476653..bec79babdc043 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -25,15 +25,14 @@ import { NextUrlWithParsedQuery, getRequestMeta } from '../request-meta' import { pathHasPrefix } from '../../shared/lib/router/utils/path-has-prefix' import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-prefix' import setupCompression from 'next/dist/compiled/compression' +import { NoFallbackError } from '../base-server' import { signalFromNodeResponse } from '../web/spec-extension/adapters/next-request' -import { filterReqHeaders, ipcForbiddenHeaders } from './server-ipc/utils' import { PHASE_PRODUCTION_SERVER, PHASE_DEVELOPMENT_SERVER, PERMANENT_REDIRECT_STATUS, } from '../../shared/lib/constants' -import { toNodeOutgoingHttpHeaders } from '../web/utils' const debug = setupDebug('next:router-server:main') @@ -315,68 +314,16 @@ export async function initialize(opts: { renderWorkerOpts ) - let bodyStream: ReadableStream | undefined = undefined - let readableController: ReadableStreamController - const { req: mockedReq, res: mockedRes } = - await createRequestResponseMocks({ - url: req.url || '/', - method: req.method || 'GET', - headers: filterReqHeaders(invokeHeaders, ipcForbiddenHeaders), - bodyReadable: getRequestMeta( - req, - '__NEXT_CLONABLE_BODY' - )?.cloneBodyStream(), - resWriter(chunk) { - readableController.enqueue(Buffer.from(chunk)) - return true - }, - }) - - bodyStream = new ReadableStream({ - start(controller) { - readableController = controller - }, - }) - - mockedRes.on('close', () => { - readableController.close() - }) - initResult?.requestHandler(mockedReq, mockedRes) - await mockedRes.headPromise - - // when we receive x-no-fallback we restart - if (mockedRes.getHeader('x-no-fallback')) { - // eslint-disable-next-line - await handleRequest(handleIndex + 1) - return - } - - for (const [key, value] of Object.entries({ - ...filterReqHeaders( - toNodeOutgoingHttpHeaders(mockedRes.headers), - ipcForbiddenHeaders - ), - })) { - if ( - [ - 'content-length', - 'x-middleware-rewrite', - 'x-middleware-redirect', - 'x-middleware-refresh', - 'x-middleware-invoke', - 'x-invoke-path', - 'x-invoke-query', - ].includes(key) - ) { - continue - } - if (value) { - res.setHeader(key, value) + try { + await initResult?.requestHandler(req, res) + } catch (err) { + if (err instanceof NoFallbackError) { + // eslint-disable-next-line + await handleRequest(handleIndex + 1) + return } + throw err } - res.statusCode = mockedRes.statusCode - res.statusMessage = mockedRes.statusMessage - await pipeReadable(bodyStream, res) return } catch (e) { // If the client aborts before we can receive a response object (when diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 7c23358621868..e9241daa107dc 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -894,11 +894,7 @@ export default class NextNodeServer extends BaseServer { } catch (err: any) { if (err instanceof NoFallbackError) { if (this.isRenderWorker) { - res.setHeader('x-no-fallback', '1') - res.send() - return { - finished: true, - } + throw err } return { diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index 3a6b1959d3aad..8dccea469c99b 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -12,8 +12,8 @@ import { spawnSync } from 'child_process' // we can't do this if imported in jest test file otherwise // it duplicates tests if ( - !process.env.NEXT_PRIVATE_WORKER && typeof jest === 'undefined' && + !process.env.NEXT_PRIVATE_WORKER && process.env.__NEXT_PRIVATE_PREBUNDLED_REACT ) { for (let i = 0; i < 1; i++) { diff --git a/test/e2e/app-dir/app/index.test.ts b/test/e2e/app-dir/app/index.test.ts index 54cf83cec0c7f..9b4404f888462 100644 --- a/test/e2e/app-dir/app/index.test.ts +++ b/test/e2e/app-dir/app/index.test.ts @@ -310,7 +310,9 @@ createNextDescribe( const res = await next.fetch('/dashboard') expect(res.headers.get('x-edge-runtime')).toBe('1') expect(res.headers.get('vary')).toBe( - 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url' + isNextDeploy + ? 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url' + : 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url, Accept-Encoding' ) }) diff --git a/test/integration/api-body-parser/pages/api/index.js b/test/integration/api-body-parser/pages/api/index.js index 28ef9244fcefe..bb894d917ddfc 100644 --- a/test/integration/api-body-parser/pages/api/index.js +++ b/test/integration/api-body-parser/pages/api/index.js @@ -1,5 +1,12 @@ -export default ({ method, body }, res) => { - if (method === 'POST') { - res.status(200).json(body) +export default (req, res) => { + if ( + process.env.CUSTOM_SERVER && + typeof req.fromCustomServer === 'undefined' + ) { + throw new Error('missing custom req field') + } + + if (req.method === 'POST') { + res.status(200).json(req.body) } } diff --git a/test/integration/api-body-parser/server.js b/test/integration/api-body-parser/server.js index 0f786557e5fce..f37ee6845f7db 100644 --- a/test/integration/api-body-parser/server.js +++ b/test/integration/api-body-parser/server.js @@ -14,6 +14,7 @@ app.prepare().then(() => { server.use(express.json({ limit: '5mb' })) server.all('*', (req, res) => { + req.fromCustomServer = true handleNextRequests(req, res) }) diff --git a/test/integration/api-body-parser/test/index.test.js b/test/integration/api-body-parser/test/index.test.js index ddca1496237bd..ae6d540c14462 100644 --- a/test/integration/api-body-parser/test/index.test.js +++ b/test/integration/api-body-parser/test/index.test.js @@ -69,7 +69,11 @@ async function makeRequestWithInvalidContentType() { const startServer = async (optEnv = {}, opts) => { const scriptPath = join(appDir, 'server.js') context.appPort = appPort = await getPort() - const env = Object.assign({ ...process.env }, { PORT: `${appPort}` }, optEnv) + const env = Object.assign( + { ...process.env }, + { PORT: `${appPort}`, CUSTOM_SERVER: 'true' }, + optEnv + ) server = await initNextServerScript( scriptPath, From fe58674a7f68b5d28f0f9be869226b529e8e2109 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 09:15:53 -0700 Subject: [PATCH 10/54] middleware and config case --- packages/next/src/server/config.ts | 8 ++---- packages/next/src/server/lib/mock-request.ts | 8 +++--- .../server/lib/router-utils/resolve-routes.ts | 27 ++++++++----------- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index cfdf251330d82..20fad55e535ce 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -313,12 +313,8 @@ function assignDefaults( } // Append trailing slash for non-default loaders and when trailingSlash is set - if (images.path) { - if ( - (images.loader !== 'default' && - images.path[images.path.length - 1] !== '/') || - result.trailingSlash - ) { + if (images.path && result.trailingSlash) { + if (images.path[images.path.length - 1] !== '/') { images.path += '/' } } diff --git a/packages/next/src/server/lib/mock-request.ts b/packages/next/src/server/lib/mock-request.ts index 563583db9bf0c..90a643328ac42 100644 --- a/packages/next/src/server/lib/mock-request.ts +++ b/packages/next/src/server/lib/mock-request.ts @@ -185,6 +185,10 @@ export class MockedResponse extends Stream.Writable implements ServerResponse { ? fromNodeOutgoingHttpHeaders(res.headers) : new Headers() + this.headPromise = new Promise((resolve) => { + this.headPromiseResolve = resolve + }) + // Attach listeners for the `finish`, `end`, and `error` events to the // `MockedResponse` instance. this.hasStreamed = new Promise((resolve, reject) => { @@ -196,10 +200,6 @@ export class MockedResponse extends Stream.Writable implements ServerResponse { return val }) - this.headPromise = new Promise((resolve) => { - this.headPromiseResolve = resolve - }) - if (res.resWriter) { this.resWriter = res.resWriter } diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index 313e0adaf2369..33eaabd659555 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -446,12 +446,12 @@ export function getResolveRoutes( } const invokeHeaders: typeof req.headers = { - ...req.headers, 'x-invoke-path': '', 'x-invoke-query': '', 'x-invoke-output': '', 'x-middleware-invoke': '1', } + Object.assign(req.headers, invokeHeaders) debug('invoking middleware', req.url, invokeHeaders) @@ -459,20 +459,15 @@ export function getResolveRoutes( let bodyStream: ReadableStream | undefined = undefined try { let readableController: ReadableStreamController - const { req: mockedReq, res: mockedRes } = - await createRequestResponseMocks({ - url: req.url || '/', - method: req.method || 'GET', - headers: filterReqHeaders(invokeHeaders, ipcForbiddenHeaders), - bodyReadable: getRequestMeta( - req, - '__NEXT_CLONABLE_BODY' - )?.cloneBodyStream(), - resWriter(chunk) { - readableController.enqueue(Buffer.from(chunk)) - return true - }, - }) + const { res: mockedRes } = await createRequestResponseMocks({ + url: req.url || '/', + method: req.method || 'GET', + headers: filterReqHeaders(invokeHeaders, ipcForbiddenHeaders), + resWriter(chunk) { + readableController.enqueue(Buffer.from(chunk)) + return true + }, + }) bodyStream = new ReadableStream({ start(controller) { @@ -487,7 +482,7 @@ export function getResolveRoutes( mockedRes.on('close', () => { readableController.close() }) - initResult?.requestHandler(mockedReq, mockedRes) + initResult?.requestHandler(req, mockedRes) await mockedRes.headPromise middlewareRes = mockedRes } catch (e) { From a5be36c8529ded56ceb96e4b65b63df4dd00f17c Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 09:31:00 -0700 Subject: [PATCH 11/54] fix pages react aliasing with app dir --- packages/next/src/build/webpack-config.ts | 13 ++++++++----- packages/next/src/server/config.ts | 7 ++++++- test/e2e/app-dir/rsc-basic/rsc-basic.test.ts | 10 +++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 5ca9c091340c4..428247c7d4c0b 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -1113,6 +1113,14 @@ export default async function getBaseWebpackConfig( '@opentelemetry/api': 'next/dist/compiled/@opentelemetry/api', }), + ...(hasAppDir + ? createRSCAliases(bundledReactChannel, { + reactSharedSubset: false, + reactDomServerRenderingStub: false, + reactProductionProfiling, + }) + : {}), + ...(config.images.loaderFile ? { 'next/dist/shared/lib/image-loader': config.images.loaderFile, @@ -2025,11 +2033,6 @@ export default async function getBaseWebpackConfig( [require.resolve('next/dynamic')]: require.resolve( 'next/dist/shared/lib/app-dynamic' ), - ...createRSCAliases(bundledReactChannel, { - reactSharedSubset: false, - reactDomServerRenderingStub: false, - reactProductionProfiling, - }), }, }, }, diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index 20fad55e535ce..03631ab93ecaa 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -20,6 +20,7 @@ import { loadEnvConfig, updateInitialEnv } from '@next/env' import { flushAndExit } from '../telemetry/flush-and-exit' import { findRootDir } from '../lib/find-root' import { setHttpClientAndAgentOptions } from './setup-http-agent-env' +import { pathHasPrefix } from '../shared/lib/router/utils/path-has-prefix' export { DomainLocale, NextConfig, normalizeConfig } from './config-shared' @@ -308,7 +309,11 @@ function assignDefaults( ) } - if (images.path === imageConfigDefault.path && result.basePath) { + if ( + result.basePath && + images.path && + !pathHasPrefix(images.path, result.basePath) + ) { images.path = `${result.basePath}${images.path}` } diff --git a/test/e2e/app-dir/rsc-basic/rsc-basic.test.ts b/test/e2e/app-dir/rsc-basic/rsc-basic.test.ts index 1f83176fa745f..fc1916897c808 100644 --- a/test/e2e/app-dir/rsc-basic/rsc-basic.test.ts +++ b/test/e2e/app-dir/rsc-basic/rsc-basic.test.ts @@ -450,8 +450,8 @@ createNextDescribe( expect(await res.text()).toBe('Hello from import-test.js') }) - it('should use stable react for pages', async () => { - const ssrPaths = ['/pages-react', '/pages-react-edge'] + it('should use bundled react for pages with app', async () => { + const ssrPaths = ['/pages-react', '/edge-pages-react'] const promises = ssrPaths.map(async (pathname) => { const resPages$ = await next.render$(pathname) const ssrPagesReactVersions = [ @@ -461,7 +461,7 @@ createNextDescribe( ] ssrPagesReactVersions.forEach((version) => { - expect(version).not.toMatch('-canary-') + expect(version).toMatch('-canary-') }) }) await Promise.all(promises) @@ -496,10 +496,10 @@ createNextDescribe( `) browserPagesReactVersions.forEach((version) => - expect(version).not.toMatch('-canary-') + expect(version).toMatch('-canary-') ) browserEdgePagesReactVersions.forEach((version) => - expect(version).not.toMatch('-canary-') + expect(version).toMatch('-canary-') ) }) From 7cc363fbf0001e4e9fbdb6b4f27b875f21d77056 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 09:32:39 -0700 Subject: [PATCH 12/54] lint --- packages/next/src/server/lib/router-utils/resolve-routes.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index 33eaabd659555..14072a1c94180 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -27,11 +27,7 @@ import { detectDomainLocale } from '../../../shared/lib/i18n/detect-domain-local import { normalizeLocalePath } from '../../../shared/lib/i18n/normalize-locale-path' import { removePathPrefix } from '../../../shared/lib/router/utils/remove-path-prefix' -import { - NextUrlWithParsedQuery, - addRequestMeta, - getRequestMeta, -} from '../../request-meta' +import { NextUrlWithParsedQuery, addRequestMeta } from '../../request-meta' import { compileNonPath, matchHas, From 6c79f694028ea68bcba94f3cbde8d80d506d6b55 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 10:04:35 -0700 Subject: [PATCH 13/54] config file watching --- packages/next/src/cli/next-dev.ts | 27 +--------------- packages/next/src/server/lib/start-server.ts | 32 ++++++++++++++++++- packages/next/src/server/next.ts | 33 +++++++++++--------- 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index 7badb769eecfe..f7bd1bc4a61c5 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -4,7 +4,7 @@ import { getPort, printAndExit } from '../server/lib/utils' import * as Log from '../build/output/log' import { CliCommand } from '../lib/commands' import { getProjectDir } from '../lib/get-project-dir' -import { CONFIG_FILES, PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' +import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' import path from 'path' import { NextConfigComplete } from '../server/config-shared' import { traceGlobals } from '../trace/shared' @@ -14,7 +14,6 @@ import { findPagesDir } from '../lib/find-pages-dir' import { findRootDir } from '../lib/find-root' import { fileExists, FileType } from '../lib/file-exists' import { getNpxCommand } from '../lib/helpers/get-npx-command' -import Watchpack from 'watchpack' import { resetEnv } from '@next/env' import { createSelfSignedCertificate } from '../lib/mkcert' import uploadTrace from '../trace/upload-trace' @@ -107,15 +106,6 @@ const handleSessionStop = async () => { process.on('SIGINT', handleSessionStop) process.on('SIGTERM', handleSessionStop) -function watchConfigFiles( - dirToWatch: string, - onChange: (filename: string) => void -) { - const wp = new Watchpack() - wp.watch({ files: CONFIG_FILES.map((file) => path.join(dirToWatch, file)) }) - wp.on('change', onChange) -} - const nextDev: CliCommand = async (args) => { if (args['--help']) { console.log(` @@ -322,21 +312,6 @@ const nextDev: CliCommand = async (args) => { } } - // TODO: move config file watching to inner process - watchConfigFiles(devServerOptions.dir, async (filename) => { - if (process.env.__NEXT_DISABLE_MEMORY_WATCHER) { - Log.info( - `Detected change, manual restart required due to '__NEXT_DISABLE_MEMORY_WATCHER' usage` - ) - return - } - Log.warn( - `\n> Found a change in ${path.basename( - filename - )}. Restarting the server to apply the changes...` - ) - }) - await runDevServer(false) } } diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 2641d9afadb0f..3a54c1699498a 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -4,19 +4,22 @@ import '../require-hook' import type { IncomingMessage, ServerResponse } from 'http' +import fs from 'fs' +import path from 'path' import http from 'http' import https from 'https' +import Watchpack from 'watchpack' import * as Log from '../../build/output/log' import setupDebug from 'next/dist/compiled/debug' import { getDebugPort } from './utils' import { formatHostname } from './format-hostname' import { initialize } from './router-server' -import fs from 'fs' import { WorkerRequestHandler, WorkerUpgradeHandler, } from './setup-server-worker' import { checkIsNodeDebugging } from './is-node-debugging' +import { CONFIG_FILES } from '../../shared/lib/constants' const debug = setupDebug('next:start-server') if (process.env.NEXT_CPU_PROF) { @@ -265,4 +268,31 @@ export async function startServer({ }) server.listen(port, hostname) }) + + if (isDev) { + function watchConfigFiles( + dirToWatch: string, + onChange: (filename: string) => void + ) { + const wp = new Watchpack() + wp.watch({ + files: CONFIG_FILES.map((file) => path.join(dirToWatch, file)), + }) + wp.on('change', onChange) + } + watchConfigFiles(dir, async (filename) => { + if (process.env.__NEXT_DISABLE_MEMORY_WATCHER) { + Log.info( + `Detected change, manual restart required due to '__NEXT_DISABLE_MEMORY_WATCHER' usage` + ) + return + } + Log.warn( + `\n> Found a change in ${path.basename( + filename + )}. Restarting the server to apply the changes...` + ) + process.exit(0) + }) + } } diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index 8dccea469c99b..1dee5d2cb253b 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -11,30 +11,35 @@ import { spawnSync } from 'child_process' // worker we need to re-spawn with correct args // we can't do this if imported in jest test file otherwise // it duplicates tests -if ( - typeof jest === 'undefined' && - !process.env.NEXT_PRIVATE_WORKER && - process.env.__NEXT_PRIVATE_PREBUNDLED_REACT -) { - for (let i = 0; i < 1; i++) { +if (typeof jest === 'undefined' && !process.env.NEXT_PRIVATE_WORKER) { + const nodePath = process.argv0 + const newArgs = [ + '--experimental-loader', + 'next/dist/esm/server/esm-loader.mjs', + '--no-warnings', + ...process.argv.splice(1), + ] + function startWorker() { try { - const newArgs = [ - '--experimental-loader', - 'next/dist/esm/server/esm-loader.mjs', - '--no-warnings', - ...process.argv.splice(1), - ] - spawnSync(process.argv[0], newArgs, { + console.log(process.argv[0], newArgs) + const result = spawnSync(nodePath, newArgs, { stdio: 'inherit', env: { ...process.env, NEXT_PRIVATE_WORKER: '1', }, }) + console.log('got result', result) + + if (!(result.signal || result.status)) { + startWorker() + } } catch (err) { - console.error(`spawn failed retrying`, err) + console.error(err) + process.exit(1) } } + startWorker() } import './require-hook' From 2c7d48e36358c995ef012109e8b8b8c873362472 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 10:32:37 -0700 Subject: [PATCH 14/54] fix build workers aliasing and config restart --- packages/next/src/bin/next.ts | 5 ++++- packages/next/src/build/index.ts | 11 +++++------ packages/next/src/server/next.ts | 8 +++++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index a9b06e5f466b9..68b3476da4e64 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -136,7 +136,10 @@ async function main() { // un-necessarily const config = await loadConfig( command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, - dir + dir, + undefined, + undefined, + true ) let dirsResult: ReturnType | undefined = undefined diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 94f5ebc8178b0..eb4badf3fbcdc 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1252,12 +1252,11 @@ export default async function build( ...process.env, __NEXT_INCREMENTAL_CACHE_IPC_PORT: ipcPort + '', __NEXT_INCREMENTAL_CACHE_IPC_KEY: ipcValidationKey, - __NEXT_PRIVATE_PREBUNDLED_REACT: - type === 'app' - ? config.experimental.serverActions - ? 'experimental' - : 'next' - : '', + __NEXT_PRIVATE_PREBUNDLED_REACT: hasAppDir + ? config.experimental.serverActions + ? 'experimental' + : 'next' + : '', }, }, enableWorkerThreads: config.experimental.workerThreads, diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index 1dee5d2cb253b..f046d58dbf604 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -21,7 +21,6 @@ if (typeof jest === 'undefined' && !process.env.NEXT_PRIVATE_WORKER) { ] function startWorker() { try { - console.log(process.argv[0], newArgs) const result = spawnSync(nodePath, newArgs, { stdio: 'inherit', env: { @@ -29,11 +28,14 @@ if (typeof jest === 'undefined' && !process.env.NEXT_PRIVATE_WORKER) { NEXT_PRIVATE_WORKER: '1', }, }) - console.log('got result', result) - if (!(result.signal || result.status)) { + if ( + !(result.signal || result.status) && + process.env.NODE_ENV === 'development' + ) { startWorker() } + process.exit(0) } catch (err) { console.error(err) process.exit(1) From d7164a86d69b34a1958b815a04ba42f125331f0b Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 10:35:56 -0700 Subject: [PATCH 15/54] lint fix --- packages/next/src/build/index.ts | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index eb4badf3fbcdc..04a2ada2b54b0 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1208,11 +1208,7 @@ export default async function build( ) : config.experimental.cpus || 4 - function createStaticWorker( - type: 'app' | 'pages', - ipcPort: number, - ipcValidationKey: string - ) { + function createStaticWorker(ipcPort: number, ipcValidationKey: string) { let infoPrinted = false return new Worker(staticWorkerPath, { @@ -1315,13 +1311,9 @@ export default async function build( config.experimental.allowedRevalidateHeaderKeys, }) - const pagesStaticWorkers = createStaticWorker( - 'pages', - ipcPort, - ipcValidationKey - ) + const pagesStaticWorkers = createStaticWorker(ipcPort, ipcValidationKey) const appStaticWorkers = isAppDirEnabled - ? createStaticWorker('app', ipcPort, ipcValidationKey) + ? createStaticWorker(ipcPort, ipcValidationKey) : undefined const analysisBegin = process.hrtime() @@ -3301,12 +3293,8 @@ export default async function build( const exportApp: typeof import('../export').default = require('../export').default - const pagesWorker = createStaticWorker( - 'pages', - ipcPort, - ipcValidationKey - ) - const appWorker = createStaticWorker('app', ipcPort, ipcValidationKey) + const pagesWorker = createStaticWorker(ipcPort, ipcValidationKey) + const appWorker = createStaticWorker(ipcPort, ipcValidationKey) const options: ExportOptions = { isInvokedFromCli: false, From 36621d76f9c78acb4a483c6fed05779760c7dde6 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 10:43:35 -0700 Subject: [PATCH 16/54] fix env resetting --- packages/next/src/bin/next.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 68b3476da4e64..31752c8fdb93e 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -148,12 +148,12 @@ async function main() { } catch (_) { // handle this error further down } + process.env = origEnv if (dirsResult?.appDir) { // we need to reset env if we are going to create // the worker process with the esm loader so that the // initial env state is correct - process.env = origEnv process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = config.experimental .serverActions ? 'experimental' From 6e2cee331747ae9dcd1d909f84209ffe6f715eec Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 11:03:27 -0700 Subject: [PATCH 17/54] fix env handling --- packages/next/src/bin/next.ts | 10 ++++++++-- packages/next/src/server/lib/router-server.ts | 5 +---- packages/next/src/server/next.ts | 7 ++++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 31752c8fdb93e..0084b11361ea7 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -126,7 +126,10 @@ async function main() { const currentArgsSpec = commandArgs[command]() const validatedArgs = getValidatedArgs(currentArgsSpec, forwardedArgs) - if (command === 'start' || command === 'dev') { + if ( + (command === 'start' || command === 'dev') && + !process.env.NEXT_PRIVATE_WORKER + ) { const dir = getProjectDir( process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0] ) @@ -148,7 +151,10 @@ async function main() { } catch (_) { // handle this error further down } - process.env = origEnv + + if (dirsResult?.appDir || process.env.NODE_ENV === 'development') { + process.env = origEnv + } if (dirsResult?.appDir) { // we need to reset env if we are going to create diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index bec79babdc043..c8bb4901e8e07 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -72,10 +72,7 @@ export async function initialize(opts: { const config = await loadConfig( opts.dev ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, - opts.dir, - undefined, - undefined, - true + opts.dir ) let compress: ReturnType | undefined diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index f046d58dbf604..75f4ff1b42640 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -11,7 +11,12 @@ import { spawnSync } from 'child_process' // worker we need to re-spawn with correct args // we can't do this if imported in jest test file otherwise // it duplicates tests -if (typeof jest === 'undefined' && !process.env.NEXT_PRIVATE_WORKER) { +if ( + typeof jest === 'undefined' && + !process.env.NEXT_PRIVATE_WORKER && + (process.env.__NEXT_PRIVATE_PREBUNDLED_REACT || + process.env.NODE_ENV === 'development') +) { const nodePath = process.argv0 const newArgs = [ '--experimental-loader', From d525bf6fc0e630bdcf528e4dadcc8532dd9b85f6 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 31 Aug 2023 12:35:01 -0700 Subject: [PATCH 18/54] apply esm loader for build workers --- packages/next/src/build/index.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 04a2ada2b54b0..cb58208262fde 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1244,6 +1244,11 @@ export default async function build( }, numWorkers, forkOptions: { + execArgv: [ + '--experimental-loader', + 'next/dist/esm/server/esm-loader.mjs', + '--no-warnings', + ], env: { ...process.env, __NEXT_INCREMENTAL_CACHE_IPC_PORT: ipcPort + '', From 97e85bb0a7cdc495b20dbfbe505d36b547a52330 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Fri, 1 Sep 2023 14:14:17 +0200 Subject: [PATCH 19/54] Revert change to config.ts --- packages/next/src/server/config.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index 03631ab93ecaa..cfdf251330d82 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -20,7 +20,6 @@ import { loadEnvConfig, updateInitialEnv } from '@next/env' import { flushAndExit } from '../telemetry/flush-and-exit' import { findRootDir } from '../lib/find-root' import { setHttpClientAndAgentOptions } from './setup-http-agent-env' -import { pathHasPrefix } from '../shared/lib/router/utils/path-has-prefix' export { DomainLocale, NextConfig, normalizeConfig } from './config-shared' @@ -309,17 +308,17 @@ function assignDefaults( ) } - if ( - result.basePath && - images.path && - !pathHasPrefix(images.path, result.basePath) - ) { + if (images.path === imageConfigDefault.path && result.basePath) { images.path = `${result.basePath}${images.path}` } // Append trailing slash for non-default loaders and when trailingSlash is set - if (images.path && result.trailingSlash) { - if (images.path[images.path.length - 1] !== '/') { + if (images.path) { + if ( + (images.loader !== 'default' && + images.path[images.path.length - 1] !== '/') || + result.trailingSlash + ) { images.path += '/' } } From ca336524261af09b68304ed6ebd9a8016a441a5f Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Fri, 1 Sep 2023 16:49:54 -0700 Subject: [PATCH 20/54] prevent next.config defaults from being constantly assigned --- packages/next/src/bin/next.ts | 21 ++++++++++++--------- packages/next/src/server/config.ts | 3 +++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 0084b11361ea7..59e52a2514c62 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -135,15 +135,6 @@ async function main() { ) const origEnv = Object.assign({}, process.env) - // TODO: set config to env variable to be re-used so we don't reload - // un-necessarily - const config = await loadConfig( - command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, - dir, - undefined, - undefined, - true - ) let dirsResult: ReturnType | undefined = undefined try { @@ -156,6 +147,18 @@ async function main() { process.env = origEnv } + // TODO: set config to env variable to be re-used so we don't reload + // un-necessarily + const config = await loadConfig( + command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, + dir, + undefined, + undefined, + true + ) + + process.env.__NEXT_PRIVATE_CONFIG_INITIALIZED = '1' + if (dirsResult?.appDir) { // we need to reset env if we are going to create // the worker process with the esm loader so that the diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index cfdf251330d82..507f26005a475 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -186,6 +186,9 @@ function assignDefaults( const result = { ...defaultConfig, ...config } + // avoid mutating config if it was already initialized + if (process.env.__NEXT_PRIVATE_CONFIG_INITIALIZED) return result + if (result.output === 'export') { if (result.i18n) { throw new Error( From 60d00d512985ca9e21d54e708235f3b1b32a84de Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Fri, 1 Sep 2023 16:57:06 -0700 Subject: [PATCH 21/54] fix args --- packages/next/src/bin/next.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 59e52a2514c62..65e424ba75e51 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -138,7 +138,7 @@ async function main() { let dirsResult: ReturnType | undefined = undefined try { - dirsResult = findPagesDir(dir, true) + dirsResult = findPagesDir(dir) } catch (_) { // handle this error further down } From d1f3f7393901d746fec3aff2a80ccfaedca35aa7 Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Fri, 1 Sep 2023 17:04:05 -0700 Subject: [PATCH 22/54] Revert "prevent next.config defaults from being constantly assigned" This reverts commit ca336524261af09b68304ed6ebd9a8016a441a5f. --- packages/next/src/bin/next.ts | 21 +++++++++------------ packages/next/src/server/config.ts | 3 --- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 65e424ba75e51..fc2feb7628fd9 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -135,6 +135,15 @@ async function main() { ) const origEnv = Object.assign({}, process.env) + // TODO: set config to env variable to be re-used so we don't reload + // un-necessarily + const config = await loadConfig( + command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, + dir, + undefined, + undefined, + true + ) let dirsResult: ReturnType | undefined = undefined try { @@ -147,18 +156,6 @@ async function main() { process.env = origEnv } - // TODO: set config to env variable to be re-used so we don't reload - // un-necessarily - const config = await loadConfig( - command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, - dir, - undefined, - undefined, - true - ) - - process.env.__NEXT_PRIVATE_CONFIG_INITIALIZED = '1' - if (dirsResult?.appDir) { // we need to reset env if we are going to create // the worker process with the esm loader so that the diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index d5e991052537f..3036890a1963b 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -186,9 +186,6 @@ function assignDefaults( const result = { ...defaultConfig, ...config } - // avoid mutating config if it was already initialized - if (process.env.__NEXT_PRIVATE_CONFIG_INITIALIZED) return result - if (result.output === 'export') { if (result.i18n) { throw new Error( From fef7850cda18e1c72bfdc3ccfaba4ca19f916723 Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Fri, 1 Sep 2023 17:21:47 -0700 Subject: [PATCH 23/54] attempt to fix image test --- packages/next/src/server/config.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index 3036890a1963b..c4b637f23bd8d 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -313,14 +313,12 @@ function assignDefaults( } // Append trailing slash for non-default loaders and when trailingSlash is set - if (images.path) { - if ( - (images.loader !== 'default' && - images.path[images.path.length - 1] !== '/') || - result.trailingSlash - ) { - images.path += '/' - } + if ( + images.path && + !images.path.endsWith('/') && + (images.loader !== 'default' || result.trailingSlash) + ) { + images.path += '/' } if (images.loaderFile) { From efc6ce59c939b81e2e3c2f44511ed136705ea55a Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Sat, 2 Sep 2023 08:27:20 -0700 Subject: [PATCH 24/54] check for app dir when bundling experimental react in standalone --- packages/next/src/build/index.ts | 3 ++- packages/next/src/build/utils.ts | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index be31b579e94b6..f4363f6470b82 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -2384,7 +2384,8 @@ export default async function build( outputFileTracingRoot, requiredServerFiles.config, middlewareManifest, - hasInstrumentationHook + hasInstrumentationHook, + hasAppDir ) }) } diff --git a/packages/next/src/build/utils.ts b/packages/next/src/build/utils.ts index 867429c2ba901..a4c2e9ef8634e 100644 --- a/packages/next/src/build/utils.ts +++ b/packages/next/src/build/utils.ts @@ -1804,7 +1804,8 @@ export async function copyTracedFiles( tracingRoot: string, serverConfig: { [key: string]: any }, middlewareManifest: MiddlewareManifest, - hasInstrumentationHook: boolean + hasInstrumentationHook: boolean, + hasAppDir: boolean ) { const outputPath = path.join(distDir, 'standalone') let moduleType = false @@ -1967,9 +1968,11 @@ const nextConfig = ${JSON.stringify({ })} process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(nextConfig) -process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = nextConfig.experimental && nextConfig.experimental.serverActions - ? 'experimental' - : 'next' +process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = ${hasAppDir} + ? nextConfig.experimental && nextConfig.experimental.serverActions + ? 'experimental' + : 'next' + : ''; if ( Number.isNaN(keepAliveTimeout) || From 373d58d8949405d6dc4b24d69cdac9f1dda82c1d Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Sat, 2 Sep 2023 08:27:38 -0700 Subject: [PATCH 25/54] tweak test case & re-add pathPrefix check --- packages/next/src/server/config.ts | 7 ++++++- test/e2e/app-dir/app/index.test.ts | 4 +--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/next/src/server/config.ts b/packages/next/src/server/config.ts index c4b637f23bd8d..a3909fc8ed758 100644 --- a/packages/next/src/server/config.ts +++ b/packages/next/src/server/config.ts @@ -20,6 +20,7 @@ import { loadEnvConfig, updateInitialEnv } from '@next/env' import { flushAndExit } from '../telemetry/flush-and-exit' import { findRootDir } from '../lib/find-root' import { setHttpClientAndAgentOptions } from './setup-http-agent-env' +import { pathHasPrefix } from '../shared/lib/router/utils/path-has-prefix' export { DomainLocale, NextConfig, normalizeConfig } from './config-shared' @@ -308,7 +309,11 @@ function assignDefaults( ) } - if (images.path === imageConfigDefault.path && result.basePath) { + if ( + images.path === imageConfigDefault.path && + result.basePath && + !pathHasPrefix(images.path, result.basePath) + ) { images.path = `${result.basePath}${images.path}` } diff --git a/test/e2e/app-dir/app/index.test.ts b/test/e2e/app-dir/app/index.test.ts index 9b4404f888462..54cf83cec0c7f 100644 --- a/test/e2e/app-dir/app/index.test.ts +++ b/test/e2e/app-dir/app/index.test.ts @@ -310,9 +310,7 @@ createNextDescribe( const res = await next.fetch('/dashboard') expect(res.headers.get('x-edge-runtime')).toBe('1') expect(res.headers.get('vary')).toBe( - isNextDeploy - ? 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url' - : 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url, Accept-Encoding' + 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Next-Url' ) }) From f50d86a793a5a6ac1099230b8bc5e11dc1704b32 Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Sat, 2 Sep 2023 09:44:17 -0700 Subject: [PATCH 26/54] fix cleanup exit code behavior --- packages/next/src/bin/next.ts | 5 +---- packages/next/src/server/lib/start-server.ts | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index fc2feb7628fd9..82f7b3c59cce8 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -139,10 +139,7 @@ async function main() { // un-necessarily const config = await loadConfig( command === 'dev' ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, - dir, - undefined, - undefined, - true + dir ) let dirsResult: ReturnType | undefined = undefined diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 3a54c1699498a..7d2ef87b0d1d6 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -228,10 +228,10 @@ export async function startServer({ } try { - const cleanup = () => { + const cleanup = (code: number | null) => { debug('start-server process cleanup') server.close() - process.exit(0) + process.exit(code ?? 0) } const exception = (err: Error) => { // This is the render worker, we keep the process alive From e62db558ed6f5c115464ff71b74f76015096615d Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Sun, 3 Sep 2023 13:16:36 -0700 Subject: [PATCH 27/54] tweak restart behavior --- packages/next/src/server/lib/render-server.ts | 2 -- .../src/server/lib/setup-server-worker.ts | 4 +-- packages/next/src/server/lib/start-server.ts | 3 ++- packages/next/src/server/next.ts | 12 ++++----- test/integration/cli/test/index.test.js | 26 ++++++++++++------- test/lib/next-test-utils.ts | 7 +++-- 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/packages/next/src/server/lib/render-server.ts b/packages/next/src/server/lib/render-server.ts index 2512353f1fe92..0b0186310b321 100644 --- a/packages/next/src/server/lib/render-server.ts +++ b/packages/next/src/server/lib/render-server.ts @@ -3,8 +3,6 @@ import type { NextServer, RequestHandler } from '../next' import next from '../next' import { PropagateToWorkersField } from './router-utils/types' -export const WORKER_SELF_EXIT_CODE = 77 - let result: | undefined | { diff --git a/packages/next/src/server/lib/setup-server-worker.ts b/packages/next/src/server/lib/setup-server-worker.ts index ba7d195fa4ff2..eb08e1bc5d1da 100644 --- a/packages/next/src/server/lib/setup-server-worker.ts +++ b/packages/next/src/server/lib/setup-server-worker.ts @@ -17,7 +17,7 @@ process.on('uncaughtException', (err) => { console.error(err) }) -export const WORKER_SELF_EXIT_CODE = 77 +export const RESTART_EXIT_CODE = 77 const MAXIMUM_HEAP_SIZE_ALLOWED = (v8.getHeapStatistics().heap_size_limit / 1024 / 1024) * 0.9 @@ -67,7 +67,7 @@ export async function initializeServerWorker( 'The server is running out of memory, restarting to free up memory.' ) server.close() - process.exit(WORKER_SELF_EXIT_CODE) + process.exit(RESTART_EXIT_CODE) } }) }) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 7d2ef87b0d1d6..1127673ef7560 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -15,6 +15,7 @@ import { getDebugPort } from './utils' import { formatHostname } from './format-hostname' import { initialize } from './router-server' import { + RESTART_EXIT_CODE, WorkerRequestHandler, WorkerUpgradeHandler, } from './setup-server-worker' @@ -292,7 +293,7 @@ export async function startServer({ filename )}. Restarting the server to apply the changes...` ) - process.exit(0) + process.exit(RESTART_EXIT_CODE) }) } } diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index 75f4ff1b42640..f3be1324cb368 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -4,7 +4,11 @@ import type { UrlWithParsedQuery } from 'url' import type { NextConfigComplete } from './config-shared' import type { IncomingMessage, ServerResponse } from 'http' import type { NextUrlWithParsedQuery } from './request-meta' - +import { + RESTART_EXIT_CODE, + WorkerRequestHandler, + WorkerUpgradeHandler, +} from './lib/setup-server-worker' import { spawnSync } from 'child_process' // if we are not inside of the esm loader enabled @@ -35,7 +39,7 @@ if ( }) if ( - !(result.signal || result.status) && + result.status === RESTART_EXIT_CODE && process.env.NODE_ENV === 'development' ) { startWorker() @@ -63,10 +67,6 @@ import { PHASE_PRODUCTION_SERVER } from '../shared/lib/constants' import { getTracer } from './lib/trace/tracer' import { NextServerSpan } from './lib/trace/constants' import { formatUrl } from '../shared/lib/router/utils/format-url' -import { - WorkerRequestHandler, - WorkerUpgradeHandler, -} from './lib/setup-server-worker' import { checkIsNodeDebugging } from './lib/is-node-debugging' let ServerImpl: typeof Server diff --git a/test/integration/cli/test/index.test.js b/test/integration/cli/test/index.test.js index 160bc39a0cfed..84ebeb19607c1 100644 --- a/test/integration/cli/test/index.test.js +++ b/test/integration/cli/test/index.test.js @@ -8,6 +8,7 @@ import { nextBuild, runNextCommand, runNextCommandDev, + killProcess, } from 'next-test-utils' import fs from 'fs-extra' import path, { join } from 'path' @@ -21,7 +22,8 @@ const dirDuplicateSass = join(__dirname, '../duplicate-sass') const testExitSignal = async ( killSignal = '', args = [], - readyRegex = /Creating an optimized production/ + readyRegex = /Creating an optimized production/, + expectedExitSignal ) => { let instance const killSigint = (inst) => { @@ -38,14 +40,18 @@ const testExitSignal = async ( }).catch((err) => expect.fail(err.message)) await check(() => output, readyRegex) - instance.kill(killSignal) + await killProcess(instance.pid, killSignal) const { code, signal } = await cmdPromise - // Node can only partially emulate signals on Windows. Our signal handlers won't affect the exit code. - // See: https://nodejs.org/api/process.html#process_signal_events - const expectedExitSignal = process.platform === `win32` ? killSignal : null - expect(signal).toBe(expectedExitSignal) - expect(code).toBe(0) + + if (!expectedExitSignal) { + // Node can only partially emulate signals on Windows. Our signal handlers won't affect the exit code. + // See: https://nodejs.org/api/process.html#process_signal_events + expectedExitSignal = process.platform === `win32` ? killSignal : null + expect(code).toBe(0) + } else { + expect(signal).toBe(expectedExitSignal) + } } describe('CLI Usage', () => { @@ -633,7 +639,8 @@ describe('CLI Usage', () => { await testExitSignal( 'SIGINT', ['dev', dirBasic, '-p', port], - /started server on/ + /started server on/, + 'SIGINT' ) }) test('should exit when SIGTERM is signalled', async () => { @@ -641,7 +648,8 @@ describe('CLI Usage', () => { await testExitSignal( 'SIGTERM', ['dev', dirBasic, '-p', port], - /started server on/ + /started server on/, + 'SIGTERM' ) }) diff --git a/test/lib/next-test-utils.ts b/test/lib/next-test-utils.ts index 4c723d6bde913..b2e74171b1c52 100644 --- a/test/lib/next-test-utils.ts +++ b/test/lib/next-test-utils.ts @@ -518,9 +518,12 @@ export function buildTS( }) } -export async function killProcess(pid: number): Promise { +export async function killProcess( + pid: number, + signal: string | number = 'SIGTERM' +): Promise { return await new Promise((resolve, reject) => { - treeKill(pid, (err) => { + treeKill(pid, signal, (err) => { if (err) { if ( process.platform === 'win32' && From 50567fa2115368f615335f15a7b9feeb7e75ab44 Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Mon, 4 Sep 2023 16:53:27 -0700 Subject: [PATCH 28/54] pass parsedUrl to middleware request & copy over NEXT_REQUEST_META --- packages/next/src/server/base-http/node.ts | 2 +- packages/next/src/server/lib/router-utils/resolve-routes.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/next/src/server/base-http/node.ts b/packages/next/src/server/base-http/node.ts index 7d99b8c198b1e..64d8f8cedf96e 100644 --- a/packages/next/src/server/base-http/node.ts +++ b/packages/next/src/server/base-http/node.ts @@ -17,7 +17,7 @@ type Req = IncomingMessage & { export class NodeNextRequest extends BaseNextRequest { public headers = this._req.headers; - [NEXT_REQUEST_META]: RequestMeta = {} + [NEXT_REQUEST_META]: RequestMeta = this._req[NEXT_REQUEST_META] || {} get originalRequest() { // Need to mimic these changes to the original req object for places where we use it: diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index 14072a1c94180..1382f999bed0f 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -478,7 +478,7 @@ export function getResolveRoutes( mockedRes.on('close', () => { readableController.close() }) - initResult?.requestHandler(req, mockedRes) + initResult?.requestHandler(req, mockedRes, parsedUrl) await mockedRes.headPromise middlewareRes = mockedRes } catch (e) { From 1bd7d3f7d450c26764d67f63de30a3ad94c75f8a Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Tue, 5 Sep 2023 07:11:48 -0700 Subject: [PATCH 29/54] questionably fix watch-config-file failure --- packages/next/src/server/dev/hot-reloader-webpack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/server/dev/hot-reloader-webpack.ts b/packages/next/src/server/dev/hot-reloader-webpack.ts index f480c0480d27f..33fffe4f7e8df 100644 --- a/packages/next/src/server/dev/hot-reloader-webpack.ts +++ b/packages/next/src/server/dev/hot-reloader-webpack.ts @@ -712,7 +712,7 @@ export default class HotReloader implements NextJsHotReloaderInterface { this.versionInfo = await this.getVersionInfo( startSpan, - !!process.env.NEXT_TEST_MODE || this.telemetry.isEnabled + !!process.env.__NEXT_TEST_MODE || this.telemetry.isEnabled ) await this.clean(startSpan) From 35094c2b0a36bf6fbeedc59201b9626b1e8df249 Mon Sep 17 00:00:00 2001 From: Zack Tanner Date: Tue, 5 Sep 2023 08:51:23 -0700 Subject: [PATCH 30/54] attempt a fix by adding node-polyfill-web-streams --- packages/next/src/server/lib/router-utils/resolve-routes.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index 1382f999bed0f..c77833e8d1083 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -35,6 +35,8 @@ import { } from '../../../shared/lib/router/utils/prepare-destination' import { createRequestResponseMocks } from '../mock-request' +import '../../node-polyfill-web-streams' + const debug = setupDebug('next:router-server:resolve-routes') export function getResolveRoutes( From 9be1386a253cc55282fb3966738814a5fcb8b631 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 16:25:18 -0700 Subject: [PATCH 31/54] rework middleware handling --- packages/next/src/server/base-server.ts | 2 +- .../next/src/server/lib/route-resolver.ts | 124 +++++------------- packages/next/src/server/lib/router-server.ts | 2 + .../server/lib/router-utils/resolve-routes.ts | 56 ++++++-- .../src/server/lib/router-utils/setup-dev.ts | 4 +- packages/next/src/server/next-server.ts | 24 +++- 6 files changed, 104 insertions(+), 108 deletions(-) diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 4ab6727a95f08..f69c20a1d1bd8 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -1299,7 +1299,7 @@ export default abstract class Server { return this.renderError(null, req, res, '/_error', {}) } - if (this.minimalMode || this.renderOpts.dev) { + if (this.minimalMode || this.renderOpts.dev || (err as any).bubble) { throw err } this.logError(getProperError(err)) diff --git a/packages/next/src/server/lib/route-resolver.ts b/packages/next/src/server/lib/route-resolver.ts index 47fa60aa9c36f..76378b56163ae 100644 --- a/packages/next/src/server/lib/route-resolver.ts +++ b/packages/next/src/server/lib/route-resolver.ts @@ -110,83 +110,6 @@ export async function makeResolver( } : {} - const middlewareServerAddr = await new Promise<{ - hostname: string - port: number - }>((resolve) => { - const srv = http.createServer(async (req, res) => { - const cloneableBody = getCloneableBody(req) - try { - const { run } = - require('../web/sandbox') as typeof import('../web/sandbox') - - const result = await run({ - distDir, - name: middlewareInfo.name || '/', - paths: middlewareInfo.paths || [], - edgeFunctionEntry: middlewareInfo, - request: { - headers: req.headers, - method: req.method || 'GET', - nextConfig: { - i18n: nextConfig.i18n, - basePath: nextConfig.basePath, - trailingSlash: nextConfig.trailingSlash, - }, - url: `http://${fetchHostname}:${port}${req.url}`, - body: cloneableBody, - signal: signalFromNodeResponse(res), - }, - useCache: true, - onWarning: console.warn, - }) - - for (let [key, value] of result.response.headers) { - if (key.toLowerCase() !== 'set-cookie') continue - - // Clear existing header. - result.response.headers.delete(key) - - // Append each cookie individually. - const cookies = splitCookiesString(value) - for (const cookie of cookies) { - result.response.headers.append(key, cookie) - } - } - - for (const [key, value] of Object.entries( - toNodeOutgoingHttpHeaders(result.response.headers) - )) { - if (key !== 'content-encoding' && value !== undefined) { - res.setHeader(key, value as string | string[]) - } - } - res.statusCode = result.response.status - - if (result.response.body) { - await pipeReadable(result.response.body, res) - } else { - res.end() - } - } catch (err) { - console.error(err) - res.statusCode = 500 - res.end('Internal Server Error') - } - }) - srv.on('listening', () => { - const srvAddr = srv.address() - if (!srvAddr || typeof srvAddr === 'string') { - throw new Error("Failed to determine middleware's host/port.") - } - resolve({ - hostname: srvAddr.address, - port: srvAddr.port, - }) - }) - srv.listen(0) - }) - if (middleware?.files.length) { fsChecker.middlewareMatcher = getMiddlewareRouteMatcher( middleware.matcher?.map((item) => ({ @@ -212,20 +135,42 @@ export async function makeResolver( async initialize() { return { async requestHandler(req, res) { - fetch( - `http://localhost:${middlewareServerAddr.port}${req.url || ''}` - ).then((fetchRes) => { - for (const [key, value] of fetchRes.headers) { - res.setHeader(key, value) - } - if (fetchRes.body) { - Stream.Readable.fromWeb(fetchRes.body as any).pipe(res) - } else { - res.end() - } + if (!req.headers['x-middleware-invoke']) { + throw new Error(`Invariant unexpected request handler call`) + } + + const cloneableBody = getCloneableBody(req) + const { run } = + require('../web/sandbox') as typeof import('../web/sandbox') + + const result = await run({ + distDir, + name: middlewareInfo.name || '/', + paths: middlewareInfo.paths || [], + edgeFunctionEntry: middlewareInfo, + request: { + headers: req.headers, + method: req.method || 'GET', + nextConfig: { + i18n: nextConfig.i18n, + basePath: nextConfig.basePath, + trailingSlash: nextConfig.trailingSlash, + }, + url: `http://${fetchHostname}:${port}${req.url}`, + body: cloneableBody, + signal: signalFromNodeResponse(res), + }, + useCache: true, + onWarning: console.warn, }) + + const err = new Error() + ;(err as any).result = result + throw err + }, + async upgradeHandler() { + throw new Error(`Invariant: unexpected upgrade handler call`) }, - async upgradeHandler() {}, } }, deleteAppClientCache() {}, @@ -243,6 +188,7 @@ export async function makeResolver( ): Promise { const routeResult = await resolveRoutes({ req, + res, isUpgradeReq: false, signal: signalFromNodeResponse(res), }) diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 664c057195037..a04d05ddb520e 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -368,6 +368,7 @@ export async function initialize(opts: { matchedOutput, } = await resolveRoutes({ req, + res, isUpgradeReq: false, signal: signalFromNodeResponse(res), invokedOutputs, @@ -661,6 +662,7 @@ export async function initialize(opts: { const { matchedOutput, parsedUrl } = await resolveRoutes({ req, + res: socket as any, isUpgradeReq: true, signal: signalFromNodeResponse(socket), }) diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index c77833e8d1083..81e94bbd24db2 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -1,6 +1,6 @@ import type { TLSSocket } from 'tls' import type { FsOutput } from './filesystem' -import type { IncomingMessage } from 'http' +import type { IncomingMessage, ServerResponse } from 'http' import type { NextConfigComplete } from '../../config-shared' import type { RenderWorker, initialize } from '../router-server' import type { PatchMatcher } from '../../../shared/lib/router/utils/path-match' @@ -96,10 +96,12 @@ export function getResolveRoutes( async function resolveRoutes({ req, + res, isUpgradeReq, invokedOutputs, }: { req: IncomingMessage + res: ServerResponse isUpgradeReq: boolean signal: AbortSignal invokedOutputs?: Set @@ -453,7 +455,7 @@ export function getResolveRoutes( debug('invoking middleware', req.url, invokeHeaders) - let middlewareRes + let middlewareRes: Response | undefined = undefined let bodyStream: ReadableStream | undefined = undefined try { let readableController: ReadableStreamController @@ -467,12 +469,6 @@ export function getResolveRoutes( }, }) - bodyStream = new ReadableStream({ - start(controller) { - readableController = controller - }, - }) - const initResult = await renderWorkers.pages?.initialize( renderWorkerOpts ) @@ -480,9 +476,27 @@ export function getResolveRoutes( mockedRes.on('close', () => { readableController.close() }) - initResult?.requestHandler(req, mockedRes, parsedUrl) - await mockedRes.headPromise - middlewareRes = mockedRes + + try { + await initResult?.requestHandler(req, res, parsedUrl) + } catch (err: any) { + if (!('result' in err) || !('response' in err.result)) { + throw err + } + middlewareRes = err.result.response as Response + res.statusCode = middlewareRes.status + + if (middlewareRes.body) { + bodyStream = middlewareRes.body + } else if (middlewareRes.status !== 200) { + bodyStream = new ReadableStream({ + start(controller) { + controller.enqueue('') + controller.close() + }, + }) + } + } } catch (e) { // If the client aborts before we can receive a response object // (when the headers are flushed), then we can early exit without @@ -497,11 +511,25 @@ export function getResolveRoutes( throw e } + if (res.closed || res.finished) { + return { + parsedUrl, + resHeaders, + finished: true, + } + } + + if (!middlewareRes) { + throw new Error( + `Invariant: missing middleware response ${req.url}` + ) + } + const middlewareHeaders = toNodeOutgoingHttpHeaders( middlewareRes.headers ) as Record - debug('middleware res', middlewareRes.statusCode, middlewareHeaders) + debug('middleware res', middlewareRes.status, middlewareHeaders) if (middlewareHeaders['x-middleware-override-headers']) { const overriddenHeaders: Set = new Set() @@ -613,7 +641,7 @@ export function getResolveRoutes( parsedUrl, resHeaders, finished: true, - statusCode: middlewareRes.statusCode, + statusCode: middlewareRes.status, } } @@ -623,7 +651,7 @@ export function getResolveRoutes( resHeaders, finished: true, bodyStream, - statusCode: middlewareRes.statusCode, + statusCode: middlewareRes.status, } } } diff --git a/packages/next/src/server/lib/router-utils/setup-dev.ts b/packages/next/src/server/lib/router-utils/setup-dev.ts index 506427a177463..80ef42b1419b1 100644 --- a/packages/next/src/server/lib/router-utils/setup-dev.ts +++ b/packages/next/src/server/lib/router-utils/setup-dev.ts @@ -104,6 +104,7 @@ import { TurboPackConnectedAction, } from '../../dev/hot-reloader-types' import { debounce } from '../../utils' +import { deleteCache } from '../../../build/webpack/plugins/nextjs-require-cache-hot-reloader' const wsServer = new ws.Server({ noServer: true }) @@ -363,8 +364,7 @@ async function startWatcher(opts: SetupOpts) { sendHmrDebounce() } - const clearCache = (filePath: string) => - (global as any)._nextDeleteCache?.([filePath]) + const clearCache = (filePath: string) => deleteCache(filePath) async function loadPartialManifest( name: string, diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 06e6663c84075..78ab998368475 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -44,6 +44,7 @@ import { SERVER_DIRECTORY, NEXT_FONT_MANIFEST, PHASE_PRODUCTION_BUILD, + INTERNAL_HEADERS, } from '../shared/lib/constants' import { findDir } from '../lib/find-pages-dir' import { UrlWithParsedQuery } from 'url' @@ -1437,7 +1438,9 @@ export default class NextNodeServer extends BaseServer { checkIsOnDemandRevalidate(params.request, this.renderOpts.previewProps) .isOnDemandRevalidate ) { - return { finished: false } + return { + response: new Response(null, { headers: { 'x-middleware-next': '1' } }), + } as FetchEventResult } let url: string @@ -1584,6 +1587,11 @@ export default class NextNodeServer extends BaseServer { let result: Awaited< ReturnType > + let bubblingResult = false + + for (const key of INTERNAL_HEADERS) { + delete req.headers[key] + } try { await this.ensureMiddleware() @@ -1595,7 +1603,15 @@ export default class NextNodeServer extends BaseServer { parsed: parsed, }) - if (isMiddlewareInvoke && 'response' in result) { + if (isMiddlewareInvoke) { + bubblingResult = true + const err = new Error() + ;(err as any).result = result + ;(err as any).bubble = true + throw err + } + + if ('response' in result) { for (const [key, value] of Object.entries( toNodeOutgoingHttpHeaders(result.response.headers) )) { @@ -1614,6 +1630,10 @@ export default class NextNodeServer extends BaseServer { return { finished: true } } } catch (err) { + if (bubblingResult) { + throw err + } + if (isError(err) && err.code === 'ENOENT') { await this.render404(req, res, parsed) return { finished: true } From cb998a0d74a8a600c5a555431b2ed56294b03bfa Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 16:37:43 -0700 Subject: [PATCH 32/54] fix cases --- packages/next/src/server/lib/route-resolver.ts | 3 --- packages/next/src/server/lib/router-utils/resolve-routes.ts | 2 +- .../integration/middleware-overrides-node.js-api/middleware.js | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/next/src/server/lib/route-resolver.ts b/packages/next/src/server/lib/route-resolver.ts index 76378b56163ae..45d013f819082 100644 --- a/packages/next/src/server/lib/route-resolver.ts +++ b/packages/next/src/server/lib/route-resolver.ts @@ -6,7 +6,6 @@ import '../node-polyfill-fetch' import url from 'url' import path from 'path' -import http from 'http' import { findPageFile } from './find-page-file' import { getRequestMeta } from '../request-meta' import setupDebug from 'next/dist/compiled/debug' @@ -16,13 +15,11 @@ import { setupFsCheck } from './router-utils/filesystem' import { proxyRequest } from './router-utils/proxy-request' import { getResolveRoutes } from './router-utils/resolve-routes' import { PERMANENT_REDIRECT_STATUS } from '../../shared/lib/constants' -import { splitCookiesString, toNodeOutgoingHttpHeaders } from '../web/utils' import { formatHostname } from './format-hostname' import { signalFromNodeResponse } from '../web/spec-extension/adapters/next-request' import { getMiddlewareRouteMatcher } from '../../shared/lib/router/utils/middleware-route-matcher' import type { RenderWorker } from './router-server' import { pipeReadable } from '../pipe-readable' -import { Stream } from 'stream' type RouteResult = | { diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index 81e94bbd24db2..3803de8ea790f 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -488,7 +488,7 @@ export function getResolveRoutes( if (middlewareRes.body) { bodyStream = middlewareRes.body - } else if (middlewareRes.status !== 200) { + } else if (middlewareRes.status) { bodyStream = new ReadableStream({ start(controller) { controller.enqueue('') diff --git a/test/integration/middleware-overrides-node.js-api/middleware.js b/test/integration/middleware-overrides-node.js-api/middleware.js index 1bea9d9ca3cb3..a9110fb87f022 100644 --- a/test/integration/middleware-overrides-node.js-api/middleware.js +++ b/test/integration/middleware-overrides-node.js-api/middleware.js @@ -1,5 +1,5 @@ export default function middleware() { process.cwd = () => 'fixed-value' - console.log(process.cwd(), process.env) + console.log(process.cwd(), !!process.env) return new Response() } From 31c0b7972b076266a99e91fe69b3d211a24bb377 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 17:45:04 -0700 Subject: [PATCH 33/54] fix multi-zone case --- packages/next/src/server/api-utils/node.ts | 5 ++-- packages/next/src/server/base-server.ts | 9 +++++++ .../next/src/server/dev/next-dev-server.ts | 16 ++++++------- .../future/route-modules/pages-api/module.ts | 3 +++ packages/next/src/server/lib/render-server.ts | 24 ++++++++++++------- packages/next/src/server/lib/router-server.ts | 12 +++++----- .../src/server/lib/router-utils/setup-dev.ts | 4 ++-- packages/next/src/server/next-server.ts | 2 ++ 8 files changed, 48 insertions(+), 27 deletions(-) diff --git a/packages/next/src/server/api-utils/node.ts b/packages/next/src/server/api-utils/node.ts index 44f36814bc74f..6a2d06b58c3d2 100644 --- a/packages/next/src/server/api-utils/node.ts +++ b/packages/next/src/server/api-utils/node.ts @@ -199,6 +199,8 @@ type RevalidateFn = (config: { type ApiContext = __ApiPreviewProps & { trustHostHeader?: boolean + ipcPort?: string + ipcKey?: string allowedRevalidateHeaderKeys?: string[] hostname?: string revalidate?: RevalidateFn @@ -460,10 +462,9 @@ async function revalidate( throw new Error(`Invalid response ${res.status}`) } } else if (context.revalidate) { + const { ipcPort, ipcKey } = context // We prefer to use the IPC call if running under the workers mode. - const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT if (ipcPort) { - const ipcKey = process.env.__NEXT_PRIVATE_ROUTER_IPC_KEY const res = await invokeRequest( `http://${ context.hostname || 'localhost' diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index f69c20a1d1bd8..776d24a891cef 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -194,6 +194,8 @@ export interface Options { _routerWorker?: boolean _renderWorker?: boolean + _ipcPort?: string + _ipcKey?: string isNodeDebugging?: 'brk' | boolean } @@ -295,6 +297,8 @@ export default abstract class Server { protected readonly clientReferenceManifest?: ClientReferenceManifest protected nextFontManifest?: NextFontManifest private readonly responseCache: ResponseCacheBase + protected readonly ipcPort?: string + protected readonly ipcKey?: string protected abstract getPublicDir(): string protected abstract getHasStaticDir(): boolean @@ -383,7 +387,12 @@ export default abstract class Server { customServer = true, hostname, port, + _ipcPort, + _ipcKey, } = options + + this.ipcPort = _ipcPort + this.ipcKey = _ipcKey this.serverOptions = options this.isRenderWorker = options._renderWorker diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 1eca11b7358cb..98ed2680ed27c 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -493,8 +493,8 @@ export default class DevServer extends Server { fetchHostname: this.fetchHostname, method: 'logErrorWithOriginalStack', args: [errorToJSON(err as Error), type], - ipcPort: process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT, - ipcKey: process.env.__NEXT_PRIVATE_ROUTER_IPC_KEY, + ipcPort: this.ipcPort, + ipcKey: this.ipcKey, }) return } @@ -742,8 +742,8 @@ export default class DevServer extends Server { fetchHostname: this.fetchHostname, method: 'ensurePage', args: [opts], - ipcPort: process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT, - ipcKey: process.env.__NEXT_PRIVATE_ROUTER_IPC_KEY, + ipcPort: this.ipcPort, + ipcKey: this.ipcKey, }) } @@ -806,8 +806,8 @@ export default class DevServer extends Server { fetchHostname: this.fetchHostname, method: 'getFallbackErrorComponents', args: [], - ipcPort: process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT, - ipcKey: process.env.__NEXT_PRIVATE_ROUTER_IPC_KEY, + ipcPort: this.ipcPort, + ipcKey: this.ipcKey, }) return await loadDefaultErrorComponents(this.distDir) } @@ -822,8 +822,8 @@ export default class DevServer extends Server { fetchHostname: this.fetchHostname, method: 'getCompilationError', args: [page], - ipcPort: process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT, - ipcKey: process.env.__NEXT_PRIVATE_ROUTER_IPC_KEY, + ipcPort: this.ipcPort, + ipcKey: this.ipcKey, }) return deserializeErr(err) } diff --git a/packages/next/src/server/future/route-modules/pages-api/module.ts b/packages/next/src/server/future/route-modules/pages-api/module.ts index 88dbda73b464c..e72991d112987 100644 --- a/packages/next/src/server/future/route-modules/pages-api/module.ts +++ b/packages/next/src/server/future/route-modules/pages-api/module.ts @@ -89,6 +89,9 @@ type PagesAPIRouteHandlerContext = RouteModuleHandleContext & { * The page that's being rendered. */ page: string + + ipcPort?: string + ipcKey?: string } export type PagesAPIRouteModuleOptions = RouteModuleOptions< diff --git a/packages/next/src/server/lib/render-server.ts b/packages/next/src/server/lib/render-server.ts index 0b0186310b321..03dfd056b6077 100644 --- a/packages/next/src/server/lib/render-server.ts +++ b/packages/next/src/server/lib/render-server.ts @@ -3,7 +3,8 @@ import type { NextServer, RequestHandler } from '../next' import next from '../next' import { PropagateToWorkersField } from './router-utils/types' -let result: +const result: Record< + string, | undefined | { requestHandler: ReturnType< @@ -13,8 +14,9 @@ let result: InstanceType['getUpgradeHandler'] > } +> = {} -let app: ReturnType | undefined +let apps: Record | undefined> = {} let sandboxContext: undefined | typeof import('../web/sandbox/context') let requireCacheHotReloader: @@ -41,9 +43,11 @@ export function deleteCache(filePaths: string[]) { } export async function propagateServerField( + dir: string, field: PropagateToWorkersField, value: any ) { + const app = apps[dir] if (!app) { throw new Error('Invariant cant propagate server field, no app initialized') } @@ -73,11 +77,13 @@ export async function initialize(opts: { serverFields?: any server?: any experimentalTestProxy: boolean + _ipcPort?: string + _ipcKey?: string }) { // if we already setup the server return as we only need to do // this on first worker boot - if (result) { - return result + if (result[opts.dir]) { + return result[opts.dir] } const type = process.env.__NEXT_PRIVATE_RENDER_WORKER @@ -88,7 +94,7 @@ export async function initialize(opts: { let requestHandler: RequestHandler let upgradeHandler: any - app = next({ + const app = next({ ...opts, _routerWorker: opts.workerType === 'router', _renderWorker: opts.workerType === 'render', @@ -98,15 +104,15 @@ export async function initialize(opts: { port: opts.port, isNodeDebugging: opts.isNodeDebugging, }) - + apps[opts.dir] = app requestHandler = app.getRequestHandler() upgradeHandler = app.getUpgradeHandler() + await app.prepare(opts.serverFields) - result = { + result[opts.dir] = { requestHandler, upgradeHandler, } - - return result + return result[opts.dir] } diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index a04d05ddb520e..db36fd03c15ec 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -104,6 +104,7 @@ export async function initialize(opts: { const { setupDev } = (await require('./router-utils/setup-dev')) as typeof import('./router-utils/setup-dev') + devInstance = await setupDev({ // Passed here but the initialization of this object happens below, doing the initialization before the setupDev call breaks. renderWorkers, @@ -179,10 +180,6 @@ export async function initialize(opts: { }, } as any) - // Set global environment variables for the app render server to use. - process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT = ipcPort + '' - process.env.__NEXT_PRIVATE_ROUTER_IPC_KEY = ipcValidationKey - renderWorkers.app = require('./render-server') as typeof import('./render-server') @@ -196,15 +193,18 @@ export async function initialize(opts: { minimalMode: opts.minimalMode, dev: !!opts.dev, server: opts.server, + _ipcPort: ipcPort + '', + _ipcKey: ipcValidationKey, isNodeDebugging: !!opts.isNodeDebugging, serverFields: devInstance?.serverFields || {}, experimentalTestProxy: !!opts.experimentalTestProxy, } // pre-initialize workers + const handlers = await renderWorkers.app?.initialize(renderWorkerOpts) const initialized = { - app: await renderWorkers.app?.initialize(renderWorkerOpts), - pages: await renderWorkers.pages?.initialize(renderWorkerOpts), + app: handlers, + pages: handlers, } const logError = async ( diff --git a/packages/next/src/server/lib/router-utils/setup-dev.ts b/packages/next/src/server/lib/router-utils/setup-dev.ts index 80ef42b1419b1..b5d9799bc9acb 100644 --- a/packages/next/src/server/lib/router-utils/setup-dev.ts +++ b/packages/next/src/server/lib/router-utils/setup-dev.ts @@ -157,8 +157,8 @@ async function startWatcher(opts: SetupOpts) { ) async function propagateToWorkers(field: PropagateToWorkersField, args: any) { - await opts.renderWorkers.app?.propagateServerField(field, args) - await opts.renderWorkers.pages?.propagateServerField(field, args) + await opts.renderWorkers.app?.propagateServerField(opts.dir, field, args) + await opts.renderWorkers.pages?.propagateServerField(opts.dir, field, args) } const serverFields: { diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 78ab998368475..28a6055a044cc 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -433,6 +433,8 @@ export default class NextNodeServer extends BaseServer { { previewProps: this.renderOpts.previewProps, revalidate: this.revalidate.bind(this), + ipcPort: this.ipcPort, + ipcKey: this.ipcKey, trustHostHeader: this.nextConfig.experimental.trustHostHeader, allowedRevalidateHeaderKeys: this.nextConfig.experimental.allowedRevalidateHeaderKeys, From 7fc62a3292ffe36b053504ea35beb39bf6608e4d Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 21:01:24 -0700 Subject: [PATCH 34/54] fix dev middleware cases --- packages/next/src/server/base-server.ts | 20 ++++++++++---------- packages/next/src/server/next-server.ts | 16 ++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 776d24a891cef..e95898807f02c 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -1271,17 +1271,17 @@ export default abstract class Server { if (nextDataResult.finished) { return } - const result = await this.handleCatchallMiddlewareRequest( - req, - res, - parsedUrl - ) - if (!result.finished) { - res.setHeader('x-middleware-next', '1') - res.body('') - res.send() + await this.handleCatchallMiddlewareRequest(req, res, parsedUrl) + + // if we didn't already, bubble a NextResponse.next() result + const err = new Error() + ;(err as any).result = { + response: new Response(null, { + headers: { 'x-middleware-next': '1' }, + }), } - return + ;(err as any).bubble = true + throw err } // ensure we strip the basePath when not using an invoke header diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 28a6055a044cc..ed8db1793a4a5 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1605,15 +1605,15 @@ export default class NextNodeServer extends BaseServer { parsed: parsed, }) - if (isMiddlewareInvoke) { - bubblingResult = true - const err = new Error() - ;(err as any).result = result - ;(err as any).bubble = true - throw err - } - if ('response' in result) { + if (isMiddlewareInvoke) { + bubblingResult = true + const err = new Error() + ;(err as any).result = result + ;(err as any).bubble = true + throw err + } + for (const [key, value] of Object.entries( toNodeOutgoingHttpHeaders(result.response.headers) )) { From e64ffd444a9e7dbff211a5b0f05314b7432ae2bb Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 22:28:36 -0700 Subject: [PATCH 35/54] add yarn pnp loader handling --- packages/next/src/bin/next.ts | 1 + packages/next/src/build/index.ts | 5 +-- packages/next/src/server/config-utils.ts | 2 +- packages/next/src/server/esm-loader.mts | 8 ++--- .../src/server/{lib => }/import-overrides.ts | 31 +++++++++++++------ .../src/server/lib/get-esm-loader-path.ts | 30 ++++++++++++++++++ packages/next/src/server/next.ts | 6 ++-- packages/next/src/server/require-hook.js | 2 +- 8 files changed, 66 insertions(+), 19 deletions(-) rename packages/next/src/server/{lib => }/import-overrides.ts (83%) create mode 100644 packages/next/src/server/lib/get-esm-loader-path.ts diff --git a/packages/next/src/bin/next.ts b/packages/next/src/bin/next.ts index 82f7b3c59cce8..b38e42f1e8617 100755 --- a/packages/next/src/bin/next.ts +++ b/packages/next/src/bin/next.ts @@ -133,6 +133,7 @@ async function main() { const dir = getProjectDir( process.env.NEXT_PRIVATE_DEV_DIR || validatedArgs._[0] ) + process.env.NEXT_PRIVATE_DIR = dir const origEnv = Object.assign({}, process.env) // TODO: set config to env variable to be re-used so we don't reload diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index d3d575a19e42b..db37d1d3c18d7 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -147,9 +147,10 @@ import { buildDataRoute } from '../server/lib/router-utils/build-data-route' import { baseOverrides, experimentalOverrides, -} from '../server/lib/import-overrides' +} from '../server/import-overrides' import { initialize } from '../server/lib/incremental-cache-server' import { nodeFs } from '../server/lib/node-fs-methods' +import { getEsmLoaderPath } from '../server/lib/get-esm-loader-path' export type SsgRoute = { initialRevalidateSeconds: number | false @@ -1242,7 +1243,7 @@ export default async function build( forkOptions: { execArgv: [ '--experimental-loader', - 'next/dist/esm/server/esm-loader.mjs', + getEsmLoaderPath(), '--no-warnings', ], env: { diff --git a/packages/next/src/server/config-utils.ts b/packages/next/src/server/config-utils.ts index 7014f439df03c..bf6431ad8672c 100644 --- a/packages/next/src/server/config-utils.ts +++ b/packages/next/src/server/config-utils.ts @@ -10,7 +10,7 @@ export function loadWebpackHook() { // hook the Node.js require so that webpack requires are // routed to the bundled and now initialized webpack version - require('../server/lib/import-overrides').addHookAliases( + require('../server/import-overrides').addHookAliases( [ ['webpack', 'next/dist/compiled/webpack/webpack-lib'], ['webpack/package', 'next/dist/compiled/webpack/package'], diff --git a/packages/next/src/server/esm-loader.mts b/packages/next/src/server/esm-loader.mts index f7c7b90582b14..313bde8b39aae 100644 --- a/packages/next/src/server/esm-loader.mts +++ b/packages/next/src/server/esm-loader.mts @@ -1,12 +1,12 @@ import module from 'module' const require = module.createRequire(import.meta.url) -const { - overrideReact, - hookPropertyMap, -} = require('next/dist/server/lib/import-overrides') export function resolve(specifier: string, context: any, nextResolve: any) { + const { overrideReact, hookPropertyMap } = require(process.env.NEXT_YARN_PNP + ? './import-overrides' + : 'next/dist/server/import-overrides') as typeof import('./import-overrides') + // In case the environment variable is set after the module is loaded. overrideReact() diff --git a/packages/next/src/server/lib/import-overrides.ts b/packages/next/src/server/import-overrides.ts similarity index 83% rename from packages/next/src/server/lib/import-overrides.ts rename to packages/next/src/server/import-overrides.ts index 12f0421ffa853..c513795ea9690 100644 --- a/packages/next/src/server/lib/import-overrides.ts +++ b/packages/next/src/server/import-overrides.ts @@ -1,11 +1,29 @@ -import { dirname } from 'path' +const { dirname } = require('path') as typeof import('path') +let resolve: typeof require.resolve = process.env.NEXT_MINIMAL + ? // @ts-ignore + __non_webpack_require__.resolve + : require.resolve + +let nextPaths: undefined | { paths: string[] | undefined } = undefined + +if (!process.env.NEXT_MINIMAL) { + nextPaths = { + paths: resolve.paths('next/package.json') || undefined, + } +} export const hookPropertyMap = new Map() // these must use require.resolve to be statically analyzable export const defaultOverrides = { - 'styled-jsx': dirname(require.resolve('styled-jsx/package.json')), - 'styled-jsx/style': require.resolve('styled-jsx/style'), + 'styled-jsx': dirname( + process.env.NEXT_MINIMAL + ? require.resolve('styled-jsx/package.json') + : require.resolve('styled-jsx/package.json', nextPaths) + ), + 'styled-jsx/style': process.env.NEXT_MINIMAL + ? require.resolve('styled-jsx/style') + : require.resolve('styled-jsx/style', nextPaths), } export const baseOverrides = { @@ -57,13 +75,8 @@ export const experimentalOverrides = { let aliasedPrebundledReact = false -const resolve = process.env.NEXT_MINIMAL - ? // @ts-ignore - __non_webpack_require__.resolve - : require.resolve - const toResolveMap = (map: Record): [string, string][] => - Object.entries(map).map(([key, value]) => [key, resolve(value)]) + Object.entries(map).map(([key, value]) => [key, resolve(value, nextPaths)]) export function addHookAliases(aliases: [string, string][] = []) { for (const [key, value] of aliases) { diff --git a/packages/next/src/server/lib/get-esm-loader-path.ts b/packages/next/src/server/lib/get-esm-loader-path.ts new file mode 100644 index 0000000000000..7038142d4d00d --- /dev/null +++ b/packages/next/src/server/lib/get-esm-loader-path.ts @@ -0,0 +1,30 @@ +export function getEsmLoaderPath() { + let esmLoaderPath = 'next/dist/esm/server/esm-loader.mjs' + + // Since loaders don't stack Yarn PnP's loader isn't + // applied when loading our loader so we need to move + // it outside of the PnP cache + // x-ref: https://github.com/yarnpkg/berry/issues/3700 + if (process.versions.pnp) { + process.env.NEXT_YARN_PNP = '1' + const fs = require('fs') as typeof import('fs') + const path = require('path') as typeof import('path') + const tmpDir = path.join( + process.env.NEXT_PRIVATE_DIR || process.cwd(), + `.next/loader` + ) + fs.mkdirSync(tmpDir, { recursive: true }) + + for (const file of [esmLoaderPath, 'next/dist/server/import-overrides']) { + const resolvedFile = require.resolve(file) + const newFile = path.join(tmpDir, path.basename(file)) + + fs.copyFileSync(resolvedFile, newFile) + + if (file === esmLoaderPath) { + esmLoaderPath = newFile + } + } + } + return esmLoaderPath +} diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index f3be1324cb368..0c1ddd2bc6f11 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -4,12 +4,13 @@ import type { UrlWithParsedQuery } from 'url' import type { NextConfigComplete } from './config-shared' import type { IncomingMessage, ServerResponse } from 'http' import type { NextUrlWithParsedQuery } from './request-meta' +import { spawnSync } from 'child_process' +import { getEsmLoaderPath } from './lib/get-esm-loader-path' import { RESTART_EXIT_CODE, WorkerRequestHandler, WorkerUpgradeHandler, } from './lib/setup-server-worker' -import { spawnSync } from 'child_process' // if we are not inside of the esm loader enabled // worker we need to re-spawn with correct args @@ -22,9 +23,10 @@ if ( process.env.NODE_ENV === 'development') ) { const nodePath = process.argv0 + const newArgs = [ '--experimental-loader', - 'next/dist/esm/server/esm-loader.mjs', + getEsmLoaderPath(), '--no-warnings', ...process.argv.splice(1), ] diff --git a/packages/next/src/server/require-hook.js b/packages/next/src/server/require-hook.js index 67f773de0d41a..1630adde5b0b3 100644 --- a/packages/next/src/server/require-hook.js +++ b/packages/next/src/server/require-hook.js @@ -6,7 +6,7 @@ const mod = require('module') const resolveFilename = mod._resolveFilename -const { overrideReact, hookPropertyMap } = require('./lib/import-overrides') +const { overrideReact, hookPropertyMap } = require('./import-overrides') mod._resolveFilename = function ( originalResolveFilename, From 103f011adbc76de998fd0f37f6d269ba449a96e6 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 23:20:48 -0700 Subject: [PATCH 36/54] fix more tests cases --- packages/next/src/build/index.ts | 11 +++++++--- packages/next/src/build/utils.ts | 10 ++++++---- packages/next/src/server/base-server.ts | 20 +++++++++---------- .../next/src/server/dev/next-dev-server.ts | 2 +- packages/next/src/server/lib/router-server.ts | 4 ++++ .../server/lib/router-utils/resolve-routes.ts | 8 +------- packages/next/src/server/next-server.ts | 6 +++--- .../custom-server/custom-server.test.ts | 4 ++-- test/production/custom-server/server.js | 2 +- 9 files changed, 36 insertions(+), 31 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index db37d1d3c18d7..01615f7bae18a 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -2100,9 +2100,14 @@ export default async function build( const vanillaServerEntries = [ ...sharedEntriesSet, - isStandalone - ? require.resolve('next/dist/server/lib/start-server') - : null, + ...(isStandalone + ? [ + require.resolve('next/dist/server/lib/start-server'), + require.resolve('next/dist/server/next'), + require.resolve('next/dist/esm/server/esm-loader.mjs'), + require.resolve('next/dist/server/import-overrides'), + ] + : []), require.resolve('next/dist/server/next-server'), ].filter(Boolean) as string[] diff --git a/packages/next/src/build/utils.ts b/packages/next/src/build/utils.ts index a4c2e9ef8634e..6361d49334c04 100644 --- a/packages/next/src/build/utils.ts +++ b/packages/next/src/build/utils.ts @@ -1938,12 +1938,11 @@ export async function copyTracedFiles( moduleType ? `import path from 'path' import { fileURLToPath } from 'url' +import module from 'module' +const require = module.createRequire(import.meta.url) const __dirname = fileURLToPath(new URL('.', import.meta.url)) -import { startServer } from 'next/dist/server/lib/start-server.js' ` - : ` -const path = require('path') -const { startServer } = require('next/dist/server/lib/start-server')` + : `const path = require('path')` } const dir = path.join(__dirname) @@ -1974,6 +1973,9 @@ process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = ${hasAppDir} : 'next' : ''; +require('next') +const { startServer } = require('next/dist/server/lib/start-server') + if ( Number.isNaN(keepAliveTimeout) || !Number.isFinite(keepAliveTimeout) || diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index e95898807f02c..4755a0317302b 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -1271,17 +1271,17 @@ export default abstract class Server { if (nextDataResult.finished) { return } - await this.handleCatchallMiddlewareRequest(req, res, parsedUrl) - - // if we didn't already, bubble a NextResponse.next() result - const err = new Error() - ;(err as any).result = { - response: new Response(null, { - headers: { 'x-middleware-next': '1' }, - }), + const result = await this.handleCatchallMiddlewareRequest( + req, + res, + parsedUrl + ) + + if (result.finished) { + return + } else { + throw new Error(`Invariant: middleware response was not finished`) } - ;(err as any).bubble = true - throw err } // ensure we strip the basePath when not using an invoke header diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 98ed2680ed27c..122e9177faa75 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -390,7 +390,7 @@ export default class DevServer extends Server { } response.statusCode = 500 - this.renderError(err, request, response, parsedUrl.pathname) + await this.renderError(err, request, response, parsedUrl.pathname) return { finished: true } } } diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index db36fd03c15ec..c6281b885b13f 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -374,6 +374,10 @@ export async function initialize(opts: { invokedOutputs, }) + if (res.closed || res.finished) { + return + } + if (devInstance && matchedOutput?.type === 'devVirtualFsItem') { const origUrl = req.url || '/' diff --git a/packages/next/src/server/lib/router-utils/resolve-routes.ts b/packages/next/src/server/lib/router-utils/resolve-routes.ts index 3803de8ea790f..11097b180bc8c 100644 --- a/packages/next/src/server/lib/router-utils/resolve-routes.ts +++ b/packages/next/src/server/lib/router-utils/resolve-routes.ts @@ -511,7 +511,7 @@ export function getResolveRoutes( throw e } - if (res.closed || res.finished) { + if (res.closed || res.finished || !middlewareRes) { return { parsedUrl, resHeaders, @@ -519,12 +519,6 @@ export function getResolveRoutes( } } - if (!middlewareRes) { - throw new Error( - `Invariant: missing middleware response ${req.url}` - ) - } - const middlewareHeaders = toNodeOutgoingHttpHeaders( middlewareRes.headers ) as Record diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index ed8db1793a4a5..dccff61e46ce2 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1631,7 +1631,7 @@ export default class NextNodeServer extends BaseServer { } return { finished: true } } - } catch (err) { + } catch (err: any) { if (bubblingResult) { throw err } @@ -1643,14 +1643,14 @@ export default class NextNodeServer extends BaseServer { if (err instanceof DecodeError) { res.statusCode = 400 - this.renderError(err, req, res, parsed.pathname || '') + await this.renderError(err, req, res, parsed.pathname || '') return { finished: true } } const error = getProperError(err) console.error(error) res.statusCode = 500 - this.renderError(error, req, res, parsed.pathname || '') + await this.renderError(error, req, res, parsed.pathname || '') return { finished: true } } diff --git a/test/production/custom-server/custom-server.test.ts b/test/production/custom-server/custom-server.test.ts index a26c6c1e313a2..cb643e015b545 100644 --- a/test/production/custom-server/custom-server.test.ts +++ b/test/production/custom-server/custom-server.test.ts @@ -21,10 +21,10 @@ createNextDescribe( expect($('body').text()).toMatch(/app: .+-canary/) }) - it('should render pages with react stable', async () => { + it('should render pages with react canary', async () => { const $ = await next.render$(`/2`) expect($('body').text()).toMatch(/pages:/) - expect($('body').text()).not.toMatch(/canary/) + expect($('body').text()).toMatch(/canary/) }) }) } diff --git a/test/production/custom-server/server.js b/test/production/custom-server/server.js index 113493cc75eae..81c9b3a203a2b 100644 --- a/test/production/custom-server/server.js +++ b/test/production/custom-server/server.js @@ -30,7 +30,7 @@ async function main() { res.statusCode = 500 res.end('Internal Server Error') } - }).listen(port, '0.0.0.0', (err) => { + }).listen(port, undefined, (err) => { if (err) throw err // Start mode console.log(`- Local: http://${hostname}:${port}`) From 582264054583400157fbd1ca2fffb2bb4de2aca7 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 23:35:25 -0700 Subject: [PATCH 37/54] missing await --- .../next/src/server/dev/next-dev-server.ts | 2 +- .../watch-config-file/index.test.ts | 42 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 122e9177faa75..dc413df1ddd75 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -419,7 +419,7 @@ export default class DevServer extends Server { const err = getProperError(error) const { req, res, page } = params res.statusCode = 500 - this.renderError(err, req, res, page) + await this.renderError(err, req, res, page) return null } } diff --git a/test/development/watch-config-file/index.test.ts b/test/development/watch-config-file/index.test.ts index 4051ffe312bf6..1d7ddb36f3e11 100644 --- a/test/development/watch-config-file/index.test.ts +++ b/test/development/watch-config-file/index.test.ts @@ -9,28 +9,28 @@ createNextDescribe( ({ next }) => { it('should output config file change', async () => { await check(async () => next.cliOutput, /ready/) - await next.patchFile( - 'next.config.js', - ` - const nextConfig = { - reactStrictMode: true, - async redirects() { - return [ - { - source: '/about', - destination: '/', - permanent: false, - }, - ] - }, - } - module.exports = nextConfig` - ) - await check( - async () => next.cliOutput, - /Found a change in next\.config\.js\. Restarting the server to apply the changes\.\.\./ - ) + await check(async () => { + await next.patchFile( + 'next.config.js', + ` + console.log(${Date.now()}) + const nextConfig = { + reactStrictMode: true, + async redirects() { + return [ + { + source: '/about', + destination: '/', + permanent: false, + }, + ] + }, + } + module.exports = nextConfig` + ) + return next.cliOutput + }, /Found a change in next\.config\.js\. Restarting the server to apply the changes\.\.\./) await check(() => next.fetch('/about').then((res) => res.status), 200) }) From 5ad5cb37437dff6aa9402e076780b121ce2673f6 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 5 Sep 2023 23:55:35 -0700 Subject: [PATCH 38/54] fix middleware dev static chunks case --- packages/next/src/server/base-server.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 4755a0317302b..cd02f8bb50a77 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -1280,7 +1280,16 @@ export default abstract class Server { if (result.finished) { return } else { - throw new Error(`Invariant: middleware response was not finished`) + const err = new Error() + ;(err as any).result = { + response: new Response(null, { + headers: { + 'x-middleware-next': '1', + }, + }), + } + ;(err as any).bubble = true + throw err } } From e89e23447fd6afa499edc3b3e2f52c251fab6272 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Wed, 6 Sep 2023 12:56:55 +0200 Subject: [PATCH 39/54] Remove leftover await --- .../build/webpack/plugins/nextjs-require-cache-hot-reloader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts index 93e2ad3ede3f2..f14071a5df009 100644 --- a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts +++ b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts @@ -112,7 +112,7 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance { compilation.outputOptions.path!, page + '.js' ) - await deleteCache(outputPath) + deleteCache(outputPath) } }) } From ddc0dfb6aede0594964f58ebfc88269fa8906f38 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 6 Sep 2023 16:02:21 -0700 Subject: [PATCH 40/54] remove debug fields --- packages/next/src/lib/get-project-dir.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/next/src/lib/get-project-dir.ts b/packages/next/src/lib/get-project-dir.ts index ce47c791070a2..c8530357c8823 100644 --- a/packages/next/src/lib/get-project-dir.ts +++ b/packages/next/src/lib/get-project-dir.ts @@ -35,10 +35,7 @@ export function getProjectDir(dir?: string) { Log.error( `Invalid project directory provided, no such directory: ${path.resolve( dir || '.' - )}`, - new Error().stack, - process.env.NEXT_PRIVATE_WORKER, - process.argv + )}` ) process.exit(1) } From b0dacadadd2415b4770c55cbb38fd3b33015d04f Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 6 Sep 2023 16:10:51 -0700 Subject: [PATCH 41/54] remove other log --- packages/next/src/server/require-hook.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/next/src/server/require-hook.js b/packages/next/src/server/require-hook.js index 1630adde5b0b3..d908bcbcd44ad 100644 --- a/packages/next/src/server/require-hook.js +++ b/packages/next/src/server/require-hook.js @@ -16,9 +16,6 @@ mod._resolveFilename = function ( isMain, options ) { - if (request === 'from-require' || request === 'from-import') { - console.log('require-hook', request) - } // In case the environment variable is set after the module is loaded. overrideReact() From a08ca47b919f9453b7695ade61ba414987c9e558 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 6 Sep 2023 17:37:08 -0700 Subject: [PATCH 42/54] remove un-needed IPC server --- packages/next/src/server/api-utils/node.ts | 25 --- packages/next/src/server/base-server.ts | 8 - .../next/src/server/dev/next-dev-server.ts | 30 ++-- packages/next/src/server/lib/router-server.ts | 143 ++++++++++-------- packages/next/src/server/next-server.ts | 2 - 5 files changed, 89 insertions(+), 119 deletions(-) diff --git a/packages/next/src/server/api-utils/node.ts b/packages/next/src/server/api-utils/node.ts index 6a2d06b58c3d2..7763ee6f79957 100644 --- a/packages/next/src/server/api-utils/node.ts +++ b/packages/next/src/server/api-utils/node.ts @@ -199,8 +199,6 @@ type RevalidateFn = (config: { type ApiContext = __ApiPreviewProps & { trustHostHeader?: boolean - ipcPort?: string - ipcKey?: string allowedRevalidateHeaderKeys?: string[] hostname?: string revalidate?: RevalidateFn @@ -462,29 +460,6 @@ async function revalidate( throw new Error(`Invalid response ${res.status}`) } } else if (context.revalidate) { - const { ipcPort, ipcKey } = context - // We prefer to use the IPC call if running under the workers mode. - if (ipcPort) { - const res = await invokeRequest( - `http://${ - context.hostname || 'localhost' - }:${ipcPort}?key=${ipcKey}&method=revalidate&args=${encodeURIComponent( - JSON.stringify([{ urlPath, revalidateHeaders, opts }]) - )}`, - { - method: 'GET', - headers: {}, - } - ) - const result = await res.json() - - if (result.err) { - throw new Error(result.err.message) - } - - return - } - await context.revalidate({ urlPath, revalidateHeaders, diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index cd02f8bb50a77..7a6d308449e54 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -194,8 +194,6 @@ export interface Options { _routerWorker?: boolean _renderWorker?: boolean - _ipcPort?: string - _ipcKey?: string isNodeDebugging?: 'brk' | boolean } @@ -297,8 +295,6 @@ export default abstract class Server { protected readonly clientReferenceManifest?: ClientReferenceManifest protected nextFontManifest?: NextFontManifest private readonly responseCache: ResponseCacheBase - protected readonly ipcPort?: string - protected readonly ipcKey?: string protected abstract getPublicDir(): string protected abstract getHasStaticDir(): boolean @@ -387,12 +383,8 @@ export default abstract class Server { customServer = true, hostname, port, - _ipcPort, - _ipcKey, } = options - this.ipcPort = _ipcPort - this.ipcKey = _ipcKey this.serverOptions = options this.isRenderWorker = options._renderWorker diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index dc413df1ddd75..fcf3d3704da1c 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -65,10 +65,7 @@ import { IncrementalCache } from '../lib/incremental-cache' import LRUCache from 'next/dist/compiled/lru-cache' import { errorToJSON } from '../render' import { getMiddlewareRouteMatcher } from '../../shared/lib/router/utils/middleware-route-matcher' -import { - deserializeErr, - invokeIpcMethod, -} from '../lib/server-ipc/request-utils' +import { deserializeErr } from '../lib/server-ipc/request-utils' // Load ReactDevOverlay only when needed let ReactDevOverlayImpl: FunctionComponent @@ -102,6 +99,10 @@ export default class DevServer extends Server { UnwrapPromise> > + private invokeDevMethod({ method, args }: { method: string; args: any[] }) { + return (global as any)._nextDevHandlers[method](this.dir, ...args) + } + protected staticPathsWorker?: { [key: string]: any } & { loadStaticPaths: typeof import('./static-paths-worker').loadStaticPaths } @@ -489,12 +490,9 @@ export default class DevServer extends Server { type?: 'unhandledRejection' | 'uncaughtException' | 'warning' | 'app-dir' ): Promise { if (this.isRenderWorker) { - await invokeIpcMethod({ - fetchHostname: this.fetchHostname, + await this.invokeDevMethod({ method: 'logErrorWithOriginalStack', args: [errorToJSON(err as Error), type], - ipcPort: this.ipcPort, - ipcKey: this.ipcKey, }) return } @@ -738,12 +736,9 @@ export default class DevServer extends Server { throw new Error('Invariant ensurePage called outside render worker') } - await invokeIpcMethod({ - fetchHostname: this.fetchHostname, + await this.invokeDevMethod({ method: 'ensurePage', args: [opts], - ipcPort: this.ipcPort, - ipcKey: this.ipcKey, }) } @@ -802,12 +797,9 @@ export default class DevServer extends Server { protected async getFallbackErrorComponents(): Promise { if (this.isRenderWorker) { - await invokeIpcMethod({ - fetchHostname: this.fetchHostname, + await this.invokeDevMethod({ method: 'getFallbackErrorComponents', args: [], - ipcPort: this.ipcPort, - ipcKey: this.ipcKey, }) return await loadDefaultErrorComponents(this.distDir) } @@ -818,14 +810,10 @@ export default class DevServer extends Server { async getCompilationError(page: string): Promise { if (this.isRenderWorker) { - const err = await invokeIpcMethod({ - fetchHostname: this.fetchHostname, + return await this.invokeDevMethod({ method: 'getCompilationError', args: [page], - ipcPort: this.ipcPort, - ipcKey: this.ipcKey, }) - return deserializeErr(err) } throw new Error( 'Invariant getCompilationError called outside render worker' diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 25b3820ebe8c9..ba96ef0ac8cc6 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -1,4 +1,6 @@ import type { IncomingMessage } from 'http' +import type { NextJsHotReloaderInterface } from '../dev/hot-reloader-types' +import type { createWorker } from './server-ipc' // this must come first as it includes require hooks import type { @@ -18,7 +20,6 @@ import { setupFsCheck } from './router-utils/filesystem' import { proxyRequest } from './router-utils/proxy-request' import { isAbortError, pipeReadable } from '../pipe-readable' import { createRequestResponseMocks } from './mock-request' -import { createIpcServer, createWorker } from './server-ipc' import { UnwrapPromise } from '../../lib/coalesced-function' import { getResolveRoutes } from './router-utils/resolve-routes' import { NextUrlWithParsedQuery, getRequestMeta } from '../request-meta' @@ -50,6 +51,13 @@ export interface RenderWorkers { pages?: Awaited> } +const devInstances: Record< + string, + UnwrapPromise> +> = {} + +const requestHandlers: Record = {} + export async function initialize(opts: { dir: string port: number @@ -117,68 +125,79 @@ export async function initialize(opts: { isCustomServer: opts.customServer, turbo: !!process.env.TURBOPACK, }) - } - - const { ipcPort, ipcValidationKey } = await createIpcServer({ - async ensurePage( - match: Parameters< - InstanceType< - typeof import('../dev/hot-reloader-webpack').default - >['ensurePage'] - >[0] - ) { - // TODO: remove after ensure is pulled out of server - return await devInstance?.hotReloader.ensurePage(match) - }, - async logErrorWithOriginalStack(...args: any[]) { - // @ts-ignore - return await devInstance?.logErrorWithOriginalStack(...args) - }, - async getFallbackErrorComponents() { - await devInstance?.hotReloader?.buildFallbackError() - // Build the error page to ensure the fallback is built too. - // TODO: See if this can be moved into hotReloader or removed. - await devInstance?.hotReloader.ensurePage({ - page: '/_error', - clientOnly: false, - }) - }, - async getCompilationError(page: string) { - const errors = await devInstance?.hotReloader?.getCompilationErrors(page) - if (!errors) return - - // Return the very first error we found. - return errors[0] - }, - async revalidate({ - urlPath, - revalidateHeaders, - opts: revalidateOpts, - }: { - urlPath: string - revalidateHeaders: IncomingMessage['headers'] - opts: any - }) { - const mocked = createRequestResponseMocks({ - url: urlPath, - headers: revalidateHeaders, - }) - - // eslint-disable-next-line @typescript-eslint/no-use-before-define - await requestHandler(mocked.req, mocked.res) - await mocked.res.hasStreamed - - if ( - mocked.res.getHeader('x-nextjs-cache') !== 'REVALIDATED' && - !( - mocked.res.statusCode === 404 && revalidateOpts.unstable_onlyGenerated + devInstances[opts.dir] = devInstance + ;(global as any)._nextDevHandlers = { + async ensurePage( + dir: string, + match: Parameters< + InstanceType< + typeof import('../dev/hot-reloader-webpack').default + >['ensurePage'] + >[0] + ) { + const curDevInstance = devInstances[dir] + // TODO: remove after ensure is pulled out of server + return await curDevInstance?.hotReloader.ensurePage(match) + }, + async logErrorWithOriginalStack(dir: string, ...args: any[]) { + const curDevInstance = devInstances[dir] + // @ts-ignore + return await curDevInstance?.logErrorWithOriginalStack(...args) + }, + async getFallbackErrorComponents(dir: string) { + const curDevInstance = devInstances[dir] + await curDevInstance.hotReloader.buildFallbackError() + // Build the error page to ensure the fallback is built too. + // TODO: See if this can be moved into hotReloader or removed. + await curDevInstance.hotReloader.ensurePage({ + page: '/_error', + clientOnly: false, + }) + }, + async getCompilationError(dir: string, page: string) { + const curDevInstance = devInstances[dir] + const errors = await curDevInstance?.hotReloader?.getCompilationErrors( + page ) + if (!errors) return + + // Return the very first error we found. + return errors[0] + }, + async revalidate( + dir: string, + { + urlPath, + revalidateHeaders, + opts: revalidateOpts, + }: { + urlPath: string + revalidateHeaders: IncomingMessage['headers'] + opts: any + } ) { - throw new Error(`Invalid response ${mocked.res.statusCode}`) - } - return {} - }, - } as any) + const mocked = createRequestResponseMocks({ + url: urlPath, + headers: revalidateHeaders, + }) + const curRequestHandler = requestHandlers[dir] + // eslint-disable-next-line @typescript-eslint/no-use-before-define + await curRequestHandler(mocked.req, mocked.res) + await mocked.res.hasStreamed + + if ( + mocked.res.getHeader('x-nextjs-cache') !== 'REVALIDATED' && + !( + mocked.res.statusCode === 404 && + revalidateOpts.unstable_onlyGenerated + ) + ) { + throw new Error(`Invalid response ${mocked.res.statusCode}`) + } + return {} + }, + } as any + } renderWorkers.app = require('./render-server') as typeof import('./render-server') @@ -193,8 +212,6 @@ export async function initialize(opts: { minimalMode: opts.minimalMode, dev: !!opts.dev, server: opts.server, - _ipcPort: ipcPort + '', - _ipcKey: ipcValidationKey, isNodeDebugging: !!opts.isNodeDebugging, serverFields: devInstance?.serverFields || {}, experimentalTestProxy: !!opts.experimentalTestProxy, diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index dccff61e46ce2..05ac6545c120e 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -433,8 +433,6 @@ export default class NextNodeServer extends BaseServer { { previewProps: this.renderOpts.previewProps, revalidate: this.revalidate.bind(this), - ipcPort: this.ipcPort, - ipcKey: this.ipcKey, trustHostHeader: this.nextConfig.experimental.trustHostHeader, allowedRevalidateHeaderKeys: this.nextConfig.experimental.allowedRevalidateHeaderKeys, From e8bf9c2dc9c4ab39d194fb02cddea6d37280d0f3 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 6 Sep 2023 17:58:15 -0700 Subject: [PATCH 43/54] rm extra error serialize --- packages/next/src/server/api-utils/node.ts | 1 - packages/next/src/server/dev/next-dev-server.ts | 4 +--- packages/next/src/server/lib/router-server.ts | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/next/src/server/api-utils/node.ts b/packages/next/src/server/api-utils/node.ts index 7763ee6f79957..40ea977cf9f9b 100644 --- a/packages/next/src/server/api-utils/node.ts +++ b/packages/next/src/server/api-utils/node.ts @@ -35,7 +35,6 @@ import { PRERENDER_REVALIDATE_HEADER, PRERENDER_REVALIDATE_ONLY_GENERATED_HEADER, } from '../../lib/constants' -import { invokeRequest } from '../lib/server-ipc/invoke-request' export function tryGetPreviewData( req: IncomingMessage | BaseNextRequest | Request, diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index fcf3d3704da1c..1ca090392b934 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -63,9 +63,7 @@ import { DefaultFileReader } from '../future/route-matcher-providers/dev/helpers import { NextBuildContext } from '../../build/build-context' import { IncrementalCache } from '../lib/incremental-cache' import LRUCache from 'next/dist/compiled/lru-cache' -import { errorToJSON } from '../render' import { getMiddlewareRouteMatcher } from '../../shared/lib/router/utils/middleware-route-matcher' -import { deserializeErr } from '../lib/server-ipc/request-utils' // Load ReactDevOverlay only when needed let ReactDevOverlayImpl: FunctionComponent @@ -492,7 +490,7 @@ export default class DevServer extends Server { if (this.isRenderWorker) { await this.invokeDevMethod({ method: 'logErrorWithOriginalStack', - args: [errorToJSON(err as Error), type], + args: [err, type], }) return } diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index ba96ef0ac8cc6..8a4eafed24987 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -1,5 +1,4 @@ import type { IncomingMessage } from 'http' -import type { NextJsHotReloaderInterface } from '../dev/hot-reloader-types' import type { createWorker } from './server-ipc' // this must come first as it includes require hooks @@ -663,6 +662,7 @@ export async function initialize(opts: { requestHandler = wrapRequestHandlerWorker(requestHandler) interceptTestApis() } + requestHandlers[opts.dir] = requestHandler const upgradeHandler: WorkerUpgradeHandler = async (req, socket, head) => { try { From a6f3d52aa457b1c932e2ab8c83fdeed42e2a40e2 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 6 Sep 2023 21:10:33 -0700 Subject: [PATCH 44/54] ensure vendored is in split chunks --- packages/next/src/build/webpack-config.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index c08f07e34507b..897bea70d6e62 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -1267,7 +1267,16 @@ export default async function getBaseWebpackConfig( } } - for (const packageName of ['react', 'react-dom']) { + for (const packageName of [ + 'react', + 'react-dom', + ...(hasAppDir + ? [ + `next/dist/compiled/react${bundledReactChannel}`, + `next/dist/compiled/react-dom${bundledReactChannel}`, + ] + : []), + ]) { addPackagePath(packageName, dir) } From b064f7723a983db382490ddae327218b85944c27 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 7 Sep 2023 16:06:19 -0700 Subject: [PATCH 45/54] fix lint --- packages/next/src/server/base-server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 7210fd6a55826..4196204178cf8 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -44,7 +44,6 @@ import { getRedirectStatus } from '../lib/redirect-status' import { isEdgeRuntime } from '../lib/is-edge-runtime' import { APP_PATHS_MANIFEST, - INTERNAL_HEADERS, NEXT_BUILTIN_DOCUMENT, PAGES_MANIFEST, STATIC_STATUS_PAGES, From 4a278295508c283e6ce55125c03103ab594b890e Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Sat, 9 Sep 2023 09:54:55 +0200 Subject: [PATCH 46/54] Bring back missing import that broke in merge --- packages/next/src/build/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index fa4f7f2964e7b..080b196eac285 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -147,6 +147,7 @@ import { generateInterceptionRoutesRewrites } from '../lib/generate-interception import { buildDataRoute } from '../server/lib/router-utils/build-data-route' import { baseOverrides, + defaultOverrides, experimentalOverrides, } from '../server/import-overrides' import { initialize } from '../server/lib/incremental-cache-server' From 643168a4de17b9b2439d678705d309529ac67898 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Sat, 9 Sep 2023 09:59:07 +0200 Subject: [PATCH 47/54] Clarify ipc variables --- packages/next/src/build/index.ts | 35 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 080b196eac285..4dbccc78ca65c 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1207,7 +1207,10 @@ export default async function build( ) : config.experimental.cpus || 4 - function createStaticWorker(ipcPort: number, ipcValidationKey: string) { + function createStaticWorker( + incrementalCacheIpcPort: number, + incrementalCacheIpcValidationKey: string + ) { let infoPrinted = false return new Worker(staticWorkerPath, { @@ -1250,8 +1253,9 @@ export default async function build( ], env: { ...process.env, - __NEXT_INCREMENTAL_CACHE_IPC_PORT: ipcPort + '', - __NEXT_INCREMENTAL_CACHE_IPC_KEY: ipcValidationKey, + __NEXT_INCREMENTAL_CACHE_IPC_PORT: incrementalCacheIpcPort + '', + __NEXT_INCREMENTAL_CACHE_IPC_KEY: + incrementalCacheIpcValidationKey, __NEXT_PRIVATE_PREBUNDLED_REACT: hasAppDir ? config.experimental.serverActions ? 'experimental' @@ -1291,7 +1295,10 @@ export default async function build( CacheHandler = CacheHandler.default || CacheHandler } - const { ipcPort, ipcValidationKey } = await initialize({ + const { + ipcPort: incrementalCacheIpcPort, + ipcValidationKey: incrementalCacheIpcValidationKey, + } = await initialize({ fs: nodeFs, dev: false, appDir: isAppDirEnabled, @@ -1315,9 +1322,15 @@ export default async function build( config.experimental.allowedRevalidateHeaderKeys, }) - const pagesStaticWorkers = createStaticWorker(ipcPort, ipcValidationKey) + const pagesStaticWorkers = createStaticWorker( + incrementalCacheIpcPort, + incrementalCacheIpcValidationKey + ) const appStaticWorkers = isAppDirEnabled - ? createStaticWorker(ipcPort, ipcValidationKey) + ? createStaticWorker( + incrementalCacheIpcPort, + incrementalCacheIpcValidationKey + ) : undefined const analysisBegin = process.hrtime() @@ -3317,8 +3330,14 @@ export default async function build( const exportApp: typeof import('../export').default = require('../export').default - const pagesWorker = createStaticWorker(ipcPort, ipcValidationKey) - const appWorker = createStaticWorker(ipcPort, ipcValidationKey) + const pagesWorker = createStaticWorker( + incrementalCacheIpcPort, + incrementalCacheIpcValidationKey + ) + const appWorker = createStaticWorker( + incrementalCacheIpcPort, + incrementalCacheIpcValidationKey + ) const options: ExportOptions = { isInvokedFromCli: false, From b19f410e1c50ae8551ae42671b04756c88a88d46 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Sat, 9 Sep 2023 09:59:50 +0200 Subject: [PATCH 48/54] Clarify initialize() --- packages/next/src/build/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 4dbccc78ca65c..8d92c8ae5587d 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -150,7 +150,7 @@ import { defaultOverrides, experimentalOverrides, } from '../server/import-overrides' -import { initialize } from '../server/lib/incremental-cache-server' +import { initialize as initializeIncrementalCache } from '../server/lib/incremental-cache-server' import { nodeFs } from '../server/lib/node-fs-methods' import { getEsmLoaderPath } from '../server/lib/get-esm-loader-path' @@ -1298,7 +1298,7 @@ export default async function build( const { ipcPort: incrementalCacheIpcPort, ipcValidationKey: incrementalCacheIpcValidationKey, - } = await initialize({ + } = await initializeIncrementalCache({ fs: nodeFs, dev: false, appDir: isAppDirEnabled, From f1fafd6bfd138862eca136a65298fb6e6ad5a600 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Sat, 9 Sep 2023 10:11:13 +0200 Subject: [PATCH 49/54] Remove unused properties --- .../next/src/server/future/route-modules/pages-api/module.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/next/src/server/future/route-modules/pages-api/module.ts b/packages/next/src/server/future/route-modules/pages-api/module.ts index 67f3ad6cdb60f..976daeeec4a87 100644 --- a/packages/next/src/server/future/route-modules/pages-api/module.ts +++ b/packages/next/src/server/future/route-modules/pages-api/module.ts @@ -89,9 +89,6 @@ type PagesAPIRouteHandlerContext = RouteModuleHandleContext & { * The page that's being rendered. */ page: string - - ipcPort?: string - ipcKey?: string } export type PagesAPIRouteModuleOptions = RouteModuleOptions< From b506e9891971c6017afa8fb5a81f747c9b333259 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 9 Sep 2023 11:48:38 -0700 Subject: [PATCH 50/54] fix styled-jsx resolve --- packages/next/src/server/import-overrides.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/next/src/server/import-overrides.ts b/packages/next/src/server/import-overrides.ts index 2319c7f32d819..919a158b26b6d 100644 --- a/packages/next/src/server/import-overrides.ts +++ b/packages/next/src/server/import-overrides.ts @@ -15,8 +15,12 @@ if (!process.env.NEXT_MINIMAL) { export const hookPropertyMap = new Map() export const defaultOverrides = { - 'styled-jsx': dirname(resolve('styled-jsx/package.json')), - 'styled-jsx/style': resolve('styled-jsx/style'), + 'styled-jsx': process.env.NEXT_MINIMAL + ? dirname(resolve('styled-jsx/package.json')) + : dirname(resolve('styled-jsx/package.json', nextPaths)), + 'styled-jsx/style': process.env.NEXT_MINIMAL + ? dirname(resolve('styled-jsx/style')) + : dirname(resolve('styled-jsx/style', nextPaths)), } export const baseOverrides = { From 6bbdac49b2b0b6da4a067f49b991b77efd346e58 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 11 Sep 2023 08:52:14 -0700 Subject: [PATCH 51/54] tweak require-hook handling --- packages/next/src/server/require-hook.js | 24 ++++++++++++++++++++++++ packages/next/src/server/require.ts | 16 ++++++++++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/next/src/server/require-hook.js b/packages/next/src/server/require-hook.js index d908bcbcd44ad..ce9e30e6e9ffe 100644 --- a/packages/next/src/server/require-hook.js +++ b/packages/next/src/server/require-hook.js @@ -3,7 +3,9 @@ // Individually compiled modules are as defined for the compilation in bundles/webpack/packages/*. // This module will only be loaded once per process. +const path = require('path') const mod = require('module') +const originalRequire = mod.prototype.require const resolveFilename = mod._resolveFilename const { overrideReact, hookPropertyMap } = require('./import-overrides') @@ -26,3 +28,25 @@ mod._resolveFilename = function ( // We use `bind` here to avoid referencing outside variables to create potential memory leaks. }.bind(null, resolveFilename, hookPropertyMap) + +// This is a hack to make sure that if a user requires a Next.js module that wasn't bundled +// that needs to point to the rendering runtime version, it will point to the correct one. +// This can happen on `pages` when a user requires a dependency that uses next/image for example. +// This is only needed in production as in development we fallback to the external version. +if (process.env.NODE_ENV !== 'development' && !process.env.TURBOPACK) { + mod.prototype.require = function (request) { + if (request.endsWith('.shared-runtime')) { + const isAppRequire = process.env.__NEXT_PRIVATE_RUNTIME_TYPE === 'app' + const currentRuntime = `${ + isAppRequire + ? 'next/dist/compiled/next-server/app-page.runtime' + : 'next/dist/compiled/next-server/pages.runtime' + }.prod` + const base = path.basename(request, '.shared-runtime') + const camelized = base.replace(/-([a-z])/g, (g) => g[1].toUpperCase()) + const instance = originalRequire.call(this, currentRuntime) + return instance.default.sharedModules[camelized] + } + return originalRequire.call(this, request) + } +} diff --git a/packages/next/src/server/require.ts b/packages/next/src/server/require.ts index bbc8e91dc2835..f020ea27e7eba 100644 --- a/packages/next/src/server/require.ts +++ b/packages/next/src/server/require.ts @@ -116,10 +116,18 @@ export function requirePage( }) } - return process.env.NEXT_MINIMAL - ? // @ts-ignore - __non_webpack_require__(pagePath) - : require(pagePath) + // since require is synchronous we can set the specific runtime + // we are requiring for the require-hook and then clear after + try { + process.env.__NEXT_PRIVATE_RUNTIME_TYPE = isAppPath ? 'app' : 'pages' + const mod = process.env.NEXT_MINIMAL + ? // @ts-ignore + __non_webpack_require__(pagePath) + : require(pagePath) + return mod + } finally { + process.env.__NEXT_PRIVATE_RUNTIME_TYPE = '' + } } export function requireFontManifest(distDir: string) { From b788fb5707e9d9918623d7a4a4c1bc69dc0e6893 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 11 Sep 2023 09:32:48 -0700 Subject: [PATCH 52/54] fix styled-jsx --- packages/next/src/build/webpack-config.ts | 7 ++++--- packages/next/src/server/import-overrides.ts | 4 ++-- test/integration/amphtml-fragment-style/pages/index.js | 5 ++++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 703f4375ad6b2..a5a6191365693 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -68,6 +68,7 @@ import { NextFontManifestPlugin } from './webpack/plugins/next-font-manifest-plu import { getSupportedBrowsers } from './utils' import { MemoryWithGcCachePlugin } from './webpack/plugins/memory-with-gc-cache-plugin' import { getBabelConfigFile } from './get-babel-config-file' +import { defaultOverrides } from '../server/import-overrides' type ExcludesFalse = (x: T | false) => x is T type ClientEntries = { @@ -1146,8 +1147,8 @@ export default async function getBaseWebpackConfig( next: NEXT_PROJECT_ROOT, - 'styled-jsx/style$': require.resolve(`styled-jsx/style`), - 'styled-jsx$': require.resolve(`styled-jsx`), + 'styled-jsx/style$': defaultOverrides['styled-jsx/style'], + 'styled-jsx$': defaultOverrides['styled-jsx'], ...customAppAliases, ...customErrorAlias, @@ -1558,7 +1559,7 @@ export default async function getBaseWebpackConfig( // Forcedly resolve the styled-jsx installed by next.js, // since `resolveExternal` cannot find the styled-jsx dep with pnpm if (request === 'styled-jsx/style') { - resolveResult.res = require.resolve(request) + return `commonjs ${defaultOverrides['styled-jsx/style']}` } const { res, isEsm } = resolveResult diff --git a/packages/next/src/server/import-overrides.ts b/packages/next/src/server/import-overrides.ts index 919a158b26b6d..db203e6d04644 100644 --- a/packages/next/src/server/import-overrides.ts +++ b/packages/next/src/server/import-overrides.ts @@ -19,8 +19,8 @@ export const defaultOverrides = { ? dirname(resolve('styled-jsx/package.json')) : dirname(resolve('styled-jsx/package.json', nextPaths)), 'styled-jsx/style': process.env.NEXT_MINIMAL - ? dirname(resolve('styled-jsx/style')) - : dirname(resolve('styled-jsx/style', nextPaths)), + ? resolve('styled-jsx/style') + : resolve('styled-jsx/style', nextPaths), } export const baseOverrides = { diff --git a/test/integration/amphtml-fragment-style/pages/index.js b/test/integration/amphtml-fragment-style/pages/index.js index f57bb4c1e36fe..c45defad01f66 100644 --- a/test/integration/amphtml-fragment-style/pages/index.js +++ b/test/integration/amphtml-fragment-style/pages/index.js @@ -1,6 +1,6 @@ export const config = { amp: true } -export default () => ( +const Comp = () => (

Hello world!

) + +Comp.getInitialProps = () => ({}) +export default Comp From c1cf93452c73bf6a0de518ca84c3532c7f6bd6dd Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 11 Sep 2023 09:46:57 -0700 Subject: [PATCH 53/54] extra change --- packages/next/src/build/webpack-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index a5a6191365693..00f85439e00cd 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -1559,7 +1559,7 @@ export default async function getBaseWebpackConfig( // Forcedly resolve the styled-jsx installed by next.js, // since `resolveExternal` cannot find the styled-jsx dep with pnpm if (request === 'styled-jsx/style') { - return `commonjs ${defaultOverrides['styled-jsx/style']}` + resolveResult.res = defaultOverrides['styled-jsx/style'] } const { res, isEsm } = resolveResult From 5e6442f90d94f7c6896399acaec186e73780cfe4 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 11 Sep 2023 10:59:09 -0700 Subject: [PATCH 54/54] fix turbopack cache clearing --- .../src/server/lib/router-utils/setup-dev.ts | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/next/src/server/lib/router-utils/setup-dev.ts b/packages/next/src/server/lib/router-utils/setup-dev.ts index 48f7bad6b69f3..c0019bff3ed29 100644 --- a/packages/next/src/server/lib/router-utils/setup-dev.ts +++ b/packages/next/src/server/lib/router-utils/setup-dev.ts @@ -311,21 +311,20 @@ async function startWatcher(opts: SetupOpts) { async function processResult( result: TurbopackResult ): Promise> { - await (global as any)._nextDeleteCache?.( - result.serverPaths - .map((p) => path.join(distDir, p)) - .concat([ - // We need to clear the chunk cache in react - require.resolve( - 'next/dist/compiled/react-server-dom-webpack/cjs/react-server-dom-webpack-client.edge.development.js' - ), - // And this redirecting module as well - require.resolve( - 'next/dist/compiled/react-server-dom-webpack/client.edge.js' - ), - ]) - ) - + for (const file of result.serverPaths + .map((p) => path.join(distDir, p)) + .concat([ + // We need to clear the chunk cache in react + require.resolve( + 'next/dist/compiled/react-server-dom-webpack/cjs/react-server-dom-webpack-client.edge.development.js' + ), + // And this redirecting module as well + require.resolve( + 'next/dist/compiled/react-server-dom-webpack/client.edge.js' + ), + ])) { + deleteCache(file) + } return result }