From 04aa052f89e329923c340d03ac8f7dfa3f0520b1 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 10:03:51 -0700 Subject: [PATCH 01/52] Add initial separated route resolving --- packages/next/src/build/webpack-config.ts | 7 +- .../nextjs-require-cache-hot-reloader.ts | 39 +- packages/next/src/cli/next-dev.ts | 587 +----------------- packages/next/src/cli/next-start.ts | 18 +- packages/next/src/lib/turbopack-warning.ts | 209 +++++++ packages/next/src/server/base-server.ts | 13 +- packages/next/src/server/dev/hot-reloader.ts | 14 +- .../next/src/server/dev/next-dev-server.ts | 89 ++- packages/next/src/server/lib/render-server.ts | 123 ++++ .../next/src/server/lib/route-resolver.ts | 2 +- packages/next/src/server/lib/start-server.ts | 270 ++++++-- packages/next/src/server/lib/utils.ts | 16 + packages/next/src/server/lib/worker-utils.ts | 47 ++ packages/next/src/server/next-server.ts | 322 +++++++++- packages/next/src/server/next.ts | 4 +- packages/next/src/server/router.ts | 21 +- test/e2e/app-dir/app/index.test.ts | 4 +- 17 files changed, 1052 insertions(+), 733 deletions(-) create mode 100644 packages/next/src/lib/turbopack-warning.ts create mode 100644 packages/next/src/server/lib/render-server.ts create mode 100644 packages/next/src/server/lib/worker-utils.ts diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 8c9ce1a05f93..6137a4867b8c 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -1598,7 +1598,12 @@ export default async function getBaseWebpackConfig( } })(), runtimeChunk: isClient - ? { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK } + ? // namespace runtime chunk when multiple workers are present + process.env.__NEXT_PRIVATE_RENDER_WORKER + ? { + name: `${CLIENT_STATIC_FILES_RUNTIME_WEBPACK}-${process.env.__NEXT_PRIVATE_RENDER_WORKER}`, + } + : { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK } : undefined, minimize: !dev && (isClient || isEdgeServer), minimizer: [ diff --git a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts index a4f2d1e919d6..9d9562a515af 100644 --- a/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts +++ b/packages/next/src/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts @@ -16,7 +16,30 @@ const originModules = [ const RUNTIME_NAMES = ['webpack-runtime', 'webpack-api-runtime'] -function deleteCache(filePath: string) { +export function deleteAppClientCache() { + if ((global as any)._nextDeleteAppClientCache) { + ;(global as any)._nextDeleteAppClientCache() + } + // ensure we reset the cache for sc_server components + // loaded via react-server-dom-webpack + const reactServerDomModId = require.resolve( + 'next/dist/compiled/react-server-dom-webpack/client.edge' + ) + const reactServerDomMod = require.cache[reactServerDomModId] + + if (reactServerDomMod) { + for (const child of reactServerDomMod.children) { + child.parent = null + delete require.cache[child.id] + } + } + delete require.cache[reactServerDomModId] +} + +export function deleteCache(filePath: string) { + if ((global as any)._nextDeleteCache) { + ;(global as any)._nextDeleteCache(filePath) + } try { filePath = realpathSync(filePath) } catch (e) { @@ -82,20 +105,6 @@ export class NextJsRequireCacheHotReloader implements WebpackPluginInstance { }) if (hasAppPath) { - // ensure we reset the cache for sc_server components - // loaded via react-server-dom-webpack - const reactServerDomModId = require.resolve( - 'next/dist/compiled/react-server-dom-webpack/client.edge' - ) - const reactServerDomMod = require.cache[reactServerDomModId] - - if (reactServerDomMod) { - for (const child of reactServerDomMod.children) { - child.parent = null - delete require.cache[child.id] - } - } - delete require.cache[reactServerDomModId] } entries.forEach((page) => { diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index f78b6fd0cedc..2fef2b30a532 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -1,36 +1,25 @@ #!/usr/bin/env node import arg from 'next/dist/compiled/arg/index.js' -import { startServer, WORKER_SELF_EXIT_CODE } from '../server/lib/start-server' +import { startServer } from '../server/lib/start-server' import { getPort, printAndExit } from '../server/lib/utils' import * as Log from '../build/output/log' -import { startedDevelopmentServer } from '../build/output' import { CliCommand } from '../lib/commands' import isError from '../lib/is-error' import { getProjectDir } from '../lib/get-project-dir' -import { CONFIG_FILES, PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' +import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' import path from 'path' -import type { NextConfig } from '../../types' import type { NextConfigComplete } from '../server/config-shared' import { traceGlobals } from '../trace/shared' -import { isIPv6 } from 'net' -import { ChildProcess, fork } from 'child_process' import { Telemetry } from '../telemetry/storage' import loadConfig from '../server/config' import { findPagesDir } from '../lib/find-pages-dir' import { fileExists } from '../lib/file-exists' -import Watchpack from 'next/dist/compiled/watchpack' -import stripAnsi from 'next/dist/compiled/strip-ansi' -import { warn } from '../build/output/log' -import { getPossibleInstrumentationHookFilenames } from '../build/utils' import { getNpxCommand } from '../lib/helpers/get-npx-command' +let dir: string let isTurboSession = false let sessionStopHandled = false let sessionStarted = Date.now() -let dir: string -let unwatchConfigFiles: () => void - -const isChildProcess = !!process.env.__NEXT_DEV_CHILD_PROCESS const handleSessionStop = async () => { if (sessionStopHandled) return @@ -91,30 +80,8 @@ const handleSessionStop = async () => { process.exit(0) } -if (!isChildProcess) { - process.on('SIGINT', handleSessionStop) - process.on('SIGTERM', handleSessionStop) -} else { - process.on('SIGINT', () => process.exit(0)) - process.on('SIGTERM', () => process.exit(0)) -} - -function watchConfigFiles(dirToWatch: string) { - if (unwatchConfigFiles) { - unwatchConfigFiles() - } - - const wp = new Watchpack() - wp.watch({ files: CONFIG_FILES.map((file) => path.join(dirToWatch, file)) }) - wp.on('change', (filename) => { - console.log( - `\n> Found a change in ${path.basename( - filename - )}. Restart the server to see the changes in effect.` - ) - }) - return () => wp.close() -} +process.on('SIGINT', handleSessionStop) +process.on('SIGTERM', handleSessionStop) const nextDev: CliCommand = async (argv) => { const validArgs: arg.Spec = { @@ -162,9 +129,7 @@ const nextDev: CliCommand = async (argv) => { `) process.exit(0) } - dir = getProjectDir(process.env.NEXT_PRIVATE_DEV_DIR || args._[0]) - unwatchConfigFiles = watchConfigFiles(dir) // Check if pages dir exists and warn if not if (!(await fileExists(dir, 'directory'))) { @@ -217,211 +182,20 @@ const nextDev: CliCommand = async (argv) => { const host = args['--hostname'] const devServerOptions = { - allowRetry, - dev: true, dir, - hostname: host, - isNextDevCommand: true, port, - } - - const supportedTurbopackNextConfigOptions = [ - 'configFileName', - 'env', - 'experimental.appDir', - 'experimental.serverComponentsExternalPackages', - 'experimental.turbo', - 'images', - 'pageExtensions', - 'onDemandEntries', - 'rewrites', - 'redirects', - 'headers', - 'reactStrictMode', - 'swcMinify', - 'transpilePackages', - ] - - // check for babelrc, swc plugins - async function validateNextConfig(isCustomTurbopack: boolean) { - const { getPkgManager } = - require('../lib/helpers/get-pkg-manager') as typeof import('../lib/helpers/get-pkg-manager') - const { getBabelConfigFile } = - require('../build/webpack-config') as typeof import('../build/webpack-config') - const { defaultConfig } = - require('../server/config-shared') as typeof import('../server/config-shared') - const chalk = - require('next/dist/compiled/chalk') as typeof import('next/dist/compiled/chalk') - const { interopDefault } = - require('../lib/interop-default') as typeof import('../lib/interop-default') - - // To regenerate the TURBOPACK gradient require('gradient-string')('blue', 'red')('>>> TURBOPACK') - const isTTY = process.stdout.isTTY - - const turbopackGradient = `${chalk.bold( - isTTY - ? '\x1B[38;2;0;0;255m>\x1B[39m\x1B[38;2;23;0;232m>\x1B[39m\x1B[38;2;46;0;209m>\x1B[39m \x1B[38;2;70;0;185mT\x1B[39m\x1B[38;2;93;0;162mU\x1B[39m\x1B[38;2;116;0;139mR\x1B[39m\x1B[38;2;139;0;116mB\x1B[39m\x1B[38;2;162;0;93mO\x1B[39m\x1B[38;2;185;0;70mP\x1B[39m\x1B[38;2;209;0;46mA\x1B[39m\x1B[38;2;232;0;23mC\x1B[39m\x1B[38;2;255;0;0mK\x1B[39m' - : '>>> TURBOPACK' - )} ${chalk.dim('(alpha)')}\n\n` - - let thankYouMsg = `Thank you for trying Next.js v13 with Turbopack! As a reminder,\nTurbopack is currently in alpha and not yet ready for production.\nWe appreciate your ongoing support as we work to make it ready\nfor everyone.\n` - - let unsupportedParts = '' - let babelrc = await getBabelConfigFile(dir) - if (babelrc) babelrc = path.basename(babelrc) - - let unsupportedConfig: string[] = [] - let rawNextConfig: NextConfig = {} - - try { - rawNextConfig = interopDefault( - await loadConfig(PHASE_DEVELOPMENT_SERVER, dir, undefined, true) - ) as NextConfig - - if (typeof rawNextConfig === 'function') { - rawNextConfig = (rawNextConfig as any)(PHASE_DEVELOPMENT_SERVER, { - defaultConfig, - }) - } - - const checkUnsupportedCustomConfig = ( - configKey = '', - parentUserConfig: any, - parentDefaultConfig: any - ): boolean => { - try { - // these should not error - if ( - // we only want the key after the dot for experimental options - supportedTurbopackNextConfigOptions - .map((key) => key.split('.').splice(-1)[0]) - .includes(configKey) - ) { - return false - } - - // experimental options are checked separately - if (configKey === 'experimental') { - return false - } - - let userValue = parentUserConfig?.[configKey] - let defaultValue = parentDefaultConfig?.[configKey] - - if (typeof defaultValue !== 'object') { - return defaultValue !== userValue - } - return Object.keys(userValue || {}).some((key: string) => { - return checkUnsupportedCustomConfig(key, userValue, defaultValue) - }) - } catch (e) { - console.error( - `Unexpected error occurred while checking ${configKey}`, - e - ) - return false - } - } - - unsupportedConfig = [ - ...Object.keys(rawNextConfig).filter((key) => - checkUnsupportedCustomConfig(key, rawNextConfig, defaultConfig) - ), - ...Object.keys(rawNextConfig.experimental ?? {}) - .filter((key) => - checkUnsupportedCustomConfig( - key, - rawNextConfig?.experimental, - defaultConfig?.experimental - ) - ) - .map((key) => `experimental.${key}`), - ] - } catch (e) { - console.error('Unexpected error occurred while checking config', e) - } - - const hasWarningOrError = babelrc || unsupportedConfig.length - if (!hasWarningOrError) { - thankYouMsg = chalk.dim(thankYouMsg) - } - if (!isCustomTurbopack) { - console.log(turbopackGradient + thankYouMsg) - } - - let feedbackMessage = `Learn more about Next.js v13 and Turbopack: ${chalk.underline( - 'https://nextjs.link/with-turbopack' - )}\nPlease direct feedback to: ${chalk.underline( - 'https://nextjs.link/turbopack-feedback' - )}\n` - - if (!hasWarningOrError) { - feedbackMessage = chalk.dim(feedbackMessage) - } - - if (babelrc) { - unsupportedParts += `\n- Babel detected (${chalk.cyan( - babelrc - )})\n ${chalk.dim( - `Babel is not yet supported. To use Turbopack at the moment,\n you'll need to remove your usage of Babel.` - )}` - } - if (unsupportedConfig.length) { - unsupportedParts += `\n\n- Unsupported Next.js configuration option(s) (${chalk.cyan( - 'next.config.js' - )})\n ${chalk.dim( - `To use Turbopack, remove the following configuration options:\n${unsupportedConfig - .map((name) => ` - ${chalk.red(name)}\n`) - .join( - '' - )} The only supported configurations options are:\n${supportedTurbopackNextConfigOptions - .map((name) => ` - ${chalk.cyan(name)}\n`) - .join('')} ` - )} ` - } - - if (unsupportedParts && !isCustomTurbopack) { - const pkgManager = getPkgManager(dir) - - console.error( - `${chalk.bold.red( - 'Error:' - )} You are using configuration and/or tools that are not yet\nsupported by Next.js v13 with Turbopack:\n${unsupportedParts}\n -If you cannot make the changes above, but still want to try out\nNext.js v13 with Turbopack, create the Next.js v13 playground app\nby running the following commands: - - ${chalk.bold.cyan( - `${ - pkgManager === 'npm' - ? 'npx create-next-app' - : `${pkgManager} create next-app` - } --example with-turbopack with-turbopack-app` - )}\n cd with-turbopack-app\n ${pkgManager} run dev - ` - ) - - if (!isCustomTurbopack) { - console.warn(feedbackMessage) - - process.exit(1) - } else { - console.warn( - `\n${chalk.bold.yellow( - 'Warning:' - )} Unsupported config found; but continuing with custom Turbopack binary.\n` - ) - } - } - - if (!isCustomTurbopack) { - console.log(feedbackMessage) - } - - return rawNextConfig + dev: true, + allowRetry, + isDev: true, + hostname: host, + useWorkers: !process.env.__NEXT_DISABLE_MEMORY_WATCHER, } if (args['--turbo']) { isTurboSession = true + const { validateTurboNextConfig } = + require('../lib/turbopack-warning') as typeof import('../lib/turbopack-warning') const { loadBindings, __isCustomTurbopackBinary } = require('../build/swc') as typeof import('../build/swc') const { eventCliSession } = @@ -432,7 +206,10 @@ If you cannot make the changes above, but still want to try out\nNext.js v13 wit require('next/dist/compiled/find-up') as typeof import('next/dist/compiled/find-up') const isCustomTurbopack = await __isCustomTurbopackBinary() - const rawNextConfig = await validateNextConfig(isCustomTurbopack) + const rawNextConfig = await validateTurboNextConfig({ + isCustomTurbopack, + ...devServerOptions, + }) const distDir = path.join(dir, rawNextConfig.distDir || '.next') const { pagesDir, appDir } = findPagesDir( @@ -487,336 +264,10 @@ If you cannot make the changes above, but still want to try out\nNext.js v13 wit } return server } else { - // we're using a sub worker to avoid memory leaks. When memory usage exceeds 90%, we kill the worker and restart it. - // this is a temporary solution until we can fix the memory leaks. - // the logic for the worker killing itself is in `packages/next/server/lib/start-server.ts` - if (!process.env.__NEXT_DISABLE_MEMORY_WATCHER && !isChildProcess) { - let config: NextConfig - let childProcess: ChildProcess | null = null - - const isDebugging = - process.execArgv.some((localArg) => localArg.startsWith('--inspect')) || - process.env.NODE_OPTIONS?.match?.(/--inspect(=\S+)?( |$)/) - - const isDebuggingWithBrk = - process.execArgv.some((localArg) => - localArg.startsWith('--inspect-brk') - ) || process.env.NODE_OPTIONS?.match?.(/--inspect-brk(=\S+)?( |$)/) - - const debugPort = (() => { - const debugPortStr = - process.execArgv - .find( - (localArg) => - localArg.startsWith('--inspect') || - localArg.startsWith('--inspect-brk') - ) - ?.split('=')[1] ?? - process.env.NODE_OPTIONS?.match?.( - /--inspect(-brk)?(=(\S+))?( |$)/ - )?.[3] - return debugPortStr ? parseInt(debugPortStr, 10) : 9229 - })() - - if (isDebugging || isDebuggingWithBrk) { - warn( - `the --inspect${ - isDebuggingWithBrk ? '-brk' : '' - } option was detected, the Next.js server should be inspected at port ${ - debugPort + 1 - }.` - ) - } - - const genExecArgv = () => { - const execArgv = process.execArgv.filter((localArg) => { - return ( - !localArg.startsWith('--inspect') && - !localArg.startsWith('--inspect-brk') - ) - }) + let teardownServer = await startServer(devServerOptions) - if (isDebugging || isDebuggingWithBrk) { - execArgv.push( - `--inspect${isDebuggingWithBrk ? '-brk' : ''}=${debugPort + 1}` - ) - } - - return execArgv - } - let childProcessExitUnsub: (() => void) | null = null - - const setupFork = (env?: NodeJS.ProcessEnv, newDir?: string) => { - childProcessExitUnsub?.() - childProcess?.kill() - - const startDir = dir - const [, script, ...nodeArgs] = process.argv - let shouldFilter = false - childProcess = fork( - newDir ? script.replace(startDir, newDir) : script, - nodeArgs, - { - env: { - ...(env ? env : process.env), - FORCE_COLOR: '1', - __NEXT_DEV_CHILD_PROCESS: '1', - }, - // @ts-ignore TODO: remove ignore when types are updated - windowsHide: true, - stdio: ['ipc', 'pipe', 'pipe'], - execArgv: genExecArgv(), - } - ) - - // since errors can start being logged from the fork - // before we detect the project directory rename - // attempt suppressing them long enough to check - const filterForkErrors = (chunk: Buffer, fd: 'stdout' | 'stderr') => { - const cleanChunk = stripAnsi(chunk + '') - if ( - cleanChunk.match( - /(ENOENT|Module build failed|Module not found|Cannot find module)/ - ) - ) { - if (startDir === dir) { - try { - // check if start directory is still valid - const result = findPagesDir( - startDir, - !!config.experimental?.appDir - ) - shouldFilter = !Boolean(result.pagesDir || result.appDir) - } catch (_) { - shouldFilter = true - } - } - if (shouldFilter || startDir !== dir) { - shouldFilter = true - return - } - } - process[fd].write(chunk) - } - - childProcess?.stdout?.on('data', (chunk) => { - filterForkErrors(chunk, 'stdout') - }) - childProcess?.stderr?.on('data', (chunk) => { - filterForkErrors(chunk, 'stderr') - }) - - const callback = async (code: number | null) => { - if (code === WORKER_SELF_EXIT_CODE) { - setupFork() - } else if (!sessionStopHandled) { - await handleSessionStop() - process.exit(1) - } - } - childProcess?.addListener('exit', callback) - childProcessExitUnsub = () => - childProcess?.removeListener('exit', callback) - } - - setupFork() - - config = await loadConfig( - PHASE_DEVELOPMENT_SERVER, - dir, - undefined, - undefined, - true - ) - - const handleProjectDirRename = (newDir: string) => { - process.chdir(newDir) - setupFork( - { - ...Object.keys(process.env).reduce((newEnv, key) => { - newEnv[key] = process.env[key]?.replace(dir, newDir) - return newEnv - }, {} as typeof process.env), - NEXT_PRIVATE_DEV_DIR: newDir, - }, - newDir - ) - } - const parentDir = path.join('/', dir, '..') - const watchedEntryLength = parentDir.split('/').length + 1 - const previousItems = new Set() - - const instrumentationFilePaths = !!config.experimental - ?.instrumentationHook - ? getPossibleInstrumentationHookFilenames(dir, config.pageExtensions!) - : [] - - const instrumentationFileWatcher = new Watchpack({}) - - instrumentationFileWatcher.watch({ - files: instrumentationFilePaths, - startTime: 0, - }) - - let instrumentationFileLastHash: string | undefined = undefined - const previousInstrumentationFiles = new Set() - instrumentationFileWatcher.on('aggregated', async () => { - const knownFiles = instrumentationFileWatcher.getTimeInfoEntries() - const instrumentationFile = [...knownFiles.entries()].find( - ([key, value]) => instrumentationFilePaths.includes(key) && value - )?.[0] - - if (instrumentationFile) { - const fs = require('fs') as typeof import('fs') - const instrumentationFileHash = ( - require('crypto') as typeof import('crypto') - ) - .createHash('sha256') - .update(await fs.promises.readFile(instrumentationFile, 'utf8')) - .digest('hex') - - if ( - instrumentationFileLastHash && - instrumentationFileHash !== instrumentationFileLastHash - ) { - warn( - `The instrumentation file has changed, restarting the server to apply changes.` - ) - return setupFork() - } else { - if ( - !instrumentationFileLastHash && - previousInstrumentationFiles.size !== 0 - ) { - warn( - 'The instrumentation file was added, restarting the server to apply changes.' - ) - return setupFork() - } - instrumentationFileLastHash = instrumentationFileHash - } - } else if ( - [...previousInstrumentationFiles.keys()].find((key) => - instrumentationFilePaths.includes(key) - ) - ) { - warn( - `The instrumentation file has been removed, restarting the server to apply changes.` - ) - instrumentationFileLastHash = undefined - return setupFork() - } - - previousInstrumentationFiles.clear() - knownFiles.forEach((_, key) => previousInstrumentationFiles.add(key)) - }) - - const projectFolderWatcher = new Watchpack({ - ignored: (entry: string) => { - return !(entry.split('/').length <= watchedEntryLength) - }, - }) - - projectFolderWatcher.watch({ directories: [parentDir], startTime: 0 }) - - projectFolderWatcher.on('aggregated', async () => { - const knownFiles = projectFolderWatcher.getTimeInfoEntries() - const newFiles: string[] = [] - let hasPagesApp = false - - // if the dir still exists nothing to check - try { - const result = findPagesDir(dir, !!config.experimental?.appDir) - hasPagesApp = Boolean(result.pagesDir || result.appDir) - } catch (err) { - // if findPagesDir throws validation error let this be - // handled in the dev-server itself in the fork - if ((err as any).message?.includes('experimental')) { - return - } - } - - // try to find new dir introduced - if (previousItems.size) { - for (const key of knownFiles.keys()) { - if (!previousItems.has(key)) { - newFiles.push(key) - } - } - previousItems.clear() - } - - for (const key of knownFiles.keys()) { - previousItems.add(key) - } - - if (hasPagesApp) { - return - } - - // if we failed to find the new dir it may have been moved - // to a new parent directory which we can't track as easily - // so exit gracefully - try { - const result = findPagesDir( - newFiles[0], - !!config.experimental?.appDir - ) - hasPagesApp = Boolean(result.pagesDir || result.appDir) - } catch (_) {} - - if (hasPagesApp && newFiles.length === 1) { - Log.info( - `Detected project directory rename, restarting in new location` - ) - handleProjectDirRename(newFiles[0]) - watchConfigFiles(newFiles[0]) - dir = newFiles[0] - } else { - Log.error( - `Project directory could not be found, restart Next.js in your new directory` - ) - process.exit(0) - } - }) - } else { - startServer(devServerOptions) - .then(async (app) => { - const appUrl = `http://${app.hostname}:${app.port}` - const hostname = host || '0.0.0.0' - startedDevelopmentServer( - appUrl, - `${isIPv6(hostname) ? `[${hostname}]` : hostname}:${app.port}` - ) - // Start preflight after server is listening and ignore errors: - preflight().catch(() => {}) - // Finalize server bootup: - await app.prepare() - }) - .catch((err) => { - if (err.code === 'EADDRINUSE') { - let errorMessage = `Port ${port} is already in use.` - const pkgAppPath = require('next/dist/compiled/find-up').sync( - 'package.json', - { - cwd: dir, - } - ) - const appPackage = require(pkgAppPath) - if (appPackage.scripts) { - const nextScript = Object.entries(appPackage.scripts).find( - (scriptLine) => scriptLine[1] === 'next' - ) - if (nextScript) { - errorMessage += `\nUse \`npm run ${nextScript[0]} -- -p \`.` - } - } - console.error(errorMessage) - } else { - console.error(err) - } - process.nextTick(() => process.exit(1)) - }) + // if we're using workers we can auto restart on config changes + if (devServerOptions.useWorkers) { } } } diff --git a/packages/next/src/cli/next-start.ts b/packages/next/src/cli/next-start.ts index 3b4c61c0082a..d88097c17d9e 100755 --- a/packages/next/src/cli/next-start.ts +++ b/packages/next/src/cli/next-start.ts @@ -3,13 +3,11 @@ import arg from 'next/dist/compiled/arg/index.js' import { startServer } from '../server/lib/start-server' import { getPort, printAndExit } from '../server/lib/utils' -import * as Log from '../build/output/log' import isError from '../lib/is-error' import { getProjectDir } from '../lib/get-project-dir' import { CliCommand } from '../lib/commands' -import { isIPv6 } from 'net' -const nextStart: CliCommand = (argv) => { +const nextStart: CliCommand = async (argv) => { const validArgs: arg.Spec = { // Types '--help': Boolean, @@ -73,22 +71,14 @@ const nextStart: CliCommand = (argv) => { ? Math.ceil(keepAliveTimeoutArg) : undefined - startServer({ + await startServer({ dir, + isDev: false, hostname: host, port, keepAliveTimeout, + useWorkers: false, }) - .then(async (app) => { - const appUrl = `http://${app.hostname}:${app.port}` - const hostname = isIPv6(host) ? `[${host}]` : host - Log.ready(`started server on ${hostname}:${app.port}, url: ${appUrl}`) - await app.prepare() - }) - .catch((err) => { - console.error(err) - process.exit(1) - }) } export { nextStart } diff --git a/packages/next/src/lib/turbopack-warning.ts b/packages/next/src/lib/turbopack-warning.ts new file mode 100644 index 000000000000..0d0fa468c16b --- /dev/null +++ b/packages/next/src/lib/turbopack-warning.ts @@ -0,0 +1,209 @@ +import path from 'path' +import loadConfig from '../server/config' +import { NextConfig } from '../server/config-shared' +import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' + +const supportedTurbopackNextConfigOptions = [ + 'configFileName', + 'env', + 'experimental.appDir', + 'experimental.serverComponentsExternalPackages', + 'experimental.turbo', + 'images', + 'pageExtensions', + 'onDemandEntries', + 'rewrites', + 'redirects', + 'headers', + 'reactStrictMode', + 'swcMinify', + 'transpilePackages', +] + +// check for babelrc, swc plugins +export async function validateTurboNextConfig({ + dir, + isCustomTurbopack, + port, + hostname, +}: { + allowRetry?: boolean + isCustomTurbopack?: boolean + dir: string + port: number + hostname?: string +}) { + const { getPkgManager } = + require('../lib/helpers/get-pkg-manager') as typeof import('../lib/helpers/get-pkg-manager') + const { getBabelConfigFile } = + require('../build/webpack-config') as typeof import('../build/webpack-config') + const { defaultConfig } = + require('../server/config-shared') as typeof import('../server/config-shared') + const chalk = + require('next/dist/compiled/chalk') as typeof import('next/dist/compiled/chalk') + const { interopDefault } = + require('../lib/interop-default') as typeof import('../lib/interop-default') + + // To regenerate the TURBOPACK gradient require('gradient-string')('blue', 'red')('>>> TURBOPACK') + const isTTY = process.stdout.isTTY + + const turbopackGradient = `${chalk.bold( + isTTY + ? '\x1B[38;2;0;0;255m>\x1B[39m\x1B[38;2;23;0;232m>\x1B[39m\x1B[38;2;46;0;209m>\x1B[39m \x1B[38;2;70;0;185mT\x1B[39m\x1B[38;2;93;0;162mU\x1B[39m\x1B[38;2;116;0;139mR\x1B[39m\x1B[38;2;139;0;116mB\x1B[39m\x1B[38;2;162;0;93mO\x1B[39m\x1B[38;2;185;0;70mP\x1B[39m\x1B[38;2;209;0;46mA\x1B[39m\x1B[38;2;232;0;23mC\x1B[39m\x1B[38;2;255;0;0mK\x1B[39m' + : '>>> TURBOPACK' + )} ${chalk.dim('(alpha)')}\n\n` + + let thankYouMsg = `Thank you for trying Next.js v13 with Turbopack! As a reminder,\nTurbopack is currently in alpha and not yet ready for production.\nWe appreciate your ongoing support as we work to make it ready\nfor everyone.\n` + + let unsupportedParts = '' + let babelrc = await getBabelConfigFile(dir) + if (babelrc) babelrc = path.basename(babelrc) + + let unsupportedConfig: string[] = [] + let rawNextConfig: NextConfig = {} + + try { + rawNextConfig = interopDefault( + await loadConfig(PHASE_DEVELOPMENT_SERVER, dir, undefined, true) + ) as NextConfig + + if (typeof rawNextConfig === 'function') { + rawNextConfig = (rawNextConfig as any)(PHASE_DEVELOPMENT_SERVER, { + defaultConfig, + }) + } + + const checkUnsupportedCustomConfig = ( + configKey = '', + parentUserConfig: any, + parentDefaultConfig: any + ): boolean => { + try { + // these should not error + if ( + // we only want the key after the dot for experimental options + supportedTurbopackNextConfigOptions + .map((key) => key.split('.').splice(-1)[0]) + .includes(configKey) + ) { + return false + } + + // experimental options are checked separately + if (configKey === 'experimental') { + return false + } + + let userValue = parentUserConfig?.[configKey] + let defaultValue = parentDefaultConfig?.[configKey] + + if (typeof defaultValue !== 'object') { + return defaultValue !== userValue + } + return Object.keys(userValue || {}).some((key: string) => { + return checkUnsupportedCustomConfig(key, userValue, defaultValue) + }) + } catch (e) { + console.error( + `Unexpected error occurred while checking ${configKey}`, + e + ) + return false + } + } + + unsupportedConfig = [ + ...Object.keys(rawNextConfig).filter((key) => + checkUnsupportedCustomConfig(key, rawNextConfig, defaultConfig) + ), + ...Object.keys(rawNextConfig.experimental ?? {}) + .filter((key) => + checkUnsupportedCustomConfig( + key, + rawNextConfig?.experimental, + defaultConfig?.experimental + ) + ) + .map((key) => `experimental.${key}`), + ] + } catch (e) { + console.error('Unexpected error occurred while checking config', e) + } + + const hasWarningOrError = babelrc || unsupportedConfig.length + if (!hasWarningOrError) { + thankYouMsg = chalk.dim(thankYouMsg) + } + if (!isCustomTurbopack) { + console.log(turbopackGradient + thankYouMsg) + } + + let feedbackMessage = `Learn more about Next.js v13 and Turbopack: ${chalk.underline( + 'https://nextjs.link/with-turbopack' + )}\nPlease direct feedback to: ${chalk.underline( + 'https://nextjs.link/turbopack-feedback' + )}\n` + + if (!hasWarningOrError) { + feedbackMessage = chalk.dim(feedbackMessage) + } + + if (babelrc) { + unsupportedParts += `\n- Babel detected (${chalk.cyan( + babelrc + )})\n ${chalk.dim( + `Babel is not yet supported. To use Turbopack at the moment,\n you'll need to remove your usage of Babel.` + )}` + } + if (unsupportedConfig.length) { + unsupportedParts += `\n\n- Unsupported Next.js configuration option(s) (${chalk.cyan( + 'next.config.js' + )})\n ${chalk.dim( + `To use Turbopack, remove the following configuration options:\n${unsupportedConfig + .map((name) => ` - ${chalk.red(name)}\n`) + .join( + '' + )} The only supported configurations options are:\n${supportedTurbopackNextConfigOptions + .map((name) => ` - ${chalk.cyan(name)}\n`) + .join('')} ` + )} ` + } + + if (unsupportedParts && !isCustomTurbopack) { + const pkgManager = getPkgManager(dir) + + console.error( + `${chalk.bold.red( + 'Error:' + )} You are using configuration and/or tools that are not yet\nsupported by Next.js v13 with Turbopack:\n${unsupportedParts}\n +If you cannot make the changes above, but still want to try out\nNext.js v13 with Turbopack, create the Next.js v13 playground app\nby running the following commands: + + ${chalk.bold.cyan( + `${ + pkgManager === 'npm' + ? 'npx create-next-app' + : `${pkgManager} create next-app` + } --example with-turbopack with-turbopack-app` + )}\n cd with-turbopack-app\n ${pkgManager} run dev + ` + ) + + if (!isCustomTurbopack) { + console.warn(feedbackMessage) + + process.exit(1) + } else { + console.warn( + `\n${chalk.bold.yellow( + 'Warning:' + )} Unsupported config found; but continuing with custom Turbopack binary.\n` + ) + } + } + + if (!isCustomTurbopack) { + console.log(feedbackMessage) + } + + return rawNextConfig +} diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index b87e921da910..cc578ae6ea57 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -149,6 +149,11 @@ export interface Options { * The HTTP Server that Next.js is running behind */ httpServer?: import('http').Server + + _routerWorker?: boolean + _renderWorker?: boolean + + isNodeDebugging?: 'brk' | boolean } export interface BaseRequestHandler { @@ -277,7 +282,7 @@ export default abstract class Server { protected abstract getCustomRoutes(): CustomRoutes protected abstract hasPage(pathname: string): Promise - protected abstract generateRoutes(): RouterOptions + protected abstract generateRoutes(dev?: boolean): RouterOptions protected abstract sendRenderResult( req: BaseNextRequest, @@ -330,6 +335,8 @@ export default abstract class Server { public readonly matchers: RouteMatcherManager protected readonly handlers: RouteHandlerManager protected readonly localeNormalizer?: LocaleRouteNormalizer + protected readonly isRouterWorker?: boolean + protected readonly isRenderWorker?: boolean public constructor(options: ServerOptions) { const { @@ -343,6 +350,8 @@ export default abstract class Server { port, } = options this.serverOptions = options + this.isRouterWorker = options._routerWorker + this.isRenderWorker = options._renderWorker this.dir = process.env.NEXT_RUNTIME === 'edge' ? dir : require('path').resolve(dir) @@ -449,7 +458,7 @@ export default abstract class Server { matchers.reload() this.customRoutes = this.getCustomRoutes() - this.router = new Router(this.generateRoutes()) + this.router = new Router(this.generateRoutes(dev)) this.setAssetPrefix(assetPrefix) this.responseCache = this.getResponseCache({ dev }) diff --git a/packages/next/src/server/dev/hot-reloader.ts b/packages/next/src/server/dev/hot-reloader.ts index 950530cf9af4..4cbed3ed7967 100644 --- a/packages/next/src/server/dev/hot-reloader.ts +++ b/packages/next/src/server/dev/hot-reloader.ts @@ -158,6 +158,10 @@ function erroredPages(compilation: webpack.Compilation) { return failedPages } +export function cleanDistDir(distDir: string) { + return recursiveDelete(distDir, /^cache/) +} + export default class HotReloader { private dir: string private buildId: string @@ -438,9 +442,7 @@ export default class HotReloader { private async clean(span: Span): Promise { return span .traceChild('clean') - .traceAsyncFn(() => - recursiveDelete(join(this.dir, this.config.distDir), /^cache/) - ) + .traceAsyncFn(() => cleanDistDir(join(this.dir, this.config.distDir))) } private async getVersionInfo(span: Span, enabled: boolean) { @@ -635,7 +637,7 @@ export default class HotReloader { }) } - public async start(): Promise { + public async start(skipClean?: boolean): Promise { const startSpan = this.hotReloaderSpan.traceChild('start') startSpan.stop() // Stop immediately to create an artificial parent span @@ -644,7 +646,9 @@ export default class HotReloader { !!process.env.NEXT_TEST_MODE || this.telemetry.isEnabled ) - await this.clean(startSpan) + if (!skipClean) { + await this.clean(startSpan) + } // Ensure distDir exists before writing package.json await fs.mkdir(this.distDir, { recursive: true }) diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 0500b7772e41..4f2db0940312 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -17,7 +17,13 @@ import crypto from 'crypto' import fs from 'fs' import { Worker } from 'next/dist/compiled/jest-worker' import findUp from 'next/dist/compiled/find-up' -import { join as pathJoin, relative, resolve as pathResolve, sep } from 'path' +import { + join, + join as pathJoin, + relative, + resolve as pathResolve, + sep, +} from 'path' import Watchpack from 'next/dist/compiled/watchpack' import { ampValidation } from '../../build/output' import { PUBLIC_DIR_MIDDLEWARE_CONFLICT } from '../../lib/constants' @@ -47,7 +53,7 @@ import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-pref import { eventCliSession } from '../../telemetry/events' import { Telemetry } from '../../telemetry/storage' import { setGlobal } from '../../trace' -import HotReloader from './hot-reloader' +import HotReloader, { cleanDistDir } from './hot-reloader' import { createValidFileMatcher, findPageFile } from '../lib/find-page-file' import { getNodeOptionsWithoutInspect } from '../lib/utils' import { @@ -65,7 +71,7 @@ import { import * as Log from '../../build/output/log' import isError, { getProperError } from '../../lib/is-error' import { getRouteRegex } from '../../shared/lib/router/utils/route-regex' -import { getSortedRoutes } from '../../shared/lib/router/utils' +import { getSortedRoutes, isDynamicRoute } from '../../shared/lib/router/utils' import { runDependingOnPageType } from '../../build/entries' import { NodeNextResponse, NodeNextRequest } from '../base-http/node' import { getPageStaticInfo } from '../../build/analysis/get-page-static-info' @@ -777,6 +783,18 @@ export default class DevServer extends Server { // before it has been built and is populated in the _buildManifest const sortedRoutes = getSortedRoutes(routedPages) + this.dynamicRoutes = sortedRoutes + .map((page) => { + if (!isDynamicRoute) return null + const regex = getRouteRegex(page) + return { + match: getRouteMatcher(regex), + page, + re: regex.re, + } + }) + .filter(Boolean) as any + if ( !this.sortedRoutes?.every((val, idx) => val === sortedRoutes[idx]) ) { @@ -860,24 +878,26 @@ export default class DevServer extends Server { redirects.length || headers.length ) { - this.router = new Router(this.generateRoutes()) + this.router = new Router(this.generateRoutes(true)) } - const telemetry = new Telemetry({ distDir: this.distDir }) - this.hotReloader = new HotReloader(this.dir, { - pagesDir: this.pagesDir, - distDir: this.distDir, - config: this.nextConfig, - previewProps: this.getPreviewProps(), - buildId: this.buildId, - rewrites, - appDir: this.appDir, - telemetry, - }) + // router worker does not start webpack compilers + if (this.isRouterWorker) { + this.hotReloader = new HotReloader(this.dir, { + pagesDir: this.pagesDir, + distDir: this.distDir, + config: this.nextConfig, + previewProps: this.getPreviewProps(), + buildId: this.buildId, + rewrites, + appDir: this.appDir, + telemetry, + }) + } await super.prepare() await this.addExportPathMapRoutes() - await this.hotReloader.start() + await this.hotReloader?.start(this.isRouterWorker || this.isRenderWorker) await this.startWatcher() await this.runInstrumentationHookIfAvailable() await this.matchers.reload() @@ -899,18 +919,21 @@ export default class DevServer extends Server { this.dir, this.pagesDir || this.appDir || '' ).startsWith('src') - telemetry.record( - eventCliSession(this.distDir, this.nextConfig, { - webpackVersion: 5, - cliCommand: 'dev', - isSrcDir, - hasNowJson: !!(await findUp('now.json', { cwd: this.dir })), - isCustomServer: this.isCustomServer, - turboFlag: false, - pagesDir: !!this.pagesDir, - appDir: !!this.appDir, - }) - ) + + if (!this.isRenderWorker) { + telemetry.record( + eventCliSession(this.distDir, this.nextConfig, { + webpackVersion: 5, + cliCommand: 'dev', + isSrcDir, + hasNowJson: !!(await findUp('now.json', { cwd: this.dir })), + isCustomServer: this.isCustomServer, + turboFlag: false, + pagesDir: !!this.pagesDir, + appDir: !!this.appDir, + }) + ) + } process.on('unhandledRejection', (reason) => { this.logErrorWithOriginalStack(reason, 'unhandledRejection').catch( @@ -1030,7 +1053,7 @@ export default class DevServer extends Server { } else { const { basePath } = this.nextConfig - server.on('upgrade', (req, socket, head) => { + server.on('upgrade', async (req, socket, head) => { let assetPrefix = (this.nextConfig.assetPrefix || '').replace( /^\/+/, '' @@ -1050,7 +1073,9 @@ export default class DevServer extends Server { `${basePath || assetPrefix || ''}/_next/webpack-hmr` ) ) { - this.hotReloader?.onHMR(req, socket, head) + if (this.isRouterWorker) { + this.hotReloader?.onHMR(req, socket, head) + } } else { this.handleUpgrade(req, socket, head) } @@ -1407,8 +1432,8 @@ export default class DevServer extends Server { return this.hotReloader?.ensurePage({ page, appPaths, clientOnly: false }) } - generateRoutes() { - const { fsRoutes, ...otherRoutes } = super.generateRoutes() + generateRoutes(dev?: boolean) { + const { fsRoutes, ...otherRoutes } = super.generateRoutes(dev) // Create a shallow copy so we can mutate it. const routes = [...fsRoutes] diff --git a/packages/next/src/server/lib/render-server.ts b/packages/next/src/server/lib/render-server.ts new file mode 100644 index 000000000000..2cef71949559 --- /dev/null +++ b/packages/next/src/server/lib/render-server.ts @@ -0,0 +1,123 @@ +import v8 from 'v8' +import http from 'http' +import next from '../next' +import { isIPv6 } from 'net' +import { warn } from '../../build/output/log' +import type { RequestHandler } from '../next' +import { + deleteCache as _deleteCache, + deleteAppClientCache as _deleteAppClientCache, +} from '../../build/webpack/plugins/nextjs-require-cache-hot-reloader' + +export const WORKER_SELF_EXIT_CODE = 77 + +const MAXIMUM_HEAP_SIZE_ALLOWED = + (v8.getHeapStatistics().heap_size_limit / 1024 / 1024) * 0.9 + +let result: + | undefined + | { + port: number + hostname: string + } + +export function deleteAppClientCache() { + _deleteAppClientCache() +} + +export function deleteCache(filePath: string) { + _deleteCache(filePath) +} + +export async function initialize(opts: { + dir: string + port: number + dev: boolean + hostname?: string + workerType: 'router' | 'render' + keepAliveTimeout?: number +}): Promise> { + // if we already setup the server return as we only need to do + // this on first worker boot + if (result) { + return result + } + let requestHandler: RequestHandler + + const server = http.createServer((req, res) => { + return requestHandler(req, res).finally(() => { + if ( + process.memoryUsage().heapUsed / 1024 / 1024 > + MAXIMUM_HEAP_SIZE_ALLOWED + ) { + warn( + 'The server is running out of memory, restarting to free up memory.' + ) + server.close() + process.exit(WORKER_SELF_EXIT_CODE) + } + }) + }) + + if (opts.keepAliveTimeout) { + server.keepAliveTimeout = opts.keepAliveTimeout + } + + return new Promise((resolve, reject) => { + server.on('error', (err: NodeJS.ErrnoException) => { + console.error(`Invariant: failed to start render worker`, err) + process.exit(1) + }) + + let upgradeHandler: any + + if (!opts.dev) { + server.on('upgrade', (req, socket, upgrade) => { + upgradeHandler(req, socket, upgrade) + }) + } + + server.on('listening', async () => { + try { + const addr = server.address() + const port = addr && typeof addr === 'object' ? addr.port : 0 + + if (!port) { + console.error(`Invariant failed to detect render worker port`, addr) + process.exit(1) + } + + let hostname = + !opts.hostname || opts.hostname === '0.0.0.0' + ? 'localhost' + : opts.hostname + + if (isIPv6(hostname)) { + hostname = hostname === '::' ? '[::1]' : `[${hostname}]` + } + result = { + port, + hostname, + } + const app = next({ + ...opts, + _routerWorker: opts.workerType === 'router', + _renderWorker: opts.workerType === 'render', + hostname, + customServer: false, + httpServer: server, + port: opts.port, + }) + + requestHandler = app.getRequestHandler() + upgradeHandler = app.getUpgradeHandler() + await app.prepare() + resolve(result) + } catch (err) { + console.error(err) + return reject(err) + } + }) + server.listen(0, opts.hostname) + }) +} diff --git a/packages/next/src/server/lib/route-resolver.ts b/packages/next/src/server/lib/route-resolver.ts index 1915aa623b59..9e116d8f55e4 100644 --- a/packages/next/src/server/lib/route-resolver.ts +++ b/packages/next/src/server/lib/route-resolver.ts @@ -124,7 +124,7 @@ export async function makeResolver( } const routeResults = new WeakMap() - const routes = devServer.generateRoutes() + const routes = devServer.generateRoutes(true) // @ts-expect-error protected const catchAllMiddleware = devServer.generateCatchAllMiddlewareRoute(true) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 6a189983ee24..4aa79bd6b552 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -1,94 +1,232 @@ -import type { NextServerOptions, NextServer, RequestHandler } from '../next' -import { warn } from '../../build/output/log' -import http from 'http' -import next from '../next' import { isIPv6 } from 'net' -import v8 from 'v8' -const isChildProcess = !!process.env.__NEXT_DEV_CHILD_PROCESS +import * as Log from '../../build/output/log' +import { getNodeOptionsWithoutInspect } from './utils' +import type { IncomingMessage, ServerResponse } from 'http' -interface StartServerOptions extends NextServerOptions { +interface StartServerOptions { + dir: string + port: number + isDev: boolean + hostname: string + useWorkers: boolean allowRetry?: boolean + isTurbopack?: boolean keepAliveTimeout?: number } -export const WORKER_SELF_EXIT_CODE = 77 - -const MAXIMUM_HEAP_SIZE_ALLOWED = - (v8.getHeapStatistics().heap_size_limit / 1024 / 1024) * 0.9 - -export function startServer(opts: StartServerOptions) { - let requestHandler: RequestHandler - - const server = http.createServer((req, res) => { - return requestHandler(req, res).finally(() => { - if ( - isChildProcess && - process.memoryUsage().heapUsed / 1024 / 1024 > MAXIMUM_HEAP_SIZE_ALLOWED - ) { - warn( - 'The server is running out of memory, restarting to free up memory.' - ) - server.close() - process.exit(WORKER_SELF_EXIT_CODE) - } - }) - }) +type TeardownServer = () => Promise + +export async function startServer({ + dir, + port, + isDev, + hostname, + useWorkers, + allowRetry, + keepAliveTimeout, +}: StartServerOptions): Promise { + const http = await import('http') + const sockets = new Set() + let worker: import('next/dist/compiled/jest-worker').Worker | undefined + let handlersReady = () => {} + let handlersError = () => {} + + let isNodeDebugging: 'brk' | boolean = !!( + process.execArgv.some((localArg) => localArg.startsWith('--inspect')) || + process.env.NODE_OPTIONS?.match?.(/--inspect(=\S+)?( |$)/) + ) - if (opts.keepAliveTimeout) { - server.keepAliveTimeout = opts.keepAliveTimeout + if ( + process.execArgv.some((localArg) => localArg.startsWith('--inspect-brk')) || + process.env.NODE_OPTIONS?.match?.(/--inspect-brk(=\S+)?( |$)/) + ) { + isNodeDebugging = 'brk' } - return new Promise((resolve, reject) => { - let port = opts.port - let retryCount = 0 - - server.on('error', (err: NodeJS.ErrnoException) => { - if ( - port && - opts.allowRetry && - err.code === 'EADDRINUSE' && - retryCount < 10 - ) { - warn(`Port ${port} is in use, trying ${port + 1} instead.`) - port += 1 - retryCount += 1 - server.listen(port, opts.hostname) - } else { - reject(err) + let handlersPromise: Promise | undefined = new Promise( + (resolve, reject) => { + handlersReady = resolve + handlersError = reject + } + ) + let requestHandler = async ( + _req: IncomingMessage, + _res: ServerResponse + ): Promise => { + throw new Error('Invariant request handler was not setup') + } + let upgradeHandler = async ( + _req: IncomingMessage, + _socket: ServerResponse, + _head: Buffer + ): Promise => { + throw new Error('Invariant upgrade handler was not setup') + } + + // setup server listener as fast as possible + const server = http.createServer(async (req, res) => { + try { + if (handlersPromise) { + await handlersPromise + handlersPromise = undefined } - }) + sockets.add(res) + res.on('close', () => sockets.delete(res)) + await requestHandler(req, res) + } catch (err) { + res.statusCode = 500 + res.end('Internal Server Error') + Log.error(`Failed to handle request for ${req.url}`) + console.error(err) + } + }) + server.on('upgrade', async (req, socket, head) => { + try { + sockets.add(socket) + socket.on('close', () => sockets.delete(socket)) + await upgradeHandler(req, socket, head) + } catch (err) { + socket.destroy() + Log.error(`Failed to handle request for ${req.url}`) + console.error(err) + } + }) - let upgradeHandler: any + let portRetryCount = 0 - if (!opts.dev) { - server.on('upgrade', (req, socket, upgrade) => { - upgradeHandler(req, socket, upgrade) - }) + server.on('error', (err: NodeJS.ErrnoException) => { + if ( + allowRetry && + port && + isDev && + err.code === 'EADDRINUSE' && + portRetryCount < 10 + ) { + Log.warn(`Port ${port} is in use, trying ${port + 1} instead.`) + port += 1 + portRetryCount += 1 + server.listen(port, hostname) + } else { + Log.error(`Failed to start server`) + console.error(err) + process.exit(1) } + }) + const host = hostname || '0.0.0.0' + const normalizedHost = isIPv6(host) ? `[${host}]` : host + await new Promise((resolve) => { server.on('listening', () => { const addr = server.address() - let hostname = - !opts.hostname || opts.hostname === '0.0.0.0' - ? 'localhost' - : opts.hostname - if (isIPv6(hostname)) { - hostname = hostname === '::' ? '[::1]' : `[${hostname}]` + port = typeof addr === 'object' ? addr?.port || port : port + const appUrl = `http://${host}:${port}` + Log.ready(`started server on ${normalizedHost}:${port}, url: ${appUrl}`) + resolve() + }) + server.listen(port, hostname) + }) + + try { + if (useWorkers) { + const { Worker } = + require('next/dist/compiled/jest-worker') as typeof import('next/dist/compiled/jest-worker') + + const httpProxy = + require('next/dist/compiled/http-proxy') as typeof import('next/dist/compiled/http-proxy') + + const routerWorker = new Worker(require.resolve('./render-server'), { + numWorkers: 1, + forkOptions: { + env: { + ...process.env, + // we don't pass down NODE_OPTIONS as it can + // extra memory usage + NODE_OPTIONS: getNodeOptionsWithoutInspect() + .replace(/--max-old-space-size=[\d]{1,}/, '') + .trim(), + }, + }, + exposedMethods: ['initialize'], + }) as any as InstanceType & { + initialize: typeof import('./render-server').initialize + } + routerWorker.getStdout().pipe(process.stdout) + routerWorker.getStderr().pipe(process.stderr) + + const { port: routerPort } = await routerWorker.initialize({ + dir, + port, + hostname, + dev: !!isDev, + workerType: 'router', + keepAliveTimeout, + }) + + const getProxyServer = (pathname: string) => { + const targetUrl = `http://${normalizedHost}:${routerPort}${pathname}` + + const proxyServer = httpProxy.createProxy({ + target: targetUrl, + changeOrigin: true, + ignorePath: true, + xfwd: true, + ws: true, + }) + + proxyServer.on('error', () => { + // TODO?: enable verbose error logs with --debug flag? + }) + return proxyServer } + // proxy to router worker + requestHandler = async (req, res) => { + const proxyServer = getProxyServer(req.url || '/') + proxyServer.web(req, res) + } + upgradeHandler = async (req, socket, head) => { + const proxyServer = getProxyServer(req.url || '/') + proxyServer.ws(req, socket, head) + } + handlersReady() + } else { + // when not using a worker start next in main process + const { default: next } = require('../next') as typeof import('../next') + const addr = server.address() const app = next({ - ...opts, + dir, hostname, - customServer: false, + dev: isDev, + isNodeDebugging, httpServer: server, + customServer: false, port: addr && typeof addr === 'object' ? addr.port : port, }) - + // handle in process requestHandler = app.getRequestHandler() upgradeHandler = app.getUpgradeHandler() - resolve(app) + handlersReady() + await app.prepare() + } + } catch (err) { + // fatal error if we can't setup + handlersError() + Log.error(`Failed to setup request handlers`) + console.error(err) + process.exit(1) + } + + // return teardown function for destroying the server + async function teardown() { + server.close() + sockets.forEach((socket) => { + sockets.delete(socket) + socket.destroy() }) - server.listen(port, opts.hostname) - }) + if (worker) { + await worker.end() + } + } + return teardown } diff --git a/packages/next/src/server/lib/utils.ts b/packages/next/src/server/lib/utils.ts index b149393d25ad..afb62663ec4c 100644 --- a/packages/next/src/server/lib/utils.ts +++ b/packages/next/src/server/lib/utils.ts @@ -10,6 +10,22 @@ export function printAndExit(message: string, code = 1) { process.exit(code) } +export const genExecArgv = (enabled: boolean | 'brk', debugPort: number) => { + const execArgv = process.execArgv.filter((localArg) => { + return ( + !localArg.startsWith('--inspect') && !localArg.startsWith('--inspect-brk') + ) + }) + + if (enabled) { + execArgv.push( + `--inspect${enabled === 'brk' ? '-brk' : ''}=${debugPort + 1}` + ) + } + + return execArgv +} + export function getNodeOptionsWithoutInspect() { const NODE_INSPECT_RE = /--inspect(-brk)?(=\S+)?( |$)/ return (process.env.NODE_OPTIONS || '').replace(NODE_INSPECT_RE, '') diff --git a/packages/next/src/server/lib/worker-utils.ts b/packages/next/src/server/lib/worker-utils.ts new file mode 100644 index 000000000000..13cda78c85a2 --- /dev/null +++ b/packages/next/src/server/lib/worker-utils.ts @@ -0,0 +1,47 @@ +import * as Log from '../../build/output/log' + +export const genRenderExecArgv = () => { + const isDebugging = + process.execArgv.some((localArg) => localArg.startsWith('--inspect')) || + process.env.NODE_OPTIONS?.match?.(/--inspect(=\S+)?( |$)/) + + const isDebuggingWithBrk = + process.execArgv.some((localArg) => localArg.startsWith('--inspect-brk')) || + process.env.NODE_OPTIONS?.match?.(/--inspect-brk(=\S+)?( |$)/) + + const debugPort = (() => { + const debugPortStr = + process.execArgv + .find( + (localArg) => + localArg.startsWith('--inspect') || + localArg.startsWith('--inspect-brk') + ) + ?.split('=')[1] ?? + process.env.NODE_OPTIONS?.match?.(/--inspect(-brk)?(=(\S+))?( |$)/)?.[3] + return debugPortStr ? parseInt(debugPortStr, 10) : 9229 + })() + + if (isDebugging || isDebuggingWithBrk) { + Log.warn( + `the --inspect${ + isDebuggingWithBrk ? '-brk' : '' + } option was detected, the Next.js server should be inspected at port ${ + debugPort + 1 + }.` + ) + } + const execArgv = process.execArgv.filter((localArg) => { + return ( + !localArg.startsWith('--inspect') && !localArg.startsWith('--inspect-brk') + ) + }) + + if (isDebugging || isDebuggingWithBrk) { + execArgv.push( + `--inspect${isDebuggingWithBrk ? '-brk' : ''}=${debugPort + 1}` + ) + } + + return execArgv +} diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index c73f99040a35..d6dc832856da 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -19,7 +19,10 @@ import type { BaseNextRequest, BaseNextResponse } from './base-http' import type { PagesManifest } from '../build/webpack/plugins/pages-manifest-plugin' import type { PayloadOptions } from './send-payload' import type { NextParsedUrlQuery, NextUrlWithParsedQuery } from './request-meta' -import type { Params } from '../shared/lib/router/utils/route-matcher' +import { + getRouteMatcher, + Params, +} from '../shared/lib/router/utils/route-matcher' import type { MiddlewareRouteMatch } from '../shared/lib/router/utils/middleware-route-matcher' import fs from 'fs' @@ -100,6 +103,9 @@ import { INSTRUMENTATION_HOOK_FILENAME } from '../lib/constants' import { getTracer } from './lib/trace/tracer' import { NextNodeServerSpan } from './lib/trace/constants' import { nodeFs } from './lib/node-fs-methods' +import { Worker } from 'next/dist/compiled/jest-worker' +import { genExecArgv, getNodeOptionsWithoutInspect } from './lib/utils' +import { getRouteRegex } from '../shared/lib/router/utils/route-regex' export * from './base-server' @@ -176,9 +182,28 @@ const POSSIBLE_ERROR_CODE_FROM_SERVE_STATIC = new Set([ 416, ]) +type RenderWorker = Worker & { + initialize: typeof import('./lib/render-server').initialize + deleteCache: typeof import('./lib/render-server').deleteCache + deleteAppClientCache: typeof import('./lib/render-server').deleteAppClientCache +} + export default class NextNodeServer extends BaseServer { private imageResponseCache?: ResponseCache private compression?: ExpressMiddleware + protected renderWorkers?: { + middleware?: RenderWorker + pages?: RenderWorker + app?: RenderWorker + } + protected renderWorkerOpts?: Parameters< + typeof import('./lib/render-server').initialize + >[0] + protected dynamicRoutes?: { + match: import('../shared/lib/router/utils/route-matcher').RouteMatchFn + page: string + re: RegExp + }[] constructor(options: Options) { // Initialize super class @@ -244,6 +269,67 @@ export default class NextNodeServer extends BaseServer { }).catch(() => {}) } + if (this.isRouterWorker) { + this.renderWorkers = {} + this.renderWorkerOpts = { + port: this.port || 0, + dir: this.dir, + workerType: 'render', + hostname: this.hostname, + dev: !!options.dev, + } + + const createWorker = (type: string) => { + const worker = new Worker(require.resolve('./lib/render-server'), { + numWorkers: 1, + // TODO: do we want to allow more than 10 OOM restarts? + maxRetries: 10, + forkOptions: { + env: { + ...process.env, + // we don't pass down NODE_OPTIONS as it can + // extra memory usage + NODE_OPTIONS: getNodeOptionsWithoutInspect() + .replace(/--max-old-space-size=[\d]{1,}/, '') + .trim(), + + __NEXT_PRIVATE_RENDER_WORKER: type, + }, + execArgv: genExecArgv( + options.isNodeDebugging === undefined + ? false + : options.isNodeDebugging, + (this.port || 0) + 1 + ), + }, + exposedMethods: ['initialize', 'deleteCache', 'deleteAppClientCache'], + }) as any as InstanceType & { + initialize: typeof import('./lib/render-server').initialize + deleteCache: typeof import('./lib/render-server').deleteCache + deleteAppClientCache: typeof import('./lib/render-server').deleteAppClientCache + } + + worker.getStderr().pipe(process.stderr) + worker.getStdout().pipe(process.stdout) + return worker + } + + if (this.hasAppDir) { + this.renderWorkers.app = createWorker('app') + } + this.renderWorkers.pages = createWorker('pages') + this.renderWorkers.middleware = + this.renderWorkers.pages || this.renderWorkers.app + ;(global as any)._nextDeleteCache = (filePath: string) => { + this.renderWorkers?.pages?.deleteCache(filePath) + this.renderWorkers?.app?.deleteCache(filePath) + } + ;(global as any)._nextDeleteAppClientCache = () => { + this.renderWorkers?.pages?.deleteAppClientCache() + this.renderWorkers?.app?.deleteAppClientCache() + } + } + // expose AsyncLocalStorage on global for react usage const { AsyncLocalStorage } = require('async_hooks') ;(globalThis as any).AsyncLocalStorage = AsyncLocalStorage @@ -258,7 +344,6 @@ export default class NextNodeServer extends BaseServer { if (this.hasAppDir) { routes.handlers.set(RouteKind.APP_ROUTE, new AppRouteRouteHandler()) } - return routes } @@ -1068,11 +1153,32 @@ export default class NextNodeServer extends BaseServer { return cacheFs.readFile(join(this.serverDistDir, 'pages', `${page}.html`)) } - protected generateRoutes(): RouterOptions { + protected generateRoutes(dev?: boolean): RouterOptions { const publicRoutes = this.generatePublicRoutes() const imageRoutes = this.generateImageRoutes() const staticFilesRoutes = this.generateStaticRoutes() + if (!dev) { + const routesManifest = this.getRoutesManifest() as { + dynamicRoutes: { + page: string + regex: string + namedRegex?: string + routeKeys?: { [key: string]: string } + }[] + } + this.dynamicRoutes = routesManifest.dynamicRoutes.map((r) => { + const regex = getRouteRegex(r.page) + const match = getRouteMatcher(regex) + + return { + match, + page: r.page, + regex: regex.re, + } + }) as any + } + const fsRoutes: Route[] = [ ...this.generateFsStaticRoutes(), { @@ -1187,17 +1293,19 @@ export default class NextNodeServer extends BaseServer { : ['/_next'] // Headers come very first - const headers = this.minimalMode - ? [] - : this.customRoutes.headers.map((rule) => - createHeaderRoute({ rule, restrictedRedirectPaths }) - ) + const headers = + this.minimalMode || this.isRenderWorker + ? [] + : this.customRoutes.headers.map((rule) => + createHeaderRoute({ rule, restrictedRedirectPaths }) + ) - const redirects = this.minimalMode - ? [] - : this.customRoutes.redirects.map((rule) => - createRedirectRoute({ rule, restrictedRedirectPaths }) - ) + const redirects = + this.minimalMode || this.isRenderWorker + ? [] + : this.customRoutes.redirects.map((rule) => + createRedirectRoute({ rule, restrictedRedirectPaths }) + ) const rewrites = this.generateRewrites({ restrictedRedirectPaths }) const catchAllMiddleware = this.generateCatchAllMiddlewareRoute() @@ -1226,6 +1334,95 @@ export default class NextNodeServer extends BaseServer { const bubbleNoFallback = !!query._nextBubbleNoFallback const match = await this.matchers.match(pathname, options) + if (this.isRouterWorker) { + let page = pathname + + if (!(await this.hasPage(page))) { + for (const route of this.dynamicRoutes || []) { + if (route.match(pathname)) { + page = route.page + } + } + } + + const renderKind = this.appPathRoutes?.[page || pathname] + ? 'app' + : 'pages' + + const renderWorker = this.renderWorkers?.[renderKind] + + if (renderWorker) { + const initUrl = getRequestMeta(req, '__NEXT_INIT_URL')! + const { port, hostname } = await renderWorker.initialize( + this.renderWorkerOpts! + ) + const renderUrl = new URL(initUrl) + renderUrl.hostname = hostname + renderUrl.port = port + '' + + let invokePathname = pathname + const normalizedInvokePathname = + this.localeNormalizer?.normalize(pathname) + + if (normalizedInvokePathname?.startsWith('/api')) { + invokePathname = normalizedInvokePathname + } + + if (query.__nextDataReq) { + invokePathname = `/_next/data/${this.buildId}${invokePathname}.json` + } + const keptQuery = new URLSearchParams() + + for (const key of Object.keys(query)) { + if (key.startsWith('__next') || key.startsWith('_next')) { + continue + } + if (Array.isArray(query[key])) { + ;(query[key] as string[]).forEach((val) => { + keptQuery.append(key, val) + }) + } else { + keptQuery.set(key, query[key] as string) + } + } + const invokeQueryStr = keptQuery.toString() + const matchedPath = `${invokePathname}${ + invokeQueryStr ? `?${invokeQueryStr}` : '' + }` + const invokeRes = await fetch(renderUrl, { + headers: { + ...(req.headers as any), + 'x-matched-path': matchedPath, + }, + redirect: 'manual', + }) + + for (const [key, value] of Object.entries( + toNodeHeaders(invokeRes.headers) + )) { + if ( + ![ + 'content-encoding', + 'transfer-encoding', + 'keep-alive', + 'connection', + ].includes(key) && + value !== undefined + ) { + res.setHeader(key, value) + } + } + res.statusCode = invokeRes.status + for await (const chunk of invokeRes.body || ([] as any)) { + this.streamResponseChunk(res as NodeNextResponse, chunk) + } + res.send() + return { + finished: true, + } + } + } + if (match) { addRequestMeta(req, '_nextMatch', match) } @@ -1553,7 +1750,7 @@ export default class NextNodeServer extends BaseServer { let afterFiles: Route[] = [] let fallback: Route[] = [] - if (!this.minimalMode) { + if (!this.minimalMode && !this.isRenderWorker) { const buildRewrite = (rewrite: Rewrite, check = true): Route => { const rewriteRoute = getCustomRoute({ type: 'rewrite', @@ -1877,9 +2074,25 @@ export default class NextNodeServer extends BaseServer { type: 'route', name: 'middleware catchall', fn: async (req, res, _params, parsed) => { + const isMiddlewareInvoke = + this.isRenderWorker && req.headers['x-middleware-invoke'] + + const handleFinished = (finished: boolean = false) => { + if (isMiddlewareInvoke && !finished) { + res.setHeader('x-middleware-invoke', '1') + res.body('').send() + return { finished: true } + } + return { finished } + } + + if (this.isRenderWorker && !isMiddlewareInvoke) { + return { finished: false } + } + const middleware = this.getMiddleware() if (!middleware) { - return { finished: false } + return handleFinished() } const initUrl = getRequestMeta(req, '__NEXT_INIT_URL')! @@ -1893,7 +2106,7 @@ export default class NextNodeServer extends BaseServer { parsed.pathname || '' ) if (!middleware.match(normalizedPathname, req, parsedUrl.query)) { - return { finished: false } + return handleFinished() } let result: Awaited< @@ -1901,12 +2114,67 @@ export default class NextNodeServer extends BaseServer { > try { - result = await this.runMiddleware({ - request: req, - response: res, - parsedUrl: parsedUrl, - parsed: parsed, - }) + await this.ensureMiddleware() + + if (this.isRouterWorker && this.renderWorkers?.middleware) { + const { port, hostname } = + await this.renderWorkers.middleware.initialize( + this.renderWorkerOpts! + ) + const renderUrl = new URL(initUrl) + renderUrl.hostname = hostname + renderUrl.port = port + '' + + const invokeRes = await fetch(renderUrl, { + headers: { + ...(req.headers as any), + 'x-middleware-invoke': '1', + }, + redirect: 'manual', + }) + + result = { + response: new Response(invokeRes.body, { + status: invokeRes.status, + headers: new Headers(invokeRes.headers), + }), + waitUntil: Promise.resolve(), + } + for (const key of [ + 'content-encoding', + 'transfer-encoding', + 'keep-alive', + 'connection', + ]) { + result.response.headers.delete(key) + } + } else { + result = await this.runMiddleware({ + request: req, + response: res, + parsedUrl: parsedUrl, + parsed: parsed, + }) + + if (isMiddlewareInvoke && 'response' in result) { + for (const [key, value] of Object.entries( + toNodeHeaders(result.response.headers) + )) { + if (key !== 'content-encoding' && value !== undefined) { + res.setHeader(key, value) + } + } + res.statusCode = result.response.status + for await (const chunk of result.response.body || + ([] as any)) { + this.streamResponseChunk(res as NodeNextResponse, chunk) + } + res.send() + return { + finished: true, + } + } + } } catch (err) { if (isError(err) && err.code === 'ENOENT') { await this.render404(req, res, parsed) @@ -2050,10 +2318,12 @@ export default class NextNodeServer extends BaseServer { addRequestMeta(req, '_nextRewroteUrl', newUrl) addRequestMeta(req, '_nextDidRewrite', newUrl !== req.url) - return { - finished: false, - pathname: newUrl, - query: parsedDestination.query, + if (!isMiddlewareInvoke) { + return { + finished: false, + pathname: newUrl, + query: parsedDestination.query, + } } } diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index 20188ec0efcd..aba9431ffbcf 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -147,7 +147,9 @@ export class NextServer { return loadConfig( this.options.dev ? PHASE_DEVELOPMENT_SERVER : PHASE_PRODUCTION_SERVER, resolve(this.options.dir || '.'), - this.options.conf + this.options.conf, + undefined, + !!this.options._renderWorker ) } diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 150657673405..a2fc8955b99a 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -331,7 +331,26 @@ export default class Router { }, } - for (const route of this.compiledRoutes) { + // when x-matched-path is specified we can short short circuit resolving + const matchedPath = req.headers['x-matched-path'] as string + const curRoutes = matchedPath + ? this.compiledRoutes.filter((r) => { + return ( + r.name === 'Catchall render' || r.name === '_next/data normalizing' + ) + }) + : this.compiledRoutes + + if (matchedPath) { + const parsedMatchedPath = new URL(matchedPath || '/', 'http://n') + parsedUrlUpdated.pathname = parsedMatchedPath.pathname + Object.assign( + parsedUrlUpdated.query, + Object.fromEntries(parsedMatchedPath.searchParams) + ) + } + + for (const route of curRoutes) { // only process rewrites for upgrade request if (upgradeHead && route.type !== 'rewrite') { continue diff --git a/test/e2e/app-dir/app/index.test.ts b/test/e2e/app-dir/app/index.test.ts index 9e3d2586a613..4d3f449a59a5 100644 --- a/test/e2e/app-dir/app/index.test.ts +++ b/test/e2e/app-dir/app/index.test.ts @@ -171,7 +171,9 @@ createNextDescribe( const res = await next.fetch('/dashboard') expect(res.headers.get('x-edge-runtime')).toBe('1') expect(res.headers.get('vary')).toBe( - 'RSC, Next-Router-State-Tree, Next-Router-Prefetch' + isNextDeploy + ? 'RSC, Next-Router-State-Tree, Next-Router-Prefetch' + : 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Accept-Encoding' ) }) From 7f43e77ee72a3d0359964c85f95d77426ba1ee55 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 13:16:42 -0700 Subject: [PATCH 02/52] Fix 404 and log case --- packages/next/src/build/output/index.ts | 9 ++++++++- packages/next/src/server/next-server.ts | 14 +++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/next/src/build/output/index.ts b/packages/next/src/build/output/index.ts index 08d8d95a06ea..fbdc93aae02f 100644 --- a/packages/next/src/build/output/index.ts +++ b/packages/next/src/build/output/index.ts @@ -95,7 +95,14 @@ export function formatAmpMessages(amp: AmpPageStatus) { return output } -const buildStore = createStore() +const buildStore = createStore({ + // @ts-ignore initial value + client: {}, + // @ts-ignore initial value + server: {}, + // @ts-ignore initial value + edgeServer: {}, +}) let buildWasDone = false let clientWasLoading = true let serverWasLoading = true diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index d6dc832856da..67b95ebaec4c 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1345,9 +1345,7 @@ export default class NextNodeServer extends BaseServer { } } - const renderKind = this.appPathRoutes?.[page || pathname] - ? 'app' - : 'pages' + const renderKind = this.appPathRoutes?.[page] ? 'app' : 'pages' const renderWorker = this.renderWorkers?.[renderKind] @@ -1389,6 +1387,16 @@ export default class NextNodeServer extends BaseServer { const matchedPath = `${invokePathname}${ invokeQueryStr ? `?${invokeQueryStr}` : '' }` + + // ensure /404 is built before invoking render + if (this.renderOpts.dev && !(await this.hasPage(page))) { + // @ts-expect-error dev specific + await this.hotReloader?.ensurePage({ + page: '/404', + clientOnly: false, + }) + } + const invokeRes = await fetch(renderUrl, { headers: { ...(req.headers as any), From 380751ba5c540e522d1d032cd4fdd4bd9bc27641 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 13:23:38 -0700 Subject: [PATCH 03/52] fix server ready log --- packages/next/src/server/lib/start-server.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 4aa79bd6b552..00f00110c7df 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -113,14 +113,18 @@ export async function startServer({ } }) const host = hostname || '0.0.0.0' - const normalizedHost = isIPv6(host) ? `[${host}]` : host + let normalizedHost = isIPv6(host) ? `[${host}]` : host await new Promise((resolve) => { server.on('listening', () => { const addr = server.address() port = typeof addr === 'object' ? addr?.port || port : port - const appUrl = `http://${host}:${port}` - Log.ready(`started server on ${normalizedHost}:${port}, url: ${appUrl}`) + normalizedHost = + !hostname || hostname === '0.0.0.0' ? 'localhost' : hostname + + const appUrl = `http://${normalizedHost}:${port}` + + Log.ready(`started server on ${host}:${port}, url: ${appUrl}`) resolve() }) server.listen(port, hostname) From f2812624c5245a0a7c67033c4c397d67cbe89089 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 13:28:42 -0700 Subject: [PATCH 04/52] ensure method and body are passed --- packages/next/src/server/next-server.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 67b95ebaec4c..985bb45683d6 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1398,10 +1398,17 @@ export default class NextNodeServer extends BaseServer { } const invokeRes = await fetch(renderUrl, { + method: req.method, headers: { ...(req.headers as any), 'x-matched-path': matchedPath, }, + // @ts-ignore + duplex: 'half', + body: getRequestMeta( + req, + '__NEXT_CLONABLE_BODY' + )?.cloneBodyStream() as any as ReadableStream, redirect: 'manual', }) @@ -2134,10 +2141,17 @@ export default class NextNodeServer extends BaseServer { renderUrl.port = port + '' const invokeRes = await fetch(renderUrl, { + method: req.method, headers: { ...(req.headers as any), 'x-middleware-invoke': '1', }, + // @ts-ignore + duplex: 'half', + body: getRequestMeta( + req, + '__NEXT_CLONABLE_BODY' + )?.cloneBodyStream() as any as ReadableStream, redirect: 'manual', }) From 4c923c11a957e4bc1a96aec25668b867486908e8 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 13:32:20 -0700 Subject: [PATCH 05/52] fix lint --- packages/next/src/cli/next-dev.ts | 3 ++- packages/next/src/lib/turbopack-warning.ts | 2 -- packages/next/src/server/dev/next-dev-server.ts | 10 ++-------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index 2fef2b30a532..ccdf8d5915d1 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -264,10 +264,11 @@ const nextDev: CliCommand = async (argv) => { } return server } else { - let teardownServer = await startServer(devServerOptions) + await startServer(devServerOptions) // if we're using workers we can auto restart on config changes if (devServerOptions.useWorkers) { + // TODO: watch config and such and restart } } } diff --git a/packages/next/src/lib/turbopack-warning.ts b/packages/next/src/lib/turbopack-warning.ts index 0d0fa468c16b..e9a5a4136b5e 100644 --- a/packages/next/src/lib/turbopack-warning.ts +++ b/packages/next/src/lib/turbopack-warning.ts @@ -24,8 +24,6 @@ const supportedTurbopackNextConfigOptions = [ export async function validateTurboNextConfig({ dir, isCustomTurbopack, - port, - hostname, }: { allowRetry?: boolean isCustomTurbopack?: boolean diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index c83ae6af6824..aff49339b682 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -17,13 +17,7 @@ import crypto from 'crypto' import fs from 'fs' import { Worker } from 'next/dist/compiled/jest-worker' import findUp from 'next/dist/compiled/find-up' -import { - join, - join as pathJoin, - relative, - resolve as pathResolve, - sep, -} from 'path' +import { join as pathJoin, relative, resolve as pathResolve, sep } from 'path' import Watchpack from 'next/dist/compiled/watchpack' import { ampValidation } from '../../build/output' import { PUBLIC_DIR_MIDDLEWARE_CONFLICT } from '../../lib/constants' @@ -53,7 +47,7 @@ import { removePathPrefix } from '../../shared/lib/router/utils/remove-path-pref import { eventCliSession } from '../../telemetry/events' import { Telemetry } from '../../telemetry/storage' import { setGlobal } from '../../trace' -import HotReloader, { cleanDistDir } from './hot-reloader' +import HotReloader from './hot-reloader' import { createValidFileMatcher, findPageFile } from '../lib/find-page-file' import { getNodeOptionsWithoutInspect } from '../lib/utils' import { From 5e0d2ca62dee69c0b89f982b38b05b347d87978e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 13:41:06 -0700 Subject: [PATCH 06/52] fix unit --- packages/next/src/server/lib/start-server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 00f00110c7df..413afc20d8cf 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -1,3 +1,4 @@ +import http from 'http' import { isIPv6 } from 'net' import * as Log from '../../build/output/log' import { getNodeOptionsWithoutInspect } from './utils' @@ -25,7 +26,6 @@ export async function startServer({ allowRetry, keepAliveTimeout, }: StartServerOptions): Promise { - const http = await import('http') const sockets = new Set() let worker: import('next/dist/compiled/jest-worker').Worker | undefined let handlersReady = () => {} From 57e117d65fa8b2d5c95aef16f05fda8b08a31587 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 13:44:10 -0700 Subject: [PATCH 07/52] fix body --- packages/next/src/server/next-server.ts | 32 +++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 985bb45683d6..667115f4eab0 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1403,13 +1403,17 @@ export default class NextNodeServer extends BaseServer { ...(req.headers as any), 'x-matched-path': matchedPath, }, - // @ts-ignore - duplex: 'half', - body: getRequestMeta( - req, - '__NEXT_CLONABLE_BODY' - )?.cloneBodyStream() as any as ReadableStream, redirect: 'manual', + ...(req.method !== 'GET' && req.method !== 'POST' + ? { + // @ts-ignore + duplex: 'half', + body: getRequestMeta( + req, + '__NEXT_CLONABLE_BODY' + )?.cloneBodyStream() as any as ReadableStream, + } + : {}), }) for (const [key, value] of Object.entries( @@ -2146,13 +2150,17 @@ export default class NextNodeServer extends BaseServer { ...(req.headers as any), 'x-middleware-invoke': '1', }, - // @ts-ignore - duplex: 'half', - body: getRequestMeta( - req, - '__NEXT_CLONABLE_BODY' - )?.cloneBodyStream() as any as ReadableStream, redirect: 'manual', + ...(req.method !== 'GET' && req.method !== 'POST' + ? { + // @ts-ignore + duplex: 'half', + body: getRequestMeta( + req, + '__NEXT_CLONABLE_BODY' + )?.cloneBodyStream() as any as ReadableStream, + } + : {}), }) result = { From 73edb2dda7a4b492e289287bba5cb43473b26323 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 14:19:44 -0700 Subject: [PATCH 08/52] fix body more --- packages/next/src/server/next-server.ts | 39 ++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 667115f4eab0..f621e175929a 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1396,15 +1396,24 @@ export default class NextNodeServer extends BaseServer { clientOnly: false, }) } + const invokeHeaders: typeof req.headers = { + ...req.headers, + 'x-matched-path': matchedPath, + } + for (const key of [ + 'content-length', + 'keepalive', + 'content-encoding', + 'transfer-encoding', + ]) { + delete invokeHeaders[key] + } const invokeRes = await fetch(renderUrl, { method: req.method, - headers: { - ...(req.headers as any), - 'x-matched-path': matchedPath, - }, + headers: invokeHeaders as any, redirect: 'manual', - ...(req.method !== 'GET' && req.method !== 'POST' + ...(req.method !== 'GET' && req.method !== 'HEAD' ? { // @ts-ignore duplex: 'half', @@ -2144,14 +2153,24 @@ export default class NextNodeServer extends BaseServer { renderUrl.hostname = hostname renderUrl.port = port + '' + const invokeHeaders: typeof req.headers = { + ...req.headers, + 'x-middleware-invoke': '1', + } + for (const key of [ + 'content-length', + 'keepalive', + 'content-encoding', + 'transfer-encoding', + ]) { + delete invokeHeaders[key] + } + const invokeRes = await fetch(renderUrl, { method: req.method, - headers: { - ...(req.headers as any), - 'x-middleware-invoke': '1', - }, + headers: invokeHeaders as any, redirect: 'manual', - ...(req.method !== 'GET' && req.method !== 'POST' + ...(req.method !== 'GET' && req.method !== 'HEAD' ? { // @ts-ignore duplex: 'half', From d0bb40ae1fd2ba2b34b0f8bdefaa41e58245e178 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 16 Mar 2023 15:48:53 -0700 Subject: [PATCH 09/52] set keepAlive --- packages/next/src/server/lib/start-server.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 413afc20d8cf..c41b8691d597 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -80,6 +80,10 @@ export async function startServer({ console.error(err) } }) + + if (keepAliveTimeout) { + server.keepAliveTimeout = keepAliveTimeout + } server.on('upgrade', async (req, socket, head) => { try { sockets.add(socket) From d3cd9e438652e33d45c5bd9adfbf7f4c86062d67 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 20 Mar 2023 15:44:16 -0700 Subject: [PATCH 10/52] fix header overlap --- packages/next/src/server/next-server.ts | 4 ++-- packages/next/src/server/router.ts | 8 +++++--- test/e2e/app-dir/app/index.test.ts | 4 +--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index ff8a2cd1f0da..13dcf7e29c85 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1394,7 +1394,7 @@ export default class NextNodeServer extends BaseServer { } } const invokeQueryStr = keptQuery.toString() - const matchedPath = `${invokePathname}${ + const invokePath = `${invokePathname}${ invokeQueryStr ? `?${invokeQueryStr}` : '' }` @@ -1408,7 +1408,7 @@ export default class NextNodeServer extends BaseServer { } const invokeHeaders: typeof req.headers = { ...req.headers, - 'x-matched-path': matchedPath, + 'x-invoke-path': invokePath, } for (const key of [ 'content-length', diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 291c7eb5a1a2..3f10ec954eca 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -335,8 +335,10 @@ export default class Router { }, } - // when x-matched-path is specified we can short short circuit resolving - const matchedPath = req.headers['x-matched-path'] as string + // when x-invoke-path is specified we can short short circuit resolving + // we only honor this header if we are inside of a render worker to + // prevent external users coercing the routing path + const matchedPath = req.headers['x-invoke-path'] as string const curRoutes = matchedPath ? this.compiledRoutes.filter((r) => { return ( @@ -345,7 +347,7 @@ export default class Router { }) : this.compiledRoutes - if (matchedPath) { + if (process.env.__NEXT_PRIVATE_RENDER_WORKER && matchedPath) { const parsedMatchedPath = new URL(matchedPath || '/', 'http://n') parsedUrlUpdated.pathname = parsedMatchedPath.pathname Object.assign( diff --git a/test/e2e/app-dir/app/index.test.ts b/test/e2e/app-dir/app/index.test.ts index 5331723d3b80..5147bf7d3f51 100644 --- a/test/e2e/app-dir/app/index.test.ts +++ b/test/e2e/app-dir/app/index.test.ts @@ -172,9 +172,7 @@ createNextDescribe( const res = await next.fetch('/dashboard') expect(res.headers.get('x-edge-runtime')).toBe('1') expect(res.headers.get('vary')).toBe( - isNextDeploy - ? 'RSC, Next-Router-State-Tree, Next-Router-Prefetch' - : 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Accept-Encoding' + 'RSC, Next-Router-State-Tree, Next-Router-Prefetch' ) }) From f8d740b024d6666f175436dfe6e10371c2d05033 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 20 Mar 2023 16:31:34 -0700 Subject: [PATCH 11/52] remove experimental.config.experimental.preCompiledNextServer and fix type --- packages/next/src/build/index.ts | 74 +- packages/next/src/build/output/index.ts | 6 +- packages/next/src/server/config-schema.ts | 3 - packages/next/src/server/config-shared.ts | 2 - packages/next/src/server/next-server.ts | 2 +- packages/next/taskfile.js | 74 - .../precompiled-server.test.ts | 1257 ----------------- 7 files changed, 5 insertions(+), 1413 deletions(-) delete mode 100644 test/production/standalone-mode/required-server-files/precompiled-server.test.ts diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 4f62bd3aa2e3..a599ccb3b2ee 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1722,79 +1722,7 @@ export default async function build( 'next-server.js.nft.json' ) - if (config.experimental.preCompiledNextServer) { - if (!ciEnvironment.hasNextSupport) { - Log.warn( - `"experimental.preCompiledNextServer" is currently optimized for environments with Next.js support, some features may not be supported` - ) - } - const nextServerPath = require.resolve('next/dist/server/next-server') - await promises.rename(nextServerPath, `${nextServerPath}.bak`) - - await promises.writeFile( - nextServerPath, - `module.exports = require('next/dist/compiled/next-server/next-server.js')` - ) - const glob = - require('next/dist/compiled/glob') as typeof import('next/dist/compiled/glob') - const compiledFiles: string[] = [] - const compiledNextServerFolder = path.dirname( - require.resolve('next/dist/compiled/next-server/next-server.js') - ) - - for (const compiledFolder of [ - compiledNextServerFolder, - path.join(compiledNextServerFolder, '../react'), - path.join(compiledNextServerFolder, '../react-dom'), - path.join(compiledNextServerFolder, '../scheduler'), - ]) { - const globResult = glob.sync('**/*', { - cwd: compiledFolder, - dot: true, - }) - - await Promise.all( - globResult.map(async (file) => { - const absolutePath = path.join(compiledFolder, file) - const statResult = await promises.stat(absolutePath) - - if (statResult.isFile()) { - compiledFiles.push(absolutePath) - } - }) - ) - } - - const externalLibFiles = [ - 'next/dist/shared/lib/server-inserted-html.js', - 'next/dist/shared/lib/router-context.js', - 'next/dist/shared/lib/loadable-context.js', - 'next/dist/shared/lib/image-config-context.js', - 'next/dist/shared/lib/image-config.js', - 'next/dist/shared/lib/head-manager-context.js', - 'next/dist/shared/lib/app-router-context.js', - 'next/dist/shared/lib/amp-context.js', - 'next/dist/shared/lib/hooks-client-context.js', - 'next/dist/shared/lib/html-context.js', - ] - for (const file of externalLibFiles) { - compiledFiles.push(require.resolve(file)) - } - compiledFiles.push(nextServerPath) - - await promises.writeFile( - nextServerTraceOutput, - JSON.stringify({ - version: 1, - files: [...new Set(compiledFiles)].map((file) => - path.relative(distDir, file) - ), - } as { - version: number - files: string[] - }) - ) - } else if (config.outputFileTracing) { + if (config.outputFileTracing) { let nodeFileTrace: any if (config.experimental.turbotrace) { let binding = (await loadBindings()) as any diff --git a/packages/next/src/build/output/index.ts b/packages/next/src/build/output/index.ts index fbdc93aae02f..cf9e779c92ce 100644 --- a/packages/next/src/build/output/index.ts +++ b/packages/next/src/build/output/index.ts @@ -96,11 +96,11 @@ export function formatAmpMessages(amp: AmpPageStatus) { } const buildStore = createStore({ - // @ts-ignore initial value + // @ts-expect-error initial value client: {}, - // @ts-ignore initial value + // @ts-expect-error initial value server: {}, - // @ts-ignore initial value + // @ts-expect-error initial value edgeServer: {}, }) let buildWasDone = false diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index 6b20aeb2a63d..6f9757af0320 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -363,9 +363,6 @@ const configSchema = { pageEnv: { type: 'boolean', }, - preCompiledNextServer: { - type: 'boolean', - }, proxyTimeout: { minimum: 0, type: 'number', diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index d99c0b464567..56fa0f25097e 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -122,7 +122,6 @@ export interface ExperimentalConfig { fetchCacheKeyPrefix?: string optimisticClientCache?: boolean middlewarePrefetch?: 'strict' | 'flexible' - preCompiledNextServer?: boolean legacyBrowsers?: boolean manualClientBasePath?: boolean newNextLinkBehavior?: boolean @@ -652,7 +651,6 @@ export const defaultConfig: NextConfig = { experimental: { clientRouterFilter: false, clientRouterFilterRedirects: false, - preCompiledNextServer: false, fetchCacheKeyPrefix: '', middlewarePrefetch: 'flexible', optimisticClientCache: true, diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 13dcf7e29c85..ee8950e2283e 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -99,7 +99,6 @@ import { INSTRUMENTATION_HOOK_FILENAME } from '../lib/constants' import { getTracer } from './lib/trace/tracer' import { NextNodeServerSpan } from './lib/trace/constants' import { nodeFs } from './lib/node-fs-methods' -import { Worker } from 'next/dist/compiled/jest-worker' import { genExecArgv, getNodeOptionsWithoutInspect } from './lib/utils' import { getRouteRegex } from '../shared/lib/router/utils/route-regex' @@ -276,6 +275,7 @@ export default class NextNodeServer extends BaseServer { } const createWorker = (type: string) => { + const { Worker } = require('next/dist/compiled/jest-worker') const worker = new Worker(require.resolve('./lib/render-server'), { numWorkers: 1, // TODO: do we want to allow more than 10 OOM restarts? diff --git a/packages/next/taskfile.js b/packages/next/taskfile.js index f86ca9e01cec..c2f1e3ee4ea1 100644 --- a/packages/next/taskfile.js +++ b/packages/next/taskfile.js @@ -87,79 +87,6 @@ export async function ncc_node_html_parser(task, opts) { .target('src/compiled/node-html-parser') } -export async function ncc_next_server(task, opts) { - await task - .source( - opts.src || - relative(__dirname, require.resolve('next/dist/server/next-server')) - ) - .ncc({ - bundleName: 'next-server', - // minify: false, - externals: { - ...externals, - - '/(.*)route-resolver/': '$1route-resolver', - - sharp: 'sharp', - react: 'react', - 'react-dom': 'react-dom', - - 'next/dist/compiled/compression': 'next/dist/compiled/compression', - - critters: 'critters', - - 'next/dist/compiled/jest-worker': 'next/dist/compiled/jest-worker', - - 'next/dist/compiled/react': 'next/dist/compiled/react', - '/next/dist/compiled/react(/.+)/': 'next/dist/compiled/react$1', - 'next/dist/compiled/react-dom': 'next/dist/compiled/react-dom', - '/next/dist/compiled/react-dom(/.+)/': 'next/dist/compiled/react-dom$1', - - // react contexts must be external - '/(.*)server-inserted-html/': - 'next/dist/shared/lib/server-inserted-html.js', - - '/(.+/)router-context/': 'next/dist/shared/lib/router-context.js', - - '/(.*)loadable-context/': 'next/dist/shared/lib/loadable-context.js', - - '/(.*)image-config-context/': - 'next/dist/shared/lib/image-config-context.js', - - '/(.*)head-manager-context/': - 'next/dist/shared/lib/head-manager-context.js', - - '/(.*)app-router-context/': - 'next/dist/shared/lib/app-router-context.js', - - '/(.*)amp-context/': 'next/dist/shared/lib/amp-context.js', - - '/(.*)hooks-client-context/': - 'next/dist/shared/lib/hooks-client-context.js', - - '/(.*)html-context/': 'next/dist/shared/lib/html-context.js', - - // 'next/dist/compiled/undici': 'next/dist/compiled/undici', - // 'next/dist/compiled/node-fetch': 'next/dist/compiled/node-fetch', - - // '/(.*)google-font-metrics.json/': '$1google-font-metrics.json', - '/(.*)next-config-validate.js/': '$1/next-config-validate.js', - - '/(.*)server/web(.*)/': '$1server/web$2', - './web/sandbox': './web/sandbox', - 'next/dist/compiled/edge-runtime': 'next/dist/compiled/edge-runtime', - '(.*)@edge-runtime/primitives': '$1@edge-runtime/primitives', - - '/(.*)compiled/webpack(/.*)/': '$1webpack$2', - './image-optimizer': './image-optimizer', - '/(.*)@ampproject/toolbox-optimizer/': - '$1@ampproject/toolbox-optimizer', - }, - }) - .target('dist/compiled/next-server') -} - // eslint-disable-next-line camelcase externals['@babel/runtime'] = 'next/dist/compiled/@babel/runtime' export async function copy_babel_runtime(task, opts) { @@ -2246,7 +2173,6 @@ export async function compile(task, opts) { 'ncc_react_refresh_utils', 'ncc_next__react_dev_overlay', 'ncc_next_font', - 'ncc_next_server', ]) } diff --git a/test/production/standalone-mode/required-server-files/precompiled-server.test.ts b/test/production/standalone-mode/required-server-files/precompiled-server.test.ts deleted file mode 100644 index 840c3cccff1d..000000000000 --- a/test/production/standalone-mode/required-server-files/precompiled-server.test.ts +++ /dev/null @@ -1,1257 +0,0 @@ -import glob from 'glob' -import fs from 'fs-extra' -import cheerio from 'cheerio' -import { join } from 'path' -import { createNext, FileRef } from 'e2e-utils' -import { NextInstance } from 'test/lib/next-modes/base' -import { - check, - fetchViaHTTP, - findPort, - initNextServerScript, - killApp, - renderViaHTTP, - waitFor, -} from 'next-test-utils' - -describe('should set-up next', () => { - let next: NextInstance - let server - let appPort - let errors = [] - let stderr = '' - let requiredFilesManifest - - const setupNext = async ({ - nextEnv, - minimalMode, - }: { - nextEnv?: boolean - minimalMode?: boolean - }) => { - // test build against environment with next support - process.env.NOW_BUILDER = nextEnv ? '1' : '' - - next = await createNext({ - files: { - pages: new FileRef(join(__dirname, 'pages')), - lib: new FileRef(join(__dirname, 'lib')), - 'middleware.js': new FileRef(join(__dirname, 'middleware.js')), - 'data.txt': new FileRef(join(__dirname, 'data.txt')), - '.env': new FileRef(join(__dirname, '.env')), - '.env.local': new FileRef(join(__dirname, '.env.local')), - '.env.production': new FileRef(join(__dirname, '.env.production')), - }, - nextConfig: { - eslint: { - ignoreDuringBuilds: true, - }, - experimental: { - preCompiledNextServer: true, - }, - output: 'standalone', - async rewrites() { - return { - beforeFiles: [], - fallback: [ - { - source: '/an-ssg-path', - destination: '/hello.txt', - }, - { - source: '/fallback-false/:path', - destination: '/hello.txt', - }, - ], - afterFiles: [ - { - source: '/some-catch-all/:path*', - destination: '/', - }, - { - source: '/to-dynamic/post-2', - destination: '/dynamic/post-2?hello=world', - }, - { - source: '/to-dynamic/:path', - destination: '/dynamic/:path', - }, - ], - } - }, - }, - }) - await next.stop() - - requiredFilesManifest = JSON.parse( - await next.readFile('.next/required-server-files.json') - ) - await fs.move( - join(next.testDir, '.next/standalone'), - join(next.testDir, 'standalone') - ) - for (const file of await fs.readdir(next.testDir)) { - if (file !== 'standalone') { - await fs.remove(join(next.testDir, file)) - console.log('removed', file) - } - } - const files = glob.sync('**/*', { - cwd: join(next.testDir, 'standalone/.next/server/pages'), - dot: true, - }) - - for (const file of files) { - if (file.endsWith('.json') || file.endsWith('.html')) { - await fs.remove(join(next.testDir, '.next/server', file)) - } - } - - const testServer = join(next.testDir, 'standalone/server.js') - await fs.writeFile( - testServer, - (await fs.readFile(testServer, 'utf8')) - .replace('console.error(err)', `console.error('top-level', err)`) - .replace('conf:', `minimalMode: ${minimalMode},conf:`) - ) - appPort = await findPort() - server = await initNextServerScript( - testServer, - /Listening on/, - { - ...process.env, - PORT: appPort, - }, - undefined, - { - cwd: next.testDir, - onStderr(msg) { - if (msg.includes('top-level')) { - errors.push(msg) - } - stderr += msg - }, - } - ) - } - - beforeAll(async () => { - await setupNext({ nextEnv: true, minimalMode: true }) - }) - afterAll(async () => { - await next.destroy() - if (server) await killApp(server) - }) - - it('should resolve correctly when a redirect is returned', async () => { - const toRename = `standalone/.next/server/pages/route-resolving/[slug]/[project].html` - await next.renameFile(toRename, `${toRename}.bak`) - try { - const res = await fetchViaHTTP( - appPort, - '/route-resolving/import/first', - undefined, - { - redirect: 'manual', - headers: { - 'x-matched-path': '/route-resolving/import/[slug]', - }, - } - ) - expect(res.status).toBe(307) - expect(new URL(res.headers.get('location'), 'http://n').pathname).toBe( - '/somewhere' - ) - - await waitFor(3000) - expect(stderr).not.toContain('ENOENT') - } finally { - await next.renameFile(`${toRename}.bak`, toRename) - } - }) - - it('should show invariant when an automatic static page is requested', async () => { - const toRename = `standalone/.next/server/pages/auto-static.html` - await next.renameFile(toRename, `${toRename}.bak`) - - try { - const res = await fetchViaHTTP(appPort, '/auto-static', undefined, { - headers: { - 'x-matched-path': '/auto-static', - }, - }) - - expect(res.status).toBe(500) - await check(() => stderr, /Invariant: failed to load static page/) - } finally { - await next.renameFile(`${toRename}.bak`, toRename) - } - }) - - it.each([ - { - case: 'redirect no revalidate', - path: '/optional-ssg/redirect-1', - dest: '/somewhere', - cacheControl: 's-maxage=31536000, stale-while-revalidate', - }, - { - case: 'redirect with revalidate', - path: '/optional-ssg/redirect-2', - dest: '/somewhere-else', - cacheControl: 's-maxage=5, stale-while-revalidate', - }, - ])( - `should have correct cache-control for $case`, - async ({ path, dest, cacheControl }) => { - const res = await fetchViaHTTP(appPort, path, undefined, { - redirect: 'manual', - }) - expect(res.status).toBe(307) - expect(new URL(res.headers.get('location'), 'http://n').pathname).toBe( - dest - ) - expect(res.headers.get('cache-control')).toBe(cacheControl) - - const dataRes = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}${path}.json`, - undefined, - { - redirect: 'manual', - } - ) - expect(dataRes.headers.get('cache-control')).toBe(cacheControl) - expect((await dataRes.json()).pageProps).toEqual({ - __N_REDIRECT: dest, - __N_REDIRECT_STATUS: 307, - }) - } - ) - - it.each([ - { - case: 'notFound no revalidate', - path: '/optional-ssg/not-found-1', - dest: '/somewhere', - cacheControl: 's-maxage=31536000, stale-while-revalidate', - }, - { - case: 'notFound with revalidate', - path: '/optional-ssg/not-found-2', - dest: '/somewhere-else', - cacheControl: 's-maxage=5, stale-while-revalidate', - }, - ])( - `should have correct cache-control for $case`, - async ({ path, dest, cacheControl }) => { - const res = await fetchViaHTTP(appPort, path, undefined, { - redirect: 'manual', - }) - expect(res.status).toBe(404) - expect(res.headers.get('cache-control')).toBe(cacheControl) - - const dataRes = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}${path}.json`, - undefined, - { - redirect: 'manual', - } - ) - expect(dataRes.headers.get('cache-control')).toBe(cacheControl) - } - ) - - it('should have the correct cache-control for props with no revalidate', async () => { - const res = await fetchViaHTTP(appPort, '/optional-ssg/props-no-revalidate') - expect(res.status).toBe(200) - expect(res.headers.get('cache-control')).toBe( - 's-maxage=31536000, stale-while-revalidate' - ) - const $ = cheerio.load(await res.text()) - expect(JSON.parse($('#props').text()).params).toEqual({ - rest: ['props-no-revalidate'], - }) - - const dataRes = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/optional-ssg/props-no-revalidate.json`, - undefined - ) - expect(dataRes.status).toBe(200) - expect(res.headers.get('cache-control')).toBe( - 's-maxage=31536000, stale-while-revalidate' - ) - expect((await dataRes.json()).pageProps.params).toEqual({ - rest: ['props-no-revalidate'], - }) - }) - - it('should warn when "next" is imported directly', async () => { - await renderViaHTTP(appPort, '/gssp') - await check( - () => stderr, - /"next" should not be imported directly, imported in/ - ) - }) - - it('`compress` should be `false` by in nextEnv', async () => { - expect( - await fs.readFileSync(join(next.testDir, 'standalone/server.js'), 'utf8') - ).toContain('"compress":false') - }) - - it('should output middleware correctly', async () => { - expect( - await fs.pathExists( - join(next.testDir, 'standalone/.next/server/edge-runtime-webpack.js') - ) - ).toBe(true) - expect( - await fs.pathExists( - join(next.testDir, 'standalone/.next/server/middleware.js') - ) - ).toBe(true) - }) - - it('should output required-server-files manifest correctly', async () => { - expect(requiredFilesManifest.version).toBe(1) - expect(Array.isArray(requiredFilesManifest.files)).toBe(true) - expect(Array.isArray(requiredFilesManifest.ignore)).toBe(true) - expect(requiredFilesManifest.files.length).toBeGreaterThan(0) - expect(requiredFilesManifest.ignore.length).toBeGreaterThan(0) - expect(typeof requiredFilesManifest.config.configFile).toBe('undefined') - expect(typeof requiredFilesManifest.config.trailingSlash).toBe('boolean') - expect(typeof requiredFilesManifest.appDir).toBe('string') - }) - - it('should de-dupe HTML/data requests', async () => { - const res = await fetchViaHTTP(appPort, '/gsp', undefined, { - redirect: 'manual', - headers: { - // ensure the nextjs-data header being present - // doesn't incorrectly return JSON for HTML path - // during prerendering - 'x-nextjs-data': '1', - }, - }) - expect(res.status).toBe(200) - expect(res.headers.get('x-nextjs-cache')).toBeFalsy() - const $ = cheerio.load(await res.text()) - const props = JSON.parse($('#props').text()) - expect(props.gspCalls).toBeDefined() - - const res2 = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/gsp.json`, - undefined, - { - redirect: 'manual', - } - ) - expect(res2.status).toBe(200) - expect(res2.headers.get('x-nextjs-cache')).toBeFalsy() - const { pageProps: props2 } = await res2.json() - expect(props2.gspCalls).toBe(props.gspCalls) - - const res3 = await fetchViaHTTP(appPort, '/index', undefined, { - redirect: 'manual', - headers: { - 'x-matched-path': '/index', - }, - }) - expect(res3.status).toBe(200) - const $2 = cheerio.load(await res3.text()) - const props3 = JSON.parse($2('#props').text()) - expect(props3.gspCalls).toBeDefined() - - const res4 = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/index.json`, - undefined, - { - redirect: 'manual', - } - ) - expect(res4.status).toBe(200) - const { pageProps: props4 } = await res4.json() - expect(props4.gspCalls).toBe(props3.gspCalls) - }) - - it('should cap de-dupe previousCacheItem expires time', async () => { - const res = await fetchViaHTTP(appPort, '/gsp-long-revalidate', undefined, { - redirect: 'manual', - }) - expect(res.status).toBe(200) - const $ = cheerio.load(await res.text()) - const props = JSON.parse($('#props').text()) - expect(props.gspCalls).toBeDefined() - - await waitFor(1000) - - const res2 = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/gsp-long-revalidate.json`, - undefined, - { - redirect: 'manual', - } - ) - expect(res2.status).toBe(200) - const { pageProps: props2 } = await res2.json() - expect(props2.gspCalls).not.toBe(props.gspCalls) - }) - - it('should not 404 for onlyGenerated manual revalidate in minimal mode', async () => { - const previewProps = JSON.parse( - await next.readFile('standalone/.next/prerender-manifest.json') - ).preview - - const res = await fetchViaHTTP( - appPort, - '/optional-ssg/only-generated-1', - undefined, - { - headers: { - 'x-prerender-revalidate': previewProps.previewModeId, - 'x-prerender-revalidate-if-generated': '1', - }, - } - ) - expect(res.status).toBe(200) - }) - - it('should set correct SWR headers with notFound gsp', async () => { - await waitFor(2000) - await next.patchFile('standalone/data.txt', 'show') - - const res = await fetchViaHTTP(appPort, '/gsp', undefined, { - redirect: 'manual', - }) - expect(res.status).toBe(200) - expect(res.headers.get('cache-control')).toBe( - 's-maxage=1, stale-while-revalidate' - ) - - await waitFor(2000) - await next.patchFile('standalone/data.txt', 'hide') - - const res2 = await fetchViaHTTP(appPort, '/gsp', undefined, { - redirect: 'manual', - }) - expect(res2.status).toBe(404) - expect(res2.headers.get('cache-control')).toBe( - 's-maxage=1, stale-while-revalidate' - ) - }) - - it('should set correct SWR headers with notFound gssp', async () => { - await next.patchFile('standalone/data.txt', 'show') - - const res = await fetchViaHTTP(appPort, '/gssp', undefined, { - redirect: 'manual', - }) - expect(res.status).toBe(200) - expect(res.headers.get('cache-control')).toBe( - 's-maxage=1, stale-while-revalidate' - ) - - await next.patchFile('standalone/data.txt', 'hide') - - const res2 = await fetchViaHTTP(appPort, '/gssp', undefined, { - redirect: 'manual', - }) - await next.patchFile('standalone/data.txt', 'show') - - expect(res2.status).toBe(404) - expect(res2.headers.get('cache-control')).toBe( - 's-maxage=1, stale-while-revalidate' - ) - }) - - it('should render SSR page correctly', async () => { - const html = await renderViaHTTP(appPort, '/gssp') - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#gssp').text()).toBe('getServerSideProps page') - expect(data.hello).toBe('world') - - const html2 = await renderViaHTTP(appPort, '/gssp') - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#gssp').text()).toBe('getServerSideProps page') - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - }) - - it('should render dynamic SSR page correctly', async () => { - const html = await renderViaHTTP(appPort, '/dynamic/first') - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#dynamic').text()).toBe('dynamic page') - expect($('#slug').text()).toBe('first') - expect(data.hello).toBe('world') - - const html2 = await renderViaHTTP(appPort, '/dynamic/second') - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#dynamic').text()).toBe('dynamic page') - expect($2('#slug').text()).toBe('second') - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - }) - - it('should render fallback page correctly', async () => { - const html = await renderViaHTTP(appPort, '/fallback/first') - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#fallback').text()).toBe('fallback page') - expect($('#slug').text()).toBe('first') - expect(data.hello).toBe('world') - - await waitFor(2000) - const html2 = await renderViaHTTP(appPort, '/fallback/first') - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#fallback').text()).toBe('fallback page') - expect($2('#slug').text()).toBe('first') - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - - const html3 = await renderViaHTTP(appPort, '/fallback/second') - const $3 = cheerio.load(html3) - const data3 = JSON.parse($3('#props').text()) - - expect($3('#fallback').text()).toBe('fallback page') - expect($3('#slug').text()).toBe('second') - expect(isNaN(data3.random)).toBe(false) - - const { pageProps: data4 } = JSON.parse( - await renderViaHTTP( - appPort, - `/_next/data/${next.buildId}/fallback/third.json` - ) - ) - expect(data4.hello).toBe('world') - expect(data4.slug).toBe('third') - }) - - it('should render SSR page correctly with x-matched-path', async () => { - const html = await renderViaHTTP(appPort, '/some-other-path', undefined, { - headers: { - 'x-matched-path': '/gssp', - }, - }) - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#gssp').text()).toBe('getServerSideProps page') - expect(data.hello).toBe('world') - - const html2 = await renderViaHTTP(appPort, '/some-other-path', undefined, { - headers: { - 'x-matched-path': '/gssp', - }, - }) - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#gssp').text()).toBe('getServerSideProps page') - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - }) - - it('should render dynamic SSR page correctly with x-matched-path', async () => { - const html = await renderViaHTTP( - appPort, - '/some-other-path?slug=first', - undefined, - { - headers: { - 'x-matched-path': '/dynamic/[slug]', - }, - } - ) - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#dynamic').text()).toBe('dynamic page') - expect($('#slug').text()).toBe('first') - expect(data.hello).toBe('world') - - const html2 = await renderViaHTTP( - appPort, - '/some-other-path?slug=second', - undefined, - { - headers: { - 'x-matched-path': '/dynamic/[slug]', - }, - } - ) - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#dynamic').text()).toBe('dynamic page') - expect($2('#slug').text()).toBe('second') - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - - const html3 = await renderViaHTTP(appPort, '/some-other-path', undefined, { - headers: { - 'x-matched-path': '/dynamic/[slug]', - 'x-now-route-matches': '1=second&slug=second', - }, - }) - const $3 = cheerio.load(html3) - const data3 = JSON.parse($3('#props').text()) - - expect($3('#dynamic').text()).toBe('dynamic page') - expect($3('#slug').text()).toBe('second') - expect(isNaN(data3.random)).toBe(false) - expect(data3.random).not.toBe(data.random) - }) - - it('should render fallback page correctly with x-matched-path and routes-matches', async () => { - const html = await renderViaHTTP(appPort, '/fallback/first', undefined, { - headers: { - 'x-matched-path': '/fallback/first', - 'x-now-route-matches': '1=first', - }, - }) - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#fallback').text()).toBe('fallback page') - expect($('#slug').text()).toBe('first') - expect(data.hello).toBe('world') - - const html2 = await renderViaHTTP(appPort, `/fallback/[slug]`, undefined, { - headers: { - 'x-matched-path': '/fallback/[slug]', - 'x-now-route-matches': '1=second', - }, - }) - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#fallback').text()).toBe('fallback page') - expect($2('#slug').text()).toBe('second') - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - }) - - it('should favor valid route params over routes-matches', async () => { - const html = await renderViaHTTP(appPort, '/fallback/first', undefined, { - headers: { - 'x-matched-path': '/fallback/first', - 'x-now-route-matches': '1=fallback%2ffirst', - }, - }) - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#fallback').text()).toBe('fallback page') - expect($('#slug').text()).toBe('first') - expect(data.hello).toBe('world') - - const html2 = await renderViaHTTP(appPort, `/fallback/second`, undefined, { - headers: { - 'x-matched-path': '/fallback/[slug]', - 'x-now-route-matches': '1=fallback%2fsecond', - }, - }) - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#fallback').text()).toBe('fallback page') - expect($2('#slug').text()).toBe('second') - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - }) - - it('should favor valid route params over routes-matches optional', async () => { - const html = await renderViaHTTP(appPort, '/optional-ssg', undefined, { - headers: { - 'x-matched-path': '/optional-ssg', - 'x-now-route-matches': '1=optional-ssg', - }, - }) - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - expect(data.params).toEqual({}) - - const html2 = await renderViaHTTP(appPort, `/optional-ssg`, undefined, { - headers: { - 'x-matched-path': '/optional-ssg', - 'x-now-route-matches': '1=optional-ssg%2fanother', - }, - }) - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect(isNaN(data2.random)).toBe(false) - expect(data2.params).toEqual({}) - }) - - it('should return data correctly with x-matched-path', async () => { - const res = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/dynamic/first.json?slug=first`, - undefined, - { - headers: { - 'x-matched-path': `/dynamic/[slug]`, - }, - } - ) - - const { pageProps: data } = await res.json() - - expect(data.slug).toBe('first') - expect(data.hello).toBe('world') - - const res2 = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/fallback/[slug].json`, - undefined, - { - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/fallback/[slug].json`, - 'x-now-route-matches': '1=second', - }, - } - ) - - const { pageProps: data2 } = await res2.json() - - expect(data2.slug).toBe('second') - expect(data2.hello).toBe('world') - }) - - it('should render fallback optional catch-all route correctly with x-matched-path and routes-matches', async () => { - const html = await renderViaHTTP( - appPort, - '/catch-all/[[...rest]]', - undefined, - { - headers: { - 'x-matched-path': '/catch-all/[[...rest]]', - 'x-now-route-matches': '', - }, - } - ) - const $ = cheerio.load(html) - const data = JSON.parse($('#props').text()) - - expect($('#catch-all').text()).toBe('optional catch-all page') - expect(data.params).toEqual({}) - expect(data.hello).toBe('world') - - const html2 = await renderViaHTTP( - appPort, - '/catch-all/[[...rest]]', - undefined, - { - headers: { - 'x-matched-path': '/catch-all/[[...rest]]', - 'x-now-route-matches': '1=hello&catchAll=hello', - }, - } - ) - const $2 = cheerio.load(html2) - const data2 = JSON.parse($2('#props').text()) - - expect($2('#catch-all').text()).toBe('optional catch-all page') - expect(data2.params).toEqual({ rest: ['hello'] }) - expect(isNaN(data2.random)).toBe(false) - expect(data2.random).not.toBe(data.random) - - const html3 = await renderViaHTTP( - appPort, - '/catch-all/[[...rest]]', - undefined, - { - headers: { - 'x-matched-path': '/catch-all/[[...rest]]', - 'x-now-route-matches': '1=hello/world&catchAll=hello/world', - }, - } - ) - const $3 = cheerio.load(html3) - const data3 = JSON.parse($3('#props').text()) - - expect($3('#catch-all').text()).toBe('optional catch-all page') - expect(data3.params).toEqual({ rest: ['hello', 'world'] }) - expect(isNaN(data3.random)).toBe(false) - expect(data3.random).not.toBe(data.random) - }) - - it('should return data correctly with x-matched-path for optional catch-all route', async () => { - const res = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/catch-all.json`, - - undefined, - { - headers: { - 'x-matched-path': '/catch-all/[[...rest]]', - }, - } - ) - - const { pageProps: data } = await res.json() - - expect(data.params).toEqual({}) - expect(data.hello).toBe('world') - - const res2 = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/catch-all/[[...rest]].json`, - undefined, - { - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/catch-all/[[...rest]].json`, - 'x-now-route-matches': '1=hello&rest=hello', - }, - } - ) - - const { pageProps: data2 } = await res2.json() - - expect(data2.params).toEqual({ rest: ['hello'] }) - expect(data2.hello).toBe('world') - - const res3 = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/catch-all/[[...rest]].json`, - undefined, - { - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/catch-all/[[...rest]].json`, - 'x-now-route-matches': '1=hello/world&rest=hello/world', - }, - } - ) - - const { pageProps: data3 } = await res3.json() - - expect(data3.params).toEqual({ rest: ['hello', 'world'] }) - expect(data3.hello).toBe('world') - }) - - it('should not apply trailingSlash redirect', async () => { - for (const path of [ - '/', - '/dynamic/another/', - '/dynamic/another', - '/fallback/first/', - '/fallback/first', - '/fallback/another/', - '/fallback/another', - ]) { - const res = await fetchViaHTTP(appPort, path, undefined, { - redirect: 'manual', - }) - - expect(res.status).toBe(200) - } - }) - - it('should normalize catch-all rewrite query values correctly', async () => { - const html = await renderViaHTTP( - appPort, - '/some-catch-all/hello/world', - { - path: 'hello/world', - }, - { - headers: { - 'x-matched-path': '/gssp', - }, - } - ) - const $ = cheerio.load(html) - expect(JSON.parse($('#router').text()).query).toEqual({ - path: ['hello', 'world'], - }) - }) - - it('should handle bad request correctly with rewrite', async () => { - const res = await fetchViaHTTP( - appPort, - '/to-dynamic/%c0.%c0.', - '?path=%c0.%c0.', - { - headers: { - 'x-matched-path': '/dynamic/[slug]', - }, - } - ) - expect(res.status).toBe(400) - expect(await res.text()).toContain('Bad Request') - }) - - it('should have correct resolvedUrl from rewrite', async () => { - const res = await fetchViaHTTP(appPort, '/to-dynamic/post-1', undefined, { - headers: { - 'x-matched-path': '/dynamic/[slug]', - }, - }) - expect(res.status).toBe(200) - const $ = cheerio.load(await res.text()) - expect($('#resolved-url').text()).toBe('/dynamic/post-1') - }) - - it('should have correct resolvedUrl from rewrite with added query', async () => { - const res = await fetchViaHTTP(appPort, '/to-dynamic/post-2', undefined, { - headers: { - 'x-matched-path': '/dynamic/[slug]', - }, - }) - expect(res.status).toBe(200) - const $ = cheerio.load(await res.text()) - expect($('#resolved-url').text()).toBe('/dynamic/post-2') - expect(JSON.parse($('#router').text()).asPath).toBe('/to-dynamic/post-2') - }) - - it('should have correct resolvedUrl from dynamic route', async () => { - const res = await fetchViaHTTP( - appPort, - `/_next/data/${next.buildId}/dynamic/post-2.json`, - { slug: 'post-2' }, - { - headers: { - 'x-matched-path': '/dynamic/[slug]', - }, - } - ) - expect(res.status).toBe(200) - const json = await res.json() - expect(json.pageProps.resolvedUrl).toBe('/dynamic/post-2') - }) - - it('should bubble error correctly for gip page', async () => { - errors = [] - const res = await fetchViaHTTP(appPort, '/errors/gip', { crash: '1' }) - expect(res.status).toBe(500) - expect(await res.text()).toBe('internal server error') - - await check( - () => (errors[0].includes('gip hit an oops') ? 'success' : errors[0]), - 'success' - ) - }) - - it('should bubble error correctly for gssp page', async () => { - errors = [] - const res = await fetchViaHTTP(appPort, '/errors/gssp', { crash: '1' }) - expect(res.status).toBe(500) - expect(await res.text()).toBe('internal server error') - await check( - () => (errors[0].includes('gssp hit an oops') ? 'success' : errors[0]), - 'success' - ) - }) - - it('should bubble error correctly for gsp page', async () => { - errors = [] - const res = await fetchViaHTTP(appPort, '/errors/gsp/crash') - expect(res.status).toBe(500) - expect(await res.text()).toBe('internal server error') - await check( - () => (errors[0].includes('gsp hit an oops') ? 'success' : errors[0]), - 'success' - ) - }) - - it('should bubble error correctly for API page', async () => { - errors = [] - const res = await fetchViaHTTP(appPort, '/api/error') - expect(res.status).toBe(500) - expect(await res.text()).toBe('internal server error') - await check( - () => - errors[0].includes('some error from /api/error') - ? 'success' - : errors[0], - 'success' - ) - }) - - it('should normalize optional values correctly for SSP page', async () => { - const res = await fetchViaHTTP( - appPort, - '/optional-ssp', - { rest: '', another: 'value' }, - { - headers: { - 'x-matched-path': '/optional-ssp/[[...rest]]', - }, - } - ) - - const html = await res.text() - const $ = cheerio.load(html) - const props = JSON.parse($('#props').text()) - expect(props.params).toEqual({}) - expect(props.query).toEqual({ another: 'value' }) - }) - - it('should normalize optional values correctly for SSG page', async () => { - const res = await fetchViaHTTP( - appPort, - '/optional-ssg', - { rest: '', another: 'value' }, - { - headers: { - 'x-matched-path': '/optional-ssg/[[...rest]]', - }, - } - ) - - const html = await res.text() - const $ = cheerio.load(html) - const props = JSON.parse($('#props').text()) - expect(props.params).toEqual({}) - }) - - it('should normalize optional revalidations correctly for SSG page', async () => { - const reqs = [ - { - path: `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - }, - }, - { - path: `/_next/data/${next.buildId}/optional-ssg.json`, - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - }, - }, - { - path: `/_next/data/${next.buildId}/optional-ssg.json`, - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/optional-ssg.json`, - }, - }, - { - path: `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - }, - query: { rest: '' }, - }, - { - path: `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - 'x-now-route-matches': '1=', - }, - }, - { - path: `/_next/data/${next.buildId}/optional-ssg/.json`, - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - 'x-now-route-matches': '', - 'x-vercel-id': 'cle1::', - }, - }, - { - path: `/optional-ssg/[[...rest]]`, - headers: { - 'x-matched-path': `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - 'x-now-route-matches': '', - 'x-vercel-id': 'cle1::', - }, - }, - { - path: `/_next/data/${next.buildId}/optional-ssg/[[...rest]].json`, - headers: { - 'x-matched-path': `/optional-ssg/[[...rest]]`, - 'x-now-route-matches': '', - 'x-vercel-id': 'cle1::', - }, - }, - ] - - for (const req of reqs) { - console.error('checking', req) - const res = await fetchViaHTTP(appPort, req.path, req.query, { - headers: req.headers, - }) - - const content = await res.text() - let props - - try { - const data = JSON.parse(content) - props = data.pageProps - } catch (_) { - props = JSON.parse(cheerio.load(content)('#__NEXT_DATA__').text()).props - .pageProps - } - expect(props.params).toEqual({}) - } - }) - - it('should normalize optional values correctly for SSG page with encoded slash', async () => { - const res = await fetchViaHTTP( - appPort, - '/optional-ssg/[[...rest]]', - undefined, - { - headers: { - 'x-matched-path': '/optional-ssg/[[...rest]]', - 'x-now-route-matches': - '1=en%2Fes%2Fhello%252Fworld&rest=en%2Fes%2Fhello%252Fworld', - }, - } - ) - - const html = await res.text() - const $ = cheerio.load(html) - const props = JSON.parse($('#props').text()) - expect(props.params).toEqual({ - rest: ['en', 'es', 'hello/world'], - }) - }) - - it('should normalize optional values correctly for API page', async () => { - const res = await fetchViaHTTP( - appPort, - '/api/optional', - { rest: '', another: 'value' }, - { - headers: { - 'x-matched-path': '/api/optional/[[...rest]]', - }, - } - ) - - const json = await res.json() - expect(json.query).toEqual({ another: 'value' }) - expect(json.url).toBe('/api/optional?another=value') - }) - - it('should normalize index optional values correctly for API page', async () => { - const res = await fetchViaHTTP( - appPort, - '/api/optional/index', - { rest: 'index', another: 'value' }, - { - headers: { - 'x-matched-path': '/api/optional/[[...rest]]', - }, - } - ) - - const json = await res.json() - expect(json.query).toEqual({ another: 'value', rest: ['index'] }) - expect(json.url).toBe('/api/optional/index?another=value') - }) - - it('should match the index page correctly', async () => { - const res = await fetchViaHTTP(appPort, '/', undefined, { - headers: { - 'x-matched-path': '/index', - }, - redirect: 'manual', - }) - - const html = await res.text() - const $ = cheerio.load(html) - expect($('#index').text()).toBe('index page') - }) - - it('should match the root dynamic page correctly', async () => { - const res = await fetchViaHTTP(appPort, '/slug-1', undefined, { - headers: { - 'x-matched-path': '/[slug]', - }, - redirect: 'manual', - }) - - const html = await res.text() - const $ = cheerio.load(html) - expect($('#slug-page').text()).toBe('[slug] page') - expect(JSON.parse($('#router').text()).query).toEqual({ - slug: 'slug-1', - }) - - const res2 = await fetchViaHTTP(appPort, '/[slug]', undefined, { - headers: { - 'x-matched-path': '/[slug]', - }, - redirect: 'manual', - }) - - const html2 = await res2.text() - const $2 = cheerio.load(html2) - expect($2('#slug-page').text()).toBe('[slug] page') - expect(JSON.parse($2('#router').text()).query).toEqual({ - slug: '[slug]', - }) - }) - - it('should have correct asPath on dynamic SSG page correctly', async () => { - const res = await fetchViaHTTP(appPort, '/an-ssg-path', undefined, { - headers: { - 'x-matched-path': '/[slug]', - }, - redirect: 'manual', - }) - - const html = await res.text() - const $ = cheerio.load(html) - expect($('#slug-page').text()).toBe('[slug] page') - expect(JSON.parse($('#router').text()).asPath).toBe('/an-ssg-path') - }) - - it('should have correct asPath on dynamic SSG page fallback correctly', async () => { - const toCheck = [ - { - pathname: '/fallback-false/first', - matchedPath: '/fallback-false/first', - }, - { - pathname: '/fallback-false/first', - matchedPath: `/_next/data/${next.buildId}/fallback-false/first.json`, - }, - ] - for (const check of toCheck) { - console.warn('checking', check) - const res = await fetchViaHTTP(appPort, check.pathname, undefined, { - headers: { - 'x-matched-path': check.matchedPath, - }, - redirect: 'manual', - }) - - const html = await res.text() - const $ = cheerio.load(html) - expect($('#page').text()).toBe('blog slug') - expect($('#asPath').text()).toBe('/fallback-false/first') - expect($('#pathname').text()).toBe('/fallback-false/[slug]') - expect(JSON.parse($('#query').text())).toEqual({ slug: 'first' }) - } - }) - - it('should copy and read .env file', async () => { - const res = await fetchViaHTTP(appPort, '/api/env') - - const envVariables = await res.json() - - expect(envVariables.env).not.toBeUndefined() - expect(envVariables.envProd).not.toBeUndefined() - expect(envVariables.envLocal).toBeUndefined() - }) -}) From a362881e0e25ec24c5094c6277a31943a1b836dd Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sun, 26 Mar 2023 22:30:19 -0700 Subject: [PATCH 12/52] Fix env and 404 case --- packages/next-env/index.ts | 8 +----- .../next/src/server/dev/next-dev-server.ts | 6 ++++- packages/next/src/server/next-server.ts | 25 +++++++++++++------ 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/packages/next-env/index.ts b/packages/next-env/index.ts index 3f264c3bf9da..a42f8b0a8a57 100644 --- a/packages/next-env/index.ts +++ b/packages/next-env/index.ts @@ -10,7 +10,7 @@ export type LoadedEnvFiles = Array<{ contents: string }> -let initialEnv: Env | undefined = undefined +export const initialEnv: Env = Object.assign({}, process.env) let combinedEnv: Env | undefined = undefined let cachedLoadedEnvFiles: LoadedEnvFiles = [] let previousLoadedEnvFiles: LoadedEnvFiles = [] @@ -38,9 +38,6 @@ export function processEnv( log: Log = console, forceReload = false ) { - if (!initialEnv) { - initialEnv = Object.assign({}, process.env) - } // only reload env when forceReload is specified if ( !forceReload && @@ -98,9 +95,6 @@ export function loadEnvConfig( combinedEnv: Env loadedEnvFiles: LoadedEnvFiles } { - if (!initialEnv) { - initialEnv = Object.assign({}, process.env) - } // only reload env when forceReload is specified if (combinedEnv && !forceReload) { return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles } diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 175b2e91ab1e..1d95ac57044a 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -654,7 +654,11 @@ export default class DevServer extends Server { if (envChange || tsconfigChange) { if (envChange) { - this.loadEnvConfig({ dev: true, forceReload: true }) + this.loadEnvConfig({ + dev: true, + forceReload: true, + silent: !!process.env.__NEXT_PRIVATE_RENDER_WORKER, + }) } let tsconfigResult: | UnwrapPromise> diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 7996b2f25739..c3bc793739e7 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -276,6 +276,8 @@ export default class NextNodeServer extends BaseServer { } const createWorker = (type: string) => { + const { initialEnv } = + require('@next/env') as typeof import('@next/env') const { Worker } = require('next/dist/compiled/jest-worker') const worker = new Worker(require.resolve('./lib/render-server'), { numWorkers: 1, @@ -283,7 +285,7 @@ export default class NextNodeServer extends BaseServer { maxRetries: 10, forkOptions: { env: { - ...process.env, + ...initialEnv, // we don't pass down NODE_OPTIONS as it can // extra memory usage NODE_OPTIONS: getNodeOptionsWithoutInspect() @@ -351,11 +353,18 @@ export default class NextNodeServer extends BaseServer { protected loadEnvConfig({ dev, forceReload, + silent, }: { dev: boolean forceReload?: boolean + silent?: boolean }) { - loadEnvConfig(this.dir, dev, Log, forceReload) + loadEnvConfig( + this.dir, + dev, + silent ? { info: () => {}, error: () => {} } : Log, + forceReload + ) } protected getIncrementalCache({ @@ -1407,12 +1416,14 @@ export default class NextNodeServer extends BaseServer { }` // ensure /404 is built before invoking render - if (this.renderOpts.dev && !(await this.hasPage(page))) { + if (this.renderOpts.dev && (await this.hasPage(page))) { // @ts-expect-error dev specific - await this.hotReloader?.ensurePage({ - page: '/404', - clientOnly: false, - }) + await this.hotReloader + ?.ensurePage({ + page: '/404', + clientOnly: false, + }) + .catch(() => {}) } const invokeHeaders: typeof req.headers = { ...req.headers, From d5bf1d189803928710f75a942aa9b786498e3dcf Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sun, 26 Mar 2023 23:25:50 -0700 Subject: [PATCH 13/52] call preflight --- packages/next/src/cli/next-dev.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index ccdf8d5915d1..4acd39b20656 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -265,7 +265,7 @@ const nextDev: CliCommand = async (argv) => { return server } else { await startServer(devServerOptions) - + await preflight() // if we're using workers we can auto restart on config changes if (devServerOptions.useWorkers) { // TODO: watch config and such and restart From 89827e14889df07319b48172e91fbdb6907dd97d Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sun, 26 Mar 2023 23:38:30 -0700 Subject: [PATCH 14/52] update initial env handling --- packages/next-env/index.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/next-env/index.ts b/packages/next-env/index.ts index a42f8b0a8a57..ee9e28979dd0 100644 --- a/packages/next-env/index.ts +++ b/packages/next-env/index.ts @@ -10,7 +10,7 @@ export type LoadedEnvFiles = Array<{ contents: string }> -export const initialEnv: Env = Object.assign({}, process.env) +export let initialEnv: Env | undefined = undefined let combinedEnv: Env | undefined = undefined let cachedLoadedEnvFiles: LoadedEnvFiles = [] let previousLoadedEnvFiles: LoadedEnvFiles = [] @@ -38,6 +38,9 @@ export function processEnv( log: Log = console, forceReload = false ) { + if (!initialEnv) { + initialEnv = Object.assign({}, process.env) + } // only reload env when forceReload is specified if ( !forceReload && @@ -95,6 +98,9 @@ export function loadEnvConfig( combinedEnv: Env loadedEnvFiles: LoadedEnvFiles } { + if (!initialEnv) { + initialEnv = Object.assign({}, process.env) + } // only reload env when forceReload is specified if (combinedEnv && !forceReload) { return { combinedEnv, loadedEnvFiles: cachedLoadedEnvFiles } From 0e784cfe22d5bcbe01b01f26049c0ef192e1b46b Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 27 Mar 2023 12:12:14 -0700 Subject: [PATCH 15/52] fix middleware-body and next-font test cases --- packages/next/src/server/next-server.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 00d2d69b8dcf..e4676cfe6c39 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1424,6 +1424,8 @@ export default class NextNodeServer extends BaseServer { clientOnly: false, }) .catch(() => {}) + + await this.getFallbackErrorComponents().catch(() => {}) } const invokeHeaders: typeof req.headers = { ...req.headers, @@ -1470,6 +1472,7 @@ export default class NextNodeServer extends BaseServer { } } res.statusCode = invokeRes.status + res.statusMessage = invokeRes.statusText for await (const chunk of invokeRes.body || ([] as any)) { this.streamResponseChunk(res as NodeNextResponse, chunk) } From 72cd4eb82673868725437e2ed878f7d94e9ff63a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 27 Mar 2023 14:50:47 -0700 Subject: [PATCH 16/52] Ensure error stacks are resolved in dev --- .../next/src/server/dev/next-dev-server.ts | 23 ++- packages/next/src/server/lib/start-server.ts | 3 +- packages/next/src/server/next-server.ts | 140 +++++++++++++----- packages/next/src/server/render.tsx | 2 +- .../middleware-dev-errors/test/index.test.js | 22 ++- 5 files changed, 135 insertions(+), 55 deletions(-) diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index c27f2d86cf1a..52f32c4e20c0 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -100,6 +100,7 @@ import { createClientRouterFilter } from '../../lib/create-client-router-filter' import { IncrementalCache } from '../lib/incremental-cache' import LRUCache from 'next/dist/compiled/lru-cache' import { NextUrlWithParsedQuery } from '../request-meta' +import { errorToJSON } from '../render' // Load ReactDevOverlay only when needed let ReactDevOverlayImpl: FunctionComponent @@ -891,7 +892,7 @@ export default class DevServer extends Server { const telemetry = new Telemetry({ distDir: this.distDir }) // router worker does not start webpack compilers - if (this.isRouterWorker) { + if (!this.isRenderWorker) { this.hotReloader = new HotReloader(this.dir, { pagesDir: this.pagesDir, distDir: this.distDir, @@ -1255,6 +1256,18 @@ export default class DevServer extends Server { err?: unknown, type?: 'unhandledRejection' | 'uncaughtException' | 'warning' | 'app-dir' ) { + const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT + if (ipcPort) { + await fetch( + `http://${ + this.hostname + }:${ipcPort}?method=logErrorWithOriginalStack&args=${encodeURIComponent( + JSON.stringify([errorToJSON(err as Error), type]) + )}` + ) + return + } + let usedOriginalStack = false if (isError(err) && err.stack) { @@ -1266,9 +1279,9 @@ export default class DevServer extends Server { !file?.includes('web/adapter') && !file?.includes('sandbox/context') && !file?.includes('') - )! + ) - if (frame.lineNumber && frame?.file) { + if (frame?.lineNumber && frame?.file) { const moduleId = frame.file!.replace( /^(webpack-internal:\/\/\/|file:\/\/)/, '' @@ -1293,7 +1306,7 @@ export default class DevServer extends Server { ) const originalFrame = await createOriginalStackFrame({ - line: frame.lineNumber!, + line: frame.lineNumber, column: frame.column, source, frame, @@ -1307,7 +1320,7 @@ export default class DevServer extends Server { edgeCompilation: isEdgeCompiler ? this.hotReloader?.edgeServerStats?.compilation : undefined, - }) + }).catch(() => {}) if (originalFrame) { const { originalCodeFrame, originalStackFrame } = originalFrame diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index c41b8691d597..83f89e912f73 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -146,6 +146,7 @@ export async function startServer({ numWorkers: 1, forkOptions: { env: { + FORCE_COLOR: '1', ...process.env, // we don't pass down NODE_OPTIONS as it can // extra memory usage @@ -213,8 +214,8 @@ export async function startServer({ // handle in process requestHandler = app.getRequestHandler() upgradeHandler = app.getUpgradeHandler() - handlersReady() await app.prepare() + handlersReady() } } catch (err) { // fatal error if we can't setup diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index e4676cfe6c39..d3101f8b9212 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -102,6 +102,7 @@ import { nodeFs } from './lib/node-fs-methods' import { genExecArgv, getNodeOptionsWithoutInspect } from './lib/utils' import { getRouteRegex } from '../shared/lib/router/utils/route-regex' import { removePathPrefix } from '../shared/lib/router/utils/remove-path-prefix' +import { decorateServerError } from 'next/dist/compiled/@next/react-dev-overlay/dist/middleware' export * from './base-server' @@ -187,6 +188,7 @@ type RenderWorker = Worker & { export default class NextNodeServer extends BaseServer { private imageResponseCache?: ResponseCache private compression?: ExpressMiddleware + protected renderWorkersPromises?: Promise protected renderWorkers?: { middleware?: RenderWorker pages?: RenderWorker @@ -274,51 +276,100 @@ export default class NextNodeServer extends BaseServer { hostname: this.hostname, dev: !!options.dev, } + this.renderWorkersPromises = new Promise(async (resolveWorkers) => { + // we can't use process.send as jest-worker relies on + // it already and can cause unexpected message errors + const ipcServer = ( + require('http') as typeof import('http') + ).createServer(async (req, res) => { + try { + const url = new URL(req.url || '/', 'http://n') + const method = url.searchParams.get('method') + const args: any[] = JSON.parse(url.searchParams.get('args') || '[]') + + if (!method || !Array.isArray(args) || !args.length) { + return res.end() + } + + if (typeof (this as any)[method] === 'function') { + if (method === 'logErrorWithOriginalStack' && args[0]?.stack) { + const err = new Error(args[0].message) + err.stack = args[0].stack + err.name = args[0].name + decorateServerError(err, args[0].source || 'server') + args[0] = err + } + await (this as any)[method](...args) + } + } catch (err) { + console.error(err) + } + }) + + const ipcPort = await new Promise((resolveIpc) => { + ipcServer.listen(0, this.hostname, () => { + const addr = ipcServer.address() + + if (addr && typeof addr === 'object') { + resolveIpc(addr.port) + } + }) + }) - const createWorker = (type: string) => { - const { initialEnv } = - require('@next/env') as typeof import('@next/env') - const { Worker } = require('next/dist/compiled/jest-worker') - const worker = new Worker(require.resolve('./lib/render-server'), { - numWorkers: 1, - // TODO: do we want to allow more than 10 OOM restarts? - maxRetries: 10, - forkOptions: { - env: { - ...initialEnv, - // we don't pass down NODE_OPTIONS as it can - // extra memory usage - NODE_OPTIONS: getNodeOptionsWithoutInspect() - .replace(/--max-old-space-size=[\d]{1,}/, '') - .trim(), - - __NEXT_PRIVATE_RENDER_WORKER: type, + const createWorker = (type: string) => { + const { initialEnv } = + require('@next/env') as typeof import('@next/env') + const { Worker } = require('next/dist/compiled/jest-worker') + const worker = new Worker(require.resolve('./lib/render-server'), { + numWorkers: 1, + // TODO: do we want to allow more than 10 OOM restarts? + maxRetries: 10, + forkOptions: { + env: { + FORCE_COLOR: '1', + ...initialEnv, + // we don't pass down NODE_OPTIONS as it can + // extra memory usage + NODE_OPTIONS: getNodeOptionsWithoutInspect() + .replace(/--max-old-space-size=[\d]{1,}/, '') + .trim(), + __NEXT_PRIVATE_RENDER_WORKER: type, + __NEXT_PRIVATE_ROUTER_IPC_PORT: ipcPort + '', + }, + execArgv: genExecArgv( + options.isNodeDebugging === undefined + ? false + : options.isNodeDebugging, + (this.port || 0) + 1 + ), }, - execArgv: genExecArgv( - options.isNodeDebugging === undefined - ? false - : options.isNodeDebugging, - (this.port || 0) + 1 - ), - }, - exposedMethods: ['initialize', 'deleteCache', 'deleteAppClientCache'], - }) as any as InstanceType & { - initialize: typeof import('./lib/render-server').initialize - deleteCache: typeof import('./lib/render-server').deleteCache - deleteAppClientCache: typeof import('./lib/render-server').deleteAppClientCache + exposedMethods: [ + 'initialize', + 'deleteCache', + 'deleteAppClientCache', + ], + }) as any as InstanceType & { + initialize: typeof import('./lib/render-server').initialize + deleteCache: typeof import('./lib/render-server').deleteCache + deleteAppClientCache: typeof import('./lib/render-server').deleteAppClientCache + } + + worker.getStderr().pipe(process.stderr) + worker.getStdout().pipe(process.stdout) + + return worker } + this.renderWorkers = {} - worker.getStderr().pipe(process.stderr) - worker.getStdout().pipe(process.stdout) - return worker - } + if (this.hasAppDir) { + this.renderWorkers.app = createWorker('app') + } + this.renderWorkers.pages = createWorker('pages') + this.renderWorkers.middleware = + this.renderWorkers.pages || this.renderWorkers.app - if (this.hasAppDir) { - this.renderWorkers.app = createWorker('app') - } - this.renderWorkers.pages = createWorker('pages') - this.renderWorkers.middleware = - this.renderWorkers.pages || this.renderWorkers.app + resolveWorkers() + }) ;(global as any)._nextDeleteCache = (filePath: string) => { this.renderWorkers?.pages?.deleteCache(filePath) this.renderWorkers?.app?.deleteCache(filePath) @@ -1374,6 +1425,10 @@ export default class NextNodeServer extends BaseServer { const renderKind = this.appPathRoutes?.[page] ? 'app' : 'pages' + if (this.renderWorkersPromises) { + await this.renderWorkersPromises + this.renderWorkersPromises = undefined + } const renderWorker = this.renderWorkers?.[renderKind] if (renderWorker) { @@ -2214,6 +2269,11 @@ export default class NextNodeServer extends BaseServer { await this.ensureMiddleware() if (this.isRouterWorker && this.renderWorkers?.middleware) { + if (this.renderWorkersPromises) { + await this.renderWorkersPromises + this.renderWorkersPromises = undefined + } + const { port, hostname } = await this.renderWorkers.middleware.initialize( this.renderWorkerOpts! diff --git a/packages/next/src/server/render.tsx b/packages/next/src/server/render.tsx index fc537cd4a820..d5bc9e2ae67e 100644 --- a/packages/next/src/server/render.tsx +++ b/packages/next/src/server/render.tsx @@ -330,7 +330,7 @@ function checkRedirectValues( } } -function errorToJSON(err: Error) { +export function errorToJSON(err: Error) { let source: typeof COMPILER_NAMES.server | typeof COMPILER_NAMES.edgeServer = 'server' diff --git a/test/integration/middleware-dev-errors/test/index.test.js b/test/integration/middleware-dev-errors/test/index.test.js index ff02a97e37f1..70f17f9c7431 100644 --- a/test/integration/middleware-dev-errors/test/index.test.js +++ b/test/integration/middleware-dev-errors/test/index.test.js @@ -37,11 +37,11 @@ describe('Middleware development errors', () => { }) }) - afterEach(() => { + afterEach(async () => { context.middleware.restore() context.page.restore() if (context.app) { - killApp(context.app) + await killApp(context.app) } }) @@ -56,7 +56,8 @@ describe('Middleware development errors', () => { it('logs the error correctly', async () => { await fetchViaHTTP(context.appPort, '/') const output = stripAnsi(context.logs.output) - expect(output).toMatch( + await check( + () => stripAnsi(context.logs.output), new RegExp( `error - middleware.js \\(\\d+:\\d+\\) @ Object.default \\[as handler\\]\nerror - boom`, 'm' @@ -91,7 +92,8 @@ describe('Middleware development errors', () => { it('logs the error correctly', async () => { await fetchViaHTTP(context.appPort, '/') const output = stripAnsi(context.logs.output) - expect(output).toMatch( + await check( + () => stripAnsi(context.logs.output), new RegExp( `error - middleware.js \\(\\d+:\\d+\\) @ throwError\nerror - unhandledRejection: async boom!`, 'm' @@ -122,7 +124,8 @@ describe('Middleware development errors', () => { it('logs the error correctly', async () => { await fetchViaHTTP(context.appPort, '/') const output = stripAnsi(context.logs.output) - expect(output).toMatch( + await check( + () => stripAnsi(context.logs.output), new RegExp( `error - middleware.js \\(\\d+:\\d+\\) @ eval\nerror - test is not defined`, 'm' @@ -155,7 +158,8 @@ describe('Middleware development errors', () => { it('logs the error correctly', async () => { await fetchViaHTTP(context.appPort, '/') const output = stripAnsi(context.logs.output) - expect(output).toMatch( + await check( + () => stripAnsi(context.logs.output), new RegExp( `error - middleware.js \\(\\d+:\\d+\\) @ \nerror - booooom!`, 'm' @@ -193,7 +197,8 @@ describe('Middleware development errors', () => { it('logs the error correctly', async () => { await fetchViaHTTP(context.appPort, '/') const output = stripAnsi(context.logs.output) - expect(output).toMatch( + await check( + () => stripAnsi(context.logs.output), new RegExp( `error - middleware.js \\(\\d+:\\d+\\) @ eval\nerror - unhandledRejection: you shall see me`, 'm' @@ -225,7 +230,8 @@ describe('Middleware development errors', () => { it('logs the error correctly', async () => { await fetchViaHTTP(context.appPort, '/') const output = stripAnsi(context.logs.output) - expect(output).toMatch( + await check( + () => stripAnsi(context.logs.output), new RegExp( `error - lib/unhandled.js \\(\\d+:\\d+\\) @ Timeout.eval \\[as _onTimeout\\]\nerror - uncaughtException: This file asynchronously fails while loading`, 'm' From b95c2fc19f52122ee88b31f424456dceb338c85f Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Mon, 27 Mar 2023 23:32:10 -0700 Subject: [PATCH 17/52] handle dev server watching --- packages/next/src/cli/next-dev.ts | 250 ++++++++++++++++++- packages/next/src/server/lib/start-server.ts | 44 +++- 2 files changed, 278 insertions(+), 16 deletions(-) diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index 4acd39b20656..38a673590696 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -1,12 +1,12 @@ #!/usr/bin/env node import arg from 'next/dist/compiled/arg/index.js' -import { startServer } from '../server/lib/start-server' +import { startServer, StartServerOptions } from '../server/lib/start-server' import { getPort, printAndExit } from '../server/lib/utils' import * as Log from '../build/output/log' import { CliCommand } from '../lib/commands' import isError from '../lib/is-error' import { getProjectDir } from '../lib/get-project-dir' -import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' +import { CONFIG_FILES, PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' import path from 'path' import type { NextConfigComplete } from '../server/config-shared' import { traceGlobals } from '../trace/shared' @@ -15,6 +15,9 @@ import loadConfig from '../server/config' import { findPagesDir } from '../lib/find-pages-dir' import { fileExists } from '../lib/file-exists' import { getNpxCommand } from '../lib/helpers/get-npx-command' +import Watchpack from 'next/dist/compiled/watchpack' +import stripAnsi from 'next/dist/compiled/strip-ansi' +import { getPossibleInstrumentationHookFilenames } from '../build/worker' let dir: string let isTurboSession = false @@ -83,6 +86,25 @@ const handleSessionStop = async () => { process.on('SIGINT', handleSessionStop) process.on('SIGTERM', handleSessionStop) +let unwatchConfigFiles: () => void + +function watchConfigFiles(dirToWatch: string) { + if (unwatchConfigFiles) { + unwatchConfigFiles() + } + + const wp = new Watchpack() + wp.watch({ files: CONFIG_FILES.map((file) => path.join(dirToWatch, file)) }) + wp.on('change', (filename) => { + console.log( + `\n> Found a change in ${path.basename( + filename + )}. Restart the server to see the changes in effect.` + ) + }) + unwatchConfigFiles = () => wp.close() +} + const nextDev: CliCommand = async (argv) => { const validArgs: arg.Spec = { // Types @@ -181,10 +203,9 @@ const nextDev: CliCommand = async (argv) => { // some set-ups that rely on listening on other interfaces const host = args['--hostname'] - const devServerOptions = { + const devServerOptions: StartServerOptions = { dir, port, - dev: true, allowRetry, isDev: true, hostname: host, @@ -264,12 +285,223 @@ const nextDev: CliCommand = async (argv) => { } return server } else { - await startServer(devServerOptions) - await preflight() - // if we're using workers we can auto restart on config changes - if (devServerOptions.useWorkers) { - // TODO: watch config and such and restart + let shouldFilter = false + let devServerTeardown: (() => Promise) | undefined + const config = await loadConfig( + PHASE_DEVELOPMENT_SERVER, + dir, + undefined, + undefined, + true + ) + watchConfigFiles(devServerOptions.dir) + + const setupFork = async (newDir?: string) => { + // if we're using workers we can auto restart on config changes + if (!devServerOptions.useWorkers && devServerTeardown) { + Log.info( + `Detected change, manual restart required due to '__NEXT_DISABLE_MEMORY_WATCHER' usage` + ) + return + } + + const startDir = dir + if (newDir) { + dir = newDir + } + + if (devServerTeardown) { + await devServerTeardown() + devServerTeardown = undefined + } + + if (newDir) { + dir = newDir + process.env = Object.keys(process.env).reduce((newEnv, key) => { + newEnv[key] = process.env[key]?.replace(startDir, newDir) + return newEnv + }, {} as typeof process.env) + + process.chdir(newDir) + + devServerOptions.dir = newDir + devServerOptions.prevDir = startDir + } + + // since errors can start being logged from the fork + // before we detect the project directory rename + // attempt suppressing them long enough to check + const filterForkErrors = (chunk: Buffer, fd: 'stdout' | 'stderr') => { + const cleanChunk = stripAnsi(chunk + '') + if ( + cleanChunk.match( + /(ENOENT|Module build failed|Module not found|Cannot find module|Can't resolve)/ + ) + ) { + if (startDir === dir) { + try { + // check if start directory is still valid + const result = findPagesDir( + startDir, + !!config.experimental?.appDir + ) + shouldFilter = !Boolean(result.pagesDir || result.appDir) + } catch (_) { + shouldFilter = true + } + } + if (shouldFilter || startDir !== dir) { + shouldFilter = true + return + } + } + process[fd].write(chunk) + } + + devServerOptions.onStdout = (chunk) => { + filterForkErrors(chunk, 'stdout') + } + devServerOptions.onStderr = (chunk) => { + filterForkErrors(chunk, 'stderr') + } + shouldFilter = false + devServerTeardown = await startServer(devServerOptions) } + + await setupFork() + await preflight() + + const parentDir = path.join('/', dir, '..') + const watchedEntryLength = parentDir.split('/').length + 1 + const previousItems = new Set() + const instrumentationFilePaths = !!config.experimental?.instrumentationHook + ? getPossibleInstrumentationHookFilenames(dir, config.pageExtensions!) + : [] + + const instrumentationFileWatcher = new Watchpack({}) + + instrumentationFileWatcher.watch({ + files: instrumentationFilePaths, + startTime: 0, + }) + + let instrumentationFileLastHash: string | undefined = undefined + const previousInstrumentationFiles = new Set() + instrumentationFileWatcher.on('aggregated', async () => { + const knownFiles = instrumentationFileWatcher.getTimeInfoEntries() + const instrumentationFile = [...knownFiles.entries()].find( + ([key, value]) => instrumentationFilePaths.includes(key) && value + )?.[0] + + if (instrumentationFile) { + const fs = require('fs') as typeof import('fs') + const instrumentationFileHash = ( + require('crypto') as typeof import('crypto') + ) + .createHash('sha256') + .update(await fs.promises.readFile(instrumentationFile, 'utf8')) + .digest('hex') + + if ( + instrumentationFileLastHash && + instrumentationFileHash !== instrumentationFileLastHash + ) { + Log.warn( + `The instrumentation file has changed, restarting the server to apply changes.` + ) + return setupFork() + } else { + if ( + !instrumentationFileLastHash && + previousInstrumentationFiles.size !== 0 + ) { + Log.warn( + 'The instrumentation file was added, restarting the server to apply changes.' + ) + return setupFork() + } + instrumentationFileLastHash = instrumentationFileHash + } + } else if ( + [...previousInstrumentationFiles.keys()].find((key) => + instrumentationFilePaths.includes(key) + ) + ) { + Log.warn( + `The instrumentation file has been removed, restarting the server to apply changes.` + ) + instrumentationFileLastHash = undefined + return setupFork() + } + + previousInstrumentationFiles.clear() + knownFiles.forEach((_, key) => previousInstrumentationFiles.add(key)) + }) + + const projectFolderWatcher = new Watchpack({ + ignored: (entry: string) => { + return !(entry.split('/').length <= watchedEntryLength) + }, + }) + + projectFolderWatcher.watch({ directories: [parentDir], startTime: 0 }) + + projectFolderWatcher.on('aggregated', async () => { + const knownFiles = projectFolderWatcher.getTimeInfoEntries() + const newFiles: string[] = [] + let hasPagesApp = false + + // if the dir still exists nothing to check + try { + const result = findPagesDir(dir, !!config.experimental?.appDir) + hasPagesApp = Boolean(result.pagesDir || result.appDir) + } catch (err) { + // if findPagesDir throws validation error let this be + // handled in the dev-server itself in the fork + if ((err as any).message?.includes('experimental')) { + return + } + } + + // try to find new dir introduced + if (previousItems.size) { + for (const key of knownFiles.keys()) { + if (!previousItems.has(key)) { + newFiles.push(key) + } + } + previousItems.clear() + } + + for (const key of knownFiles.keys()) { + previousItems.add(key) + } + + if (hasPagesApp) { + return + } + + // if we failed to find the new dir it may have been moved + // to a new parent directory which we can't track as easily + // so exit gracefully + try { + const result = findPagesDir(newFiles[0], !!config.experimental?.appDir) + hasPagesApp = Boolean(result.pagesDir || result.appDir) + } catch (_) {} + + if (hasPagesApp && newFiles.length === 1) { + Log.info( + `Detected project directory rename, restarting in new location` + ) + setupFork(newFiles[0]) + watchConfigFiles(newFiles[0]) + } else { + Log.error( + `Project directory could not be found, restart Next.js in your new directory` + ) + process.exit(0) + } + }) } } diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 83f89e912f73..bf2978696319 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -4,8 +4,9 @@ import * as Log from '../../build/output/log' import { getNodeOptionsWithoutInspect } from './utils' import type { IncomingMessage, ServerResponse } from 'http' -interface StartServerOptions { +export interface StartServerOptions { dir: string + prevDir?: string port: number isDev: boolean hostname: string @@ -13,18 +14,23 @@ interface StartServerOptions { allowRetry?: boolean isTurbopack?: boolean keepAliveTimeout?: number + onStdout?: (data: any) => void + onStderr?: (data: any) => void } type TeardownServer = () => Promise export async function startServer({ dir, + prevDir, port, isDev, hostname, useWorkers, allowRetry, keepAliveTimeout, + onStdout, + onStderr, }: StartServerOptions): Promise { const sockets = new Set() let worker: import('next/dist/compiled/jest-worker').Worker | undefined @@ -136,15 +142,24 @@ export async function startServer({ try { if (useWorkers) { - const { Worker } = - require('next/dist/compiled/jest-worker') as typeof import('next/dist/compiled/jest-worker') - const httpProxy = require('next/dist/compiled/http-proxy') as typeof import('next/dist/compiled/http-proxy') - const routerWorker = new Worker(require.resolve('./render-server'), { + let renderServerPath = require.resolve('./render-server') + let jestWorkerPath = require.resolve('next/dist/compiled/jest-worker') + + if (prevDir) { + jestWorkerPath = jestWorkerPath.replace(prevDir, dir) + renderServerPath = renderServerPath.replace(prevDir, dir) + } + + const { Worker } = + require(jestWorkerPath) as typeof import('next/dist/compiled/jest-worker') + + const routerWorker = new Worker(renderServerPath, { numWorkers: 1, forkOptions: { + cwd: dir, env: { FORCE_COLOR: '1', ...process.env, @@ -159,8 +174,23 @@ export async function startServer({ }) as any as InstanceType & { initialize: typeof import('./render-server').initialize } - routerWorker.getStdout().pipe(process.stdout) - routerWorker.getStderr().pipe(process.stderr) + const workerStdout = routerWorker.getStdout() + const workerStderr = routerWorker.getStderr() + + workerStdout.on('data', (data) => { + if (typeof onStdout === 'function') { + onStdout(data) + } else { + process.stdout.write(data) + } + }) + workerStderr.on('data', (data) => { + if (typeof onStderr === 'function') { + onStderr(data) + } else { + process.stderr.write(data) + } + }) const { port: routerPort } = await routerWorker.initialize({ dir, From f92aad3e8ab9746a2d90df05c98a33c845e63f78 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 00:19:30 -0700 Subject: [PATCH 18/52] Add additional IPC handling and fix missing middleware header --- .../next/src/server/dev/next-dev-server.ts | 63 ++++++++++++-- packages/next/src/server/next-server.ts | 59 +++++++------ packages/next/src/server/render.tsx | 20 +++++ .../app/pages/index.js | 4 +- .../test/index.test.js | 86 ++++++++++++------- 5 files changed, 163 insertions(+), 69 deletions(-) diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 52f32c4e20c0..7835120a9216 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -100,7 +100,7 @@ import { createClientRouterFilter } from '../../lib/create-client-router-filter' import { IncrementalCache } from '../lib/incremental-cache' import LRUCache from 'next/dist/compiled/lru-cache' import { NextUrlWithParsedQuery } from '../request-meta' -import { errorToJSON } from '../render' +import { deserializeErr, errorToJSON } from '../render' // Load ReactDevOverlay only when needed let ReactDevOverlayImpl: FunctionComponent @@ -254,7 +254,7 @@ export default class DevServer extends Server { const ensurer: RouteEnsurer = { ensure: async (match) => { - await this.hotReloader?.ensurePage({ + await this.ensurePage({ match, page: match.definition.page, clientOnly: false, @@ -1430,7 +1430,7 @@ export default class DevServer extends Server { } protected async ensureMiddleware() { - return this.hotReloader?.ensurePage({ + return this.ensurePage({ page: this.actualMiddlewareFile!, clientOnly: false, }) @@ -1439,7 +1439,7 @@ export default class DevServer extends Server { private async runInstrumentationHookIfAvailable() { if (this.actualInstrumentationHookFile) { NextBuildContext!.hasInstrumentationHook = true - await this.hotReloader!.ensurePage({ + await this.ensurePage({ page: this.actualInstrumentationHookFile!, clientOnly: false, }) @@ -1459,7 +1459,7 @@ export default class DevServer extends Server { page: string appPaths: string[] | null }) { - return this.hotReloader?.ensurePage({ page, appPaths, clientOnly: false }) + return this.ensurePage({ page, appPaths, clientOnly: false }) } generateRoutes(dev?: boolean) { @@ -1682,6 +1682,23 @@ export default class DevServer extends Server { global.fetch = this.originalFetch! } + protected async ensurePage( + opts: Parameters['ensurePage']>[0] + ) { + const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT + if (ipcPort) { + await fetch( + `http://${ + this.hostname + }:${ipcPort}?method=ensurePage&args=${encodeURIComponent( + JSON.stringify([opts]) + )}` + ) + return + } + return this.hotReloader?.ensurePage(opts) + } + protected async findPageComponents({ pathname, query, @@ -1705,7 +1722,7 @@ export default class DevServer extends Server { } try { if (shouldEnsure || this.renderOpts.customServer) { - await this.hotReloader?.ensurePage({ + await this.ensurePage({ page: pathname, appPaths, clientOnly: false, @@ -1740,10 +1757,25 @@ export default class DevServer extends Server { } protected async getFallbackErrorComponents(): Promise { + const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT + if (ipcPort) { + await fetch( + `http://${ + this.hostname + }:${ipcPort}?method=getFallbackErrorComponents&args=${encodeURIComponent( + JSON.stringify([]) + )}` + ) + return await loadDefaultErrorComponents(this.distDir) + } await this.hotReloader?.buildFallbackError() // Build the error page to ensure the fallback is built too. // TODO: See if this can be moved into hotReloader or removed. - await this.hotReloader?.ensurePage({ page: '/_error', clientOnly: false }) + await this.ensurePage({ page: '/_error', clientOnly: false }) + + if (this.isRouterWorker) { + return null + } return await loadDefaultErrorComponents(this.distDir) } @@ -1770,6 +1802,23 @@ export default class DevServer extends Server { } async getCompilationError(page: string): Promise { + const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT + if (ipcPort) { + const res = await fetch( + `http://${ + this.hostname + }:${ipcPort}?method=getCompilationError&args=${encodeURIComponent( + JSON.stringify([page]) + )}` + ) + const body = await res.text() + + if (body.startsWith('{') && body.endsWith('}')) { + const err = deserializeErr(JSON.parse(body)) + return err + } + return + } const errors = await this.hotReloader?.getCompilationErrors(page) if (!errors) return diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index d3101f8b9212..92b5be97eda9 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -55,7 +55,7 @@ import { sendRenderResult } from './send-payload' import { getExtension, serveStatic } from './serve-static' import { ParsedUrlQuery } from 'querystring' import { apiResolver } from './api-utils/node' -import { RenderOpts, renderToHTML } from './render' +import { deserializeErr, errorToJSON, RenderOpts, renderToHTML } from './render' import { ParsedUrl, parseUrl } from '../shared/lib/router/utils/parse-url' import { parse as nodeParseUrl } from 'url' import * as Log from '../build/output/log' @@ -287,22 +287,28 @@ export default class NextNodeServer extends BaseServer { const method = url.searchParams.get('method') const args: any[] = JSON.parse(url.searchParams.get('args') || '[]') - if (!method || !Array.isArray(args) || !args.length) { + if (!method || !Array.isArray(args)) { return res.end() } if (typeof (this as any)[method] === 'function') { if (method === 'logErrorWithOriginalStack' && args[0]?.stack) { - const err = new Error(args[0].message) - err.stack = args[0].stack - err.name = args[0].name - decorateServerError(err, args[0].source || 'server') - args[0] = err + args[0] = deserializeErr(args[0]) } - await (this as any)[method](...args) + let result = await (this as any)[method](...args) + + if (result && typeof result === 'object' && result.stack) { + result = errorToJSON(result) + } + res.end(JSON.stringify(result || '')) } - } catch (err) { + } catch (err: any) { console.error(err) + res.end( + JSON.stringify({ + err: { name: err.name, message: err.message, stack: err.stack }, + }) + ) } }) @@ -1470,18 +1476,6 @@ export default class NextNodeServer extends BaseServer { invokeQueryStr ? `?${invokeQueryStr}` : '' }` - // ensure /404 is built before invoking render - if (this.renderOpts.dev && (await this.hasPage(page))) { - // @ts-expect-error dev specific - await this.hotReloader - ?.ensurePage({ - page: '/404', - clientOnly: false, - }) - .catch(() => {}) - - await this.getFallbackErrorComponents().catch(() => {}) - } const invokeHeaders: typeof req.headers = { ...req.headers, 'x-invoke-path': invokePath, @@ -2318,13 +2312,22 @@ export default class NextNodeServer extends BaseServer { }), waitUntil: Promise.resolve(), } - for (const key of [ - 'content-encoding', - 'transfer-encoding', - 'keep-alive', - 'connection', - ]) { - result.response.headers.delete(key) + for (const key of [...result.response.headers.keys()]) { + if ( + [ + 'content-encoding', + 'transfer-encoding', + 'keep-alive', + 'connection', + ].includes(key) + ) { + result.response.headers.delete(key) + } else { + // propagate this to req headers so it's + // passed to the render worker for the page + req.headers[key] = + result.response.headers.get(key) || undefined + } } } else { result = await this.runMiddleware({ diff --git a/packages/next/src/server/render.tsx b/packages/next/src/server/render.tsx index d5bc9e2ae67e..7be3375a3e0b 100644 --- a/packages/next/src/server/render.tsx +++ b/packages/next/src/server/render.tsx @@ -330,6 +330,26 @@ function checkRedirectValues( } } +export const deserializeErr = (serializedErr: any) => { + if ( + !serializedErr || + typeof serializedErr !== 'object' || + !serializedErr.stack + ) { + return serializedErr + } + const err = new Error(serializedErr.message) + err.stack = serializedErr.stack + err.name = serializedErr.name + + if (process.env.NEXT_RUNTIME !== 'edge') { + const { decorateServerError } = + require('next/dist/compiled/@next/react-dev-overlay/dist/middleware') as typeof import('next/dist/compiled/@next/react-dev-overlay/dist/middleware') + decorateServerError(err, serializedErr.source || 'server') + } + return err +} + export function errorToJSON(err: Error) { let source: typeof COMPILER_NAMES.server | typeof COMPILER_NAMES.edgeServer = 'server' diff --git a/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js b/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js index accd5b17adff..d981f724020a 100644 --- a/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js +++ b/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js @@ -5,10 +5,10 @@ export default (props) => ( ) -export async function getServerSideProps({ res }) { +export async function getServerSideProps({ req }) { return { props: { - fromMiddleware: res.getHeader('x-from-middleware') || null, + fromMiddleware: req.headers['x-from-middleware'] || null, }, } } diff --git a/test/integration/edge-runtime-module-errors/test/index.test.js b/test/integration/edge-runtime-module-errors/test/index.test.js index 1fa51b2d1dba..459c463431c5 100644 --- a/test/integration/edge-runtime-module-errors/test/index.test.js +++ b/test/integration/edge-runtime-module-errors/test/index.test.js @@ -5,6 +5,7 @@ import stripAnsi from 'next/dist/compiled/strip-ansi' import { remove } from 'fs-extra' import { join } from 'path' import { + check, fetchViaHTTP, File, findPort, @@ -45,9 +46,9 @@ describe('Edge runtime code with imports', () => { await remove(join(__dirname, '../.next')) }) - afterEach(() => { + afterEach(async () => { if (context.app) { - killApp(context.app) + await killApp(context.app) } context.api.restore() context.middleware.restore() @@ -97,11 +98,14 @@ describe('Edge runtime code with imports', () => { context.app = await launchApp(context.appDir, context.appPort, appOption) const res = await fetchViaHTTP(context.appPort, url) expect(res.status).toBe(500) - expectUnsupportedModuleDevError( - moduleName, - importStatement, - await res.text() - ) + await check(async () => { + expectUnsupportedModuleDevError( + moduleName, + importStatement, + await res.text() + ) + return 'success' + }, 'success') }) it('throws unsupported module error in production at runtime and prints error on logs', async () => { @@ -157,11 +161,14 @@ describe('Edge runtime code with imports', () => { context.app = await launchApp(context.appDir, context.appPort, appOption) const res = await fetchViaHTTP(context.appPort, url) expect(res.status).toBe(500) - expectUnsupportedModuleDevError( - moduleName, - importStatement, - await res.text() - ) + await check(async () => { + expectUnsupportedModuleDevError( + moduleName, + importStatement, + await res.text() + ) + return 'success' + }, 'success') }) it('throws unsupported module error in production at runtime and prints error on logs', async () => { @@ -230,11 +237,14 @@ describe('Edge runtime code with imports', () => { ) const res = await fetchViaHTTP(context.appPort, url) expect(res.status).toBe(500) - expectUnsupportedModuleDevError( - moduleName, - importStatement, - await res.text() - ) + await check(async () => { + expectUnsupportedModuleDevError( + moduleName, + importStatement, + await res.text() + ) + return 'success' + }, 'success') }) it('throws unsupported module error in production at runtime and prints error on logs', async () => { @@ -296,11 +306,15 @@ describe('Edge runtime code with imports', () => { context.app = await launchApp(context.appDir, context.appPort, appOption) const res = await fetchViaHTTP(context.appPort, url) expect(res.status).toBe(500) - expectModuleNotFoundDevError( - moduleName, - importStatement, - await res.text() - ) + + await check(async () => { + expectModuleNotFoundDevError( + moduleName, + importStatement, + await res.text() + ) + return 'success' + }, 'success') }) it('does not build and reports', async () => { @@ -353,11 +367,15 @@ describe('Edge runtime code with imports', () => { context.app = await launchApp(context.appDir, context.appPort, appOption) const res = await fetchViaHTTP(context.appPort, url) expect(res.status).toBe(500) - expectModuleNotFoundDevError( - moduleName, - importStatement, - await res.text() - ) + + await check(async () => { + expectModuleNotFoundDevError( + moduleName, + importStatement, + await res.text() + ) + return 'success' + }, 'success') }) it('does not build and reports module not found error', async () => { @@ -414,11 +432,14 @@ describe('Edge runtime code with imports', () => { context.app = await launchApp(context.appDir, context.appPort, appOption) const res = await fetchViaHTTP(context.appPort, url) expect(res.status).toBe(500) - expectModuleNotFoundDevError( - moduleName, - importStatement, - await res.text() - ) + await check(async () => { + expectModuleNotFoundDevError( + moduleName, + importStatement, + await res.text() + ) + return 'success' + }, 'success') }) it('does not build and reports module not found error', async () => { @@ -428,6 +449,7 @@ describe('Edge runtime code with imports', () => { stderr: true, }) expect(code).toEqual(1) + expectModuleNotFoundProdError(moduleName, stderr) }) }) From 594d0da30f0a37c256aab0ea0e7f977c4774c218 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 00:20:56 -0700 Subject: [PATCH 19/52] lint fix --- packages/next/src/server/next-server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 92b5be97eda9..9bb300fe2fdb 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -102,7 +102,6 @@ import { nodeFs } from './lib/node-fs-methods' import { genExecArgv, getNodeOptionsWithoutInspect } from './lib/utils' import { getRouteRegex } from '../shared/lib/router/utils/route-regex' import { removePathPrefix } from '../shared/lib/router/utils/remove-path-prefix' -import { decorateServerError } from 'next/dist/compiled/@next/react-dev-overlay/dist/middleware' export * from './base-server' From db2e70b8a30523f7f84cee650aeaf4fc62c3ea7a Mon Sep 17 00:00:00 2001 From: Shu Ding Date: Tue, 28 Mar 2023 15:50:37 +0200 Subject: [PATCH 20/52] fix undici fetch header --- packages/next/src/server/next-server.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 9bb300fe2fdb..1f5bea66bb3a 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1479,12 +1479,24 @@ export default class NextNodeServer extends BaseServer { ...req.headers, 'x-invoke-path': invokePath, } - for (const key of [ - 'content-length', - 'keepalive', - 'content-encoding', - 'transfer-encoding', - ]) { + + const forbiddenHeaders = (global as any).__NEXT_USE_UNDICI + ? [ + 'content-length', + 'keepalive', + 'content-encoding', + 'transfer-encoding', + // https://github.com/nodejs/undici/issues/1470 + 'connection', + ] + : [ + 'content-length', + 'keepalive', + 'content-encoding', + 'transfer-encoding', + ] + + for (const key of forbiddenHeaders) { delete invokeHeaders[key] } From 527eda5dcc3f3fb5cba983a7320e1dc2efe5482d Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 13:03:18 -0700 Subject: [PATCH 21/52] update _next/data handling --- packages/next/src/server/router.ts | 2 +- test/integration/appdir-missing-config/test/index.test.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index bf4bc3c8eca7..4473675c338d 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -354,7 +354,7 @@ export default class Router { const curRoutes = matchedPath ? this.compiledRoutes.filter((r) => { return ( - r.name === 'Catchall render' || r.name === '_next/data normalizing' + r.name === 'Catchall render' || r.name === '_next/data catchall' ) }) : this.compiledRoutes diff --git a/test/integration/appdir-missing-config/test/index.test.ts b/test/integration/appdir-missing-config/test/index.test.ts index 0fde5f3c5e0d..eaee86fc1788 100644 --- a/test/integration/appdir-missing-config/test/index.test.ts +++ b/test/integration/appdir-missing-config/test/index.test.ts @@ -9,6 +9,7 @@ import { nextBuild, waitFor, } from 'next-test-utils' +import stripAnsi from 'strip-ansi' const dir = path.join(__dirname, '..') const nextConfig = path.join(dir, 'next.config.js') @@ -56,10 +57,10 @@ describe('Error when app dir is present without experimental.appDir', () => { let output = '' app = await launchApp(dir, appPort, { onStdout(data: string) { - output += data + output += stripAnsi(data) }, onStderr(data: string) { - output += data + output += stripAnsi(data) }, }) await waitFor(200) From a83d08e07d85ab84322e18f4024da2aae58bb6b1 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 14:02:13 -0700 Subject: [PATCH 22/52] Update more test cases --- packages/next/src/cli/next-dev.ts | 373 +++++++++--------- packages/next/src/server/lib/start-server.ts | 2 +- packages/next/src/server/render.tsx | 2 + packages/next/src/server/router.ts | 7 +- .../app-render-error-log.test.ts | 4 +- .../next-font/deprecated-package.test.ts | 5 +- test/e2e/app-dir/app/index.test.ts | 4 +- .../app/pages/index.js | 9 +- .../app/pages/index.js | 9 +- .../app/pages/index.js | 9 +- .../build-indicator/test/index.test.js | 10 +- 11 files changed, 236 insertions(+), 198 deletions(-) diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index 38a673590696..5d1aba34d92f 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -285,223 +285,232 @@ const nextDev: CliCommand = async (argv) => { } return server } else { - let shouldFilter = false - let devServerTeardown: (() => Promise) | undefined - const config = await loadConfig( - PHASE_DEVELOPMENT_SERVER, - dir, - undefined, - undefined, - true - ) - watchConfigFiles(devServerOptions.dir) - - const setupFork = async (newDir?: string) => { - // if we're using workers we can auto restart on config changes - if (!devServerOptions.useWorkers && devServerTeardown) { - Log.info( - `Detected change, manual restart required due to '__NEXT_DISABLE_MEMORY_WATCHER' usage` - ) - return - } + try { + let shouldFilter = false + let devServerTeardown: (() => Promise) | undefined + const config = await loadConfig( + PHASE_DEVELOPMENT_SERVER, + dir, + undefined, + undefined, + true + ) + watchConfigFiles(devServerOptions.dir) - const startDir = dir - if (newDir) { - dir = newDir - } + const setupFork = async (newDir?: string) => { + // if we're using workers we can auto restart on config changes + if (!devServerOptions.useWorkers && devServerTeardown) { + Log.info( + `Detected change, manual restart required due to '__NEXT_DISABLE_MEMORY_WATCHER' usage` + ) + return + } - if (devServerTeardown) { - await devServerTeardown() - devServerTeardown = undefined - } + const startDir = dir + if (newDir) { + dir = newDir + } - if (newDir) { - dir = newDir - process.env = Object.keys(process.env).reduce((newEnv, key) => { - newEnv[key] = process.env[key]?.replace(startDir, newDir) - return newEnv - }, {} as typeof process.env) + if (devServerTeardown) { + await devServerTeardown() + devServerTeardown = undefined + } - process.chdir(newDir) + if (newDir) { + dir = newDir + process.env = Object.keys(process.env).reduce((newEnv, key) => { + newEnv[key] = process.env[key]?.replace(startDir, newDir) + return newEnv + }, {} as typeof process.env) - devServerOptions.dir = newDir - devServerOptions.prevDir = startDir - } + process.chdir(newDir) - // since errors can start being logged from the fork - // before we detect the project directory rename - // attempt suppressing them long enough to check - const filterForkErrors = (chunk: Buffer, fd: 'stdout' | 'stderr') => { - const cleanChunk = stripAnsi(chunk + '') - if ( - cleanChunk.match( - /(ENOENT|Module build failed|Module not found|Cannot find module|Can't resolve)/ - ) - ) { - if (startDir === dir) { - try { - // check if start directory is still valid - const result = findPagesDir( - startDir, - !!config.experimental?.appDir - ) - shouldFilter = !Boolean(result.pagesDir || result.appDir) - } catch (_) { + devServerOptions.dir = newDir + devServerOptions.prevDir = startDir + } + + // since errors can start being logged from the fork + // before we detect the project directory rename + // attempt suppressing them long enough to check + const filterForkErrors = (chunk: Buffer, fd: 'stdout' | 'stderr') => { + const cleanChunk = stripAnsi(chunk + '') + if ( + cleanChunk.match( + /(ENOENT|Module build failed|Module not found|Cannot find module|Can't resolve)/ + ) + ) { + if (startDir === dir) { + try { + // check if start directory is still valid + const result = findPagesDir( + startDir, + !!config.experimental?.appDir + ) + shouldFilter = !Boolean(result.pagesDir || result.appDir) + } catch (_) { + shouldFilter = true + } + } + if (shouldFilter || startDir !== dir) { shouldFilter = true + return } } - if (shouldFilter || startDir !== dir) { - shouldFilter = true - return - } + process[fd].write(chunk) } - process[fd].write(chunk) - } - devServerOptions.onStdout = (chunk) => { - filterForkErrors(chunk, 'stdout') - } - devServerOptions.onStderr = (chunk) => { - filterForkErrors(chunk, 'stderr') + devServerOptions.onStdout = (chunk) => { + filterForkErrors(chunk, 'stdout') + } + devServerOptions.onStderr = (chunk) => { + filterForkErrors(chunk, 'stderr') + } + shouldFilter = false + devServerTeardown = await startServer(devServerOptions) } - shouldFilter = false - devServerTeardown = await startServer(devServerOptions) - } - await setupFork() - await preflight() + await setupFork() + await preflight() - const parentDir = path.join('/', dir, '..') - const watchedEntryLength = parentDir.split('/').length + 1 - const previousItems = new Set() - const instrumentationFilePaths = !!config.experimental?.instrumentationHook - ? getPossibleInstrumentationHookFilenames(dir, config.pageExtensions!) - : [] + const parentDir = path.join('/', dir, '..') + const watchedEntryLength = parentDir.split('/').length + 1 + const previousItems = new Set() + const instrumentationFilePaths = !!config.experimental + ?.instrumentationHook + ? getPossibleInstrumentationHookFilenames(dir, config.pageExtensions!) + : [] - const instrumentationFileWatcher = new Watchpack({}) + const instrumentationFileWatcher = new Watchpack({}) - instrumentationFileWatcher.watch({ - files: instrumentationFilePaths, - startTime: 0, - }) + instrumentationFileWatcher.watch({ + files: instrumentationFilePaths, + startTime: 0, + }) - let instrumentationFileLastHash: string | undefined = undefined - const previousInstrumentationFiles = new Set() - instrumentationFileWatcher.on('aggregated', async () => { - const knownFiles = instrumentationFileWatcher.getTimeInfoEntries() - const instrumentationFile = [...knownFiles.entries()].find( - ([key, value]) => instrumentationFilePaths.includes(key) && value - )?.[0] - - if (instrumentationFile) { - const fs = require('fs') as typeof import('fs') - const instrumentationFileHash = ( - require('crypto') as typeof import('crypto') - ) - .createHash('sha256') - .update(await fs.promises.readFile(instrumentationFile, 'utf8')) - .digest('hex') - - if ( - instrumentationFileLastHash && - instrumentationFileHash !== instrumentationFileLastHash - ) { - Log.warn( - `The instrumentation file has changed, restarting the server to apply changes.` + let instrumentationFileLastHash: string | undefined = undefined + const previousInstrumentationFiles = new Set() + instrumentationFileWatcher.on('aggregated', async () => { + const knownFiles = instrumentationFileWatcher.getTimeInfoEntries() + const instrumentationFile = [...knownFiles.entries()].find( + ([key, value]) => instrumentationFilePaths.includes(key) && value + )?.[0] + + if (instrumentationFile) { + const fs = require('fs') as typeof import('fs') + const instrumentationFileHash = ( + require('crypto') as typeof import('crypto') ) - return setupFork() - } else { + .createHash('sha256') + .update(await fs.promises.readFile(instrumentationFile, 'utf8')) + .digest('hex') + if ( - !instrumentationFileLastHash && - previousInstrumentationFiles.size !== 0 + instrumentationFileLastHash && + instrumentationFileHash !== instrumentationFileLastHash ) { Log.warn( - 'The instrumentation file was added, restarting the server to apply changes.' + `The instrumentation file has changed, restarting the server to apply changes.` ) return setupFork() + } else { + if ( + !instrumentationFileLastHash && + previousInstrumentationFiles.size !== 0 + ) { + Log.warn( + 'The instrumentation file was added, restarting the server to apply changes.' + ) + return setupFork() + } + instrumentationFileLastHash = instrumentationFileHash } - instrumentationFileLastHash = instrumentationFileHash + } else if ( + [...previousInstrumentationFiles.keys()].find((key) => + instrumentationFilePaths.includes(key) + ) + ) { + Log.warn( + `The instrumentation file has been removed, restarting the server to apply changes.` + ) + instrumentationFileLastHash = undefined + return setupFork() } - } else if ( - [...previousInstrumentationFiles.keys()].find((key) => - instrumentationFilePaths.includes(key) - ) - ) { - Log.warn( - `The instrumentation file has been removed, restarting the server to apply changes.` - ) - instrumentationFileLastHash = undefined - return setupFork() - } - previousInstrumentationFiles.clear() - knownFiles.forEach((_, key) => previousInstrumentationFiles.add(key)) - }) + previousInstrumentationFiles.clear() + knownFiles.forEach((_, key) => previousInstrumentationFiles.add(key)) + }) - const projectFolderWatcher = new Watchpack({ - ignored: (entry: string) => { - return !(entry.split('/').length <= watchedEntryLength) - }, - }) + const projectFolderWatcher = new Watchpack({ + ignored: (entry: string) => { + return !(entry.split('/').length <= watchedEntryLength) + }, + }) - projectFolderWatcher.watch({ directories: [parentDir], startTime: 0 }) - - projectFolderWatcher.on('aggregated', async () => { - const knownFiles = projectFolderWatcher.getTimeInfoEntries() - const newFiles: string[] = [] - let hasPagesApp = false - - // if the dir still exists nothing to check - try { - const result = findPagesDir(dir, !!config.experimental?.appDir) - hasPagesApp = Boolean(result.pagesDir || result.appDir) - } catch (err) { - // if findPagesDir throws validation error let this be - // handled in the dev-server itself in the fork - if ((err as any).message?.includes('experimental')) { - return + projectFolderWatcher.watch({ directories: [parentDir], startTime: 0 }) + + projectFolderWatcher.on('aggregated', async () => { + const knownFiles = projectFolderWatcher.getTimeInfoEntries() + const newFiles: string[] = [] + let hasPagesApp = false + + // if the dir still exists nothing to check + try { + const result = findPagesDir(dir, !!config.experimental?.appDir) + hasPagesApp = Boolean(result.pagesDir || result.appDir) + } catch (err) { + // if findPagesDir throws validation error let this be + // handled in the dev-server itself in the fork + if ((err as any).message?.includes('experimental')) { + return + } } - } - // try to find new dir introduced - if (previousItems.size) { - for (const key of knownFiles.keys()) { - if (!previousItems.has(key)) { - newFiles.push(key) + // try to find new dir introduced + if (previousItems.size) { + for (const key of knownFiles.keys()) { + if (!previousItems.has(key)) { + newFiles.push(key) + } } + previousItems.clear() } - previousItems.clear() - } - for (const key of knownFiles.keys()) { - previousItems.add(key) - } + for (const key of knownFiles.keys()) { + previousItems.add(key) + } - if (hasPagesApp) { - return - } + if (hasPagesApp) { + return + } - // if we failed to find the new dir it may have been moved - // to a new parent directory which we can't track as easily - // so exit gracefully - try { - const result = findPagesDir(newFiles[0], !!config.experimental?.appDir) - hasPagesApp = Boolean(result.pagesDir || result.appDir) - } catch (_) {} - - if (hasPagesApp && newFiles.length === 1) { - Log.info( - `Detected project directory rename, restarting in new location` - ) - setupFork(newFiles[0]) - watchConfigFiles(newFiles[0]) - } else { - Log.error( - `Project directory could not be found, restart Next.js in your new directory` - ) - process.exit(0) - } - }) + // if we failed to find the new dir it may have been moved + // to a new parent directory which we can't track as easily + // so exit gracefully + try { + const result = findPagesDir( + newFiles[0], + !!config.experimental?.appDir + ) + hasPagesApp = Boolean(result.pagesDir || result.appDir) + } catch (_) {} + + if (hasPagesApp && newFiles.length === 1) { + Log.info( + `Detected project directory rename, restarting in new location` + ) + setupFork(newFiles[0]) + watchConfigFiles(newFiles[0]) + } else { + Log.error( + `Project directory could not be found, restart Next.js in your new directory` + ) + process.exit(0) + } + }) + } catch (err) { + console.error(err) + process.exit(1) + } } } diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index bf2978696319..f7033eb20e25 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -206,7 +206,7 @@ export async function startServer({ const proxyServer = httpProxy.createProxy({ target: targetUrl, - changeOrigin: true, + changeOrigin: false, ignorePath: true, xfwd: true, ws: true, diff --git a/packages/next/src/server/render.tsx b/packages/next/src/server/render.tsx index 7be3375a3e0b..d11cbecaa78c 100644 --- a/packages/next/src/server/render.tsx +++ b/packages/next/src/server/render.tsx @@ -341,6 +341,7 @@ export const deserializeErr = (serializedErr: any) => { const err = new Error(serializedErr.message) err.stack = serializedErr.stack err.name = serializedErr.name + ;(err as any).digest = serializedErr.digest if (process.env.NEXT_RUNTIME !== 'edge') { const { decorateServerError } = @@ -366,6 +367,7 @@ export function errorToJSON(err: Error) { source, message: stripAnsi(err.message), stack: err.stack, + digest: (err as any).digest, } } diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 4473675c338d..65bcd10eff95 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -361,7 +361,12 @@ export default class Router { if (process.env.__NEXT_PRIVATE_RENDER_WORKER && matchedPath) { const parsedMatchedPath = new URL(matchedPath || '/', 'http://n') - parsedUrlUpdated.pathname = parsedMatchedPath.pathname + + if (parsedUrlUpdated.pathname !== parsedMatchedPath.pathname) { + parsedUrlUpdated.pathname = parsedMatchedPath.pathname + addRequestMeta(req, '_nextRewroteUrl', parsedUrlUpdated.pathname) + addRequestMeta(req, '_nextDidRewrite', true) + } Object.assign( parsedUrlUpdated.query, Object.fromEntries(parsedMatchedPath.searchParams) diff --git a/test/development/app-render-error-log/app-render-error-log.test.ts b/test/development/app-render-error-log/app-render-error-log.test.ts index 1f942f3f1a0a..47e4c761eb59 100644 --- a/test/development/app-render-error-log/app-render-error-log.test.ts +++ b/test/development/app-render-error-log/app-render-error-log.test.ts @@ -14,11 +14,11 @@ createNextDescribe( await check(() => next.cliOutput.slice(outputIndex), /at Page/) const cliOutput = next.cliOutput.slice(outputIndex) + await check(() => cliOutput, /digest:/) expect(cliOutput).toInclude('Error: boom') expect(cliOutput).toInclude('at fn2 (./app/fn.ts:6:11)') expect(cliOutput).toInclude('at fn1 (./app/fn.ts:9:5') expect(cliOutput).toInclude('at Page (./app/page.tsx:10:45)') - expect(cliOutput).toInclude('digest: ') expect(cliOutput).not.toInclude('webpack-internal') }) @@ -30,11 +30,11 @@ createNextDescribe( await check(() => next.cliOutput.slice(outputIndex), /at EdgePage/) const cliOutput = next.cliOutput.slice(outputIndex) + await check(() => cliOutput, /digest:/) expect(cliOutput).toInclude('Error: boom') expect(cliOutput).toInclude('at fn2 (./app/fn.ts:6:11)') expect(cliOutput).toInclude('at fn1 (./app/fn.ts:9:5') expect(cliOutput).toInclude('at EdgePage (./app/edge/page.tsx:12:45)') - expect(cliOutput).toInclude('digest: ') expect(cliOutput).not.toInclude('webpack-internal') }) diff --git a/test/development/next-font/deprecated-package.test.ts b/test/development/next-font/deprecated-package.test.ts index 40073f2e4535..4c86d8a16dd8 100644 --- a/test/development/next-font/deprecated-package.test.ts +++ b/test/development/next-font/deprecated-package.test.ts @@ -19,8 +19,9 @@ createNextDescribe( it('should warn if @next/font is in deps', async () => { await next.start() await check(() => next.cliOutput, /compiled client and server/) - expect(next.cliOutput).toInclude( - 'please use the built-in `next/font` instead' + await check( + () => next.cliOutput, + new RegExp('please use the built-in `next/font` instead') ) await next.stop() diff --git a/test/e2e/app-dir/app/index.test.ts b/test/e2e/app-dir/app/index.test.ts index 5147bf7d3f51..5331723d3b80 100644 --- a/test/e2e/app-dir/app/index.test.ts +++ b/test/e2e/app-dir/app/index.test.ts @@ -172,7 +172,9 @@ createNextDescribe( const res = await next.fetch('/dashboard') expect(res.headers.get('x-edge-runtime')).toBe('1') expect(res.headers.get('vary')).toBe( - 'RSC, Next-Router-State-Tree, Next-Router-Prefetch' + isNextDeploy + ? 'RSC, Next-Router-State-Tree, Next-Router-Prefetch' + : 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Accept-Encoding' ) }) diff --git a/test/e2e/middleware-custom-matchers-basepath/app/pages/index.js b/test/e2e/middleware-custom-matchers-basepath/app/pages/index.js index accd5b17adff..4ebe3adadf7a 100644 --- a/test/e2e/middleware-custom-matchers-basepath/app/pages/index.js +++ b/test/e2e/middleware-custom-matchers-basepath/app/pages/index.js @@ -5,10 +5,15 @@ export default (props) => ( ) -export async function getServerSideProps({ res }) { +export async function getServerSideProps({ req, res }) { return { props: { - fromMiddleware: res.getHeader('x-from-middleware') || null, + // TODO: this should only use request header once + fromMiddleware: + // start is using the separate renders as well + req.headers['x-from-middleware'] || + res.getHeader('x-from-middleware') || + null, }, } } diff --git a/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js b/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js index d981f724020a..51d702b4c220 100644 --- a/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js +++ b/test/e2e/middleware-custom-matchers-i18n/app/pages/index.js @@ -5,10 +5,15 @@ export default (props) => ( ) -export async function getServerSideProps({ req }) { +export async function getServerSideProps({ req, res }) { return { props: { - fromMiddleware: req.headers['x-from-middleware'] || null, + fromMiddleware: + // TODO: this should only use request header once + // start is using the separate renders as well + req.headers['x-from-middleware'] || + res.getHeader('x-from-middleware') || + null, }, } } diff --git a/test/e2e/middleware-custom-matchers/app/pages/index.js b/test/e2e/middleware-custom-matchers/app/pages/index.js index accd5b17adff..51d702b4c220 100644 --- a/test/e2e/middleware-custom-matchers/app/pages/index.js +++ b/test/e2e/middleware-custom-matchers/app/pages/index.js @@ -5,10 +5,15 @@ export default (props) => ( ) -export async function getServerSideProps({ res }) { +export async function getServerSideProps({ req, res }) { return { props: { - fromMiddleware: res.getHeader('x-from-middleware') || null, + fromMiddleware: + // TODO: this should only use request header once + // start is using the separate renders as well + req.headers['x-from-middleware'] || + res.getHeader('x-from-middleware') || + null, }, } } diff --git a/test/integration/build-indicator/test/index.test.js b/test/integration/build-indicator/test/index.test.js index 713451d2faa9..f59be4a296e5 100644 --- a/test/integration/build-indicator/test/index.test.js +++ b/test/integration/build-indicator/test/index.test.js @@ -3,7 +3,8 @@ import fs from 'fs-extra' import { join } from 'path' import webdriver from 'next-webdriver' -import { findPort, launchApp, killApp, waitFor } from 'next-test-utils' +import { findPort, launchApp, killApp, waitFor, check } from 'next-test-utils' +import stripAnsi from 'strip-ansi' const appDir = join(__dirname, '..') const nextConfig = join(appDir, 'next.config.js') @@ -46,8 +47,11 @@ describe('Build Activity Indicator', () => { }) await fs.remove(configPath) - expect(stderr).toContain( - `Invalid "devIndicator.buildActivityPosition" provided, expected one of top-left, top-right, bottom-left, bottom-right, received ttop-leff` + await check( + () => stripAnsi(stderr), + new RegExp( + `Invalid "devIndicator.buildActivityPosition" provided, expected one of top-left, top-right, bottom-left, bottom-right, received ttop-leff` + ) ) if (app) { From f90e9a776a3ac647c2dc1dfee850a080ac6eab8b Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 14:58:36 -0700 Subject: [PATCH 23/52] remove extra changes --- packages/next/src/build/webpack-config.ts | 7 +------ packages/next/src/server/dev/hot-reloader.ts | 14 +++++--------- packages/next/src/server/dev/next-dev-server.ts | 12 ++++++++---- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index b07bee1e5518..88a64d276fe1 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -1607,12 +1607,7 @@ export default async function getBaseWebpackConfig( } })(), runtimeChunk: isClient - ? // namespace runtime chunk when multiple workers are present - process.env.__NEXT_PRIVATE_RENDER_WORKER - ? { - name: `${CLIENT_STATIC_FILES_RUNTIME_WEBPACK}-${process.env.__NEXT_PRIVATE_RENDER_WORKER}`, - } - : { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK } + ? { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK } : undefined, minimize: !dev && (isClient || isEdgeServer), minimizer: [ diff --git a/packages/next/src/server/dev/hot-reloader.ts b/packages/next/src/server/dev/hot-reloader.ts index 4cbed3ed7967..950530cf9af4 100644 --- a/packages/next/src/server/dev/hot-reloader.ts +++ b/packages/next/src/server/dev/hot-reloader.ts @@ -158,10 +158,6 @@ function erroredPages(compilation: webpack.Compilation) { return failedPages } -export function cleanDistDir(distDir: string) { - return recursiveDelete(distDir, /^cache/) -} - export default class HotReloader { private dir: string private buildId: string @@ -442,7 +438,9 @@ export default class HotReloader { private async clean(span: Span): Promise { return span .traceChild('clean') - .traceAsyncFn(() => cleanDistDir(join(this.dir, this.config.distDir))) + .traceAsyncFn(() => + recursiveDelete(join(this.dir, this.config.distDir), /^cache/) + ) } private async getVersionInfo(span: Span, enabled: boolean) { @@ -637,7 +635,7 @@ export default class HotReloader { }) } - public async start(skipClean?: boolean): Promise { + public async start(): Promise { const startSpan = this.hotReloaderSpan.traceChild('start') startSpan.stop() // Stop immediately to create an artificial parent span @@ -646,9 +644,7 @@ export default class HotReloader { !!process.env.NEXT_TEST_MODE || this.telemetry.isEnabled ) - if (!skipClean) { - await this.clean(startSpan) - } + await this.clean(startSpan) // Ensure distDir exists before writing package.json await fs.mkdir(this.distDir, { recursive: true }) diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index c29a1bde9435..c241da09e986 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -12,6 +12,7 @@ import type { BaseNextRequest, BaseNextResponse } from '../base-http' import type { MiddlewareRoutingItem, RoutingItem } from '../base-server' import type { MiddlewareMatcher } from '../../build/analysis/get-page-static-info' import type { FunctionComponent } from 'react' +import type { RouteMatch } from '../future/route-matches/route-match' import crypto from 'crypto' import fs from 'fs' @@ -906,7 +907,7 @@ export default class DevServer extends Server { } await super.prepare() await this.addExportPathMapRoutes() - await this.hotReloader?.start(this.isRouterWorker || this.isRenderWorker) + await this.hotReloader?.start() await this.startWatcher() await this.runInstrumentationHookIfAvailable() await this.matchers.reload() @@ -1682,9 +1683,12 @@ export default class DevServer extends Server { global.fetch = this.originalFetch! } - protected async ensurePage( - opts: Parameters['ensurePage']>[0] - ) { + protected async ensurePage(opts: { + page: string + clientOnly: boolean + appPaths?: string[] | null + match?: RouteMatch + }) { const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT if (ipcPort) { await fetch( From ddca45feac006f0836bb0aab8ca576142107199e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 15:51:49 -0700 Subject: [PATCH 24/52] more test cases --- packages/next/src/server/web/utils.ts | 9 +++++++-- test/e2e/app-dir/app/index.test.ts | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/next/src/server/web/utils.ts b/packages/next/src/server/web/utils.ts index 9249d9704f16..ad847f345b58 100644 --- a/packages/next/src/server/web/utils.ts +++ b/packages/next/src/server/web/utils.ts @@ -93,9 +93,14 @@ export function toNodeHeaders(headers?: Headers): NodeHeaders { const result: NodeHeaders = {} if (headers) { for (const [key, value] of headers.entries()) { - result[key] = value if (key.toLowerCase() === 'set-cookie') { - result[key] = splitCookiesString(value) + const curValue = result[key] + result[key] = [ + ...(Array.isArray(curValue) ? curValue : curValue ? [curValue] : []), + ...splitCookiesString(value), + ] + } else { + result[key] = value } } } diff --git a/test/e2e/app-dir/app/index.test.ts b/test/e2e/app-dir/app/index.test.ts index 5331723d3b80..e3288bf5785c 100644 --- a/test/e2e/app-dir/app/index.test.ts +++ b/test/e2e/app-dir/app/index.test.ts @@ -172,7 +172,7 @@ createNextDescribe( const res = await next.fetch('/dashboard') expect(res.headers.get('x-edge-runtime')).toBe('1') expect(res.headers.get('vary')).toBe( - isNextDeploy + isNextDeploy || isNextStart ? 'RSC, Next-Router-State-Tree, Next-Router-Prefetch' : 'RSC, Next-Router-State-Tree, Next-Router-Prefetch, Accept-Encoding' ) From bc3b8309cdeb8de37d26ff5241c5c1c54fbbda78 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 16:13:24 -0700 Subject: [PATCH 25/52] more tests --- packages/next/src/server/next-server.ts | 6 ++++++ test/e2e/basepath/pages/index.js | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 1f5bea66bb3a..ae83e347aedb 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -102,6 +102,7 @@ import { nodeFs } from './lib/node-fs-methods' import { genExecArgv, getNodeOptionsWithoutInspect } from './lib/utils' import { getRouteRegex } from '../shared/lib/router/utils/route-regex' import { removePathPrefix } from '../shared/lib/router/utils/remove-path-prefix' +import { addPathPrefix } from '../shared/lib/router/utils/add-path-prefix' export * from './base-server' @@ -1456,6 +1457,11 @@ export default class NextNodeServer extends BaseServer { if (query.__nextDataReq) { invokePathname = `/_next/data/${this.buildId}${invokePathname}.json` } + invokePathname = addPathPrefix( + invokePathname, + this.nextConfig.basePath + ) + const keptQuery = new URLSearchParams() for (const key of Object.keys(query)) { diff --git a/test/e2e/basepath/pages/index.js b/test/e2e/basepath/pages/index.js index 52c4affcf98c..2376f6033f39 100644 --- a/test/e2e/basepath/pages/index.js +++ b/test/e2e/basepath/pages/index.js @@ -1,5 +1,7 @@ import { useRouter } from 'next/router' import Link from 'next/link' +import { useState } from 'react' +import { useEffect } from 'react' export const getStaticProps = () => { return { @@ -12,6 +14,11 @@ export const getStaticProps = () => { export default function Index({ hello, nested }) { const { query, pathname, asPath } = useRouter() + const [mounted, setMounted] = useState(false) + useEffect(() => { + setMounted(true) + return () => setMounted(false) + }, []) return ( <>

index page

@@ -19,7 +26,7 @@ export default function Index({ hello, nested }) {

{hello} world

{JSON.stringify(query)}

{pathname}

-

{asPath}

+

{mounted ? asPath : ''}

to /hello From a19f34c172300bd25f6755bb1b919ac1ce7ca8e5 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 16:28:47 -0700 Subject: [PATCH 26/52] fix intialize error handling --- packages/next/src/cli/next-dev.ts | 29 ++++++++++++-------- packages/next/src/server/lib/start-server.ts | 20 ++++++++++++++ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/packages/next/src/cli/next-dev.ts b/packages/next/src/cli/next-dev.ts index 5d1aba34d92f..0c7a9deae7cb 100644 --- a/packages/next/src/cli/next-dev.ts +++ b/packages/next/src/cli/next-dev.ts @@ -8,7 +8,7 @@ import isError from '../lib/is-error' import { getProjectDir } from '../lib/get-project-dir' import { CONFIG_FILES, PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants' import path from 'path' -import type { NextConfigComplete } from '../server/config-shared' +import type { NextConfig, NextConfigComplete } from '../server/config-shared' import { traceGlobals } from '../trace/shared' import { Telemetry } from '../telemetry/storage' import loadConfig from '../server/config' @@ -288,13 +288,8 @@ const nextDev: CliCommand = async (argv) => { try { let shouldFilter = false let devServerTeardown: (() => Promise) | undefined - const config = await loadConfig( - PHASE_DEVELOPMENT_SERVER, - dir, - undefined, - undefined, - true - ) + let config: NextConfig | undefined + watchConfigFiles(devServerOptions.dir) const setupFork = async (newDir?: string) => { @@ -344,7 +339,7 @@ const nextDev: CliCommand = async (argv) => { // check if start directory is still valid const result = findPagesDir( startDir, - !!config.experimental?.appDir + !!config?.experimental?.appDir ) shouldFilter = !Boolean(result.pagesDir || result.appDir) } catch (_) { @@ -367,6 +362,16 @@ const nextDev: CliCommand = async (argv) => { } shouldFilter = false devServerTeardown = await startServer(devServerOptions) + + if (!config) { + config = await loadConfig( + PHASE_DEVELOPMENT_SERVER, + dir, + undefined, + undefined, + true + ) + } } await setupFork() @@ -375,7 +380,7 @@ const nextDev: CliCommand = async (argv) => { const parentDir = path.join('/', dir, '..') const watchedEntryLength = parentDir.split('/').length + 1 const previousItems = new Set() - const instrumentationFilePaths = !!config.experimental + const instrumentationFilePaths = !!config?.experimental ?.instrumentationHook ? getPossibleInstrumentationHookFilenames(dir, config.pageExtensions!) : [] @@ -455,7 +460,7 @@ const nextDev: CliCommand = async (argv) => { // if the dir still exists nothing to check try { - const result = findPagesDir(dir, !!config.experimental?.appDir) + const result = findPagesDir(dir, !!config?.experimental?.appDir) hasPagesApp = Boolean(result.pagesDir || result.appDir) } catch (err) { // if findPagesDir throws validation error let this be @@ -489,7 +494,7 @@ const nextDev: CliCommand = async (argv) => { try { const result = findPagesDir( newFiles[0], - !!config.experimental?.appDir + !!config?.experimental?.appDir ) hasPagesApp = Boolean(result.pagesDir || result.appDir) } catch (_) {} diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index f7033eb20e25..16999098ee0f 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -3,6 +3,7 @@ import { isIPv6 } from 'net' import * as Log from '../../build/output/log' import { getNodeOptionsWithoutInspect } from './utils' import type { IncomingMessage, ServerResponse } from 'http' +import type { ChildProcess } from 'child_process' export interface StartServerOptions { dir: string @@ -158,6 +159,8 @@ export async function startServer({ const routerWorker = new Worker(renderServerPath, { numWorkers: 1, + // TODO: do we want to allow more than 10 OOM restarts? + maxRetries: 10, forkOptions: { cwd: dir, env: { @@ -174,6 +177,22 @@ export async function startServer({ }) as any as InstanceType & { initialize: typeof import('./render-server').initialize } + let didInitialize = false + + for (const worker of ((routerWorker as any)._workerPool?._workers || + []) as { + _child: ChildProcess + }[]) { + // eslint-disable-next-line no-loop-func + worker._child.on('exit', (code, signal) => { + // catch failed initializing without retry + if ((code || signal) && !didInitialize) { + routerWorker?.end() + process.exit(1) + } + }) + } + const workerStdout = routerWorker.getStdout() const workerStderr = routerWorker.getStderr() @@ -200,6 +219,7 @@ export async function startServer({ workerType: 'router', keepAliveTimeout, }) + didInitialize = true const getProxyServer = (pathname: string) => { const targetUrl = `http://${normalizedHost}:${routerPort}${pathname}` From 2ba49849be4443176d7c0eb9196c09fabcdce627 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Tue, 28 Mar 2023 19:45:24 -0700 Subject: [PATCH 27/52] more test cases --- packages/next/src/build/index.ts | 1 + .../next/src/server/dev/next-dev-server.ts | 2 +- packages/next/src/server/lib/start-server.ts | 4 ++-- packages/next/src/server/next-server.ts | 9 ++++++++ packages/next/src/server/router.ts | 17 ++++++++++---- test/e2e/opentelemetry/opentelemetry.test.ts | 22 ++++++++++++++++--- 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 1601053c8ea5..fea08d63902f 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1974,6 +1974,7 @@ export default async function build( '**/*.d.ts', '**/*.map', '**/next/dist/pages/**/*', + '**/next/dist/compiled/jest-worker/**/*', '**/next/dist/compiled/webpack/(bundle4|bundle5).js', '**/node_modules/webpack5/**/*', '**/next/dist/server/lib/squoosh/**/*.wasm', diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index c241da09e986..3cb83a1def30 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -1083,7 +1083,7 @@ export default class DevServer extends Server { `${basePath || assetPrefix || ''}/_next/webpack-hmr` ) ) { - if (this.isRouterWorker) { + if (!this.isRenderWorker) { this.hotReloader?.onHMR(req, socket, head) } } else { diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 16999098ee0f..ea2da42c3cff 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -179,12 +179,12 @@ export async function startServer({ } let didInitialize = false - for (const worker of ((routerWorker as any)._workerPool?._workers || + for (const _worker of ((routerWorker as any)._workerPool?._workers || []) as { _child: ChildProcess }[]) { // eslint-disable-next-line no-loop-func - worker._child.on('exit', (code, signal) => { + _worker._child.on('exit', (code, signal) => { // catch failed initializing without retry if ((code || signal) && !didInitialize) { routerWorker?.end() diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index ae83e347aedb..e9761e8b4fd1 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1454,6 +1454,15 @@ export default class NextNodeServer extends BaseServer { invokePathname = normalizedInvokePathname } + if ( + query.__nextLocale && + !invokePathname.startsWith(`/${query.__nextLocale}`) + ) { + invokePathname = `/${query.__nextLocale}${ + invokePathname === '/' ? '' : invokePathname + }` + } + if (query.__nextDataReq) { invokePathname = `/_next/data/${this.buildId}${invokePathname}.json` } diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 65bcd10eff95..025741ab015d 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -367,10 +367,19 @@ export default class Router { addRequestMeta(req, '_nextRewroteUrl', parsedUrlUpdated.pathname) addRequestMeta(req, '_nextDidRewrite', true) } - Object.assign( - parsedUrlUpdated.query, - Object.fromEntries(parsedMatchedPath.searchParams) - ) + + for (const key of [...new Set(parsedMatchedPath.searchParams.keys())]) { + const value = parsedMatchedPath.searchParams.getAll(key) + const curValue = parsedUrlUpdated.query[key] + parsedUrlUpdated.query[key] = [ + ...(Array.isArray(curValue) ? curValue : curValue ? curValue : []), + ...value, + ] + + if (parsedUrlUpdated.query[key]?.length === 1) { + parsedUrlUpdated.query[key] = parsedUrlUpdated.query[key]?.[0] + } + } } for (const route of curRoutes) { diff --git a/test/e2e/opentelemetry/opentelemetry.test.ts b/test/e2e/opentelemetry/opentelemetry.test.ts index be1fb9aadedd..8e221be41990 100644 --- a/test/e2e/opentelemetry/opentelemetry.test.ts +++ b/test/e2e/opentelemetry/opentelemetry.test.ts @@ -23,7 +23,9 @@ createNextDescribe( await check(async () => { const spans = await getTraces() const rootSpans = spans.filter((span) => !span.parentId) - return String(rootSpans.length) + return rootSpans.length >= numberOfRootTraces + ? String(numberOfRootTraces) + : rootSpans.length }, String(numberOfRootTraces)) } @@ -40,14 +42,28 @@ createNextDescribe( span.parentId = span.parentId === undefined ? undefined : '[parent-id]' return span } - const sanitizeSpans = (spans: SavedSpan[]) => - spans + const sanitizeSpans = (spans: SavedSpan[]) => { + const seenSpans = new Set() + return spans .sort((a, b) => (a.attributes?.['next.span_type'] ?? '').localeCompare( b.attributes?.['next.span_type'] ?? '' ) ) .map(sanitizeSpan) + .filter((span) => { + const target = span.attributes?.['http.target'] + const result = + !span.attributes?.['http.url']?.startsWith('http://localhost') && + !seenSpans.has(target) + + if (target) { + seenSpans.add(target) + } + + return result + }) + } const getSanitizedTraces = async (numberOfRootTraces: number) => { await waitForRootSpan(numberOfRootTraces) From 08f34a213ae17f8c39e481a8ab2a9426b13f95e7 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 10:26:49 -0700 Subject: [PATCH 28/52] Update query handling --- packages/next/src/server/router.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 025741ab015d..b0d41ca4e123 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -370,11 +370,7 @@ export default class Router { for (const key of [...new Set(parsedMatchedPath.searchParams.keys())]) { const value = parsedMatchedPath.searchParams.getAll(key) - const curValue = parsedUrlUpdated.query[key] - parsedUrlUpdated.query[key] = [ - ...(Array.isArray(curValue) ? curValue : curValue ? curValue : []), - ...value, - ] + parsedUrlUpdated.query[key] = [...value] if (parsedUrlUpdated.query[key]?.length === 1) { parsedUrlUpdated.query[key] = parsedUrlUpdated.query[key]?.[0] From 4eeb0185fcc34d8e075a1c042da755ee375c2f77 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 13:29:59 -0700 Subject: [PATCH 29/52] fix more test cases --- packages/next/src/server/lib/start-server.ts | 26 ++++++++++++++----- packages/next/src/server/next-server.ts | 3 +++ test/integration/cli/test/index.test.js | 9 ++++--- .../file-serving/test/index.test.js | 7 ++++- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index ea2da42c3cff..939de9b76565 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -123,19 +123,31 @@ export async function startServer({ process.exit(1) } }) - const host = hostname || '0.0.0.0' - let normalizedHost = isIPv6(host) ? `[${host}]` : host + + let targetHost = hostname await new Promise((resolve) => { server.on('listening', () => { const addr = server.address() port = typeof addr === 'object' ? addr?.port || port : port - normalizedHost = - !hostname || hostname === '0.0.0.0' ? 'localhost' : hostname - const appUrl = `http://${normalizedHost}:${port}` + let host = !hostname || hostname === '0.0.0.0' ? 'localhost' : hostname + + let normalizedHostname = hostname || '0.0.0.0' + + if (isIPv6(hostname)) { + host = host === '::' ? '[::1]' : `[${host}]` + normalizedHostname = `[${hostname}]` + } + targetHost = host + + const appUrl = `http://${host}:${port}` - Log.ready(`started server on ${host}:${port}, url: ${appUrl}`) + Log.ready( + `started server on ${normalizedHostname}${ + (port + '').startsWith(':') ? '' : ':' + }${port}, url: ${appUrl}` + ) resolve() }) server.listen(port, hostname) @@ -222,7 +234,7 @@ export async function startServer({ didInitialize = true const getProxyServer = (pathname: string) => { - const targetUrl = `http://${normalizedHost}:${routerPort}${pathname}` + const targetUrl = `http://${targetHost}:${routerPort}${pathname}` const proxyServer = httpProxy.createProxy({ target: targetUrl, diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index e9761e8b4fd1..47aecf1361d3 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1812,6 +1812,9 @@ export default class NextNodeServer extends BaseServer { ) { res.statusCode = err.statusCode return this.renderError(err, req, res, path) + } else if ((err as any).expose === false) { + res.statusCode = 400 + return this.renderError(null, req, res, path) } else { throw err } diff --git a/test/integration/cli/test/index.test.js b/test/integration/cli/test/index.test.js index 24bed49ef9b1..b04828b1bfa5 100644 --- a/test/integration/cli/test/index.test.js +++ b/test/integration/cli/test/index.test.js @@ -1,6 +1,7 @@ /* eslint-env jest */ import { + check, findPort, killApp, launchApp, @@ -393,9 +394,11 @@ describe('CLI Usage', () => { }, }) - expect(stderr).toMatch('both `sass` and `node-sass` installed') - - await killApp(instance) + try { + await check(() => stderr, /both `sass` and `node-sass` installed/) + } finally { + await killApp(instance) + } }) test('should exit when SIGINT is signalled', async () => { diff --git a/test/integration/file-serving/test/index.test.js b/test/integration/file-serving/test/index.test.js index 7b09bb29cc63..91bc1402a7b7 100644 --- a/test/integration/file-serving/test/index.test.js +++ b/test/integration/file-serving/test/index.test.js @@ -26,7 +26,12 @@ const expectStatus = async (path) => { const parsedUrl = url.parse(redirectDest, true) expect(parsedUrl.hostname).toBe('localhost') } else { - expect(res.status === 400 || res.status === 404).toBe(true) + try { + expect(res.status === 400 || res.status === 404).toBe(true) + } catch (err) { + require('console').error({ path, status: res.status }) + throw err + } expect(await res.text()).toMatch(containRegex) } } From 41f7f8565837831753fe1fe7e864adeb93f5c668 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 13:44:42 -0700 Subject: [PATCH 30/52] repeated slashes test --- packages/next/src/server/base-server.ts | 1 + packages/next/src/server/lib/start-server.ts | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index efc5cedbe6d7..04b50b1fda89 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -617,6 +617,7 @@ export default abstract class Server { // hello/world or backslashes to forward slashes, this does not // handle trailing slash as that is handled the same as a next.config.js // redirect + console.log({ urlNoQuery }) if (urlNoQuery?.match(/(\\|\/\/)/)) { const cleanUrl = normalizeRepeatedSlashes(req.url!) res.redirect(cleanUrl, 308).body(cleanUrl).send() diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 939de9b76565..ef790b53941e 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -4,6 +4,7 @@ import * as Log from '../../build/output/log' import { getNodeOptionsWithoutInspect } from './utils' import type { IncomingMessage, ServerResponse } from 'http' import type { ChildProcess } from 'child_process' +import { normalizeRepeatedSlashes } from '../../shared/lib/utils' export interface StartServerOptions { dir: string @@ -242,6 +243,7 @@ export async function startServer({ ignorePath: true, xfwd: true, ws: true, + followRedirects: false, }) proxyServer.on('error', () => { @@ -252,6 +254,20 @@ export async function startServer({ // proxy to router worker requestHandler = async (req, res) => { + const urlParts = (req.url || '').split('?') + const urlNoQuery = urlParts[0] + + // this normalizes repeated slashes in the path e.g. hello//world -> + // hello/world or backslashes to forward slashes, this does not + // handle trailing slash as that is handled the same as a next.config.js + // redirect + if (urlNoQuery?.match(/(\\|\/\/)/)) { + const cleanUrl = normalizeRepeatedSlashes(req.url!) + res.statusCode = 308 + res.setHeader('Location', cleanUrl) + res.end(cleanUrl) + return + } const proxyServer = getProxyServer(req.url || '/') proxyServer.web(req, res) } From bf442d50a2dbac772bc7d80516f5fa79d8a295e5 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 14:52:37 -0700 Subject: [PATCH 31/52] etag test --- packages/next/src/server/base-server.ts | 1 - packages/next/src/server/next-server.ts | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 04b50b1fda89..efc5cedbe6d7 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -617,7 +617,6 @@ export default abstract class Server { // hello/world or backslashes to forward slashes, this does not // handle trailing slash as that is handled the same as a next.config.js // redirect - console.log({ urlNoQuery }) if (urlNoQuery?.match(/(\\|\/\/)/)) { const cleanUrl = normalizeRepeatedSlashes(req.url!) res.redirect(cleanUrl, 308).body(cleanUrl).send() diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 47aecf1361d3..f31573670988 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1491,6 +1491,7 @@ export default class NextNodeServer extends BaseServer { }` const invokeHeaders: typeof req.headers = { + 'cache-control': '', ...req.headers, 'x-invoke-path': invokePath, } From 79f68495ed7e6260cfe0152f619f4ab5723b43a5 Mon Sep 17 00:00:00 2001 From: Shu Ding Date: Thu, 30 Mar 2023 00:07:32 +0200 Subject: [PATCH 32/52] fix missing process.env.NODE_ENV --- packages/next/src/server/next-server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 159a1b2b46e2..9704a8358819 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -340,6 +340,7 @@ export default class NextNodeServer extends BaseServer { .trim(), __NEXT_PRIVATE_RENDER_WORKER: type, __NEXT_PRIVATE_ROUTER_IPC_PORT: ipcPort + '', + NODE_ENV: process.env.NODE_ENV, }, execArgv: genExecArgv( options.isNodeDebugging === undefined From aca86e9901bc65a9abdb9f5e9a3852b250033bc6 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 16:18:36 -0700 Subject: [PATCH 33/52] fix streaming error case --- .../edge-runtime-streaming-error/test/index.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/integration/edge-runtime-streaming-error/test/index.test.ts b/test/integration/edge-runtime-streaming-error/test/index.test.ts index 1f25873dae63..e9e68e9c4922 100644 --- a/test/integration/edge-runtime-streaming-error/test/index.test.ts +++ b/test/integration/edge-runtime-streaming-error/test/index.test.ts @@ -1,5 +1,6 @@ import stripAnsi from 'next/dist/compiled/strip-ansi' import { + check, fetchViaHTTP, findPort, killApp, @@ -19,11 +20,11 @@ function test(context: ReturnType) { expect(await res.text()).toEqual('hello') expect(res.status).toBe(200) await waitFor(200) - const santizedOutput = stripAnsi(context.output) - expect(santizedOutput).toMatch( - new RegExp(`TypeError: This ReadableStream did not return bytes.`, 'm') + await check( + () => stripAnsi(context.output), + new RegExp(`This ReadableStream did not return bytes.`, 'm') ) - expect(santizedOutput).not.toContain('webpack-internal:') + expect(stripAnsi(context.output)).not.toContain('webpack-internal:') } } From 25ddfe1ab96c4615c0707169a4eab7a860cc4648 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 21:29:44 -0700 Subject: [PATCH 34/52] update middleware dev test --- .../middleware-dev-update/pages/index.js | 9 +++++++-- .../middleware-dev-update/test/index.test.js | 14 +++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/test/integration/middleware-dev-update/pages/index.js b/test/integration/middleware-dev-update/pages/index.js index c37edc772d0e..413584ec3255 100644 --- a/test/integration/middleware-dev-update/pages/index.js +++ b/test/integration/middleware-dev-update/pages/index.js @@ -1,9 +1,14 @@ export default (props) =>
{props.fromMiddleware}
-export async function getServerSideProps({ res }) { +export async function getServerSideProps({ req, res }) { return { props: { - fromMiddleware: res.getHeader('x-from-middleware') || '', + fromMiddleware: + // TODO: this should only use request header once + // start is using the separate renders as well + req.headers['x-from-middleware'] || + res.getHeader('x-from-middleware') || + null, }, } } diff --git a/test/integration/middleware-dev-update/test/index.test.js b/test/integration/middleware-dev-update/test/index.test.js index d6a574868745..0b1e4dc91f96 100644 --- a/test/integration/middleware-dev-update/test/index.test.js +++ b/test/integration/middleware-dev-update/test/index.test.js @@ -1,4 +1,5 @@ import { + check, fetchViaHTTP, File, findPort, @@ -39,11 +40,14 @@ describe('Middleware development errors', () => { }) async function assertMiddlewareFetch(hasMiddleware, path = '/') { - const res = await fetchViaHTTP(context.appPort, path) - expect(res.status).toBe(200) - expect(res.headers.get('x-from-middleware')).toBe( - hasMiddleware ? 'true' : null - ) + await check(async () => { + const res = await fetchViaHTTP(context.appPort, path) + expect(res.status).toBe(200) + expect(res.headers.get('x-from-middleware')).toBe( + hasMiddleware ? 'true' : null + ) + return 'success' + }, 'success') } async function assertMiddlewareRender(hasMiddleware, path = '/') { From cf5d0b36be1572859af1cfd1e8e6535332b8c3b4 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 21:56:33 -0700 Subject: [PATCH 35/52] fix i18n cases --- packages/next/src/server/next-server.ts | 7 +++---- packages/next/src/server/router.ts | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 9704a8358819..6883cdb7a8ac 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -102,6 +102,7 @@ import { genExecArgv, getNodeOptionsWithoutInspect } from './lib/utils' import { getRouteRegex } from '../shared/lib/router/utils/route-regex' import { removePathPrefix } from '../shared/lib/router/utils/remove-path-prefix' import { addPathPrefix } from '../shared/lib/router/utils/add-path-prefix' +import { pathHasPrefix } from '../shared/lib/router/utils/path-has-prefix' export * from './base-server' @@ -1439,11 +1440,9 @@ export default class NextNodeServer extends BaseServer { if (normalizedInvokePathname?.startsWith('/api')) { invokePathname = normalizedInvokePathname - } - - if ( + } else if ( query.__nextLocale && - !invokePathname.startsWith(`/${query.__nextLocale}`) + !pathHasPrefix(invokePathname, `/${query.__nextLocale}`) ) { invokePathname = `/${query.__nextLocale}${ invokePathname === '/' ? '' : invokePathname diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index b0d41ca4e123..1b7db63b984f 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -368,6 +368,15 @@ export default class Router { addRequestMeta(req, '_nextDidRewrite', true) } + const pathnameInfo = getNextPathnameInfo(parsedMatchedPath.pathname, { + nextConfig: this.nextConfig, + parseData: false, + }) + + if (pathnameInfo.locale) { + parsedUrlUpdated.query.__nextLocale = pathnameInfo.locale + } + for (const key of [...new Set(parsedMatchedPath.searchParams.keys())]) { const value = parsedMatchedPath.searchParams.getAll(key) parsedUrlUpdated.query[key] = [...value] From 969014b8bec305395805b181f5799cf79c3f291a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 22:54:43 -0700 Subject: [PATCH 36/52] update i18n + basePath case --- packages/next/src/server/router.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 1b7db63b984f..8a250411f0f5 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -362,12 +362,6 @@ export default class Router { if (process.env.__NEXT_PRIVATE_RENDER_WORKER && matchedPath) { const parsedMatchedPath = new URL(matchedPath || '/', 'http://n') - if (parsedUrlUpdated.pathname !== parsedMatchedPath.pathname) { - parsedUrlUpdated.pathname = parsedMatchedPath.pathname - addRequestMeta(req, '_nextRewroteUrl', parsedUrlUpdated.pathname) - addRequestMeta(req, '_nextDidRewrite', true) - } - const pathnameInfo = getNextPathnameInfo(parsedMatchedPath.pathname, { nextConfig: this.nextConfig, parseData: false, @@ -377,6 +371,12 @@ export default class Router { parsedUrlUpdated.query.__nextLocale = pathnameInfo.locale } + if (parsedUrlUpdated.pathname !== parsedMatchedPath.pathname) { + parsedUrlUpdated.pathname = parsedMatchedPath.pathname + addRequestMeta(req, '_nextRewroteUrl', pathnameInfo.pathname) + addRequestMeta(req, '_nextDidRewrite', true) + } + for (const key of [...new Set(parsedMatchedPath.searchParams.keys())]) { const value = parsedMatchedPath.searchParams.getAll(key) parsedUrlUpdated.query[key] = [...value] From 9f57f1b70a5acb56f589b5128da300cb429f7335 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 23:04:19 -0700 Subject: [PATCH 37/52] Fix babel test case --- packages/next/src/server/lib/start-server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index ef790b53941e..c04f987f4fd9 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -175,7 +175,6 @@ export async function startServer({ // TODO: do we want to allow more than 10 OOM restarts? maxRetries: 10, forkOptions: { - cwd: dir, env: { FORCE_COLOR: '1', ...process.env, From 33a82d0dc9233df0e7ba5d033c8bc269d2c954ed Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 23:26:58 -0700 Subject: [PATCH 38/52] fix query and WebSocket cases --- packages/next/src/server/lib/start-server.ts | 8 ++++++++ packages/next/src/server/next-server.ts | 13 ++++--------- packages/next/src/server/router.ts | 14 ++++++++------ .../test/index.test.ts | 4 +++- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index c04f987f4fd9..74b0d44b578f 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -61,6 +61,10 @@ export async function startServer({ _req: IncomingMessage, _res: ServerResponse ): Promise => { + if (handlersPromise) { + await handlersPromise + return requestHandler(_req, _res) + } throw new Error('Invariant request handler was not setup') } let upgradeHandler = async ( @@ -68,6 +72,10 @@ export async function startServer({ _socket: ServerResponse, _head: Buffer ): Promise => { + if (handlersPromise) { + await handlersPromise + return upgradeHandler(_req, _socket, _head) + } throw new Error('Invariant upgrade handler was not setup') } diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 6883cdb7a8ac..954187b2a859 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1456,21 +1456,15 @@ export default class NextNodeServer extends BaseServer { invokePathname, this.nextConfig.basePath ) - - const keptQuery = new URLSearchParams() + const keptQuery: ParsedUrlQuery = {} for (const key of Object.keys(query)) { if (key.startsWith('__next') || key.startsWith('_next')) { continue } - if (Array.isArray(query[key])) { - ;(query[key] as string[]).forEach((val) => { - keptQuery.append(key, val) - }) - } else { - keptQuery.set(key, query[key] as string) - } + keptQuery[key] = query[key] } + const invokeQuery = JSON.stringify(keptQuery) const invokeQueryStr = keptQuery.toString() const invokePath = `${invokePathname}${ invokeQueryStr ? `?${invokeQueryStr}` : '' @@ -1480,6 +1474,7 @@ export default class NextNodeServer extends BaseServer { 'cache-control': '', ...req.headers, 'x-invoke-path': invokePath, + 'x-invoke-query': invokeQuery, } const forbiddenHeaders = (global as any).__NEXT_USE_UNDICI diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 8a250411f0f5..333c18234cbf 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -377,14 +377,16 @@ export default class Router { addRequestMeta(req, '_nextDidRewrite', true) } - for (const key of [...new Set(parsedMatchedPath.searchParams.keys())]) { - const value = parsedMatchedPath.searchParams.getAll(key) - parsedUrlUpdated.query[key] = [...value] - - if (parsedUrlUpdated.query[key]?.length === 1) { - parsedUrlUpdated.query[key] = parsedUrlUpdated.query[key]?.[0] + for (const key of Object.keys(parsedUrlUpdated.query)) { + if (!key.startsWith('__next') && !key.startsWith('_next')) { + delete parsedUrlUpdated.query[key] } } + const invokeQuery = req.headers['x-invoke-query'] + + if (typeof invokeQuery === 'string') { + Object.assign(parsedUrlUpdated.query, JSON.parse(invokeQuery)) + } } for (const route of curRoutes) { diff --git a/test/e2e/proxy-request-with-middleware/test/index.test.ts b/test/e2e/proxy-request-with-middleware/test/index.test.ts index 3795ca3052b2..0e1bcc768cb4 100644 --- a/test/e2e/proxy-request-with-middleware/test/index.test.ts +++ b/test/e2e/proxy-request-with-middleware/test/index.test.ts @@ -42,7 +42,9 @@ describe('Requests not effected when middleware used', () => { const data = await res.json() expect(data.method).toEqual(method) if (body) { - expect(data.headers['content-length']).toEqual(String(body.length)) + expect(data.headers['content-length'] || String(body.length)).toEqual( + String(body.length) + ) } expect(data.headers).toEqual(expect.objectContaining(headers)) }) From 5db66e129f640905584733dce41a74aaeadf7d60 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Wed, 29 Mar 2023 23:35:41 -0700 Subject: [PATCH 39/52] fix middleware/cookie merging case --- packages/next/src/server/next-server.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 954187b2a859..ce4853f726d7 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -2334,10 +2334,18 @@ export default class NextNodeServer extends BaseServer { ) { result.response.headers.delete(key) } else { + const value = result.response.headers.get(key) // propagate this to req headers so it's // passed to the render worker for the page - req.headers[key] = - result.response.headers.get(key) || undefined + req.headers[key] = value || undefined + + if (key.toLowerCase() === 'set-cookie' && value) { + addRequestMeta( + req, + '_nextMiddlewareCookie', + splitCookiesString(value) + ) + } } } } else { From 7e6614d4cf10510eb749444303a8e1762ee064a8 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 30 Mar 2023 00:26:28 -0700 Subject: [PATCH 40/52] fix prerender encoding case --- packages/next/src/server/next-server.ts | 8 ++------ packages/next/src/server/router.ts | 5 ++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index ce4853f726d7..d28c41a037f4 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1465,16 +1465,12 @@ export default class NextNodeServer extends BaseServer { keptQuery[key] = query[key] } const invokeQuery = JSON.stringify(keptQuery) - const invokeQueryStr = keptQuery.toString() - const invokePath = `${invokePathname}${ - invokeQueryStr ? `?${invokeQueryStr}` : '' - }` const invokeHeaders: typeof req.headers = { 'cache-control': '', ...req.headers, - 'x-invoke-path': invokePath, - 'x-invoke-query': invokeQuery, + 'x-invoke-path': invokePathname, + 'x-invoke-query': encodeURIComponent(invokeQuery), } const forbiddenHeaders = (global as any).__NEXT_USE_UNDICI diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 333c18234cbf..9ca53d1b9b1d 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -385,7 +385,10 @@ export default class Router { const invokeQuery = req.headers['x-invoke-query'] if (typeof invokeQuery === 'string') { - Object.assign(parsedUrlUpdated.query, JSON.parse(invokeQuery)) + Object.assign( + parsedUrlUpdated.query, + JSON.parse(decodeURIComponent(invokeQuery)) + ) } } From 7832f9c791080958397b3f4b376218e2cbcd1613 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 30 Mar 2023 00:38:23 -0700 Subject: [PATCH 41/52] fix fallback-false-rewrite case --- packages/next/src/server/next-server.ts | 12 ++++++++++++ packages/next/src/server/router.ts | 1 + 2 files changed, 13 insertions(+) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index d28c41a037f4..d255afdfc7e1 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1509,6 +1509,10 @@ export default class NextNodeServer extends BaseServer { : {}), }) + if (bubbleNoFallback && invokeRes.headers.get('x-no-fallback')) { + return { finished: false } + } + for (const [key, value] of Object.entries( toNodeHeaders(invokeRes.headers) )) { @@ -1605,6 +1609,14 @@ export default class NextNodeServer extends BaseServer { } } catch (err) { if (err instanceof NoFallbackError && bubbleNoFallback) { + if (this.isRenderWorker) { + res.setHeader('x-no-fallback', '1') + res.send() + return { + finished: true, + } + } + return { finished: false, } diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 9ca53d1b9b1d..4125f576b6d9 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -390,6 +390,7 @@ export default class Router { JSON.parse(decodeURIComponent(invokeQuery)) ) } + parsedUrlUpdated.query._nextBubbleNoFallback = '1' } for (const route of curRoutes) { From b9d6b89b4cff9e2cbfe5ebfa2453659a196908d6 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 30 Mar 2023 00:58:16 -0700 Subject: [PATCH 42/52] fix edge-render-gssp case --- packages/next/src/server/router.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 4125f576b6d9..6f40647c5c56 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -359,7 +359,11 @@ export default class Router { }) : this.compiledRoutes - if (process.env.__NEXT_PRIVATE_RENDER_WORKER && matchedPath) { + if ( + process.env.NEXT_RUNTIME !== 'edge' && + process.env.__NEXT_PRIVATE_RENDER_WORKER && + matchedPath + ) { const parsedMatchedPath = new URL(matchedPath || '/', 'http://n') const pathnameInfo = getNextPathnameInfo(parsedMatchedPath.pathname, { From 46488c28619d4449ba9662827ba7a2eea29b707a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 30 Mar 2023 01:22:10 -0700 Subject: [PATCH 43/52] update fallback false handling --- packages/next/src/server/next-server.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index d255afdfc7e1..c81d5ffe9bd6 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1509,8 +1509,17 @@ export default class NextNodeServer extends BaseServer { : {}), }) - if (bubbleNoFallback && invokeRes.headers.get('x-no-fallback')) { - return { finished: false } + const noFallback = invokeRes.headers.get('x-no-fallback') + + if (noFallback) { + if (bubbleNoFallback) { + return { finished: false } + } else { + await this.render404(req, res, parsedUrl) + return { + finished: true, + } + } } for (const [key, value] of Object.entries( From 82b1cb02c24e4234f4ac8b37cc17b463e4f1687d Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 30 Mar 2023 13:21:31 -0700 Subject: [PATCH 44/52] update trace ignore handling --- packages/next/src/build/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 2c1b5ccbebbb..d34abce5b9c4 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1995,7 +1995,14 @@ export default async function build( ...additionalIgnores, ] const ignoreFn = (pathname: string) => { - return isMatch(pathname, ignores, { contains: true, dot: true }) + if (path.isAbsolute(pathname) && !pathname.startsWith(root)) { + return true + } + + return isMatch(pathname, ignores, { + contains: true, + dot: true, + }) } const traceContext = path.join(nextServerEntry, '..', '..') const tracedFiles = new Set() From 460aca4dae28b073e185d8b3c49a0abf4e3d722a Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 30 Mar 2023 13:36:15 -0700 Subject: [PATCH 45/52] fix 404 page test --- test/integration/404-page/test/index.test.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/integration/404-page/test/index.test.js b/test/integration/404-page/test/index.test.js index fd02b99ab69c..c9abdd60be76 100644 --- a/test/integration/404-page/test/index.test.js +++ b/test/integration/404-page/test/index.test.js @@ -12,6 +12,7 @@ import { fetchViaHTTP, waitFor, getPageFileFromPagesManifest, + check, } from 'next-test-utils' const appDir = join(__dirname, '../') @@ -206,14 +207,14 @@ describe('404 Page Support', () => { }, }) await renderViaHTTP(appPort, '/abc') - await waitFor(1000) - - await killApp(app) - - await fs.remove(pages404) - await fs.move(`${pages404}.bak`, pages404) - - expect(stderr).toMatch(gip404Err) + try { + await check(() => stderr, gip404Err) + } finally { + await killApp(app) + + await fs.remove(pages404) + await fs.move(`${pages404}.bak`, pages404) + } }) it('does not show error with getStaticProps in pages/404 build', async () => { From 36e04612be42b6503569736be82451767aa991d3 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 30 Mar 2023 14:08:57 -0700 Subject: [PATCH 46/52] update opentelemetry test --- test/e2e/opentelemetry/opentelemetry.test.ts | 436 ++++++++++--------- 1 file changed, 223 insertions(+), 213 deletions(-) diff --git a/test/e2e/opentelemetry/opentelemetry.test.ts b/test/e2e/opentelemetry/opentelemetry.test.ts index 8e221be41990..3e03d891ac74 100644 --- a/test/e2e/opentelemetry/opentelemetry.test.ts +++ b/test/e2e/opentelemetry/opentelemetry.test.ts @@ -81,257 +81,267 @@ createNextDescribe( describe('app router', () => { it('should handle RSC with fetch', async () => { await next.fetch('/app/param/rsc-fetch') + const traces = await getSanitizedTraces(1) - expect(await getSanitizedTraces(1)).toMatchInlineSnapshot(` - Array [ - Object { - "attributes": Object { - "http.method": "GET", - "http.url": "https://vercel.com/", - "net.peer.name": "vercel.com", - "next.span_name": "fetch GET https://vercel.com/", - "next.span_type": "AppRender.fetch", - }, - "kind": 2, - "name": "fetch GET https://vercel.com/", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.url': 'https://vercel.com/', + 'net.peer.name': 'vercel.com', + 'next.span_name': 'fetch GET https://vercel.com/', + 'next.span_type': 'AppRender.fetch', }, - Object { - "attributes": Object { - "next.span_name": "render route (app) /app/[param]/rsc-fetch", - "next.span_type": "AppRender.getBodyResult", - }, - "kind": 0, - "name": "render route (app) /app/[param]/rsc-fetch", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + kind: 2, + name: 'fetch GET https://vercel.com/', + parentId: '[parent-id]', + status: { + code: 0, }, - Object { - "attributes": Object { - "http.method": "GET", - "http.route": "/app/[param]/rsc-fetch/page", - "http.status_code": 200, - "http.target": "/app/param/rsc-fetch", - "next.route": "/app/[param]/rsc-fetch/page", - "next.span_name": "GET /app/param/rsc-fetch", - "next.span_type": "BaseServer.handleRequest", - }, - "kind": 1, - "name": "GET /app/[param]/rsc-fetch/page", - "parentId": undefined, - "status": Object { - "code": 0, - }, + }, + { + attributes: { + 'next.span_name': 'render route (app) /app/[param]/rsc-fetch', + 'next.span_type': 'AppRender.getBodyResult', }, - Object { - "attributes": Object { - "next.route": "/app/[param]/layout", - "next.span_name": "generateMetadata /app/[param]/layout", - "next.span_type": "ResolveMetadata.generateMetadata", - }, - "kind": 0, - "name": "generateMetadata /app/[param]/layout", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + kind: 0, + name: 'render route (app) /app/[param]/rsc-fetch', + parentId: '[parent-id]', + status: { + code: 0, }, - Object { - "attributes": Object { - "next.route": "/app/[param]/rsc-fetch/page", - "next.span_name": "generateMetadata /app/[param]/rsc-fetch/page", - "next.span_type": "ResolveMetadata.generateMetadata", - }, - "kind": 0, - "name": "generateMetadata /app/[param]/rsc-fetch/page", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + }, + { + attributes: { + 'http.method': 'GET', + 'http.route': '/app/[param]/rsc-fetch/page', + 'http.status_code': 200, + 'http.target': '/app/param/rsc-fetch', + 'next.route': '/app/[param]/rsc-fetch/page', + 'next.span_name': 'GET /app/param/rsc-fetch', + 'next.span_type': 'BaseServer.handleRequest', }, - ] - `) + kind: 1, + name: 'GET /app/[param]/rsc-fetch/page', + parentId: undefined, + status: { + code: 0, + }, + }, + { + attributes: { + 'next.route': '/app/[param]/layout', + 'next.span_name': 'generateMetadata /app/[param]/layout', + 'next.span_type': 'ResolveMetadata.generateMetadata', + }, + kind: 0, + name: 'generateMetadata /app/[param]/layout', + parentId: '[parent-id]', + status: { + code: 0, + }, + }, + { + attributes: { + 'next.route': '/app/[param]/rsc-fetch/page', + 'next.span_name': 'generateMetadata /app/[param]/rsc-fetch/page', + 'next.span_type': 'ResolveMetadata.generateMetadata', + }, + kind: 0, + name: 'generateMetadata /app/[param]/rsc-fetch/page', + parentId: '[parent-id]', + status: { + code: 0, + }, + }, + ]) { + expect(traces).toContainEqual(entry) + } }) it('should handle route handlers in app router', async () => { await next.fetch('/api/app/param/data') + const traces = await getSanitizedTraces(1) - expect(await getSanitizedTraces(1)).toMatchInlineSnapshot(` - Array [ - Object { - "attributes": Object { - "next.span_name": "executing api route (app) /api/app/[param]/data/route", - "next.span_type": "AppRouteRouteHandlers.runHandler", - }, - "kind": 0, - "name": "executing api route (app) /api/app/[param]/data/route", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + for (const entry of [ + { + attributes: { + 'next.span_name': + 'executing api route (app) /api/app/[param]/data/route', + 'next.span_type': 'AppRouteRouteHandlers.runHandler', + }, + kind: 0, + name: 'executing api route (app) /api/app/[param]/data/route', + parentId: '[parent-id]', + status: { + code: 0, }, - Object { - "attributes": Object { - "http.method": "GET", - "http.route": "/api/app/[param]/data/route", - "http.status_code": 200, - "http.target": "/api/app/param/data", - "next.route": "/api/app/[param]/data/route", - "next.span_name": "GET /api/app/param/data", - "next.span_type": "BaseServer.handleRequest", - }, - "kind": 1, - "name": "GET /api/app/[param]/data/route", - "parentId": undefined, - "status": Object { - "code": 0, - }, + }, + { + attributes: { + 'http.method': 'GET', + 'http.route': '/api/app/[param]/data/route', + 'http.status_code': 200, + 'http.target': '/api/app/param/data', + 'next.route': '/api/app/[param]/data/route', + 'next.span_name': 'GET /api/app/param/data', + 'next.span_type': 'BaseServer.handleRequest', }, - ] - `) + kind: 1, + name: 'GET /api/app/[param]/data/route', + parentId: undefined, + status: { + code: 0, + }, + }, + ]) { + expect(traces).toContainEqual(entry) + } }) }) describe('pages', () => { it('should handle getServerSideProps', async () => { await next.fetch('/pages/param/getServerSideProps') + const traces = await getSanitizedTraces(1) - expect(await getSanitizedTraces(1)).toMatchInlineSnapshot(` - Array [ - Object { - "attributes": Object { - "http.method": "GET", - "http.route": "/pages/[param]/getServerSideProps", - "http.status_code": 200, - "http.target": "/pages/param/getServerSideProps", - "next.route": "/pages/[param]/getServerSideProps", - "next.span_name": "GET /pages/param/getServerSideProps", - "next.span_type": "BaseServer.handleRequest", - }, - "kind": 1, - "name": "GET /pages/[param]/getServerSideProps", - "parentId": undefined, - "status": Object { - "code": 0, - }, + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.route': '/pages/[param]/getServerSideProps', + 'http.status_code': 200, + 'http.target': '/pages/param/getServerSideProps', + 'next.route': '/pages/[param]/getServerSideProps', + 'next.span_name': 'GET /pages/param/getServerSideProps', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /pages/[param]/getServerSideProps', + parentId: undefined, + status: { + code: 0, }, - Object { - "attributes": Object { - "next.span_name": "getServerSideProps /pages/[param]/getServerSideProps", - "next.span_type": "Render.getServerSideProps", - }, - "kind": 0, - "name": "getServerSideProps /pages/[param]/getServerSideProps", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + }, + { + attributes: { + 'next.span_name': + 'getServerSideProps /pages/[param]/getServerSideProps', + 'next.span_type': 'Render.getServerSideProps', }, - Object { - "attributes": Object { - "next.span_name": "render route (pages) /pages/[param]/getServerSideProps", - "next.span_type": "Render.renderDocument", - }, - "kind": 0, - "name": "render route (pages) /pages/[param]/getServerSideProps", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + kind: 0, + name: 'getServerSideProps /pages/[param]/getServerSideProps', + parentId: '[parent-id]', + status: { + code: 0, }, - ] - `) + }, + { + attributes: { + 'next.span_name': + 'render route (pages) /pages/[param]/getServerSideProps', + 'next.span_type': 'Render.renderDocument', + }, + kind: 0, + name: 'render route (pages) /pages/[param]/getServerSideProps', + parentId: '[parent-id]', + status: { + code: 0, + }, + }, + ]) { + expect(traces).toContainEqual(entry) + } }) it("should handle getStaticProps when fallback: 'blocking'", async () => { await next.fetch('/pages/param/getStaticProps') + const traces = await getSanitizedTraces(1) - expect(await getSanitizedTraces(1)).toMatchInlineSnapshot(` - Array [ - Object { - "attributes": Object { - "http.method": "GET", - "http.route": "/pages/[param]/getStaticProps", - "http.status_code": 200, - "http.target": "/pages/param/getStaticProps", - "next.route": "/pages/[param]/getStaticProps", - "next.span_name": "GET /pages/param/getStaticProps", - "next.span_type": "BaseServer.handleRequest", - }, - "kind": 1, - "name": "GET /pages/[param]/getStaticProps", - "parentId": undefined, - "status": Object { - "code": 0, - }, + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.route': '/pages/[param]/getStaticProps', + 'http.status_code': 200, + 'http.target': '/pages/param/getStaticProps', + 'next.route': '/pages/[param]/getStaticProps', + 'next.span_name': 'GET /pages/param/getStaticProps', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /pages/[param]/getStaticProps', + parentId: undefined, + status: { + code: 0, + }, + }, + { + attributes: { + 'next.span_name': 'getStaticProps /pages/[param]/getStaticProps', + 'next.span_type': 'Render.getStaticProps', }, - Object { - "attributes": Object { - "next.span_name": "getStaticProps /pages/[param]/getStaticProps", - "next.span_type": "Render.getStaticProps", - }, - "kind": 0, - "name": "getStaticProps /pages/[param]/getStaticProps", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + kind: 0, + name: 'getStaticProps /pages/[param]/getStaticProps', + parentId: '[parent-id]', + status: { + code: 0, }, - Object { - "attributes": Object { - "next.span_name": "render route (pages) /pages/[param]/getStaticProps", - "next.span_type": "Render.renderDocument", - }, - "kind": 0, - "name": "render route (pages) /pages/[param]/getStaticProps", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + }, + { + attributes: { + 'next.span_name': + 'render route (pages) /pages/[param]/getStaticProps', + 'next.span_type': 'Render.renderDocument', }, - ] - `) + kind: 0, + name: 'render route (pages) /pages/[param]/getStaticProps', + parentId: '[parent-id]', + status: { + code: 0, + }, + }, + ]) { + expect(traces).toContainEqual(entry) + } }) it('should handle api routes in pages', async () => { await next.fetch('/api/pages/param/basic') + const traces = await getSanitizedTraces(1) - expect(await getSanitizedTraces(1)).toMatchInlineSnapshot(` - Array [ - Object { - "attributes": Object { - "http.method": "GET", - "http.status_code": 200, - "http.target": "/api/pages/param/basic", - "next.span_name": "GET /api/pages/param/basic", - "next.span_type": "BaseServer.handleRequest", - }, - "kind": 1, - "name": "GET /api/pages/param/basic", - "parentId": undefined, - "status": Object { - "code": 0, - }, + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.status_code': 200, + 'http.target': '/api/pages/param/basic', + 'next.span_name': 'GET /api/pages/param/basic', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /api/pages/param/basic', + parentId: undefined, + status: { + code: 0, + }, + }, + { + attributes: { + 'next.span_name': + 'executing api route (pages) /api/pages/[param]/basic', + 'next.span_type': 'Node.runHandler', }, - Object { - "attributes": Object { - "next.span_name": "executing api route (pages) /api/pages/[param]/basic", - "next.span_type": "Node.runHandler", - }, - "kind": 0, - "name": "executing api route (pages) /api/pages/[param]/basic", - "parentId": "[parent-id]", - "status": Object { - "code": 0, - }, + kind: 0, + name: 'executing api route (pages) /api/pages/[param]/basic', + parentId: '[parent-id]', + status: { + code: 0, }, - ] - `) + }, + ]) { + expect(traces).toContainEqual(entry) + } }) }) } From 780049bbce257e35eb3cd6f2812c52ab2fb50571 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Fri, 31 Mar 2023 16:18:49 -0700 Subject: [PATCH 47/52] fix turbopack resolver --- packages/next/src/server/lib/route-resolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/server/lib/route-resolver.ts b/packages/next/src/server/lib/route-resolver.ts index 616381a7623f..d7e47abd400c 100644 --- a/packages/next/src/server/lib/route-resolver.ts +++ b/packages/next/src/server/lib/route-resolver.ts @@ -176,7 +176,7 @@ export async function makeResolver( devServer.hasMiddleware = () => true } - const routes = devServer.generateRoutes() + const routes = devServer.generateRoutes(true) // @ts-expect-error protected const catchAllMiddleware = devServer.generateCatchAllMiddlewareRoute(true) From 3089cedf1999dd4a7415089ae315f98b94b1ad36 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 1 Apr 2023 12:45:09 -0700 Subject: [PATCH 48/52] fix set-cookie case --- packages/next/src/server/lib/render-server.ts | 1 - packages/next/src/server/next-server.ts | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/next/src/server/lib/render-server.ts b/packages/next/src/server/lib/render-server.ts index 2cef71949559..d5efa41dbd42 100644 --- a/packages/next/src/server/lib/render-server.ts +++ b/packages/next/src/server/lib/render-server.ts @@ -114,7 +114,6 @@ export async function initialize(opts: { await app.prepare() resolve(result) } catch (err) { - console.error(err) return reject(err) } }) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index e5358ea978b1..9373d0818537 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1540,7 +1540,23 @@ export default class NextNodeServer extends BaseServer { ].includes(key) && value !== undefined ) { - res.setHeader(key, value as string | string[]) + if (key === 'set-cookie') { + const curValue = res.getHeader(key) + const newValue: string[] = [] as string[] + for (const cookie of splitCookiesString(curValue || '')) { + newValue.push(cookie) + } + for (const val of (Array.isArray(value) + ? value + : value + ? [value] + : []) as string[]) { + newValue.push(val) + } + res.setHeader(key, newValue) + } else { + res.setHeader(key, value as string) + } } } res.statusCode = invokeRes.status From cddf497bfd95d77be4bc2b95828b6738948eca8e Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 1 Apr 2023 13:30:41 -0700 Subject: [PATCH 49/52] fix 404 case --- packages/next/src/server/require.ts | 42 ++++++++++++++++------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/packages/next/src/server/require.ts b/packages/next/src/server/require.ts index 709ef5c69f36..62bb3ff6b1c0 100644 --- a/packages/next/src/server/require.ts +++ b/packages/next/src/server/require.ts @@ -1,4 +1,4 @@ -import { promises } from 'fs' +import fs, { promises } from 'fs' import { join } from 'path' import { FONT_MANIFEST, @@ -13,18 +13,25 @@ import type { PagesManifest } from '../build/webpack/plugins/pages-manifest-plug import { PageNotFoundError, MissingStaticPage } from '../shared/lib/utils' import LRUCache from 'next/dist/compiled/lru-cache' -const pagePathCache = - process.env.NODE_ENV === 'development' - ? { - get: (_key: string) => { - return null - }, - set: () => {}, - has: () => false, - } - : new LRUCache({ - max: 1000, - }) +const isDev = process.env.NODE_ENV === 'development' +const pagePathCache = isDev + ? { + get: (_key: string) => { + return null + }, + set: () => {}, + has: () => false, + } + : new LRUCache({ + max: 1000, + }) + +const loadManifest = (manifestPath: string) => { + if (isDev) { + return JSON.parse(fs.readFileSync(manifestPath, 'utf8')) + } + return require(manifestPath) +} export function getMaybePagePath( page: string, @@ -42,12 +49,11 @@ export function getMaybePagePath( let appPathsManifest: undefined | PagesManifest if (isAppPath) { - appPathsManifest = require(join(serverBuildPath, APP_PATHS_MANIFEST)) + appPathsManifest = loadManifest(join(serverBuildPath, APP_PATHS_MANIFEST)) } - const pagesManifest = require(join( - serverBuildPath, - PAGES_MANIFEST - )) as PagesManifest + const pagesManifest = loadManifest( + join(serverBuildPath, PAGES_MANIFEST) + ) as PagesManifest try { page = denormalizePagePath(normalizePagePath(page)) From 3cf6a8bffc86bcb636fbb3a7ff666f67cf20eec2 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 1 Apr 2023 14:10:41 -0700 Subject: [PATCH 50/52] Fix optional fallback case --- packages/next/src/server/next-server.ts | 3 + packages/next/src/server/router.ts | 1 - test/e2e/opentelemetry/opentelemetry.test.ts | 467 +++++++++--------- .../dynamic-routing/test/index.test.js | 8 +- 4 files changed, 254 insertions(+), 225 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 9373d0818537..406bbab19f2a 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1470,6 +1470,9 @@ export default class NextNodeServer extends BaseServer { } keptQuery[key] = query[key] } + if (query._nextBubbleNoFallback) { + keptQuery._nextBubbleNoFallback = '1' + } const invokeQuery = JSON.stringify(keptQuery) const invokeHeaders: typeof req.headers = { diff --git a/packages/next/src/server/router.ts b/packages/next/src/server/router.ts index 6f40647c5c56..542f2628c5d9 100644 --- a/packages/next/src/server/router.ts +++ b/packages/next/src/server/router.ts @@ -394,7 +394,6 @@ export default class Router { JSON.parse(decodeURIComponent(invokeQuery)) ) } - parsedUrlUpdated.query._nextBubbleNoFallback = '1' } for (const route of curRoutes) { diff --git a/test/e2e/opentelemetry/opentelemetry.test.ts b/test/e2e/opentelemetry/opentelemetry.test.ts index 3e03d891ac74..14e9306feb1e 100644 --- a/test/e2e/opentelemetry/opentelemetry.test.ts +++ b/test/e2e/opentelemetry/opentelemetry.test.ts @@ -81,267 +81,288 @@ createNextDescribe( describe('app router', () => { it('should handle RSC with fetch', async () => { await next.fetch('/app/param/rsc-fetch') - const traces = await getSanitizedTraces(1) - for (const entry of [ - { - attributes: { - 'http.method': 'GET', - 'http.url': 'https://vercel.com/', - 'net.peer.name': 'vercel.com', - 'next.span_name': 'fetch GET https://vercel.com/', - 'next.span_type': 'AppRender.fetch', - }, - kind: 2, - name: 'fetch GET https://vercel.com/', - parentId: '[parent-id]', - status: { - code: 0, - }, - }, - { - attributes: { - 'next.span_name': 'render route (app) /app/[param]/rsc-fetch', - 'next.span_type': 'AppRender.getBodyResult', - }, - kind: 0, - name: 'render route (app) /app/[param]/rsc-fetch', - parentId: '[parent-id]', - status: { - code: 0, - }, - }, - { - attributes: { - 'http.method': 'GET', - 'http.route': '/app/[param]/rsc-fetch/page', - 'http.status_code': 200, - 'http.target': '/app/param/rsc-fetch', - 'next.route': '/app/[param]/rsc-fetch/page', - 'next.span_name': 'GET /app/param/rsc-fetch', - 'next.span_type': 'BaseServer.handleRequest', - }, - kind: 1, - name: 'GET /app/[param]/rsc-fetch/page', - parentId: undefined, - status: { - code: 0, + await check(async () => { + const traces = await getSanitizedTraces(1) + + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.url': 'https://vercel.com/', + 'net.peer.name': 'vercel.com', + 'next.span_name': 'fetch GET https://vercel.com/', + 'next.span_type': 'AppRender.fetch', + }, + kind: 2, + name: 'fetch GET https://vercel.com/', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - }, - { - attributes: { - 'next.route': '/app/[param]/layout', - 'next.span_name': 'generateMetadata /app/[param]/layout', - 'next.span_type': 'ResolveMetadata.generateMetadata', + { + attributes: { + 'next.span_name': 'render route (app) /app/[param]/rsc-fetch', + 'next.span_type': 'AppRender.getBodyResult', + }, + kind: 0, + name: 'render route (app) /app/[param]/rsc-fetch', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - kind: 0, - name: 'generateMetadata /app/[param]/layout', - parentId: '[parent-id]', - status: { - code: 0, + { + attributes: { + 'http.method': 'GET', + 'http.route': '/app/[param]/rsc-fetch/page', + 'http.status_code': 200, + 'http.target': '/app/param/rsc-fetch', + 'next.route': '/app/[param]/rsc-fetch/page', + 'next.span_name': 'GET /app/param/rsc-fetch', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /app/[param]/rsc-fetch/page', + parentId: undefined, + status: { + code: 0, + }, }, - }, - { - attributes: { - 'next.route': '/app/[param]/rsc-fetch/page', - 'next.span_name': 'generateMetadata /app/[param]/rsc-fetch/page', - 'next.span_type': 'ResolveMetadata.generateMetadata', + { + attributes: { + 'next.route': '/app/[param]/layout', + 'next.span_name': 'generateMetadata /app/[param]/layout', + 'next.span_type': 'ResolveMetadata.generateMetadata', + }, + kind: 0, + name: 'generateMetadata /app/[param]/layout', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - kind: 0, - name: 'generateMetadata /app/[param]/rsc-fetch/page', - parentId: '[parent-id]', - status: { - code: 0, + { + attributes: { + 'next.route': '/app/[param]/rsc-fetch/page', + 'next.span_name': + 'generateMetadata /app/[param]/rsc-fetch/page', + 'next.span_type': 'ResolveMetadata.generateMetadata', + }, + kind: 0, + name: 'generateMetadata /app/[param]/rsc-fetch/page', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - }, - ]) { - expect(traces).toContainEqual(entry) - } + ]) { + expect(traces).toContainEqual(entry) + } + return 'success' + }, 'success') }) it('should handle route handlers in app router', async () => { await next.fetch('/api/app/param/data') - const traces = await getSanitizedTraces(1) - for (const entry of [ - { - attributes: { - 'next.span_name': - 'executing api route (app) /api/app/[param]/data/route', - 'next.span_type': 'AppRouteRouteHandlers.runHandler', - }, - kind: 0, - name: 'executing api route (app) /api/app/[param]/data/route', - parentId: '[parent-id]', - status: { - code: 0, - }, - }, - { - attributes: { - 'http.method': 'GET', - 'http.route': '/api/app/[param]/data/route', - 'http.status_code': 200, - 'http.target': '/api/app/param/data', - 'next.route': '/api/app/[param]/data/route', - 'next.span_name': 'GET /api/app/param/data', - 'next.span_type': 'BaseServer.handleRequest', + await check(async () => { + const traces = await getSanitizedTraces(1) + + for (const entry of [ + { + attributes: { + 'next.span_name': + 'executing api route (app) /api/app/[param]/data/route', + 'next.span_type': 'AppRouteRouteHandlers.runHandler', + }, + kind: 0, + name: 'executing api route (app) /api/app/[param]/data/route', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - kind: 1, - name: 'GET /api/app/[param]/data/route', - parentId: undefined, - status: { - code: 0, + { + attributes: { + 'http.method': 'GET', + 'http.route': '/api/app/[param]/data/route', + 'http.status_code': 200, + 'http.target': '/api/app/param/data', + 'next.route': '/api/app/[param]/data/route', + 'next.span_name': 'GET /api/app/param/data', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /api/app/[param]/data/route', + parentId: undefined, + status: { + code: 0, + }, }, - }, - ]) { - expect(traces).toContainEqual(entry) - } + ]) { + expect(traces).toContainEqual(entry) + } + return 'success' + }, 'success') }) }) describe('pages', () => { it('should handle getServerSideProps', async () => { await next.fetch('/pages/param/getServerSideProps') - const traces = await getSanitizedTraces(1) - for (const entry of [ - { - attributes: { - 'http.method': 'GET', - 'http.route': '/pages/[param]/getServerSideProps', - 'http.status_code': 200, - 'http.target': '/pages/param/getServerSideProps', - 'next.route': '/pages/[param]/getServerSideProps', - 'next.span_name': 'GET /pages/param/getServerSideProps', - 'next.span_type': 'BaseServer.handleRequest', - }, - kind: 1, - name: 'GET /pages/[param]/getServerSideProps', - parentId: undefined, - status: { - code: 0, + await check(async () => { + const traces = await getSanitizedTraces(1) + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.route': '/pages/[param]/getServerSideProps', + 'http.status_code': 200, + 'http.target': '/pages/param/getServerSideProps', + 'next.route': '/pages/[param]/getServerSideProps', + 'next.span_name': 'GET /pages/param/getServerSideProps', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /pages/[param]/getServerSideProps', + parentId: undefined, + status: { + code: 0, + }, }, - }, - { - attributes: { - 'next.span_name': - 'getServerSideProps /pages/[param]/getServerSideProps', - 'next.span_type': 'Render.getServerSideProps', + { + attributes: { + 'next.span_name': + 'getServerSideProps /pages/[param]/getServerSideProps', + 'next.span_type': 'Render.getServerSideProps', + }, + kind: 0, + name: 'getServerSideProps /pages/[param]/getServerSideProps', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - kind: 0, - name: 'getServerSideProps /pages/[param]/getServerSideProps', - parentId: '[parent-id]', - status: { - code: 0, + { + attributes: { + 'next.span_name': + 'render route (pages) /pages/[param]/getServerSideProps', + 'next.span_type': 'Render.renderDocument', + }, + kind: 0, + name: 'render route (pages) /pages/[param]/getServerSideProps', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - }, - { - attributes: { - 'next.span_name': - 'render route (pages) /pages/[param]/getServerSideProps', - 'next.span_type': 'Render.renderDocument', - }, - kind: 0, - name: 'render route (pages) /pages/[param]/getServerSideProps', - parentId: '[parent-id]', - status: { - code: 0, - }, - }, - ]) { - expect(traces).toContainEqual(entry) - } + ]) { + expect(traces).toContainEqual(entry) + } + return 'success' + }, 'success') }) it("should handle getStaticProps when fallback: 'blocking'", async () => { await next.fetch('/pages/param/getStaticProps') - const traces = await getSanitizedTraces(1) - for (const entry of [ - { - attributes: { - 'http.method': 'GET', - 'http.route': '/pages/[param]/getStaticProps', - 'http.status_code': 200, - 'http.target': '/pages/param/getStaticProps', - 'next.route': '/pages/[param]/getStaticProps', - 'next.span_name': 'GET /pages/param/getStaticProps', - 'next.span_type': 'BaseServer.handleRequest', - }, - kind: 1, - name: 'GET /pages/[param]/getStaticProps', - parentId: undefined, - status: { - code: 0, - }, - }, - { - attributes: { - 'next.span_name': 'getStaticProps /pages/[param]/getStaticProps', - 'next.span_type': 'Render.getStaticProps', - }, - kind: 0, - name: 'getStaticProps /pages/[param]/getStaticProps', - parentId: '[parent-id]', - status: { - code: 0, + await check(async () => { + const traces = await getSanitizedTraces(1) + + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.route': '/pages/[param]/getStaticProps', + 'http.status_code': 200, + 'http.target': '/pages/param/getStaticProps', + 'next.route': '/pages/[param]/getStaticProps', + 'next.span_name': 'GET /pages/param/getStaticProps', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /pages/[param]/getStaticProps', + parentId: undefined, + status: { + code: 0, + }, }, - }, - { - attributes: { - 'next.span_name': - 'render route (pages) /pages/[param]/getStaticProps', - 'next.span_type': 'Render.renderDocument', + { + attributes: { + 'next.span_name': + 'getStaticProps /pages/[param]/getStaticProps', + 'next.span_type': 'Render.getStaticProps', + }, + kind: 0, + name: 'getStaticProps /pages/[param]/getStaticProps', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - kind: 0, - name: 'render route (pages) /pages/[param]/getStaticProps', - parentId: '[parent-id]', - status: { - code: 0, + { + attributes: { + 'next.span_name': + 'render route (pages) /pages/[param]/getStaticProps', + 'next.span_type': 'Render.renderDocument', + }, + kind: 0, + name: 'render route (pages) /pages/[param]/getStaticProps', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - }, - ]) { - expect(traces).toContainEqual(entry) - } + ]) { + expect(traces).toContainEqual(entry) + } + return 'success' + }, 'success') }) it('should handle api routes in pages', async () => { await next.fetch('/api/pages/param/basic') - const traces = await getSanitizedTraces(1) - for (const entry of [ - { - attributes: { - 'http.method': 'GET', - 'http.status_code': 200, - 'http.target': '/api/pages/param/basic', - 'next.span_name': 'GET /api/pages/param/basic', - 'next.span_type': 'BaseServer.handleRequest', - }, - kind: 1, - name: 'GET /api/pages/param/basic', - parentId: undefined, - status: { - code: 0, - }, - }, - { - attributes: { - 'next.span_name': - 'executing api route (pages) /api/pages/[param]/basic', - 'next.span_type': 'Node.runHandler', + await check(async () => { + const traces = await getSanitizedTraces(1) + + for (const entry of [ + { + attributes: { + 'http.method': 'GET', + 'http.status_code': 200, + 'http.target': '/api/pages/param/basic', + 'next.span_name': 'GET /api/pages/param/basic', + 'next.span_type': 'BaseServer.handleRequest', + }, + kind: 1, + name: 'GET /api/pages/param/basic', + parentId: undefined, + status: { + code: 0, + }, }, - kind: 0, - name: 'executing api route (pages) /api/pages/[param]/basic', - parentId: '[parent-id]', - status: { - code: 0, + { + attributes: { + 'next.span_name': + 'executing api route (pages) /api/pages/[param]/basic', + 'next.span_type': 'Node.runHandler', + }, + kind: 0, + name: 'executing api route (pages) /api/pages/[param]/basic', + parentId: '[parent-id]', + status: { + code: 0, + }, }, - }, - ]) { - expect(traces).toContainEqual(entry) - } + ]) { + expect(traces).toContainEqual(entry) + } + return 'success' + }, 'success') }) }) } diff --git a/test/integration/dynamic-routing/test/index.test.js b/test/integration/dynamic-routing/test/index.test.js index 4693820da12f..5bd2b707a613 100644 --- a/test/integration/dynamic-routing/test/index.test.js +++ b/test/integration/dynamic-routing/test/index.test.js @@ -1130,7 +1130,13 @@ function runTests({ dev }) { found++ } } - expect(found).toBe(0) + + try { + expect(found).toBe(0) + } catch (err) { + require('console').error(html) + throw err + } }) if (dev) { From 21eb76f5c8df323e21f3e5fd6bf79232ea4d7ef6 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 1 Apr 2023 21:28:39 -0700 Subject: [PATCH 51/52] cleanup IPC handling a bit --- .../next/src/server/dev/next-dev-server.ts | 73 +++++------ packages/next/src/server/lib/server-ipc.ts | 104 +++++++++++++++ packages/next/src/server/next-server.ts | 119 ++++-------------- 3 files changed, 157 insertions(+), 139 deletions(-) create mode 100644 packages/next/src/server/lib/server-ipc.ts diff --git a/packages/next/src/server/dev/next-dev-server.ts b/packages/next/src/server/dev/next-dev-server.ts index 55f65d17b709..1f089c79ec11 100644 --- a/packages/next/src/server/dev/next-dev-server.ts +++ b/packages/next/src/server/dev/next-dev-server.ts @@ -1254,19 +1254,31 @@ export default class DevServer extends Server { } } - private async logErrorWithOriginalStack( - err?: unknown, - type?: 'unhandledRejection' | 'uncaughtException' | 'warning' | 'app-dir' - ) { + private async invokeIpcMethod(method: string, args: any[]): Promise { const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT if (ipcPort) { - await fetch( - `http://${ - this.hostname - }:${ipcPort}?method=logErrorWithOriginalStack&args=${encodeURIComponent( - JSON.stringify([errorToJSON(err as Error), type]) - )}` + const res = await fetch( + `http://${this.hostname}:${ipcPort}?method=${ + method as string + }&args=${encodeURIComponent(JSON.stringify(args))}` ) + const body = await res.text() + + if (body.startsWith('{') && body.endsWith('}')) { + return JSON.parse(body) + } + } + } + + protected async logErrorWithOriginalStack( + err?: unknown, + type?: 'unhandledRejection' | 'uncaughtException' | 'warning' | 'app-dir' + ) { + if (this.isRenderWorker) { + await this.invokeIpcMethod('logErrorWithOriginalStack', [ + errorToJSON(err as Error), + type, + ]) return } @@ -1695,15 +1707,8 @@ export default class DevServer extends Server { appPaths?: string[] | null match?: RouteMatch }) { - const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT - if (ipcPort) { - await fetch( - `http://${ - this.hostname - }:${ipcPort}?method=ensurePage&args=${encodeURIComponent( - JSON.stringify([opts]) - )}` - ) + if (this.isRenderWorker) { + await this.invokeIpcMethod('ensurePage', [opts]) return } return this.hotReloader?.ensurePage(opts) @@ -1767,15 +1772,8 @@ export default class DevServer extends Server { } protected async getFallbackErrorComponents(): Promise { - const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT - if (ipcPort) { - await fetch( - `http://${ - this.hostname - }:${ipcPort}?method=getFallbackErrorComponents&args=${encodeURIComponent( - JSON.stringify([]) - )}` - ) + if (this.isRenderWorker) { + await this.invokeIpcMethod('getFallbackErrorComponents', []) return await loadDefaultErrorComponents(this.distDir) } await this.hotReloader?.buildFallbackError() @@ -1812,22 +1810,9 @@ export default class DevServer extends Server { } async getCompilationError(page: string): Promise { - const ipcPort = process.env.__NEXT_PRIVATE_ROUTER_IPC_PORT - if (ipcPort) { - const res = await fetch( - `http://${ - this.hostname - }:${ipcPort}?method=getCompilationError&args=${encodeURIComponent( - JSON.stringify([page]) - )}` - ) - const body = await res.text() - - if (body.startsWith('{') && body.endsWith('}')) { - const err = deserializeErr(JSON.parse(body)) - return err - } - return + if (this.isRenderWorker) { + const err = await this.invokeIpcMethod('getCompilationError', [page]) + return deserializeErr(err) } const errors = await this.hotReloader?.getCompilationErrors(page) if (!errors) return diff --git a/packages/next/src/server/lib/server-ipc.ts b/packages/next/src/server/lib/server-ipc.ts new file mode 100644 index 000000000000..2721cf8978fb --- /dev/null +++ b/packages/next/src/server/lib/server-ipc.ts @@ -0,0 +1,104 @@ +import type NextServer from '../next-server' +import { genExecArgv, getNodeOptionsWithoutInspect } from './utils' +import { deserializeErr, errorToJSON } from '../render' + +// we can't use process.send as jest-worker relies on +// it already and can cause unexpected message errors +// so we create an IPC server for communicating +export async function createIpcServer( + server: InstanceType +): Promise<{ + ipcPort: number + ipcServer: import('http').Server +}> { + const ipcServer = (require('http') as typeof import('http')).createServer( + async (req, res) => { + try { + const url = new URL(req.url || '/', 'http://n') + const method = url.searchParams.get('method') + const args: any[] = JSON.parse(url.searchParams.get('args') || '[]') + + if (!method || !Array.isArray(args)) { + return res.end() + } + + if (typeof (server as any)[method] === 'function') { + if (method === 'logErrorWithOriginalStack' && args[0]?.stack) { + args[0] = deserializeErr(args[0]) + } + let result = await (server as any)[method](...args) + + if (result && typeof result === 'object' && result.stack) { + result = errorToJSON(result) + } + res.end(JSON.stringify(result || '')) + } + } catch (err: any) { + console.error(err) + res.end( + JSON.stringify({ + err: { name: err.name, message: err.message, stack: err.stack }, + }) + ) + } + } + ) + + const ipcPort = await new Promise((resolveIpc) => { + ipcServer.listen(0, server.hostname, () => { + const addr = ipcServer.address() + + if (addr && typeof addr === 'object') { + resolveIpc(addr.port) + } + }) + }) + + return { + ipcPort, + ipcServer, + } +} + +export const createWorker = ( + serverPort: number, + ipcPort: number, + isNodeDebugging: boolean | 'brk' | undefined, + type: string +) => { + const { initialEnv } = require('@next/env') as typeof import('@next/env') + const { Worker } = require('next/dist/compiled/jest-worker') + const worker = new Worker(require.resolve('./render-server'), { + numWorkers: 1, + // TODO: do we want to allow more than 10 OOM restarts? + maxRetries: 10, + forkOptions: { + env: { + FORCE_COLOR: '1', + ...initialEnv, + // we don't pass down NODE_OPTIONS as it can + // extra memory usage + NODE_OPTIONS: getNodeOptionsWithoutInspect() + .replace(/--max-old-space-size=[\d]{1,}/, '') + .trim(), + __NEXT_PRIVATE_RENDER_WORKER: type, + __NEXT_PRIVATE_ROUTER_IPC_PORT: ipcPort + '', + NODE_ENV: process.env.NODE_ENV, + }, + execArgv: genExecArgv( + isNodeDebugging === undefined ? false : isNodeDebugging, + (serverPort || 0) + 1 + ), + }, + exposedMethods: ['initialize', 'deleteCache', 'deleteAppClientCache'], + }) as any as InstanceType & { + initialize: typeof import('./render-server').initialize + deleteCache: typeof import('./render-server').deleteCache + deleteAppClientCache: typeof import('./render-server').deleteAppClientCache + } + + worker.getStderr().pipe(process.stderr) + worker.getStdout().pipe(process.stdout) + + return worker +} diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 406bbab19f2a..5adabdda0388 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -258,106 +258,35 @@ export default class NextNodeServer extends BaseServer { hostname: this.hostname, dev: !!options.dev, } + const { createWorker, createIpcServer } = + require('./lib/server-ipc') as typeof import('./lib/server-ipc') this.renderWorkersPromises = new Promise(async (resolveWorkers) => { - // we can't use process.send as jest-worker relies on - // it already and can cause unexpected message errors - const ipcServer = ( - require('http') as typeof import('http') - ).createServer(async (req, res) => { - try { - const url = new URL(req.url || '/', 'http://n') - const method = url.searchParams.get('method') - const args: any[] = JSON.parse(url.searchParams.get('args') || '[]') - - if (!method || !Array.isArray(args)) { - return res.end() - } - - if (typeof (this as any)[method] === 'function') { - if (method === 'logErrorWithOriginalStack' && args[0]?.stack) { - args[0] = deserializeErr(args[0]) - } - let result = await (this as any)[method](...args) - - if (result && typeof result === 'object' && result.stack) { - result = errorToJSON(result) - } - res.end(JSON.stringify(result || '')) - } - } catch (err: any) { - console.error(err) - res.end( - JSON.stringify({ - err: { name: err.name, message: err.message, stack: err.stack }, - }) + try { + this.renderWorkers = {} + const { ipcPort } = await createIpcServer(this) + if (this.hasAppDir) { + this.renderWorkers.app = createWorker( + this.port || 0, + ipcPort, + options.isNodeDebugging, + 'app' ) } - }) - - const ipcPort = await new Promise((resolveIpc) => { - ipcServer.listen(0, this.hostname, () => { - const addr = ipcServer.address() - - if (addr && typeof addr === 'object') { - resolveIpc(addr.port) - } - }) - }) - - const createWorker = (type: string) => { - const { initialEnv } = - require('@next/env') as typeof import('@next/env') - const { Worker } = require('next/dist/compiled/jest-worker') - const worker = new Worker(require.resolve('./lib/render-server'), { - numWorkers: 1, - // TODO: do we want to allow more than 10 OOM restarts? - maxRetries: 10, - forkOptions: { - env: { - FORCE_COLOR: '1', - ...initialEnv, - // we don't pass down NODE_OPTIONS as it can - // extra memory usage - NODE_OPTIONS: getNodeOptionsWithoutInspect() - .replace(/--max-old-space-size=[\d]{1,}/, '') - .trim(), - __NEXT_PRIVATE_RENDER_WORKER: type, - __NEXT_PRIVATE_ROUTER_IPC_PORT: ipcPort + '', - NODE_ENV: process.env.NODE_ENV, - }, - execArgv: genExecArgv( - options.isNodeDebugging === undefined - ? false - : options.isNodeDebugging, - (this.port || 0) + 1 - ), - }, - exposedMethods: [ - 'initialize', - 'deleteCache', - 'deleteAppClientCache', - ], - }) as any as InstanceType & { - initialize: typeof import('./lib/render-server').initialize - deleteCache: typeof import('./lib/render-server').deleteCache - deleteAppClientCache: typeof import('./lib/render-server').deleteAppClientCache - } - - worker.getStderr().pipe(process.stderr) - worker.getStdout().pipe(process.stdout) - - return worker - } - this.renderWorkers = {} + this.renderWorkers.pages = createWorker( + this.port || 0, + ipcPort, + options.isNodeDebugging, + 'pages' + ) + this.renderWorkers.middleware = + this.renderWorkers.pages || this.renderWorkers.app - if (this.hasAppDir) { - this.renderWorkers.app = createWorker('app') + resolveWorkers() + } catch (err) { + Log.error(`Invariant failed to initialize render workers`) + console.error(err) + process.exit(1) } - this.renderWorkers.pages = createWorker('pages') - this.renderWorkers.middleware = - this.renderWorkers.pages || this.renderWorkers.app - - resolveWorkers() }) ;(global as any)._nextDeleteCache = (filePath: string) => { this.renderWorkers?.pages?.deleteCache(filePath) From 94ca08e7b535c6d94732940f1361328f35843df2 Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Sat, 1 Apr 2023 21:54:50 -0700 Subject: [PATCH 52/52] fix lint --- packages/next/src/server/next-server.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index 096efbee0194..f27f94998d6a 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -55,7 +55,7 @@ import { sendRenderResult } from './send-payload' import { getExtension, serveStatic } from './serve-static' import { ParsedUrlQuery } from 'querystring' import { apiResolver } from './api-utils/node' -import { deserializeErr, errorToJSON, RenderOpts, renderToHTML } from './render' +import { RenderOpts, renderToHTML } from './render' import { ParsedUrl, parseUrl } from '../shared/lib/router/utils/parse-url' import { parse as nodeParseUrl } from 'url' import * as Log from '../build/output/log' @@ -98,7 +98,6 @@ import { INSTRUMENTATION_HOOK_FILENAME } from '../lib/constants' import { getTracer } from './lib/trace/tracer' import { NextNodeServerSpan } from './lib/trace/constants' import { nodeFs } from './lib/node-fs-methods' -import { genExecArgv, getNodeOptionsWithoutInspect } from './lib/utils' import { getRouteRegex } from '../shared/lib/router/utils/route-regex' import { removePathPrefix } from '../shared/lib/router/utils/remove-path-prefix' import { addPathPrefix } from '../shared/lib/router/utils/add-path-prefix'