diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 57a3696e1b35..e7ab97e1e9a7 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -11,8 +11,10 @@ import chalk from 'next/dist/compiled/chalk' import crypto from 'crypto' import { isMatch, makeRe } from 'next/dist/compiled/micromatch' import { promises, writeFileSync } from 'fs' +import os from 'os' import { Worker as JestWorker } from 'next/dist/compiled/jest-worker' import { Worker } from '../lib/worker' +import { defaultConfig } from '../server/config-shared' import devalue from 'next/dist/compiled/devalue' import { escapeStringRegexp } from '../shared/lib/escape-regexp' import findUp from 'next/dist/compiled/find-up' @@ -1239,6 +1241,20 @@ export default async function build( process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD + // We limit the number of workers used based on the number of CPUs and + // the current available memory. This is to prevent the system from + // running out of memory as well as maximize speed. We assume that + // each worker will consume ~1GB of memory in a production build. + // For example, if the system has 10 CPU cores and 8GB of remaining memory + // we will use 8 workers. + const numWorkers = + config.experimental.cpus !== defaultConfig.experimental!.cpus + ? config.experimental.cpus + : Math.min( + config.experimental.cpus || 1, + Math.floor(os.freemem() / 1e9) + ) + const staticWorkers = new Worker(staticWorker, { timeout: timeout * 1000, onRestart: (method, [arg], attempts) => { @@ -1270,7 +1286,7 @@ export default async function build( infoPrinted = true } }, - numWorkers: config.experimental.cpus, + numWorkers, enableWorkerThreads: config.experimental.workerThreads, computeWorkerKey(method, ...args) { if (method === 'exportPage') { diff --git a/packages/next/src/build/webpack-build.ts b/packages/next/src/build/webpack-build/impl.ts similarity index 74% rename from packages/next/src/build/webpack-build.ts rename to packages/next/src/build/webpack-build/impl.ts index 93db01857527..2bc6d3a7bb9f 100644 --- a/packages/next/src/build/webpack-build.ts +++ b/packages/next/src/build/webpack-build/impl.ts @@ -1,37 +1,36 @@ import type { webpack } from 'next/dist/compiled/webpack/webpack' import chalk from 'next/dist/compiled/chalk' -import formatWebpackMessages from '../client/dev/error-overlay/format-webpack-messages' -import { nonNullable } from '../lib/non-nullable' +import formatWebpackMessages from '../../client/dev/error-overlay/format-webpack-messages' +import { nonNullable } from '../../lib/non-nullable' import { COMPILER_NAMES, CLIENT_STATIC_FILES_RUNTIME_MAIN_APP, APP_CLIENT_INTERNALS, PHASE_PRODUCTION_BUILD, COMPILER_INDEXES, -} from '../shared/lib/constants' -import { runCompiler } from './compiler' -import * as Log from './output/log' -import getBaseWebpackConfig, { loadProjectInfo } from './webpack-config' -import { NextError } from '../lib/is-error' -import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin' +} from '../../shared/lib/constants' +import { runCompiler } from '../compiler' +import * as Log from '../output/log' +import getBaseWebpackConfig, { loadProjectInfo } from '../webpack-config' +import { NextError } from '../../lib/is-error' +import { TelemetryPlugin } from '../webpack/plugins/telemetry-plugin' import { NextBuildContext, resumePluginState, getPluginState, -} from './build-context' -import { createEntrypoints } from './entries' -import loadConfig from '../server/config' -import { trace } from '../trace' -import { WEBPACK_LAYERS } from '../lib/constants' +} from '../build-context' +import { createEntrypoints } from '../entries' +import loadConfig from '../../server/config' +import { trace } from '../../trace' +import { WEBPACK_LAYERS } from '../../lib/constants' import { TraceEntryPointsPlugin, TurbotraceContext, -} from './webpack/plugins/next-trace-entrypoints-plugin' -import { UnwrapPromise } from '../lib/coalesced-function' -import * as pagesPluginModule from './webpack/plugins/pages-manifest-plugin' -import { Worker } from 'next/dist/compiled/jest-worker' +} from '../webpack/plugins/next-trace-entrypoints-plugin' +import { UnwrapPromise } from '../../lib/coalesced-function' +import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin' + import origDebug from 'next/dist/compiled/debug' -import { ChildProcess } from 'child_process' const debug = origDebug('next:build:webpack-build') @@ -57,7 +56,7 @@ function isTraceEntryPointsPlugin( return plugin instanceof TraceEntryPointsPlugin } -async function webpackBuildImpl( +export async function webpackBuildImpl( compilerName?: keyof typeof COMPILER_INDEXES ): Promise<{ duration: number @@ -367,108 +366,3 @@ export async function workerMain(workerData: { } return result } - -async function webpackBuildWithWorker() { - const { - config, - telemetryPlugin, - buildSpinner, - nextBuildSpan, - ...prunedBuildContext - } = NextBuildContext - - const getWorker = (compilerName: string) => { - const _worker = new Worker(__filename, { - exposedMethods: ['workerMain'], - numWorkers: 1, - maxRetries: 0, - forkOptions: { - env: { - ...process.env, - NEXT_PRIVATE_BUILD_WORKER: '1', - }, - }, - }) as Worker & { workerMain: typeof workerMain } - _worker.getStderr().pipe(process.stderr) - _worker.getStdout().pipe(process.stdout) - - for (const worker of ((_worker as any)._workerPool?._workers || []) as { - _child: ChildProcess - }[]) { - worker._child.on('exit', (code, signal) => { - if (code || signal) { - console.error( - `Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}` - ) - } - }) - } - - return _worker - } - - const combinedResult = { - duration: 0, - turbotraceContext: {} as TurbotraceContext, - } - // order matters here - const ORDERED_COMPILER_NAMES = [ - 'server', - 'edge-server', - 'client', - ] as (keyof typeof COMPILER_INDEXES)[] - - for (const compilerName of ORDERED_COMPILER_NAMES) { - const worker = getWorker(compilerName) - - const curResult = await worker.workerMain({ - buildContext: prunedBuildContext, - compilerName, - }) - // destroy worker so it's not sticking around using memory - await worker.end() - - // Update plugin state - prunedBuildContext.pluginState = curResult.pluginState - - prunedBuildContext.serializedPagesManifestEntries = { - edgeServerAppPaths: - curResult.serializedPagesManifestEntries?.edgeServerAppPaths, - edgeServerPages: - curResult.serializedPagesManifestEntries?.edgeServerPages, - nodeServerAppPaths: - curResult.serializedPagesManifestEntries?.nodeServerAppPaths, - nodeServerPages: - curResult.serializedPagesManifestEntries?.nodeServerPages, - } - - combinedResult.duration += curResult.duration - - if (curResult.turbotraceContext?.entriesTrace) { - combinedResult.turbotraceContext = curResult.turbotraceContext - - const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace! - if (entryNameMap) { - combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map( - entryNameMap - ) - } - } - } - buildSpinner?.stopAndPersist() - Log.info('Compiled successfully') - - return combinedResult -} - -export async function webpackBuild() { - const config = NextBuildContext.config! - - if (config.experimental.webpackBuildWorker) { - debug('using separate compiler workers') - return await webpackBuildWithWorker() - } else { - debug('building all compilers in same process') - return await webpackBuildImpl() - } -} diff --git a/packages/next/src/build/webpack-build/index.ts b/packages/next/src/build/webpack-build/index.ts new file mode 100644 index 000000000000..efbe0fb57587 --- /dev/null +++ b/packages/next/src/build/webpack-build/index.ts @@ -0,0 +1,116 @@ +import { COMPILER_INDEXES } from '../../shared/lib/constants' +import * as Log from '../output/log' +import { NextBuildContext } from '../build-context' +import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin' +import { Worker } from 'next/dist/compiled/jest-worker' +import origDebug from 'next/dist/compiled/debug' +import { ChildProcess } from 'child_process' +import path from 'path' + +const debug = origDebug('next:build:webpack-build') + +async function webpackBuildWithWorker() { + const { + config, + telemetryPlugin, + buildSpinner, + nextBuildSpan, + ...prunedBuildContext + } = NextBuildContext + + const getWorker = (compilerName: string) => { + const _worker = new Worker(path.join(__dirname, 'impl.js'), { + exposedMethods: ['workerMain'], + numWorkers: 1, + maxRetries: 0, + forkOptions: { + env: { + ...process.env, + NEXT_PRIVATE_BUILD_WORKER: '1', + }, + }, + }) as Worker & typeof import('./impl') + _worker.getStderr().pipe(process.stderr) + _worker.getStdout().pipe(process.stdout) + + for (const worker of ((_worker as any)._workerPool?._workers || []) as { + _child: ChildProcess + }[]) { + worker._child.on('exit', (code, signal) => { + if (code || signal) { + console.error( + `Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}` + ) + } + }) + } + + return _worker + } + + const combinedResult = { + duration: 0, + turbotraceContext: {} as TurbotraceContext, + } + // order matters here + const ORDERED_COMPILER_NAMES = [ + 'server', + 'edge-server', + 'client', + ] as (keyof typeof COMPILER_INDEXES)[] + + for (const compilerName of ORDERED_COMPILER_NAMES) { + const worker = getWorker(compilerName) + + const curResult = await worker.workerMain({ + buildContext: prunedBuildContext, + compilerName, + }) + // destroy worker so it's not sticking around using memory + await worker.end() + + // Update plugin state + prunedBuildContext.pluginState = curResult.pluginState + + prunedBuildContext.serializedPagesManifestEntries = { + edgeServerAppPaths: + curResult.serializedPagesManifestEntries?.edgeServerAppPaths, + edgeServerPages: + curResult.serializedPagesManifestEntries?.edgeServerPages, + nodeServerAppPaths: + curResult.serializedPagesManifestEntries?.nodeServerAppPaths, + nodeServerPages: + curResult.serializedPagesManifestEntries?.nodeServerPages, + } + + combinedResult.duration += curResult.duration + + if (curResult.turbotraceContext?.entriesTrace) { + combinedResult.turbotraceContext = curResult.turbotraceContext + + const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace! + if (entryNameMap) { + combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map( + entryNameMap + ) + } + } + } + buildSpinner?.stopAndPersist() + Log.info('Compiled successfully') + + return combinedResult +} + +export async function webpackBuild() { + const config = NextBuildContext.config! + + if (config.experimental.webpackBuildWorker) { + debug('using separate compiler workers') + return await webpackBuildWithWorker() + } else { + debug('building all compilers in same process') + const webpackBuildImpl = require('./impl').webpackBuildImpl + return await webpackBuildImpl() + } +} diff --git a/packages/next/src/lib/metadata/resolvers/resolve-url.ts b/packages/next/src/lib/metadata/resolvers/resolve-url.ts index b9d53e42a226..964a1590f9df 100644 --- a/packages/next/src/lib/metadata/resolvers/resolve-url.ts +++ b/packages/next/src/lib/metadata/resolvers/resolve-url.ts @@ -1,5 +1,5 @@ import path from '../../../shared/lib/isomorphic/path' -import { warnOnce } from '../../../shared/lib/utils/warn-once' +import * as Log from '../../../build/output/log' function isStringOrURL(icon: any): icon is string | URL { return typeof icon === 'string' || icon instanceof URL @@ -28,18 +28,18 @@ function resolveUrl( if (process.env.NODE_ENV !== 'production') { metadataBase = new URL(`http://localhost:${process.env.PORT || 3000}`) // Development mode warning - warnOnce( - `metadata.metadataBase is not set and fallbacks to "${metadataBase.origin}", please specify it in root layout to resolve absolute urls.` + Log.warn( + `metadata.metadataBase is not set for resolving url "${url}", fallbacks to "${metadataBase.origin}". See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase` ) } else { throw new Error( - `metadata.metadataBase needs to be provided for resolving absolute URL: ${url}` + `metadata.metadataBase needs to be set for resolving url "${url}". See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase\n` ) } } // Handle relative or absolute paths - const basePath = metadataBase.pathname || '/' + const basePath = metadataBase.pathname || '' const joinedPath = path.join(basePath, url) return new URL(joinedPath, metadataBase) diff --git a/test/e2e/app-dir/metadata-missing-metadata-base/index.test.ts b/test/e2e/app-dir/metadata-missing-metadata-base/index.test.ts index 3aba77bddba3..9787dbc07d64 100644 --- a/test/e2e/app-dir/metadata-missing-metadata-base/index.test.ts +++ b/test/e2e/app-dir/metadata-missing-metadata-base/index.test.ts @@ -25,17 +25,21 @@ describe('app dir - metadata missing metadataBase', () => { await next.start() await fetchViaHTTP(next.url, '/blog') expect(next.cliOutput).toInclude( - 'metadata.metadataBase is not set and fallbacks to "http://localhost:' + 'metadata.metadataBase is not set for resolving url "/blog/opengraph-image?' ) + expect(next.cliOutput).toInclude(', fallbacks to "http://localhost:') expect(next.cliOutput).toInclude( - 'please specify it in root layout to resolve absolute urls.' + '. See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase' ) }) } else { it('should error in production', async () => { await expect(next.start()).rejects.toThrow('next build failed') expect(next.cliOutput).toInclude( - 'metadata.metadataBase needs to be provided for resolving absolute URL: /blog/opengraph-image?' + 'metadata.metadataBase needs to be set for resolving url "/blog/opengraph-image?' + ) + expect(next.cliOutput).toInclude( + '. See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase' ) }) }