Skip to content

Commit

Permalink
Merge branch 'canary' into jankaifer/fix-duplicated-otel-spans
Browse files Browse the repository at this point in the history
  • Loading branch information
jankaifer committed Apr 3, 2023
2 parents f0560a9 + da37c01 commit bbbbd6c
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 133 deletions.
18 changes: 17 additions & 1 deletion packages/next/src/build/index.ts
Expand Up @@ -11,8 +11,10 @@ import chalk from 'next/dist/compiled/chalk'
import crypto from 'crypto'
import { isMatch, makeRe } from 'next/dist/compiled/micromatch'
import { promises, writeFileSync } from 'fs'
import os from 'os'
import { Worker as JestWorker } from 'next/dist/compiled/jest-worker'
import { Worker } from '../lib/worker'
import { defaultConfig } from '../server/config-shared'
import devalue from 'next/dist/compiled/devalue'
import { escapeStringRegexp } from '../shared/lib/escape-regexp'
import findUp from 'next/dist/compiled/find-up'
Expand Down Expand Up @@ -1239,6 +1241,20 @@ export default async function build(

process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD

// We limit the number of workers used based on the number of CPUs and
// the current available memory. This is to prevent the system from
// running out of memory as well as maximize speed. We assume that
// each worker will consume ~1GB of memory in a production build.
// For example, if the system has 10 CPU cores and 8GB of remaining memory
// we will use 8 workers.
const numWorkers =
config.experimental.cpus !== defaultConfig.experimental!.cpus
? config.experimental.cpus
: Math.min(
config.experimental.cpus || 1,
Math.floor(os.freemem() / 1e9)
)

const staticWorkers = new Worker(staticWorker, {
timeout: timeout * 1000,
onRestart: (method, [arg], attempts) => {
Expand Down Expand Up @@ -1270,7 +1286,7 @@ export default async function build(
infoPrinted = true
}
},
numWorkers: config.experimental.cpus,
numWorkers,
enableWorkerThreads: config.experimental.workerThreads,
computeWorkerKey(method, ...args) {
if (method === 'exportPage') {
Expand Down
@@ -1,37 +1,36 @@
import type { webpack } from 'next/dist/compiled/webpack/webpack'
import chalk from 'next/dist/compiled/chalk'
import formatWebpackMessages from '../client/dev/error-overlay/format-webpack-messages'
import { nonNullable } from '../lib/non-nullable'
import formatWebpackMessages from '../../client/dev/error-overlay/format-webpack-messages'
import { nonNullable } from '../../lib/non-nullable'
import {
COMPILER_NAMES,
CLIENT_STATIC_FILES_RUNTIME_MAIN_APP,
APP_CLIENT_INTERNALS,
PHASE_PRODUCTION_BUILD,
COMPILER_INDEXES,
} from '../shared/lib/constants'
import { runCompiler } from './compiler'
import * as Log from './output/log'
import getBaseWebpackConfig, { loadProjectInfo } from './webpack-config'
import { NextError } from '../lib/is-error'
import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin'
} from '../../shared/lib/constants'
import { runCompiler } from '../compiler'
import * as Log from '../output/log'
import getBaseWebpackConfig, { loadProjectInfo } from '../webpack-config'
import { NextError } from '../../lib/is-error'
import { TelemetryPlugin } from '../webpack/plugins/telemetry-plugin'
import {
NextBuildContext,
resumePluginState,
getPluginState,
} from './build-context'
import { createEntrypoints } from './entries'
import loadConfig from '../server/config'
import { trace } from '../trace'
import { WEBPACK_LAYERS } from '../lib/constants'
} from '../build-context'
import { createEntrypoints } from '../entries'
import loadConfig from '../../server/config'
import { trace } from '../../trace'
import { WEBPACK_LAYERS } from '../../lib/constants'
import {
TraceEntryPointsPlugin,
TurbotraceContext,
} from './webpack/plugins/next-trace-entrypoints-plugin'
import { UnwrapPromise } from '../lib/coalesced-function'
import * as pagesPluginModule from './webpack/plugins/pages-manifest-plugin'
import { Worker } from 'next/dist/compiled/jest-worker'
} from '../webpack/plugins/next-trace-entrypoints-plugin'
import { UnwrapPromise } from '../../lib/coalesced-function'
import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin'

import origDebug from 'next/dist/compiled/debug'
import { ChildProcess } from 'child_process'

const debug = origDebug('next:build:webpack-build')

Expand All @@ -57,7 +56,7 @@ function isTraceEntryPointsPlugin(
return plugin instanceof TraceEntryPointsPlugin
}

async function webpackBuildImpl(
export async function webpackBuildImpl(
compilerName?: keyof typeof COMPILER_INDEXES
): Promise<{
duration: number
Expand Down Expand Up @@ -367,108 +366,3 @@ export async function workerMain(workerData: {
}
return result
}

async function webpackBuildWithWorker() {
const {
config,
telemetryPlugin,
buildSpinner,
nextBuildSpan,
...prunedBuildContext
} = NextBuildContext

const getWorker = (compilerName: string) => {
const _worker = new Worker(__filename, {
exposedMethods: ['workerMain'],
numWorkers: 1,
maxRetries: 0,
forkOptions: {
env: {
...process.env,
NEXT_PRIVATE_BUILD_WORKER: '1',
},
},
}) as Worker & { workerMain: typeof workerMain }
_worker.getStderr().pipe(process.stderr)
_worker.getStdout().pipe(process.stdout)

for (const worker of ((_worker as any)._workerPool?._workers || []) as {
_child: ChildProcess
}[]) {
worker._child.on('exit', (code, signal) => {
if (code || signal) {
console.error(
`Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}`
)
}
})
}

return _worker
}

const combinedResult = {
duration: 0,
turbotraceContext: {} as TurbotraceContext,
}
// order matters here
const ORDERED_COMPILER_NAMES = [
'server',
'edge-server',
'client',
] as (keyof typeof COMPILER_INDEXES)[]

for (const compilerName of ORDERED_COMPILER_NAMES) {
const worker = getWorker(compilerName)

const curResult = await worker.workerMain({
buildContext: prunedBuildContext,
compilerName,
})
// destroy worker so it's not sticking around using memory
await worker.end()

// Update plugin state
prunedBuildContext.pluginState = curResult.pluginState

prunedBuildContext.serializedPagesManifestEntries = {
edgeServerAppPaths:
curResult.serializedPagesManifestEntries?.edgeServerAppPaths,
edgeServerPages:
curResult.serializedPagesManifestEntries?.edgeServerPages,
nodeServerAppPaths:
curResult.serializedPagesManifestEntries?.nodeServerAppPaths,
nodeServerPages:
curResult.serializedPagesManifestEntries?.nodeServerPages,
}

combinedResult.duration += curResult.duration

if (curResult.turbotraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext

const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap
)
}
}
}
buildSpinner?.stopAndPersist()
Log.info('Compiled successfully')

return combinedResult
}

export async function webpackBuild() {
const config = NextBuildContext.config!

if (config.experimental.webpackBuildWorker) {
debug('using separate compiler workers')
return await webpackBuildWithWorker()
} else {
debug('building all compilers in same process')
return await webpackBuildImpl()
}
}
116 changes: 116 additions & 0 deletions packages/next/src/build/webpack-build/index.ts
@@ -0,0 +1,116 @@
import { COMPILER_INDEXES } from '../../shared/lib/constants'
import * as Log from '../output/log'
import { NextBuildContext } from '../build-context'
import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
import { Worker } from 'next/dist/compiled/jest-worker'
import origDebug from 'next/dist/compiled/debug'
import { ChildProcess } from 'child_process'
import path from 'path'

const debug = origDebug('next:build:webpack-build')

async function webpackBuildWithWorker() {
const {
config,
telemetryPlugin,
buildSpinner,
nextBuildSpan,
...prunedBuildContext
} = NextBuildContext

const getWorker = (compilerName: string) => {
const _worker = new Worker(path.join(__dirname, 'impl.js'), {
exposedMethods: ['workerMain'],
numWorkers: 1,
maxRetries: 0,
forkOptions: {
env: {
...process.env,
NEXT_PRIVATE_BUILD_WORKER: '1',
},
},
}) as Worker & typeof import('./impl')
_worker.getStderr().pipe(process.stderr)
_worker.getStdout().pipe(process.stdout)

for (const worker of ((_worker as any)._workerPool?._workers || []) as {
_child: ChildProcess
}[]) {
worker._child.on('exit', (code, signal) => {
if (code || signal) {
console.error(
`Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}`
)
}
})
}

return _worker
}

const combinedResult = {
duration: 0,
turbotraceContext: {} as TurbotraceContext,
}
// order matters here
const ORDERED_COMPILER_NAMES = [
'server',
'edge-server',
'client',
] as (keyof typeof COMPILER_INDEXES)[]

for (const compilerName of ORDERED_COMPILER_NAMES) {
const worker = getWorker(compilerName)

const curResult = await worker.workerMain({
buildContext: prunedBuildContext,
compilerName,
})
// destroy worker so it's not sticking around using memory
await worker.end()

// Update plugin state
prunedBuildContext.pluginState = curResult.pluginState

prunedBuildContext.serializedPagesManifestEntries = {
edgeServerAppPaths:
curResult.serializedPagesManifestEntries?.edgeServerAppPaths,
edgeServerPages:
curResult.serializedPagesManifestEntries?.edgeServerPages,
nodeServerAppPaths:
curResult.serializedPagesManifestEntries?.nodeServerAppPaths,
nodeServerPages:
curResult.serializedPagesManifestEntries?.nodeServerPages,
}

combinedResult.duration += curResult.duration

if (curResult.turbotraceContext?.entriesTrace) {
combinedResult.turbotraceContext = curResult.turbotraceContext

const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
if (entryNameMap) {
combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
entryNameMap
)
}
}
}
buildSpinner?.stopAndPersist()
Log.info('Compiled successfully')

return combinedResult
}

export async function webpackBuild() {
const config = NextBuildContext.config!

if (config.experimental.webpackBuildWorker) {
debug('using separate compiler workers')
return await webpackBuildWithWorker()
} else {
debug('building all compilers in same process')
const webpackBuildImpl = require('./impl').webpackBuildImpl
return await webpackBuildImpl()
}
}
10 changes: 5 additions & 5 deletions packages/next/src/lib/metadata/resolvers/resolve-url.ts
@@ -1,5 +1,5 @@
import path from '../../../shared/lib/isomorphic/path'
import { warnOnce } from '../../../shared/lib/utils/warn-once'
import * as Log from '../../../build/output/log'

function isStringOrURL(icon: any): icon is string | URL {
return typeof icon === 'string' || icon instanceof URL
Expand Down Expand Up @@ -28,18 +28,18 @@ function resolveUrl(
if (process.env.NODE_ENV !== 'production') {
metadataBase = new URL(`http://localhost:${process.env.PORT || 3000}`)
// Development mode warning
warnOnce(
`metadata.metadataBase is not set and fallbacks to "${metadataBase.origin}", please specify it in root layout to resolve absolute urls.`
Log.warn(
`metadata.metadataBase is not set for resolving url "${url}", fallbacks to "${metadataBase.origin}". See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase`
)
} else {
throw new Error(
`metadata.metadataBase needs to be provided for resolving absolute URL: ${url}`
`metadata.metadataBase needs to be set for resolving url "${url}". See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase\n`
)
}
}

// Handle relative or absolute paths
const basePath = metadataBase.pathname || '/'
const basePath = metadataBase.pathname || ''
const joinedPath = path.join(basePath, url)

return new URL(joinedPath, metadataBase)
Expand Down
10 changes: 7 additions & 3 deletions test/e2e/app-dir/metadata-missing-metadata-base/index.test.ts
Expand Up @@ -25,17 +25,21 @@ describe('app dir - metadata missing metadataBase', () => {
await next.start()
await fetchViaHTTP(next.url, '/blog')
expect(next.cliOutput).toInclude(
'metadata.metadataBase is not set and fallbacks to "http://localhost:'
'metadata.metadataBase is not set for resolving url "/blog/opengraph-image?'
)
expect(next.cliOutput).toInclude(', fallbacks to "http://localhost:')
expect(next.cliOutput).toInclude(
'please specify it in root layout to resolve absolute urls.'
'. See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase'
)
})
} else {
it('should error in production', async () => {
await expect(next.start()).rejects.toThrow('next build failed')
expect(next.cliOutput).toInclude(
'metadata.metadataBase needs to be provided for resolving absolute URL: /blog/opengraph-image?'
'metadata.metadataBase needs to be set for resolving url "/blog/opengraph-image?'
)
expect(next.cliOutput).toInclude(
'. See https://beta.nextjs.org/docs/api-reference/metadata#metadatabase'
)
})
}
Expand Down

0 comments on commit bbbbd6c

Please sign in to comment.