Update parallelizing tasks with webpackBuildWorker config

packages/next/src/build/build-context.ts

@@ -5,7 +5,6 @@ import type { __ApiPreviewProps } from '../server/api-utils'
 import type { NextConfigComplete } from '../server/config-shared'
 import type { Span } from '../trace'
 import type getBaseWebpackConfig from './webpack-config'
-import type { PagesManifest } from './webpack/plugins/pages-manifest-plugin'
 import type { TelemetryPlugin } from './webpack/plugins/telemetry-plugin'
 
 // A layer for storing data that is used by plugins to communicate with each
@@ -48,12 +47,6 @@ export function getPluginState() {
 export const NextBuildContext: Partial<{
   compilerIdx?: number
   pluginState: Record<string, any>
-  serializedPagesManifestEntries: {
-    edgeServerPages?: PagesManifest
-    nodeServerPages?: PagesManifest
-    edgeServerAppPaths?: PagesManifest
-    nodeServerAppPaths?: PagesManifest
-  }
   // core fields
   dir: string
   buildId: string

packages/next/src/build/collect-build-traces.ts

@@ -1,4 +1,4 @@
-import type { Span } from '../trace'
+import { Span } from '../trace'
 import type { NextConfigComplete } from '../server/config-shared'
 
 import {
@@ -18,20 +18,23 @@ import { PageInfo } from './utils'
 import { loadBindings } from './swc'
 import { nonNullable } from '../lib/non-nullable'
 import * as ciEnvironment from '../telemetry/ci-info'
+import debugOriginal from 'next/dist/compiled/debug'
 import { isMatch } from 'next/dist/compiled/micromatch'
 import { defaultOverrides } from '../server/require-hook'
 import { nodeFileTrace } from 'next/dist/compiled/@vercel/nft'
 import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
 import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
 
+const debug = debugOriginal('next:build:build-traces')
+
 export async function collectBuildTraces({
   dir,
   config,
   distDir,
   pageKeys,
   pageInfos,
   staticPages,
-  nextBuildSpan,
+  nextBuildSpan = new Span({ name: 'build' }),
   hasSsrAmpPages,
   buildTraceContext,
   outputFileTracingRoot,
@@ -42,14 +45,16 @@ export async function collectBuildTraces({
     app?: string[]
     pages: string[]
   }
-  staticPages: Set<string>
+  staticPages: string[]
   hasSsrAmpPages: boolean
   outputFileTracingRoot: string
-  pageInfos: Map<string, PageInfo>
-  nextBuildSpan: Span
+  pageInfos: [string, PageInfo][]
+  nextBuildSpan?: Span
   config: NextConfigComplete
   buildTraceContext?: BuildTraceContext
 }) {
+  const startTime = Date.now()
+  debug('starting build traces')
   let turboTasksForTrace: unknown
   let bindings = await loadBindings()
 
@@ -99,7 +104,7 @@ export async function collectBuildTraces({
           // The turbo trace doesn't provide the traced file type and reason at present
           // let's write the traced files into the first [entry].nft.json
           const [[, entryName]] = Array.from<[string, string]>(
-            entryNameMap.entries()
+            Object.entries(entryNameMap)
           ).filter(([k]) => k.startsWith(buildTraceContextAppDir))
           const traceOutputPath = path.join(
             outputPath,
@@ -119,7 +124,7 @@ export async function collectBuildTraces({
           const outputPagesPath = path.join(outputPath, '..', 'pages')
           return (
             !f.startsWith(outputPagesPath) ||
-            !staticPages.has(
+            !staticPages.includes(
               // strip `outputPagesPath` and file ext from absolute
               f.substring(outputPagesPath.length, f.length - 3)
             )
@@ -369,10 +374,13 @@ export async function collectBuildTraces({
           }
         }
 
+        const { entryNameFilesMap } = buildTraceContext?.chunksTrace || {}
+
         await Promise.all(
           [
-            ...(buildTraceContext?.chunksTrace?.entryNameFilesMap.entries() ||
-              new Map()),
+            ...(entryNameFilesMap
+              ? Object.entries(entryNameFilesMap)
+              : new Map()),
           ].map(async ([entryName, entryNameFiles]) => {
             const isApp = entryName.startsWith('app/')
             const isPages = entryName.startsWith('pages/')
@@ -387,7 +395,7 @@ export async function collectBuildTraces({
 
             // we don't need to trace for automatically statically optimized
             // pages as they don't have server bundles
-            if (staticPages.has(route)) {
+            if (staticPages.includes(route)) {
               return
             }
             const entryOutputPath = path.join(
@@ -504,7 +512,7 @@ export async function collectBuildTraces({
 
     for (let page of pageKeys.pages) {
       // edge routes have no trace files
-      const pageInfo = pageInfos.get(page)
+      const [, pageInfo] = pageInfos.find((item) => item[0] === page) || []
       if (pageInfo?.runtime === 'edge') {
         continue
       }
@@ -584,4 +592,6 @@ export async function collectBuildTraces({
       )
     }
   })
+
+  debug(`finished build tracing ${Date.now() - startTime}ms`)
 }

packages/next/src/build/index.ts

@@ -1018,20 +1018,72 @@ export default async function build(
         return { duration, buildTraceContext: null }
       }
       let buildTraceContext: undefined | BuildTraceContext
+      let buildTracesPromise: Promise<any> | undefined = undefined
 
       if (!isGenerate) {
-        const { duration: webpackBuildDuration, ...rest } = turboNextBuild
-          ? await turbopackBuild()
-          : await webpackBuild()
+        if (isCompile && config.experimental.webpackBuildWorker) {
+          let durationInSeconds = 0
+
+          await webpackBuild(['server']).then((res) => {
+            buildTraceContext = res.buildTraceContext
+            durationInSeconds += res.duration
+            const buildTraceWorker = new Worker(
+              require.resolve('./collect-build-traces'),
+              {
+                numWorkers: 1,
+                exposedMethods: ['collectBuildTraces'],
+              }
+            ) as Worker & typeof import('./collect-build-traces')
 
-        buildTraceContext = rest.buildTraceContext
+            buildTracesPromise = buildTraceWorker
+              .collectBuildTraces({
+                dir,
+                config,
+                distDir,
+                pageKeys,
+                pageInfos: [],
+                staticPages: [],
+                hasSsrAmpPages: false,
+                buildTraceContext,
+                outputFileTracingRoot,
+              })
+              .catch((err) => {
+                console.error(err)
+                process.exit(1)
+              })
+          })
 
-        telemetry.record(
-          eventBuildCompleted(pagesPaths, {
-            durationInSeconds: webpackBuildDuration,
-            totalAppPagesCount,
+          await webpackBuild(['edge-server']).then((res) => {
+            durationInSeconds += res.duration
           })
-        )
+
+          await webpackBuild(['client']).then((res) => {
+            durationInSeconds += res.duration
+          })
+
+          buildSpinner?.stopAndPersist()
+          Log.event('Compiled successfully')
+
+          telemetry.record(
+            eventBuildCompleted(pagesPaths, {
+              durationInSeconds,
+              totalAppPagesCount,
+            })
+          )
+        } else {
+          const { duration: webpackBuildDuration, ...rest } = turboNextBuild
+            ? await turbopackBuild()
+            : await webpackBuild()
+
+          buildTraceContext = rest.buildTraceContext
+
+          telemetry.record(
+            eventBuildCompleted(pagesPaths, {
+              durationInSeconds: webpackBuildDuration,
+              totalAppPagesCount,
+            })
+          )
+        }
       }
 
       // For app directory, we run type checking after build.
@@ -1761,16 +1813,14 @@ export default async function build(
         )
       }
 
-      let buildTracesPromise: Promise<any> | undefined = undefined
-
-      if (!isGenerate && config.outputFileTracing) {
+      if (!isGenerate && config.outputFileTracing && !buildTracesPromise) {
         buildTracesPromise = collectBuildTraces({
           dir,
           config,
           distDir,
           pageKeys,
-          pageInfos,
-          staticPages,
+          pageInfos: Object.entries(pageInfos),
+          staticPages: [...staticPages],
           nextBuildSpan,
           hasSsrAmpPages,
           buildTraceContext,

packages/next/src/build/webpack-build/impl.ts

@@ -28,7 +28,6 @@ import {
   TraceEntryPointsPlugin,
 } from '../webpack/plugins/next-trace-entrypoints-plugin'
 import { UnwrapPromise } from '../../lib/coalesced-function'
-import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin'
 
 import origDebug from 'next/dist/compiled/debug'
 
@@ -62,7 +61,6 @@ export async function webpackBuildImpl(
   duration: number
   pluginState: any
   buildTraceContext?: BuildTraceContext
-  serializedPagesManifestEntries?: (typeof NextBuildContext)['serializedPagesManifestEntries']
 }> {
   let result: CompilerResult | null = {
     warnings: [],
@@ -327,12 +325,6 @@ export async function webpackBuildImpl(
       duration: webpackBuildEnd[0],
       buildTraceContext: traceEntryPointsPlugin?.buildTraceContext,
       pluginState: getPluginState(),
-      serializedPagesManifestEntries: {
-        edgeServerPages: pagesPluginModule.edgeServerPages,
-        edgeServerAppPaths: pagesPluginModule.edgeServerAppPaths,
-        nodeServerPages: pagesPluginModule.nodeServerPages,
-        nodeServerAppPaths: pagesPluginModule.nodeServerAppPaths,
-      },
     }
   }
 }
@@ -348,16 +340,6 @@ export async function workerMain(workerData: {
   // Resume plugin state
   resumePluginState(NextBuildContext.pluginState)
 
-  // restore module scope maps for flight plugins
-  const { serializedPagesManifestEntries } = NextBuildContext
-
-  for (const key of Object.keys(serializedPagesManifestEntries || {})) {
-    Object.assign(
-      (pagesPluginModule as any)[key],
-      (serializedPagesManifestEntries as any)?.[key]
-    )
-  }
-
   /// load the config because it's not serializable
   NextBuildContext.config = await loadConfig(
     PHASE_PRODUCTION_BUILD,
@@ -373,17 +355,12 @@ export async function workerMain(workerData: {
       result.buildTraceContext!.entriesTrace!.depModArray = depModArray
     }
     if (entryNameMap) {
-      const entryEntries = Array.from(entryNameMap?.entries() ?? [])
-      // @ts-expect-error
-      result.buildTraceContext.entriesTrace.entryNameMap = entryEntries
+      const entryEntries = entryNameMap
+      result.buildTraceContext!.entriesTrace!.entryNameMap = entryEntries
     }
   }
   if (chunksTrace?.entryNameFilesMap) {
-    const entryNameFilesMap = Array.from(
-      chunksTrace.entryNameFilesMap.entries() ?? []
-    )
-
-    // @ts-expect-error
+    const entryNameFilesMap = chunksTrace.entryNameFilesMap
     result.buildTraceContext!.chunksTrace!.entryNameFilesMap = entryNameFilesMap
   }
   return result