From 4055bb937bb8182bb834195d778ef245436b1a1e Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 29 Oct 2021 15:57:30 -0500 Subject: [PATCH 1/2] Update output tracing to do separate passes --- .../plugins/next-trace-entrypoints-plugin.ts | 317 +++++++++++------- 1 file changed, 192 insertions(+), 125 deletions(-) diff --git a/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts b/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts index 73ada1b583b5d..d7a85fd56d630 100644 --- a/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts +++ b/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts @@ -24,6 +24,7 @@ const TRACE_IGNORES = [ '**/*/next/dist/server/next.js', '**/*/next/dist/bin/next', ] +const root = nodePath.parse(process.cwd()).root function getModuleFromDependency( compilation: any, @@ -32,6 +33,54 @@ function getModuleFromDependency( return compilation.moduleGraph.getModule(dep) } +function getFilesMapFromReasons( + fileList: Set, + reasons: NodeFileTraceReasons +) { + // this uses the reasons tree to collect files specific to a + // certain parent allowing us to not have to trace each parent + // separately + const parentFilesMap = new Map>() + + function propagateToParents( + parents: Set, + file: string, + seen = new Set() + ) { + for (const parent of parents || []) { + if (!seen.has(parent)) { + seen.add(parent) + let parentFiles = parentFilesMap.get(parent) + + if (!parentFiles) { + parentFiles = new Set() + parentFilesMap.set(parent, parentFiles) + } + parentFiles.add(file) + const parentReason = reasons.get(parent) + + if (parentReason?.parents) { + propagateToParents(parentReason.parents, file, seen) + } + } + } + } + + for (const file of fileList!) { + const reason = reasons!.get(file) + + if ( + !reason || + !reason.parents || + (reason.type === 'initial' && reason.parents.size === 0) + ) { + continue + } + propagateToParents(reason.parents, file) + } + return parentFilesMap +} + export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { private appDir: string private entryTraces: Map> @@ -59,11 +108,20 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { // Here we output all traced assets and webpack chunks to a // ${page}.js.nft.json file - createTraceAssets(compilation: any, assets: any, span: Span) { + async createTraceAssets( + compilation: any, + assets: any, + span: Span, + readlink: any, + stat: any, + doResolve: any + ) { const outputPath = compilation.outputOptions.path - const nodeFileTraceSpan = span.traceChild('create-trace-assets') - nodeFileTraceSpan.traceFn(() => { + await span.traceChild('create-trace-assets').traceAsyncFn(async () => { + const entryFilesMap = new Map>() + const chunksToTrace = new Set() + for (const entrypoint of compilation.entrypoints.values()) { const entryFiles = new Set() @@ -71,24 +129,84 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { .getEntrypointChunk() .getAllReferencedChunks()) { for (const file of chunk.files) { - entryFiles.add(nodePath.join(outputPath, file)) + const filePath = nodePath.join(outputPath, file) + chunksToTrace.add(filePath) + entryFiles.add(filePath) } for (const file of chunk.auxiliaryFiles) { - entryFiles.add(nodePath.join(outputPath, file)) + const filePath = nodePath.join(outputPath, file) + chunksToTrace.add(filePath) + entryFiles.add(filePath) } } - // don't include the entry itself in the trace - entryFiles.delete(nodePath.join(outputPath, `../${entrypoint.name}.js`)) + entryFilesMap.set(entrypoint, entryFiles) + } + + const result = await nodeFileTrace([...chunksToTrace], { + base: root, + processCwd: this.appDir, + readFile: async (path) => { + if (chunksToTrace.has(path)) { + const source = + assets[nodePath.relative(outputPath, path)]?.source?.() + if (source) return source + } + try { + return await new Promise((resolve, reject) => { + ;( + compilation.inputFileSystem + .readFile as typeof import('fs').readFile + )(path, (err, data) => { + if (err) return reject(err) + resolve(data) + }) + }) + } catch (e) { + if (isError(e) && (e.code === 'ENOENT' || e.code === 'EISDIR')) { + return null + } + throw e + } + }, + readlink, + stat, + resolve: doResolve + ? (id, parent, job, isCjs) => { + return doResolve(id, parent, job, !isCjs) + } + : undefined, + ignore: [...TRACE_IGNORES, ...this.excludeFiles], + mixedModules: true, + }) + const reasons = result.reasons + const fileList = result.fileList + result.esmFileList.forEach((file) => fileList.add(file)) + + const parentFilesMap = getFilesMapFromReasons(fileList, reasons) + + for (const [entrypoint, entryFiles] of entryFilesMap) { const traceOutputName = `../${entrypoint.name}.js.nft.json` const traceOutputPath = nodePath.dirname( nodePath.join(outputPath, traceOutputName) ) + const allEntryFiles = new Set() + + entryFiles.forEach((file) => { + parentFilesMap + .get(nodePath.relative(root, file)) + ?.forEach((child) => { + allEntryFiles.add(nodePath.join(root, child)) + }) + }) + // don't include the entry itself in the trace + entryFiles.delete(nodePath.join(outputPath, `../${entrypoint.name}.js`)) assets[traceOutputName] = new sources.RawSource( JSON.stringify({ version: TRACE_OUTPUT_VERSION, files: [ ...entryFiles, + ...allEntryFiles, ...(this.entryTraces.get(entrypoint.name) || []), ].map((file) => { return nodePath @@ -104,12 +222,14 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { tapfinishModules( compilation: webpack5.Compilation, traceEntrypointsPluginSpan: Span, - doResolve?: ( + doResolve: ( request: string, parent: string, job: import('@vercel/nft/out/node-file-trace').Job, isEsmRequested: boolean - ) => Promise + ) => Promise, + readlink: any, + stat: any ) { compilation.hooks.finishModules.tapAsync( PLUGIN_NAME, @@ -168,71 +288,9 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { if (source) { return source.buffer() } - - try { - return await new Promise((resolve, reject) => { - ;( - compilation.inputFileSystem - .readFile as typeof import('fs').readFile - )(path, (err, data) => { - if (err) return reject(err) - resolve(data) - }) - }) - } catch (e) { - if ( - isError(e) && - (e.code === 'ENOENT' || e.code === 'EISDIR') - ) { - return null - } - throw e - } - } - const readlink = async (path: string): Promise => { - try { - return await new Promise((resolve, reject) => { - ;( - compilation.inputFileSystem - .readlink as typeof import('fs').readlink - )(path, (err, link) => { - if (err) return reject(err) - resolve(link) - }) - }) - } catch (e) { - if ( - isError(e) && - (e.code === 'EINVAL' || - e.code === 'ENOENT' || - e.code === 'UNKNOWN') - ) { - return null - } - throw e - } - } - const stat = async ( - path: string - ): Promise => { - try { - return await new Promise((resolve, reject) => { - ;( - compilation.inputFileSystem.stat as typeof import('fs').stat - )(path, (err, stats) => { - if (err) return reject(err) - resolve(stats) - }) - }) - } catch (e) { - if ( - isError(e) && - (e.code === 'ENOENT' || e.code === 'ENOTDIR') - ) { - return null - } - throw e - } + // we don't want to analyze non-transpiled + // files here, that is done against webpack output + return '' } const entryPaths = Array.from(entryModMap.keys()) @@ -262,8 +320,6 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { }) let fileList: Set let reasons: NodeFileTraceReasons - const root = nodePath.parse(process.cwd()).root - await finishModulesSpan .traceChild('node-file-trace', { traceEntryCount: entriesToTrace.length + '', @@ -276,11 +332,15 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { readlink, stat, resolve: doResolve - ? (id, parent, job, isCjs) => - // @ts-ignore - doResolve(id, parent, job, !isCjs) + ? async (id, parent, job, isCjs) => { + return doResolve(id, parent, job, !isCjs) + } : undefined, - ignore: [...TRACE_IGNORES, ...this.excludeFiles], + ignore: [ + ...TRACE_IGNORES, + ...this.excludeFiles, + '**/node_modules/**', + ], mixedModules: true, }) // @ts-ignore @@ -289,50 +349,10 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { reasons = result.reasons }) - // this uses the reasons tree to collect files specific to a certain - // parent allowing us to not have to trace each parent separately - const parentFilesMap = new Map>() - - function propagateToParents( - parents: Set, - file: string, - seen = new Set() - ) { - for (const parent of parents || []) { - if (!seen.has(parent)) { - seen.add(parent) - let parentFiles = parentFilesMap.get(parent) - - if (!parentFiles) { - parentFiles = new Set() - parentFilesMap.set(parent, parentFiles) - } - parentFiles.add(file) - const parentReason = reasons.get(parent) - - if (parentReason?.parents) { - propagateToParents(parentReason.parents, file, seen) - } - } - } - } - await finishModulesSpan .traceChild('collect-traced-files') .traceAsyncFn(() => { - for (const file of fileList!) { - const reason = reasons!.get(file) - - if ( - !reason || - !reason.parents || - (reason.type === 'initial' && reason.parents.size === 0) - ) { - continue - } - propagateToParents(reason.parents, file) - } - + const parentFilesMap = getFilesMapFromReasons(fileList, reasons) entryPaths.forEach((entry) => { const entryName = entryNameMap.get(entry)! const normalizedEntry = nodePath.relative(root, entry) @@ -371,24 +391,69 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { apply(compiler: webpack5.Compiler) { compiler.hooks.compilation.tap(PLUGIN_NAME, (compilation) => { + const readlink = async (path: string): Promise => { + try { + return await new Promise((resolve, reject) => { + ;( + compilation.inputFileSystem + .readlink as typeof import('fs').readlink + )(path, (err, link) => { + if (err) return reject(err) + resolve(link) + }) + }) + } catch (e) { + if ( + isError(e) && + (e.code === 'EINVAL' || e.code === 'ENOENT' || e.code === 'UNKNOWN') + ) { + return null + } + throw e + } + } + const stat = async (path: string): Promise => { + try { + return await new Promise((resolve, reject) => { + ;(compilation.inputFileSystem.stat as typeof import('fs').stat)( + path, + (err, stats) => { + if (err) return reject(err) + resolve(stats) + } + ) + }) + } catch (e) { + if (isError(e) && (e.code === 'ENOENT' || e.code === 'ENOTDIR')) { + return null + } + throw e + } + } + const compilationSpan = spans.get(compilation) || spans.get(compiler)! const traceEntrypointsPluginSpan = compilationSpan.traceChild( 'next-trace-entrypoint-plugin' ) traceEntrypointsPluginSpan.traceFn(() => { // @ts-ignore TODO: Remove ignore when webpack 5 is stable - compilation.hooks.processAssets.tap( + compilation.hooks.processAssets.tapAsync( { name: PLUGIN_NAME, // @ts-ignore TODO: Remove ignore when webpack 5 is stable stage: webpack.Compilation.PROCESS_ASSETS_STAGE_SUMMARIZE, }, - (assets: any) => { + (assets: any, callback: any) => { this.createTraceAssets( compilation, assets, - traceEntrypointsPluginSpan + traceEntrypointsPluginSpan, + readlink, + stat, + doResolve ) + .then(() => callback()) + .catch((err) => callback(err)) } ) let resolver = compilation.resolverFactory.get('normal') @@ -540,7 +605,9 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { this.tapfinishModules( compilation, traceEntrypointsPluginSpan, - doResolve + doResolve, + readlink, + stat ) }) }) From df320e23698d65c361e51aff008fe5c63c938656 Mon Sep 17 00:00:00 2001 From: "jj@jjsweb.site" Date: Fri, 29 Oct 2021 16:40:19 -0500 Subject: [PATCH 2/2] fix windows backslashes --- .../build/webpack/plugins/next-trace-entrypoints-plugin.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts b/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts index d7a85fd56d630..d8fa9ad76053d 100644 --- a/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts +++ b/packages/next/build/webpack/plugins/next-trace-entrypoints-plugin.ts @@ -148,7 +148,9 @@ export class TraceEntryPointsPlugin implements webpack5.WebpackPluginInstance { readFile: async (path) => { if (chunksToTrace.has(path)) { const source = - assets[nodePath.relative(outputPath, path)]?.source?.() + assets[ + nodePath.relative(outputPath, path).replace(/\\/g, '/') + ]?.source?.() if (source) return source } try {