From d0f8b1248890c1eea0b81b1d6f61a81d3dc6e44a Mon Sep 17 00:00:00 2001 From: Evan You Date: Wed, 27 Jan 2021 01:00:57 -0500 Subject: [PATCH] feat: use esbuild to scan imports --- packages/vite/src/node/logger.ts | 1 + .../src/node/optimizer/esbuildDepPlugin.ts | 15 +- packages/vite/src/node/optimizer/index.ts | 255 +----------------- packages/vite/src/node/optimizer/scan.ts | 232 ++++++++++++++++ packages/vite/src/node/plugins/html.ts | 2 +- packages/vite/src/node/plugins/resolve.ts | 49 +--- packages/vite/src/node/server/hmr.ts | 32 +-- 7 files changed, 252 insertions(+), 334 deletions(-) create mode 100644 packages/vite/src/node/optimizer/scan.ts diff --git a/packages/vite/src/node/logger.ts b/packages/vite/src/node/logger.ts index 1ced609135c252..98913cb28829fc 100644 --- a/packages/vite/src/node/logger.ts +++ b/packages/vite/src/node/logger.ts @@ -85,6 +85,7 @@ export function createLogger( output('warn', msg, opts) }, error(msg, opts) { + logger.hasWarned = true output('error', msg, opts) }, clearScreen(type) { diff --git a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts index 0f236a5a43ca38..849d50be46c108 100644 --- a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts +++ b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts @@ -1,9 +1,9 @@ import path from 'path' import { Plugin } from 'esbuild' import { knownAssetTypes } from '../constants' -import { ResolvedConfig, ResolveFn } from '..' +import { ResolvedConfig } from '..' import chalk from 'chalk' -import { deepImportRE, isBuiltin, isRunningWithYarnPnp } from '../utils' +import { isBuiltin, isRunningWithYarnPnp } from '../utils' const externalTypes = [ 'css', @@ -13,6 +13,7 @@ const externalTypes = [ 'scss', 'style', 'stylus', + 'postcss', // known SFC types 'vue', 'svelte', @@ -21,10 +22,10 @@ const externalTypes = [ export function esbuildDepPlugin( qualified: Record, - config: ResolvedConfig, - transitiveOptimized: Record, - resolve: ResolveFn + config: ResolvedConfig ): Plugin { + const resolve = config.createResolver({ asSrc: false }) + return { name: 'vite:dep-pre-bundle', setup(build) { @@ -60,10 +61,6 @@ export function esbuildDepPlugin( path: path.resolve(qualified[id]) } } else if (!isBuiltin(id)) { - // record transitive deps - const deepMatch = id.match(deepImportRE) - const pkgId = deepMatch ? deepMatch[1] || deepMatch[2] : id - transitiveOptimized[pkgId] = true // use vite resolver const resolved = await resolve(id, importer) if (resolved) { diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 28d4b6db2686cc..f216d23f2e45b0 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -3,58 +3,29 @@ import path from 'path' import chalk from 'chalk' import { createHash } from 'crypto' import { ResolvedConfig } from '../config' -import { SUPPORTED_EXTS } from '../constants' import { createDebugger, emptyDir, lookupFile, normalizePath, - resolveFrom, writeFile } from '../utils' -import { createFilter } from '@rollup/pluginutils' -import { prompt } from 'enquirer' import { build } from 'esbuild' import { esbuildDepPlugin } from './esbuildDepPlugin' import { init, parse } from 'es-module-lexer' -import { ResolveFn } from '..' +import { scanImports } from './scan' const debug = createDebugger('vite:optimize') -const KNOWN_WARN_LIST = new Set([ - 'vite', - 'vitepress', - 'tailwindcss', - 'sass', - 'less', - 'stylus', - 'postcss', - 'autoprefixer', - 'pug', - 'jest', - 'typescript' -]) - -const WARN_RE = /^(@vitejs\/|@rollup\/|vite-|rollup-|postcss-|babel-)plugin-|^@babel\/|^@tailwindcss\// - export interface DepOptimizationOptions { /** * Force optimize listed dependencies (supports deep paths). */ - include?: string[] + include?: string | RegExp | (string | RegExp)[] /** * Do not optimize these dependencies. */ exclude?: string | RegExp | (string | RegExp)[] - /** - * A list of linked dependencies that should be treated as source code. - */ - link?: string[] - /** - * Automatically run `vite optimize` on server start? - * @default true - */ - auto?: boolean } export interface DepOptimizationMetadata { @@ -66,8 +37,6 @@ export interface DepOptimizationMetadata { needsInterop: boolean } > - transitiveOptimized: Record - dependencies: Record } export async function optimizeDeps( @@ -94,9 +63,7 @@ export async function optimizeDeps( const data: DepOptimizationMetadata = { hash: getDepHash(root, pkg, config), - optimized: {}, - transitiveOptimized: {}, - dependencies: pkg.dependencies + optimized: {} } if (!force) { @@ -117,88 +84,8 @@ export async function optimizeDeps( fs.mkdirSync(cacheDir, { recursive: true }) } - const options = config.optimizeDeps || {} - const resolve = config.createResolver({ asSrc: false }) - - // Determine deps to optimize. The goal is to only pre-bundle deps that falls - // under one of the following categories: - // 1. Has imports to relative files (e.g. lodash-es, lit-html) - // 2. Has imports to bare modules that are not in the project's own deps - // (i.e. esm that imports its own dependencies, e.g. styled-components) - const { qualified, external } = await resolveQualifiedDeps( - root, - config, - resolve - ) - - // Resolve deps from linked packages in a monorepo - if (options.link) { - for (const linkedDep of options.link) { - await resolveLinkedDeps( - config.root, - linkedDep, - qualified, - external, - config, - resolve - ) - } - } - - // Force included deps - these can also be deep paths - if (options.include) { - for (let id of options.include) { - const filePath = await resolve(id) - if (filePath) { - qualified[id] = filePath - } - } - } - - let qualifiedIds = Object.keys(qualified) - const invalidIds = qualifiedIds.filter( - (id) => KNOWN_WARN_LIST.has(id) || WARN_RE.test(id) - ) - - if (invalidIds.length) { - const msg = - `It seems your dependencies contain packages that are not meant to\n` + - `be used in the browser, e.g. ${chalk.cyan(invalidIds.join(', '))}. ` + - `\nSince vite pre-bundles eligible dependencies to improve performance,\n` + - `they should probably be moved to devDependencies instead.` - - if (process.env.CI) { - logger.error(msg) - process.exit(1) - } - - const { yes } = (await prompt({ - type: 'confirm', - name: 'yes', - initial: true, - message: chalk.yellow( - msg + `\nAuto-update package.json and continue without these deps?` - ) - })) as { yes: boolean } - if (yes) { - invalidIds.forEach((id) => { - delete qualified[id] - }) - qualifiedIds = qualifiedIds.filter((id) => !invalidIds.includes(id)) - const pkgPath = lookupFile(root, ['package.json'], true)! - const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')) - invalidIds.forEach((id) => { - const v = pkg.dependencies[id] - delete pkg.dependencies[id] - ;(pkg.devDependencies || (pkg.devDependencies = {}))[id] = v - }) - fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2)) - // udpate data hash - data.hash = getDepHash(root, pkg, config) - } else { - process.exit(1) - } - } + const qualified = await scanImports(config) + const qualifiedIds = Object.keys(qualified) if (!qualifiedIds.length) { writeFile(dataPath, JSON.stringify(data, null, 2)) @@ -210,12 +97,9 @@ export async function optimizeDeps( if (!asCommand) { // This is auto run on server start - let the user know that we are // pre-optimizing deps + logger.info(chalk.greenBright(`Pre-bundling dependencies:\n${depsString}`)) logger.info( - chalk.greenBright(`Optimizable dependencies detected:\n${depsString}`) - ) - logger.info( - `Pre-bundling them to speed up dev server page load...\n` + - `(this will be run only when your dependencies or config have changed)` + `(this will be run only when your dependencies or config have changed)` ) } else { logger.info(chalk.greenBright(`Optimizing dependencies:\n${depsString}`)) @@ -227,7 +111,6 @@ export async function optimizeDeps( entryPoints: Object.values(qualified).map((p) => path.resolve(p)), bundle: true, format: 'esm', - external, logLevel: 'error', splitting: true, sourcemap: true, @@ -237,9 +120,7 @@ export async function optimizeDeps( define: { 'process.env.NODE_ENV': '"development"' }, - plugins: [ - esbuildDepPlugin(qualified, config, data.transitiveOptimized, resolve) - ] + plugins: [esbuildDepPlugin(qualified, config)] }) const meta = JSON.parse(fs.readFileSync(esbuildMetaPath, 'utf-8')) @@ -311,123 +192,6 @@ function isSingleDefaultExport(exports: string[]) { return exports.length === 1 && exports[0] === 'default' } -interface FilteredDeps { - qualified: Record - external: string[] -} - -async function resolveQualifiedDeps( - root: string, - config: ResolvedConfig, - resolve: ResolveFn -): Promise { - const { include, exclude, link } = config.optimizeDeps || {} - const qualified: Record = {} - const external: string[] = [] - - const pkgContent = lookupFile(root, ['package.json']) - if (!pkgContent) { - return { - qualified, - external - } - } - - const pkg = JSON.parse(pkgContent) - const deps = Object.keys(pkg.dependencies || {}) - const linked: string[] = [] - const excludeFilter = - exclude && createFilter(null, exclude, { resolve: false }) - - for (const id of deps) { - if (include && include.includes(id)) { - // already force included - continue - } - if (excludeFilter && !excludeFilter(id)) { - debug(`skipping ${id} (excluded)`) - continue - } - if (link && link.includes(id)) { - debug(`skipping ${id} (link)`) - continue - } - // #804 - if (id.startsWith('@types/')) { - debug(`skipping ${id} (ts declaration)`) - continue - } - let filePath - try { - filePath = await resolve(id) - } catch (e) {} - if (!filePath) { - debug(`skipping ${id} (cannot resolve entry)`) - continue - } - if (!filePath.includes('node_modules')) { - debug(`skipping ${id} (not a node_modules dep, likely linked)`) - // resolve deps of the linked module - linked.push(id) - continue - } - if (!SUPPORTED_EXTS.includes(path.extname(filePath))) { - debug(`skipping ${id} (entry is not js)`) - continue - } - // qualified! - qualified[id] = filePath - } - - // mark non-optimized deps as external - external.push( - ...(await Promise.all( - deps - .filter((id) => !qualified[id]) - // make sure aliased deps are external - // https://github.com/vitejs/vite-plugin-react/issues/4 - .map(async (id) => (await resolve(id, undefined, true)) || id) - )) - ) - - if (linked.length) { - for (const dep of linked) { - await resolveLinkedDeps(root, dep, qualified, external, config, resolve) - } - } - - return { - qualified, - external - } -} - -async function resolveLinkedDeps( - root: string, - dep: string, - qualified: Record, - external: string[], - config: ResolvedConfig, - resolve: ResolveFn -) { - const depRoot = path.dirname(resolveFrom(`${dep}/package.json`, root)) - const { qualified: q, external: e } = await resolveQualifiedDeps( - depRoot, - config, - resolve - ) - Object.keys(q).forEach((id) => { - if (!qualified[id]) { - qualified[id] = q[id] - } - }) - e.forEach((id) => { - if (!external.includes(id)) { - external.push(id) - } - }) -} - const lockfileFormats = ['package-lock.json', 'yarn.lock', 'pnpm-lock.yaml'] let cachedHash: string | undefined @@ -453,8 +217,7 @@ function getDepHash( assetsInclude: config.assetsInclude, optimizeDeps: { include: config.optimizeDeps?.include, - exclude: config.optimizeDeps?.exclude, - link: config.optimizeDeps?.link + exclude: config.optimizeDeps?.exclude } }, (_, value) => { diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts new file mode 100644 index 00000000000000..5931a4bb6f00b8 --- /dev/null +++ b/packages/vite/src/node/optimizer/scan.ts @@ -0,0 +1,232 @@ +import fs from 'fs' +import path from 'path' +import glob from 'fast-glob' +import { ResolvedConfig } from '..' +import { build, Loader, Plugin } from 'esbuild' +import { knownAssetTypes } from '../constants' +import { emptyDir, isDataUrl, isExternalUrl } from '../utils' +import { browserExternalId } from '../plugins/resolve' +import { createFilter } from '@rollup/pluginutils' +import { isCSSRequest } from '../plugins/css' +import chalk from 'chalk' +import { + createPluginContainer, + PluginContainer +} from '../server/pluginContainer' + +export async function scanImports( + config: ResolvedConfig +): Promise> { + const htmlEntries = await glob('**/index.html', { + cwd: config.root, + ignore: ['**/node_modules/**', `**/${config.build.outDir}/**`], + absolute: true + }) + + const tempDir = path.join(config.optimizeCacheDir!, 'scan') + const depImports: Record = {} + const missingImports = new Set() + const plugin = esbuildScanPlugin(config, depImports, missingImports) + + await Promise.all( + htmlEntries.map((entry) => + build({ + entryPoints: [entry], + bundle: true, + format: 'esm', + logLevel: 'error', + outdir: tempDir, + outbase: config.root, + plugins: [plugin] + }) + ) + ) + + emptyDir(tempDir) + fs.rmdirSync(tempDir) + + if (missingImports.size) { + config.logger.error( + `The following dependencies are imported but couldn't be resolved: ${[ + ...missingImports + ].join(', ')}` + ) + } + + return depImports +} + +const scriptModuleRE = /(]*type\s*=\s*(?:"module"|'module')[^>]*>)(.*?)<\/script>/gims +const scriptRE = /(]*>)(.*?)<\/script>/gims +const srcRE = /\bsrc\s*=\s*(?:"([^"]+)"|'([^']+)'|([^\s'">]+))/im +const langRE = /\blang\s*=\s*(?:"([^"]+)"|'([^']+)'|([^\s'">]+))/im + +function esbuildScanPlugin( + config: ResolvedConfig, + depImports: Record, + missingImports: Set +): Plugin { + let container: PluginContainer + + const seen = new Map() + + const resolve = async (id: string, importer?: string) => { + const key = id + importer + if (seen.has(key)) { + return seen.get(key) + } + container = container || (container = await createPluginContainer(config)) + const resolved = await container.resolveId(id, importer) + const res = resolved?.id + seen.set(key, res) + return res + } + + const include = + config.optimizeDeps?.include && + createFilter(config.optimizeDeps.include, null, { resolve: false }) + + const exclude = + config.optimizeDeps?.exclude && + createFilter(config.optimizeDeps.exclude, null, { resolve: false }) + + return { + name: 'vite:dep-scan', + setup(build) { + // bare imports: record and externalize + build.onResolve( + { + filter: /^[\w@]/ + }, + async ({ path: id, importer }) => { + if (depImports[id]) { + return { + path: id, + external: true + } + } + + if ( + isExternalUrl(id) || + isDataUrl(id) || + isCSSRequest(id) || + config.assetsInclude(id) + ) { + return { + path: id, + external: true + } + } + + const resolved = await resolve(id, importer) + if (resolved) { + // virtual id or browser external + if (id === resolved || resolved.startsWith(browserExternalId)) { + return { path: id, external: true } + } + // dep or force included, externalize and stop crawling + if (resolved.includes('node_modules') || (include && include(id))) { + if (!(exclude && exclude(id))) { + depImports[id] = resolved + } + return { + path: id, + external: true + } + } else { + // linked package, keep crawling + return { + path: resolved + } + } + } else { + config.logger.error( + chalk.red( + `Dependency ${id} not found. Is it installed? (imported by ${importer})` + ) + ) + missingImports.add(id) + } + } + ) + + const htmlTypesRe = /\.(html|vue|svelte)$/ + // html types: extract script contents + build.onResolve({ filter: htmlTypesRe }, async ({ path, importer }) => { + return { + path: await resolve(path, importer), + namespace: 'html' + } + }) + build.onLoad( + { filter: htmlTypesRe, namespace: 'html' }, + async ({ path }) => { + const raw = await fs.promises.readFile(path, 'utf-8') + const regex = path.endsWith('.html') ? scriptModuleRE : scriptRE + regex.lastIndex = 0 + let js = '' + let loader: Loader = 'js' + for (const [_, openTag, content] of raw.matchAll(regex)) { + const srcMatch = openTag.match(srcRE) + const langMatch = openTag.match(langRE) + const lang = + langMatch && (langMatch[1] || langMatch[2] || langMatch[3]) + if (lang === 'ts') { + loader = 'ts' + } + if (srcMatch) { + js += `import ${JSON.stringify( + srcMatch[1] || srcMatch[2] || srcMatch[3] + )}\n` + } else if (content.trim()) { + js += content + '\n' + } + } + js += `export default {}` + return { + loader, + contents: js + } + } + ) + + // css: externalize + build.onResolve( + { + filter: /\.(css|less|sass|scss|styl|stylus|postcss)$/ + }, + ({ path }) => ({ path, external: true }) + ) + + // known asset types: externalize + build.onResolve( + { + filter: new RegExp(`\\.(${knownAssetTypes.join('|')})$`) + }, + ({ path }) => ({ path, external: true }) + ) + + // catch all + build.onResolve( + { + filter: /.*/ + }, + async ({ path, importer }) => { + // use vite resolver to support urls + const id = await resolve(path, importer) + if (id && id !== path && !id.includes(`?worker`)) { + return { + path: id + } + } else { + // resolve failed... probably usupported type + return { + path, + external: true + } + } + } + ) + } + } +} diff --git a/packages/vite/src/node/plugins/html.ts b/packages/vite/src/node/plugins/html.ts index 3d557e7c866d18..d5a8394e67037b 100644 --- a/packages/vite/src/node/plugins/html.ts +++ b/packages/vite/src/node/plugins/html.ts @@ -21,7 +21,7 @@ const htmlProxyRE = /\?html-proxy&index=(\d+)\.js$/ export const isHTMLProxy = (id: string) => htmlProxyRE.test(id) const htmlCommentRE = //g -const scriptModuleRE = /(]*type\s*=\s*(?:"module"|'module')[^>]*>)([\s\S]*?)<\/script>/gm +const scriptModuleRE = /(]*type\s*=\s*(?:"module"|'module')[^>]*>)(.*?)<\/script>/gims export function htmlInlineScriptProxyPlugin(): Plugin { return { diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts index 6be4e2481bdac2..691101e13085f1 100644 --- a/packages/vite/src/node/plugins/resolve.ts +++ b/packages/vite/src/node/plugins/resolve.ts @@ -16,14 +16,12 @@ import { ensureVolumeInPath, resolveFrom, isDataUrl, - cleanUrl, - isJSRequest + cleanUrl } from '../utils' import { ViteDevServer } from '..' import slash from 'slash' import { createFilter } from '@rollup/pluginutils' import { PartialResolvedId } from 'rollup' -import { isCSSRequest } from './css' import { resolve as _resolveExports } from 'resolve.exports' const altMainFields = [ @@ -45,7 +43,7 @@ function resolveExports( // special id for paths marked with browser: false // https://github.com/defunctzombie/package-browser-field-spec#ignore-a-module -const browserExternalId = '__vite-browser-external' +export const browserExternalId = '__vite-browser-external' const isDebug = process.env.DEBUG const debug = createDebugger('vite:resolve-details', { @@ -321,49 +319,6 @@ export function tryNodeResolve( return } - // prevent deep imports to optimized deps. - if (server && server._optimizeDepsMetadata) { - const data = server._optimizeDepsMetadata - if ( - deepMatch && - pkg.data.name in data.optimized && - !isCSSRequest(id) && - !server.config.assetsInclude(id) - ) { - throw new Error( - chalk.yellow( - `Deep import "${chalk.cyan( - id - )}" should be avoided because dependency "${chalk.cyan( - pkg.data.name - )}" has been pre-optimized. Prefer importing directly from the module entry:\n\n` + - `${chalk.green(`import { ... } from "${pkg.data.name}"`)}\n\n` + - `If the used import is not exported from the package's main entry ` + - `and can only be attained via deep import, you can explicitly add ` + - `the deep import path to "optimizeDeps.include" in vite.config.js.\n\n` + - `If you intend to only use deep imports with this package and it ` + - `exposes valid ESM, consider adding it to "optimizeDeps.exclude".` - ) - ) - } - - if ( - pkgId in data.transitiveOptimized && - isJSRequest(id) && - basedir.startsWith(server.config.root) && - !basedir.includes('node_modules') - ) { - throw new Error( - chalk.yellow( - `dependency "${chalk.bold(pkgId)}" is imported in source code, ` + - `but was transitively pre-bundled as part of another package. ` + - `It should be explicitly listed as a dependency in package.json in ` + - `order to avoid duplicated instances of this module.` - ) - ) - } - } - let resolved = deepMatch ? resolveDeepImport(id, pkg, isProduction) : resolvePackageEntry(id, pkg, isProduction) diff --git a/packages/vite/src/node/server/hmr.ts b/packages/vite/src/node/server/hmr.ts index b2b29ea4af8c56..1c833d94034f2e 100644 --- a/packages/vite/src/node/server/hmr.ts +++ b/packages/vite/src/node/server/hmr.ts @@ -1,7 +1,7 @@ import fs from 'fs' import path from 'path' import { createServer, ViteDevServer } from '..' -import { createDebugger, lookupFile, normalizePath } from '../utils' +import { createDebugger, normalizePath } from '../utils' import { ModuleNode } from './moduleGraph' import chalk from 'chalk' import slash from 'slash' @@ -53,24 +53,6 @@ export async function handleHMRUpdate( return } - if ( - file.endsWith('package.json') && - file === - normalizePath(lookupFile(config.root, [`package.json`], true) || '') - ) { - const deps = require(file).dependencies || {} - const prevDeps = server._optimizeDepsMetadata?.dependencies || {} - // check if deps have changed - if (hasDepsChanged(deps, prevDeps)) { - config.logger.info( - chalk.green('dependencies have changed, restarting server...'), - { clear: true, timestamp: true } - ) - await restartServer(server) - } - return - } - debugHmr(`[file change] ${chalk.dim(shortFile)}`) // (dev only) the client itself cannot be hot updated. @@ -426,18 +408,6 @@ async function readModifiedFile(file: string): Promise { } } -function hasDepsChanged(deps: any, prevDeps: any): boolean { - if (Object.keys(deps).length !== Object.keys(prevDeps).length) { - return true - } - for (const key in deps) { - if (deps[key] !== prevDeps[key]) { - return true - } - } - return false -} - async function restartServer(server: ViteDevServer) { await server.close() ;(global as any).__vite_start_time = Date.now()