From 0f9e1bfdc2d228c02690f9d858e8f8cce8d93264 Mon Sep 17 00:00:00 2001 From: patak Date: Tue, 5 Dec 2023 14:34:47 +0100 Subject: [PATCH] perf: in-memory public files check (#15195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 翠 / green --- packages/vite/src/node/plugins/asset.ts | 27 +-------- packages/vite/src/node/plugins/css.ts | 2 +- packages/vite/src/node/plugins/html.ts | 2 +- .../vite/src/node/plugins/importAnalysis.ts | 3 +- packages/vite/src/node/publicDir.ts | 56 +++++++++++++++++++ packages/vite/src/node/server/index.ts | 14 ++++- .../src/node/server/middlewares/indexHtml.ts | 2 +- .../src/node/server/middlewares/static.ts | 29 +++++++--- .../vite/src/node/server/transformRequest.ts | 2 +- packages/vite/src/node/utils.ts | 24 ++++++++ 10 files changed, 121 insertions(+), 40 deletions(-) create mode 100644 packages/vite/src/node/publicDir.ts diff --git a/packages/vite/src/node/plugins/asset.ts b/packages/vite/src/node/plugins/asset.ts index 6b9f90fc77155c..4cdd846e4cab42 100644 --- a/packages/vite/src/node/plugins/asset.ts +++ b/packages/vite/src/node/plugins/asset.ts @@ -1,6 +1,5 @@ import path from 'node:path' import { parse as parseUrl } from 'node:url' -import fs from 'node:fs' import fsp from 'node:fs/promises' import { Buffer } from 'node:buffer' import * as mrmime from 'mrmime' @@ -17,6 +16,7 @@ import { } from '../build' import type { Plugin } from '../plugin' import type { ResolvedConfig } from '../config' +import { checkPublicFile } from '../publicDir' import { cleanUrl, getHash, @@ -249,31 +249,6 @@ export function assetPlugin(config: ResolvedConfig): Plugin { } } -export function checkPublicFile( - url: string, - { publicDir }: ResolvedConfig, -): string | undefined { - // note if the file is in /public, the resolver would have returned it - // as-is so it's not going to be a fully resolved path. - if (!publicDir || url[0] !== '/') { - return - } - const publicFile = path.join(publicDir, cleanUrl(url)) - if ( - !normalizePath(publicFile).startsWith( - withTrailingSlash(normalizePath(publicDir)), - ) - ) { - // can happen if URL starts with '../' - return - } - if (fs.existsSync(publicFile)) { - return publicFile - } else { - return - } -} - export async function fileToUrl( id: string, config: ResolvedConfig, diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts index 905f0b4025cb91..8b0463828e22e4 100644 --- a/packages/vite/src/node/plugins/css.ts +++ b/packages/vite/src/node/plugins/css.ts @@ -36,6 +36,7 @@ import { } from '../constants' import type { ResolvedConfig } from '../config' import type { Plugin } from '../plugin' +import { checkPublicFile } from '../publicDir' import { arraify, asyncReplace, @@ -62,7 +63,6 @@ import type { Logger } from '../logger' import { addToHTMLProxyTransformResult } from './html' import { assetUrlRE, - checkPublicFile, fileToUrl, generatedAssets, publicAssetUrlCache, diff --git a/packages/vite/src/node/plugins/html.ts b/packages/vite/src/node/plugins/html.ts index add3c14735a3ec..8440abe4221c18 100644 --- a/packages/vite/src/node/plugins/html.ts +++ b/packages/vite/src/node/plugins/html.ts @@ -24,12 +24,12 @@ import { urlCanParse, } from '../utils' import type { ResolvedConfig } from '../config' +import { checkPublicFile } from '../publicDir' import { toOutputFilePathInHtml } from '../build' import { resolveEnvPrefix } from '../env' import type { Logger } from '../logger' import { assetUrlRE, - checkPublicFile, getPublicAssetFilename, publicAssetUrlRE, urlToBuiltUrl, diff --git a/packages/vite/src/node/plugins/importAnalysis.ts b/packages/vite/src/node/plugins/importAnalysis.ts index 7a04387b7ca9c1..89b7238ddf197a 100644 --- a/packages/vite/src/node/plugins/importAnalysis.ts +++ b/packages/vite/src/node/plugins/importAnalysis.ts @@ -52,12 +52,13 @@ import { withTrailingSlash, wrapId, } from '../utils' +import { checkPublicFile } from '../publicDir' import { getDepOptimizationConfig } from '../config' import type { ResolvedConfig } from '../config' import type { Plugin } from '../plugin' import { shouldExternalizeForSSR } from '../ssr/ssrExternal' import { getDepsOptimizer, optimizedDepNeedsInterop } from '../optimizer' -import { checkPublicFile, urlRE } from './asset' +import { urlRE } from './asset' import { throwOutdatedRequest } from './optimizedDeps' import { isCSSRequest, isDirectCSSRequest } from './css' import { browserExternalId } from './resolve' diff --git a/packages/vite/src/node/publicDir.ts b/packages/vite/src/node/publicDir.ts new file mode 100644 index 00000000000000..f2055fff8afb9d --- /dev/null +++ b/packages/vite/src/node/publicDir.ts @@ -0,0 +1,56 @@ +import fs from 'node:fs' +import path from 'node:path' +import type { ResolvedConfig } from './config' +import { + cleanUrl, + normalizePath, + recursiveReaddir, + withTrailingSlash, +} from './utils' + +const publicFilesMap = new WeakMap>() + +export async function initPublicFiles( + config: ResolvedConfig, +): Promise> { + const fileNames = await recursiveReaddir(config.publicDir) + const publicFiles = new Set( + fileNames.map((fileName) => fileName.slice(config.publicDir.length)), + ) + publicFilesMap.set(config, publicFiles) + return publicFiles +} + +function getPublicFiles(config: ResolvedConfig): Set | undefined { + return publicFilesMap.get(config) +} + +export function checkPublicFile( + url: string, + config: ResolvedConfig, +): string | undefined { + // note if the file is in /public, the resolver would have returned it + // as-is so it's not going to be a fully resolved path. + const { publicDir } = config + if (!publicDir || url[0] !== '/') { + return + } + + const fileName = cleanUrl(url) + + // short-circuit if we have an in-memory publicFiles cache + const publicFiles = getPublicFiles(config) + if (publicFiles) { + return publicFiles.has(fileName) + ? normalizePath(path.join(publicDir, fileName)) + : undefined + } + + const publicFile = normalizePath(path.join(publicDir, fileName)) + if (!publicFile.startsWith(withTrailingSlash(normalizePath(publicDir)))) { + // can happen if URL starts with '../' + return + } + + return fs.existsSync(publicFile) ? publicFile : undefined +} diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts index 7aeb6cb9065593..2b799dbfd2b8e7 100644 --- a/packages/vite/src/node/server/index.ts +++ b/packages/vite/src/node/server/index.ts @@ -50,6 +50,7 @@ import { CLIENT_DIR, DEFAULT_DEV_PORT } from '../constants' import type { Logger } from '../logger' import { printServerUrls } from '../logger' import { createNoopWatcher, resolveChokidarOptions } from '../watch' +import { initPublicFiles } from '../publicDir' import type { PluginContainer } from './pluginContainer' import { ERR_CLOSED_SERVER, createPluginContainer } from './pluginContainer' import type { WebSocketServer } from './ws' @@ -378,6 +379,8 @@ export async function _createServer( ): Promise { const config = await resolveConfig(inlineConfig, 'serve') + const initPublicFilesPromise = initPublicFiles(config) + const { root, server: serverConfig } = config const httpsOptions = await resolveHttpsConfig(config.server.https) const { middlewareMode } = serverConfig @@ -623,6 +626,8 @@ export async function _createServer( } } + const publicFiles = await initPublicFilesPromise + const onHMRUpdate = async (file: string, configOnly: boolean) => { if (serverConfig.hmr !== false) { try { @@ -639,6 +644,12 @@ export async function _createServer( const onFileAddUnlink = async (file: string, isUnlink: boolean) => { file = normalizePath(file) await container.watchChange(file, { event: isUnlink ? 'delete' : 'create' }) + + if (config.publicDir && file.startsWith(config.publicDir)) { + publicFiles[isUnlink ? 'delete' : 'add']( + file.slice(config.publicDir.length), + ) + } await handleFileAddUnlink(file, server, isUnlink) await onHMRUpdate(file, true) } @@ -648,7 +659,6 @@ export async function _createServer( await container.watchChange(file, { event: 'update' }) // invalidate module graph cache on file change moduleGraph.onFileChange(file) - await onHMRUpdate(file, false) }) @@ -733,7 +743,7 @@ export async function _createServer( // this applies before the transform middleware so that these files are served // as-is without transforms. if (config.publicDir) { - middlewares.use(servePublicMiddleware(server)) + middlewares.use(servePublicMiddleware(server, publicFiles)) } // main transform middleware diff --git a/packages/vite/src/node/server/middlewares/indexHtml.ts b/packages/vite/src/node/server/middlewares/indexHtml.ts index c225c18051f414..54351a937c8e1f 100644 --- a/packages/vite/src/node/server/middlewares/indexHtml.ts +++ b/packages/vite/src/node/server/middlewares/indexHtml.ts @@ -41,8 +41,8 @@ import { unwrapId, wrapId, } from '../../utils' +import { checkPublicFile } from '../../publicDir' import { isCSSRequest } from '../../plugins/css' -import { checkPublicFile } from '../../plugins/asset' import { getCodeWithSourcemap, injectSourcesContent } from '../sourcemap' interface AssetNode { diff --git a/packages/vite/src/node/server/middlewares/static.ts b/packages/vite/src/node/server/middlewares/static.ts index 0260af1d7109c5..57f995d8e1b0f8 100644 --- a/packages/vite/src/node/server/middlewares/static.ts +++ b/packages/vite/src/node/server/middlewares/static.ts @@ -16,8 +16,8 @@ import { isParentDirectory, isSameFileUri, isWindows, + normalizePath, removeLeadingSlash, - shouldServeFile, slash, withTrailingSlash, } from '../../utils' @@ -26,10 +26,8 @@ const knownJavascriptExtensionRE = /\.[tj]sx?$/ const sirvOptions = ({ getHeaders, - shouldServe, }: { getHeaders: () => OutgoingHttpHeaders | undefined - shouldServe?: (p: string) => void }): Options => { return { dev: true, @@ -51,26 +49,43 @@ const sirvOptions = ({ } } }, - shouldServe, } } export function servePublicMiddleware( server: ViteDevServer, + publicFiles: Set, ): Connect.NextHandleFunction { const dir = server.config.publicDir const serve = sirv( dir, sirvOptions({ getHeaders: () => server.config.server.headers, - shouldServe: (filePath) => shouldServeFile(filePath, dir), }), ) + const toFilePath = (url: string) => { + let filePath = cleanUrl(url) + if (filePath.indexOf('%') !== -1) { + try { + filePath = decodeURI(filePath) + } catch (err) { + /* malform uri */ + } + } + return normalizePath(filePath) + } + // Keep the named function. The name is visible in debug logs via `DEBUG=connect:dispatcher ...` return function viteServePublicMiddleware(req, res, next) { - // skip import request and internal requests `/@fs/ /@vite-client` etc... - if (isImportRequest(req.url!) || isInternalRequest(req.url!)) { + // To avoid the performance impact of `existsSync` on every request, we check against an + // in-memory set of known public files. This set is updated on restarts. + // also skip import request and internal requests `/@fs/ /@vite-client` etc... + if ( + !publicFiles.has(toFilePath(req.url!)) || + isImportRequest(req.url!) || + isInternalRequest(req.url!) + ) { return next() } serve(req, res, next) diff --git a/packages/vite/src/node/server/transformRequest.ts b/packages/vite/src/node/server/transformRequest.ts index 0881354a6ec4e5..a37fdbe83fff7e 100644 --- a/packages/vite/src/node/server/transformRequest.ts +++ b/packages/vite/src/node/server/transformRequest.ts @@ -22,7 +22,7 @@ import { timeFrom, unwrapId, } from '../utils' -import { checkPublicFile } from '../plugins/asset' +import { checkPublicFile } from '../publicDir' import { getDepsOptimizer } from '../optimizer' import { applySourcemapIgnoreList, injectSourcesContent } from './sourcemap' import { isFileServingAllowed } from './middlewares/static' diff --git a/packages/vite/src/node/utils.ts b/packages/vite/src/node/utils.ts index 65264b14ccf4c4..395568978662ce 100644 --- a/packages/vite/src/node/utils.ts +++ b/packages/vite/src/node/utils.ts @@ -8,6 +8,7 @@ import { builtinModules, createRequire } from 'node:module' import { promises as dns } from 'node:dns' import { performance } from 'node:perf_hooks' import type { AddressInfo, Server } from 'node:net' +import fsp from 'node:fs/promises' import type { FSWatcher } from 'chokidar' import remapping from '@ampproject/remapping' import type { DecodedSourceMap, RawSourceMap } from '@ampproject/remapping' @@ -622,6 +623,29 @@ export function copyDir(srcDir: string, destDir: string): void { } } +export async function recursiveReaddir(dir: string): Promise { + if (!fs.existsSync(dir)) { + return [] + } + let dirents: fs.Dirent[] + try { + dirents = await fsp.readdir(dir, { withFileTypes: true }) + } catch (e) { + if (e.code === 'EACCES') { + // Ignore permission errors + return [] + } + throw e + } + const files = await Promise.all( + dirents.map((dirent) => { + const res = path.resolve(dir, dirent.name) + return dirent.isDirectory() ? recursiveReaddir(res) : normalizePath(res) + }), + ) + return files.flat(1) +} + // `fs.realpathSync.native` resolves differently in Windows network drive, // causing file read errors. skip for now. // https://github.com/nodejs/node/issues/37737