diff --git a/packages/vite/src/node/plugins/asset.ts b/packages/vite/src/node/plugins/asset.ts
index 6b9f90fc77155c..4cdd846e4cab42 100644
--- a/packages/vite/src/node/plugins/asset.ts
+++ b/packages/vite/src/node/plugins/asset.ts
@@ -1,6 +1,5 @@
 import path from 'node:path'
 import { parse as parseUrl } from 'node:url'
-import fs from 'node:fs'
 import fsp from 'node:fs/promises'
 import { Buffer } from 'node:buffer'
 import * as mrmime from 'mrmime'
@@ -17,6 +16,7 @@ import {
 } from '../build'
 import type { Plugin } from '../plugin'
 import type { ResolvedConfig } from '../config'
+import { checkPublicFile } from '../publicDir'
 import {
   cleanUrl,
   getHash,
@@ -249,31 +249,6 @@ export function assetPlugin(config: ResolvedConfig): Plugin {
   }
 }
 
-export function checkPublicFile(
-  url: string,
-  { publicDir }: ResolvedConfig,
-): string | undefined {
-  // note if the file is in /public, the resolver would have returned it
-  // as-is so it's not going to be a fully resolved path.
-  if (!publicDir || url[0] !== '/') {
-    return
-  }
-  const publicFile = path.join(publicDir, cleanUrl(url))
-  if (
-    !normalizePath(publicFile).startsWith(
-      withTrailingSlash(normalizePath(publicDir)),
-    )
-  ) {
-    // can happen if URL starts with '../'
-    return
-  }
-  if (fs.existsSync(publicFile)) {
-    return publicFile
-  } else {
-    return
-  }
-}
-
 export async function fileToUrl(
   id: string,
   config: ResolvedConfig,
diff --git a/packages/vite/src/node/plugins/css.ts b/packages/vite/src/node/plugins/css.ts
index 905f0b4025cb91..8b0463828e22e4 100644
--- a/packages/vite/src/node/plugins/css.ts
+++ b/packages/vite/src/node/plugins/css.ts
@@ -36,6 +36,7 @@ import {
 } from '../constants'
 import type { ResolvedConfig } from '../config'
 import type { Plugin } from '../plugin'
+import { checkPublicFile } from '../publicDir'
 import {
   arraify,
   asyncReplace,
@@ -62,7 +63,6 @@ import type { Logger } from '../logger'
 import { addToHTMLProxyTransformResult } from './html'
 import {
   assetUrlRE,
-  checkPublicFile,
   fileToUrl,
   generatedAssets,
   publicAssetUrlCache,
diff --git a/packages/vite/src/node/plugins/html.ts b/packages/vite/src/node/plugins/html.ts
index ec7132425e1287..80d2a9652cc1c9 100644
--- a/packages/vite/src/node/plugins/html.ts
+++ b/packages/vite/src/node/plugins/html.ts
@@ -24,12 +24,12 @@ import {
   urlCanParse,
 } from '../utils'
 import type { ResolvedConfig } from '../config'
+import { checkPublicFile } from '../publicDir'
 import { toOutputFilePathInHtml } from '../build'
 import { resolveEnvPrefix } from '../env'
 import type { Logger } from '../logger'
 import {
   assetUrlRE,
-  checkPublicFile,
   getPublicAssetFilename,
   publicAssetUrlRE,
   urlToBuiltUrl,
diff --git a/packages/vite/src/node/plugins/importAnalysis.ts b/packages/vite/src/node/plugins/importAnalysis.ts
index 7a04387b7ca9c1..89b7238ddf197a 100644
--- a/packages/vite/src/node/plugins/importAnalysis.ts
+++ b/packages/vite/src/node/plugins/importAnalysis.ts
@@ -52,12 +52,13 @@ import {
   withTrailingSlash,
   wrapId,
 } from '../utils'
+import { checkPublicFile } from '../publicDir'
 import { getDepOptimizationConfig } from '../config'
 import type { ResolvedConfig } from '../config'
 import type { Plugin } from '../plugin'
 import { shouldExternalizeForSSR } from '../ssr/ssrExternal'
 import { getDepsOptimizer, optimizedDepNeedsInterop } from '../optimizer'
-import { checkPublicFile, urlRE } from './asset'
+import { urlRE } from './asset'
 import { throwOutdatedRequest } from './optimizedDeps'
 import { isCSSRequest, isDirectCSSRequest } from './css'
 import { browserExternalId } from './resolve'
diff --git a/packages/vite/src/node/publicDir.ts b/packages/vite/src/node/publicDir.ts
new file mode 100644
index 00000000000000..f2055fff8afb9d
--- /dev/null
+++ b/packages/vite/src/node/publicDir.ts
@@ -0,0 +1,56 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import type { ResolvedConfig } from './config'
+import {
+  cleanUrl,
+  normalizePath,
+  recursiveReaddir,
+  withTrailingSlash,
+} from './utils'
+
+const publicFilesMap = new WeakMap<ResolvedConfig, Set<string>>()
+
+export async function initPublicFiles(
+  config: ResolvedConfig,
+): Promise<Set<string>> {
+  const fileNames = await recursiveReaddir(config.publicDir)
+  const publicFiles = new Set(
+    fileNames.map((fileName) => fileName.slice(config.publicDir.length)),
+  )
+  publicFilesMap.set(config, publicFiles)
+  return publicFiles
+}
+
+function getPublicFiles(config: ResolvedConfig): Set<string> | undefined {
+  return publicFilesMap.get(config)
+}
+
+export function checkPublicFile(
+  url: string,
+  config: ResolvedConfig,
+): string | undefined {
+  // note if the file is in /public, the resolver would have returned it
+  // as-is so it's not going to be a fully resolved path.
+  const { publicDir } = config
+  if (!publicDir || url[0] !== '/') {
+    return
+  }
+
+  const fileName = cleanUrl(url)
+
+  // short-circuit if we have an in-memory publicFiles cache
+  const publicFiles = getPublicFiles(config)
+  if (publicFiles) {
+    return publicFiles.has(fileName)
+      ? normalizePath(path.join(publicDir, fileName))
+      : undefined
+  }
+
+  const publicFile = normalizePath(path.join(publicDir, fileName))
+  if (!publicFile.startsWith(withTrailingSlash(normalizePath(publicDir)))) {
+    // can happen if URL starts with '../'
+    return
+  }
+
+  return fs.existsSync(publicFile) ? publicFile : undefined
+}
diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts
index 7aeb6cb9065593..2b799dbfd2b8e7 100644
--- a/packages/vite/src/node/server/index.ts
+++ b/packages/vite/src/node/server/index.ts
@@ -50,6 +50,7 @@ import { CLIENT_DIR, DEFAULT_DEV_PORT } from '../constants'
 import type { Logger } from '../logger'
 import { printServerUrls } from '../logger'
 import { createNoopWatcher, resolveChokidarOptions } from '../watch'
+import { initPublicFiles } from '../publicDir'
 import type { PluginContainer } from './pluginContainer'
 import { ERR_CLOSED_SERVER, createPluginContainer } from './pluginContainer'
 import type { WebSocketServer } from './ws'
@@ -378,6 +379,8 @@ export async function _createServer(
 ): Promise<ViteDevServer> {
   const config = await resolveConfig(inlineConfig, 'serve')
 
+  const initPublicFilesPromise = initPublicFiles(config)
+
   const { root, server: serverConfig } = config
   const httpsOptions = await resolveHttpsConfig(config.server.https)
   const { middlewareMode } = serverConfig
@@ -623,6 +626,8 @@ export async function _createServer(
     }
   }
 
+  const publicFiles = await initPublicFilesPromise
+
   const onHMRUpdate = async (file: string, configOnly: boolean) => {
     if (serverConfig.hmr !== false) {
       try {
@@ -639,6 +644,12 @@ export async function _createServer(
   const onFileAddUnlink = async (file: string, isUnlink: boolean) => {
     file = normalizePath(file)
     await container.watchChange(file, { event: isUnlink ? 'delete' : 'create' })
+
+    if (config.publicDir && file.startsWith(config.publicDir)) {
+      publicFiles[isUnlink ? 'delete' : 'add'](
+        file.slice(config.publicDir.length),
+      )
+    }
     await handleFileAddUnlink(file, server, isUnlink)
     await onHMRUpdate(file, true)
   }
@@ -648,7 +659,6 @@ export async function _createServer(
     await container.watchChange(file, { event: 'update' })
     // invalidate module graph cache on file change
     moduleGraph.onFileChange(file)
-
     await onHMRUpdate(file, false)
   })
 
@@ -733,7 +743,7 @@ export async function _createServer(
   // this applies before the transform middleware so that these files are served
   // as-is without transforms.
   if (config.publicDir) {
-    middlewares.use(servePublicMiddleware(server))
+    middlewares.use(servePublicMiddleware(server, publicFiles))
   }
 
   // main transform middleware
diff --git a/packages/vite/src/node/server/middlewares/indexHtml.ts b/packages/vite/src/node/server/middlewares/indexHtml.ts
index c225c18051f414..54351a937c8e1f 100644
--- a/packages/vite/src/node/server/middlewares/indexHtml.ts
+++ b/packages/vite/src/node/server/middlewares/indexHtml.ts
@@ -41,8 +41,8 @@ import {
   unwrapId,
   wrapId,
 } from '../../utils'
+import { checkPublicFile } from '../../publicDir'
 import { isCSSRequest } from '../../plugins/css'
-import { checkPublicFile } from '../../plugins/asset'
 import { getCodeWithSourcemap, injectSourcesContent } from '../sourcemap'
 
 interface AssetNode {
diff --git a/packages/vite/src/node/server/middlewares/static.ts b/packages/vite/src/node/server/middlewares/static.ts
index 0260af1d7109c5..57f995d8e1b0f8 100644
--- a/packages/vite/src/node/server/middlewares/static.ts
+++ b/packages/vite/src/node/server/middlewares/static.ts
@@ -16,8 +16,8 @@ import {
   isParentDirectory,
   isSameFileUri,
   isWindows,
+  normalizePath,
   removeLeadingSlash,
-  shouldServeFile,
   slash,
   withTrailingSlash,
 } from '../../utils'
@@ -26,10 +26,8 @@ const knownJavascriptExtensionRE = /\.[tj]sx?$/
 
 const sirvOptions = ({
   getHeaders,
-  shouldServe,
 }: {
   getHeaders: () => OutgoingHttpHeaders | undefined
-  shouldServe?: (p: string) => void
 }): Options => {
   return {
     dev: true,
@@ -51,26 +49,43 @@ const sirvOptions = ({
         }
       }
     },
-    shouldServe,
   }
 }
 
 export function servePublicMiddleware(
   server: ViteDevServer,
+  publicFiles: Set<string>,
 ): Connect.NextHandleFunction {
   const dir = server.config.publicDir
   const serve = sirv(
     dir,
     sirvOptions({
       getHeaders: () => server.config.server.headers,
-      shouldServe: (filePath) => shouldServeFile(filePath, dir),
     }),
   )
 
+  const toFilePath = (url: string) => {
+    let filePath = cleanUrl(url)
+    if (filePath.indexOf('%') !== -1) {
+      try {
+        filePath = decodeURI(filePath)
+      } catch (err) {
+        /* malform uri */
+      }
+    }
+    return normalizePath(filePath)
+  }
+
   // Keep the named function. The name is visible in debug logs via `DEBUG=connect:dispatcher ...`
   return function viteServePublicMiddleware(req, res, next) {
-    // skip import request and internal requests `/@fs/ /@vite-client` etc...
-    if (isImportRequest(req.url!) || isInternalRequest(req.url!)) {
+    // To avoid the performance impact of `existsSync` on every request, we check against an
+    // in-memory set of known public files. This set is updated on restarts.
+    // also skip import request and internal requests `/@fs/ /@vite-client` etc...
+    if (
+      !publicFiles.has(toFilePath(req.url!)) ||
+      isImportRequest(req.url!) ||
+      isInternalRequest(req.url!)
+    ) {
       return next()
     }
     serve(req, res, next)
diff --git a/packages/vite/src/node/server/transformRequest.ts b/packages/vite/src/node/server/transformRequest.ts
index 0881354a6ec4e5..a37fdbe83fff7e 100644
--- a/packages/vite/src/node/server/transformRequest.ts
+++ b/packages/vite/src/node/server/transformRequest.ts
@@ -22,7 +22,7 @@ import {
   timeFrom,
   unwrapId,
 } from '../utils'
-import { checkPublicFile } from '../plugins/asset'
+import { checkPublicFile } from '../publicDir'
 import { getDepsOptimizer } from '../optimizer'
 import { applySourcemapIgnoreList, injectSourcesContent } from './sourcemap'
 import { isFileServingAllowed } from './middlewares/static'
diff --git a/packages/vite/src/node/utils.ts b/packages/vite/src/node/utils.ts
index 65264b14ccf4c4..395568978662ce 100644
--- a/packages/vite/src/node/utils.ts
+++ b/packages/vite/src/node/utils.ts
@@ -8,6 +8,7 @@ import { builtinModules, createRequire } from 'node:module'
 import { promises as dns } from 'node:dns'
 import { performance } from 'node:perf_hooks'
 import type { AddressInfo, Server } from 'node:net'
+import fsp from 'node:fs/promises'
 import type { FSWatcher } from 'chokidar'
 import remapping from '@ampproject/remapping'
 import type { DecodedSourceMap, RawSourceMap } from '@ampproject/remapping'
@@ -622,6 +623,29 @@ export function copyDir(srcDir: string, destDir: string): void {
   }
 }
 
+export async function recursiveReaddir(dir: string): Promise<string[]> {
+  if (!fs.existsSync(dir)) {
+    return []
+  }
+  let dirents: fs.Dirent[]
+  try {
+    dirents = await fsp.readdir(dir, { withFileTypes: true })
+  } catch (e) {
+    if (e.code === 'EACCES') {
+      // Ignore permission errors
+      return []
+    }
+    throw e
+  }
+  const files = await Promise.all(
+    dirents.map((dirent) => {
+      const res = path.resolve(dir, dirent.name)
+      return dirent.isDirectory() ? recursiveReaddir(res) : normalizePath(res)
+    }),
+  )
+  return files.flat(1)
+}
+
 // `fs.realpathSync.native` resolves differently in Windows network drive,
 // causing file read errors. skip for now.
 // https://github.com/nodejs/node/issues/37737