diff --git a/jest.config.js b/jest.config.js
index 65c89e59fb18f6..56f9d45ff5a0aa 100644
--- a/jest.config.js
+++ b/jest.config.js
@@ -1,6 +1,8 @@
 module.exports = {
   preset: 'ts-jest',
-  testMatch: ['**/*.spec.[jt]s?(x)'],
+  testMatch: process.env.VITE_TEST_BUILD
+    ? ['**/playground/**/*.spec.[jt]s?(x)']
+    : ['**/*.spec.[jt]s?(x)'],
   testTimeout: process.env.CI ? 30000 : 10000,
   globalSetup: './scripts/jestGlobalSetup.js',
   globalTeardown: './scripts/jestGlobalTeardown.js',
diff --git a/package.json b/package.json
index e36a2f2131965d..1b0437597fa0e6 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
     "test": "run-s test-serve test-build",
     "test-serve": "jest",
     "debug-serve": "cross-env VITE_DEBUG_SERVE=1 node --inspect-brk ./node_modules/.bin/jest",
-    "test-build": "cross-env VITE_TEST_BUILD=1 jest playground",
+    "test-build": "cross-env VITE_TEST_BUILD=1 jest",
     "debug-build": "cross-env VITE_TEST_BUILD=1 VITE_PRESERVE_BUILD_ARTIFACTS=1 node --inspect-brk ./node_modules/.bin/jest",
     "docs": "vitepress dev docs",
     "build-docs": "vitepress build docs",
diff --git a/packages/plugin-legacy/index.js b/packages/plugin-legacy/index.js
index 4a3c90a47c4f2e..6be196f756faf6 100644
--- a/packages/plugin-legacy/index.js
+++ b/packages/plugin-legacy/index.js
@@ -418,7 +418,8 @@ async function buildPolyfillChunk(
           [name]: polyfillId
         },
         output: {
-          format: name.includes('legacy') ? 'iife' : 'es'
+          format: name.includes('legacy') ? 'iife' : 'es',
+          manualChunks: undefined
         }
       }
     }
diff --git a/packages/vite/src/node/build.ts b/packages/vite/src/node/build.ts
index a2856d1dbb57d0..2385ee60718764 100644
--- a/packages/vite/src/node/build.ts
+++ b/packages/vite/src/node/build.ts
@@ -10,7 +10,9 @@ import Rollup, {
   WarningHandler,
   OutputOptions,
   RollupOutput,
-  ExternalOption
+  ExternalOption,
+  GetManualChunk,
+  GetModuleInfo
 } from 'rollup'
 import { buildReporterPlugin } from './plugins/reporter'
 import { buildDefinePlugin } from './plugins/define'
@@ -338,8 +340,8 @@ async function doBuild(
     const generate = (output: OutputOptions = {}) => {
       return bundle[options.write ? 'write' : 'generate']({
         dir: outDir,
-        format: options.ssr ? 'cjs' : 'es',
-        exports: options.ssr ? 'named' : 'auto',
+        format: ssr ? 'cjs' : 'es',
+        exports: ssr ? 'named' : 'auto',
         sourcemap: options.sourcemap,
         name: libOptions ? libOptions.name : undefined,
         entryFileNames: ssr
@@ -359,6 +361,13 @@ async function doBuild(
         // #1048 add `Symbol.toStringTag` for module default export
         namespaceToStringTag: true,
         inlineDynamicImports: ssr && typeof input === 'string',
+        manualChunks:
+          !ssr &&
+          !libOptions &&
+          output?.format !== 'umd' &&
+          output?.format !== 'iife'
+            ? moveToVendorChunk
+            : undefined,
         ...output
       })
     }
@@ -418,6 +427,33 @@ async function doBuild(
   }
 }
 
+const moveToVendorChunk: GetManualChunk = (id, { getModuleInfo }) => {
+  if (id.includes('node_modules') && !hasDynamicImporter(id, getModuleInfo)) {
+    return 'vendor'
+  }
+}
+
+function hasDynamicImporter(
+  id: string,
+  getModuleInfo: GetModuleInfo,
+  importStack: string[] = []
+): boolean {
+  if (importStack.includes(id)) {
+    // circular deps!
+    return false
+  }
+  const mod = getModuleInfo(id)
+  if (!mod) {
+    return false
+  }
+  if (mod.dynamicImporters.length) {
+    return true
+  }
+  return mod.importers.some((importer) =>
+    hasDynamicImporter(importer, getModuleInfo, importStack.concat(id))
+  )
+}
+
 function resolveBuildOutputs(
   outputs: OutputOptions | OutputOptions[] | undefined,
   libOptions: LibraryOptions | false,