Benchmarks update (#3045)

* Update all bundlers' numbers to the latest test app and methodology * Update the benchmarks blog post following our benchmark investigation and collaboration with Vite.
vercel · Dec 22, 2022 · 03f349e · 03f349e · vercel · Dec 22, 2022
1 parent 62ef6c4
commit 03f349e
Show file tree

Hide file tree

Showing 18 changed files with 1,161 additions and 1,545 deletions.
diff --git a/docs/components/blog/Date.tsx b/docs/components/blog/Date.tsx
@@ -1,9 +1,19 @@
 import { ReactNode } from "react";
 
-function Date({ children }: { children: ReactNode }) {
+function Date({
+  children,
+  update = null,
+}: {
+  children: ReactNode;
+  update?: string;
+}) {
   return (
     <div className="text-sm mt-2 text-center text-gray-500 dark:text-gray-400 font-space-grotesk">
       {children}
+
+      {update != null && (
+        <div className="text-xs mt-1 text-center">Last updated {update}</div>
+      )}
     </div>
   );
 }

diff --git a/docs/components/pages/pack-home/benchmark-data/data.json b/docs/components/pages/pack-home/benchmark-data/data.json
@@ -1,54 +1,54 @@
 {
   "cold": {
     "1000": {
-      "next13": 1.1252525,
-      "vite": 4.76572914,
-      "next12": 3.4092260955,
-      "next11": 7.73633697
+      "next13": 1.38187759,
+      "vite": 4.19890847,
+      "next12": 3.64327949,
+      "next11": 9.19035540
     },
     "5000": {
-      "next13": 3.55898876,
-      "vite": 19.18001276,
-      "next12": 10.66688052,
-      "next11": 24.78618103
+      "next13": 3.99792562,
+      "vite": 16.59615430,
+      "next12": 12.14057345,
+      "next11": 32.89712268
     },
     "10000": {
-      "next13": 7.4692269,
-      "vite": 37.2017471,
-      "next12": 20.05620067,
-      "next11": 49.37418728
+      "next13": 7.34248178,
+      "vite": 32.25177941,
+      "next12": 23.27525035,
+      "next11": 71.80680350
     },
     "30000": {
-      "next13": 22.3009127,
-      "vite": 109.52734199,
-      "next12": 76.55617539,
-      "next11": 176.18900858
+      "next13": 21.97034306,
+      "vite": 97.74466099,
+      "next12": 89.07274544,
+      "next11": 237.61188540
     }
   },
   "file_change": {
     "1000": {
-      "next13": 0.02236675,
-      "vite": 0.10167227,
-      "next12": 0.14834910,
-      "next11": 0.21497278
+      "next13": 0.01890358,
+      "vite": 0.10476515,
+      "next12": 0.14617346,
+      "next11": 0.21155549
     },
     "5000": {
-      "next13": 0.02260578,
-      "vite": 0.10223005,
-      "next12": 0.49360912,
-      "next11": 0.89315210
+      "next13": 0.02379283,
+      "vite": 0.10963156,
+      "next12": 0.49470051,
+      "next11": 0.86600602
     },
     "10000": {
-      "next13": 0.02108141,
-      "vite": 0.10202639,
-      "next12": 1.12207728,
-      "next11": 2.29478435
+      "next13": 0.02302405,
+      "vite": 0.11295908,
+      "next12": 1.15193035,
+      "next11": 2.35675312
     },
     "30000": {
-      "next13": 0.02485774,
-      "vite": 0.11748541,
-      "next12": 2.52723383,
-      "next11": 7.32052819
+      "next13": 0.02246753,
+      "vite": 0.13328557,
+      "next12": 6.40370549,
+      "next11": 9.50431942
     }
   }
 }
diff --git a/docs/pages/blog/turbopack-benchmarks.mdx b/docs/pages/blog/turbopack-benchmarks.mdx
diff --git a/docs/pages/pack/docs/benchmarks.mdx b/docs/pages/pack/docs/benchmarks.mdx
@@ -48,46 +48,46 @@ This test measures how fast a local development server starts up on an applicati
 <ThemedImageFigure
   borderRadius={true}
   dark={{
-    source: '/images/blog/turbopack-benchmarks/bench_startup_all_dark.svg',
+    source: '/images/blog/turbopack-benchmarks/bench_startup_dark.svg',
     height: 720,
     width: 1960
   }}
   light={{
-    source: '/images/blog/turbopack-benchmarks/bench_startup_all_light.svg',
+    source: '/images/blog/turbopack-benchmarks/bench_startup_light.svg',
     height: 720,
     width: 1960
   }}
   captionSpacing={24}
-  caption="Startup time by module count. Benchmark data generated from 14” MacBook Pro 2021, M1 Max, 64GB RAM, macOS 12.6 (21G115)."
+  caption="Startup time by module count. Benchmark data generated from 16” MacBook Pro 2021, M1 Max, 32GB RAM, macOS 13.0.1 (22A400)."
 />
 
 #### Data
 
 To run this benchmark yourself, clone [`vercel/turbo`](https://github.com/vercel/turbo) and then use this command from the root:
 
 ```sh
-TURBOPACK_BENCH_COUNTS=1000,5000,10000,30000 TURBOPACK_BENCH_BUNDLERS=all cargo bench -p next-dev "startup/(Turbopack SSR|Next.js 12 SSR|Next.js 11 SSR|Vite CSR)."
+TURBOPACK_BENCH_COUNTS=1000,5000,10000,30000 TURBOPACK_BENCH_BUNDLERS=all cargo bench -p next-dev "startup/(Turbopack SSR|Next.js 12 SSR|Next.js 11 SSR|Vite SWC CSR)."
 ```
 
-Here are the numbers we were able to produce on a 14” MacBook Pro 2021, M1 Max, 64GB RAM, macOS 12.6 (21G115):
+Here are the numbers we were able to produce on a 16” MacBook Pro 2021, M1 Max, 32GB RAM, macOS 13.0.1 (22A400):
 
 ```sh
-bench_startup/Next.js 11 SSR/1000 modules              7.7±0.06s
-bench_startup/Next.js 11 SSR/5000 modules             24.8±0.11s
-bench_startup/Next.js 11 SSR/10000 modules            49.4±0.28s
-bench_startup/Next.js 11 SSR/30000 modules           176.2±1.42s
-bench_startup/Next.js 12 SSR/1000 modules              3.4±0.01s
-bench_startup/Next.js 12 SSR/5000 modules             10.7±0.02s
-bench_startup/Next.js 12 SSR/10000 modules            20.1±0.07s
-bench_startup/Next.js 12 SSR/30000 modules            76.6±0.66s
-bench_startup/Turbopack SSR/1000 modules          1125.3±44.54ms
-bench_startup/Turbopack SSR/5000 modules               3.6±0.02s
-bench_startup/Turbopack SSR/10000 modules              7.5±0.44s
-bench_startup/Turbopack SSR/30000 modules             22.3±1.29s
-bench_startup/Vite CSR/1000 modules                    4.8±0.02s
-bench_startup/Vite CSR/5000 modules                   19.2±0.15s
-bench_startup/Vite CSR/10000 modules                  37.2±0.29s
-bench_startup/Vite CSR/30000 modules                 109.5±1.14s
+bench_startup/Next.js 11 SSR/1000 modules                  9.2±0.04s
+bench_startup/Next.js 11 SSR/5000 modules                 32.9±0.67s
+bench_startup/Next.js 11 SSR/10000 modules                71.8±2.57s
+bench_startup/Next.js 11 SSR/30000 modules               237.6±6.43s
+bench_startup/Next.js 12 SSR/1000 modules                  3.6±0.02s
+bench_startup/Next.js 12 SSR/5000 modules                 12.1±0.32s
+bench_startup/Next.js 12 SSR/10000 modules                23.3±0.32s
+bench_startup/Next.js 12 SSR/30000 modules                89.1±0.21s
+bench_startup/Turbopack SSR/1000 modules               1381.9±5.62ms
+bench_startup/Turbopack SSR/5000 modules                   4.0±0.04s
+bench_startup/Turbopack SSR/10000 modules                  7.3±0.07s
+bench_startup/Turbopack SSR/30000 modules                 22.0±0.32s
+bench_startup/Vite SWC CSR/1000 modules                    4.2±0.02s
+bench_startup/Vite SWC CSR/5000 modules                   16.6±0.08s
+bench_startup/Vite SWC CSR/10000 modules                  32.3±0.12s
+bench_startup/Vite SWC CSR/30000 modules                  97.7±1.53s
 ```
 
 ### File updates (HMR)
@@ -105,35 +105,35 @@ We report the linear regression slope of the data points as the target metric. T
 <ThemedImageFigure
   borderRadius={true}
   dark={{
-    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_eval_all_dark.svg',
+    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_commit_dark.svg',
     height: 720,
     width: 1960
   }}
   light={{
-    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_eval_all_light.svg',
+    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_commit_light.svg',
     height: 720,
     width: 1960
   }}
   captionSpacing={24}
-  caption="Turbopack vs. Next.js (Webpack) vs. Vite HMR by module count. Benchmark data generated from 14” MacBook Pro 2021, M1 Max, 64GB RAM, macOS 12.6 (21G115)."
+  caption="Turbopack, Next.js (Webpack), and Vite HMR by module count. Benchmark data generated from 16” MacBook Pro 2021, M1 Max, 32GB RAM, macOS 13.0.1 (22A400)."
 />
 
 <a id="bench"/>
 
 <ThemedImageFigure
   borderRadius={true}
   dark={{
-    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_eval_vite_tp_dark.svg',
+    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_commit_turbopack_vite_dark.svg',
     height: 720,
     width: 1960
   }}
   light={{
-    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_eval_vite_tp_light.svg',
+    source: '/images/blog/turbopack-benchmarks/bench_hmr_to_commit_turbopack_vite_light.svg',
     height: 720,
     width: 1960
   }}
   captionSpacing={24}
-  caption="Turbopack vs. Vite HMR by module count. Benchmark data generated from 14” MacBook Pro 2021, M1 Max, 64GB RAM, macOS 12.6 (21G115)."
+  caption="Turbopack and Vite HMR by module count. Benchmark data generated from 16” MacBook Pro 2021, M1 Max, 32GB RAM, macOS 13.0.1 (22A400)."
 />
 
 The takeaway from these benchmarks: Turbopack performance is a function of **the size of an update**, not the size of an application.
@@ -145,27 +145,30 @@ We're excited by Turbopack's performance, but we also expect to make further spe
 To run this benchmark yourself, clone [`vercel/turbo`](https://github.com/vercel/turbo) and then use this command from the root:
 
 ```
-TURBOPACK_BENCH_COUNTS=100,500,1000,5000,10000,30000 TURBOPACK_BENCH_BUNDLERS=all cargo bench -p next-dev "hmr_to_commit/(Turbopack SSR|Next.js 12 SSR|Next.js 11 SSR|Vite SWC CSR)"
+TURBOPACK_BENCH_COUNTS=1000,5000,10000,30000 TURBOPACK_BENCH_BUNDLERS=all cargo bench -p next-dev "hmr_to_commit/(Turbopack SSR|Next.js 12 SSR|Next.js 11 SSR|Vite SWC CSR)"
 ```
 
-Here are the numbers we were able to produce on a 16” MacBook Pro 2021, M1 Max, 32GB RAM, macOS 12.6 (21G115), for Turbopack and Vite:
+Here are the numbers we were able to produce on a 16” MacBook Pro 2021, M1 Max, 32GB RAM, macOS 13.0.1 (22A400):
 
 ```sh
-bench_hmr_to_commit/Turbopack SSR/100 modules      16.8±2.73ms
-bench_hmr_to_commit/Turbopack SSR/500 modules      20.7±0.43ms
-bench_hmr_to_commit/Turbopack SSR/1000 modules     21.7±0.97ms
-bench_hmr_to_commit/Turbopack SSR/5000 modules     23.5±0.50ms
-bench_hmr_to_commit/Turbopack SSR/10000 modules    23.3±0.53ms
-bench_hmr_to_commit/Turbopack SSR/30000 modules    25.1±0.36ms
-
-bench_hmr_to_commit/Vite SWC CSR/100 modules       101.5±0.21ms
-bench_hmr_to_commit/Vite SWC CSR/500 modules       101.6±0.27ms
-bench_hmr_to_commit/Vite SWC CSR/1000 modules      101.6±0.67ms
-bench_hmr_to_commit/Vite SWC CSR/5000 modules      101.4±0.34ms
-bench_hmr_to_commit/Vite SWC CSR/10000 modules     102.5±1.21ms
-bench_hmr_to_commit/Vite SWC CSR/30000 modules     100.8±0.92ms
+bench_hmr_to_commit/Next.js 11 SSR/1000 modules         211.6±1.14ms
+bench_hmr_to_commit/Next.js 11 SSR/5000 modules        866.0±34.44ms
+bench_hmr_to_commit/Next.js 11 SSR/10000 modules           2.4±0.13s
+bench_hmr_to_commit/Next.js 11 SSR/30000 modules           9.5±3.12s
+bench_hmr_to_commit/Next.js 12 SSR/1000 modules         146.2±2.17ms
+bench_hmr_to_commit/Next.js 12 SSR/5000 modules        494.7±25.13ms
+bench_hmr_to_commit/Next.js 12 SSR/10000 modules     1151.9±280.68ms
+bench_hmr_to_commit/Next.js 12 SSR/30000 modules           6.4±2.29s
+bench_hmr_to_commit/Turbopack SSR/1000 modules           18.9±2.92ms
+bench_hmr_to_commit/Turbopack SSR/5000 modules           23.8±0.31ms
+bench_hmr_to_commit/Turbopack SSR/10000 modules          23.0±0.35ms
+bench_hmr_to_commit/Turbopack SSR/30000 modules          22.5±0.88ms
+bench_hmr_to_commit/Vite SWC CSR/1000 modules           104.8±1.52ms
+bench_hmr_to_commit/Vite SWC CSR/5000 modules           109.6±3.94ms
+bench_hmr_to_commit/Vite SWC CSR/10000 modules          113.0±1.20ms
+bench_hmr_to_commit/Vite SWC CSR/30000 modules         133.3±23.65ms
 ```
 
-Note that Vite is using an [SWC plugin](https://github.com/vitejs/vite-plugin-react-swc) for these benchmarks, which is not the default configuration.
+Note that Vite is using the official [SWC plugin](https://github.com/vitejs/vite-plugin-react-swc) for these benchmarks, which is not the default configuration.
 
 If you have questions about the benchmark, please open an [issue on GitHub](https://github.com/vercel/turbo/issues).
diff --git a/docs/pages/pack/docs/comparisons/vite.mdx b/docs/pages/pack/docs/comparisons/vite.mdx
@@ -30,6 +30,8 @@ This means that Turbopack's dev server starts up much faster than Vite's. On a 1
 
 In large applications, this differential stays consistent. In a 30,000 module application, Turbopack starts up <DocsBenchmarkStat stat="turbopack-cold-vs-vite-30000" /> faster than Vite.
 
+Note that Vite is using the official [SWC plugin](https://github.com/vitejs/vite-plugin-react-swc) for these benchmarks, which is not the default configuration.
+
 <DocsBenchmarksGraph category="cold" bars={[
   {
     label: 'Turbopack',