Add benchmark script (#13224)

Co-authored-by: fisker <lionkay@gmail.com>

.gitignore

@@ -28,3 +28,5 @@ package-lock.json
 .pnp.*
 .nyc_output
 .devcontainer
+# When installing software on gitpod.io, `core.*` files are generated
+/core.*

.gitpod.yml

@@ -2,8 +2,8 @@
 
 tasks:
   - init: |
+      brew install hyperfine
       yarn
-      yarn build
 
 github:
   # https://www.gitpod.io/docs/prebuilds/#configure-prebuilds

CONTRIBUTING.md

@@ -95,6 +95,18 @@ In the above commands:
 
 In addition to the options above, you can use [`node --prof` and `node --prof-process`](https://nodejs.org/en/docs/guides/simple-profiling/), as well as `node --trace-opt --trace-deopt`, to get more advanced performance insights.
 
+The script `scripts/benchmark/compare.sh` can be used to compare performance of two or more commits/branches using [hyperfine](https://github.com/sharkdp/hyperfine). Usage (don't forget to install hyperfine):
+
+```sh
+PRETTIER_PERF_FILENAME=my.js ./compare.sh main some-pr-branch
+```
+
+Without `PRETTIER_PERF_FILENAME`, the script uses a default JS file as input for Prettier:
+
+```sh
+./compare.sh main next
+```
+
 ## Regression testing
 
 We have a cool tool for regression testing that runs on GitHub Actions. Have a look: https://github.com/prettier/prettier-regression-testing

cspell.json

@@ -222,8 +222,8 @@
         "neoclide",
         "neoformat",
         "neovim",
-        "nocheck",
         "nnoremap",
+        "nocheck",
         "nonenumerable",
         "Nonspacing",
         "noopener",
@@ -271,9 +271,9 @@
         "Pschera",
         "quasis",
         "Raghuvir",
+        "raquo",
         "Rasmus",
         "Rattray",
-        "raquo",
         "rattrayalex",
         "readline",
         "readlines",
@@ -291,8 +291,8 @@
         "sandhose",
         "Sapegin",
         "sbdchd",
-        "sdbm",
         "scandir",
+        "sdbm",
         "Serializers",
         "setlocal",
         "setq",
@@ -330,6 +330,7 @@
         "testname",
         "tldr",
         "Tomasek",
+        "toplevel",
         "Tradeshift",
         "Transloadit",
         "trippable",

package.json

@@ -150,6 +150,7 @@
     "perf": "yarn && yarn build && cross-env NODE_ENV=production node ./dist/bin-prettier.js",
     "perf:inspect": "yarn && yarn build && cross-env NODE_ENV=production node --inspect-brk ./dist/bin-prettier.js",
     "perf:benchmark": "yarn perf --debug-benchmark",
+    "perf:compare": "./scripts/benchmark/compare.sh",
     "lint": "run-p lint:*",
     "lint:typecheck": "tsc",
     "lint:eslint": "cross-env EFF_NO_LINK_RULES=true eslint . --format friendly",

scripts/benchmark/.gitignore

@@ -0,0 +1 @@
+*/

scripts/benchmark/bench.mjs

@@ -0,0 +1,28 @@
+import { readFileSync } from "node:fs";
+
+const [, , version, method, groupCountString = 100, groupSizeString = 10] =
+  process.argv;
+const groupCount = Number(groupCountString);
+const groupSize = Number(groupSizeString);
+const { format } = await import(`./${version}/dist/index.js`);
+
+const sourceText = readFileSync(
+  process.env.PRETTIER_PERF_FILENAME || "../../src/language-js/utils/index.js",
+  "utf8"
+);
+
+for (let i = 0; i < groupCount; i++) {
+  if (method === "serial") {
+    for (let i = 0; i < groupSize; i++) {
+      await format(sourceText, { parser: "typescript" });
+    }
+  }
+
+  if (method === "parallel") {
+    await Promise.allSettled(
+      Array.from({ length: groupSize }, () =>
+        format(sourceText, { parser: "typescript" })
+      )
+    );
+  }
+}

scripts/benchmark/compare.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+commits=("${@}")
+
+root=`git rev-parse --show-toplevel`
+cd "$root/scripts/benchmark"
+
+function cleanup {
+  echo Cleaning up...
+  rm -rf "${commits[@]}"
+  echo Done!
+}
+trap cleanup EXIT
+
+args=()
+
+for commit in "${commits[@]}"; do
+  rm -rf $commit
+  git -C "$root" archive $commit --prefix $commit/ ':!tests*' ':!website' ':!docs' | tar -x
+  (cd $commit; yarn; yarn build)
+  args+=("node ./bench.mjs $commit serial")
+  args+=("node ./bench.mjs $commit parallel")
+done
+
+hyperfine --warmup 3 "${args[@]}"