perf: various performance improvements (#4959)

* perf: speed up `splitLast` function * perf: skip function wrapping if profiler is disabled * perf: index files into a tree structure to get files for a subdirectory more quickly * perf: increase max-semi-space-size for less GC interruptions at the cost of some more memory See https://github.com/nodejs/node/blob/main/doc/api/cli.md#useful-v8-options Also see https://www.alibabacloud.com/blog/better-node-application-performance-through-gc-optimization_595119 and nodejs/node#42511 for some details on impact * refactor: replace Bluebird with native Promises (part 1) * refactor: replace bluebird with native promises (part 2) * refactor: replace bluebird with native promises (part 3) * chore: make the linter happy after all the bluebird replacements * fix: splitLast behavior with no index found * chore: linter again * chore: cleanup file-tree and add header * refactor: replace pLimit with pMap
garden-io · Aug 22, 2023 · a2c5f6e · a2c5f6e
1 parent 7f67071
commit a2c5f6e
Show file tree

Hide file tree

Showing 80 changed files with 1,887 additions and 1,623 deletions.
diff --git a/bin/garden b/bin/garden
@@ -2,4 +2,4 @@
 
 repo_root=$(cd `dirname $0` && cd .. && pwd)
 
-node --max-old-space-size=4096 ${repo_root}/cli/bin/garden "$@"
+node --max-old-space-size=4096 --max-semi-space-size=64 ${repo_root}/cli/bin/garden "$@"
diff --git a/cli/bin/garden-debug b/cli/bin/garden-debug
@@ -1,3 +1,3 @@
-#!/usr/bin/env node --inspect --stack-trace-limit=1000
+#!/usr/bin/env node --inspect --stack-trace-limit=1000 --max-semi-space-size=64
 
 require("./garden")
diff --git a/cli/src/add-version-files.ts b/cli/src/add-version-files.ts
@@ -10,7 +10,6 @@ import { GitHandler } from "@garden-io/core/build/src/vcs/git"
 import { Garden } from "@garden-io/core/build/src/garden"
 import { LogLevel, RootLogger } from "@garden-io/core/build/src/logger/logger"
 import { resolve, relative } from "path"
-import Bluebird from "bluebird"
 import { STATIC_DIR, GARDEN_VERSIONFILE_NAME } from "@garden-io/core/build/src/constants"
 import { writeTreeVersionFile } from "@garden-io/core/build/src/vcs/vcs"
 import { TreeCache } from "@garden-io/core/build/src/cache"
@@ -28,24 +27,26 @@ async function addVersionFiles() {
 
   const moduleConfigs = await garden.getRawModuleConfigs()
 
-  return Bluebird.map(moduleConfigs, async (config) => {
-    const path = config.path
-    const versionFilePath = resolve(path, GARDEN_VERSIONFILE_NAME)
+  return Promise.all(
+    moduleConfigs.map(async (config) => {
+      const path = config.path
+      const versionFilePath = resolve(path, GARDEN_VERSIONFILE_NAME)
 
-    const vcsHandler = new GitHandler({
-      garden,
-      projectRoot: STATIC_DIR,
-      gardenDirPath: garden.gardenDirPath,
-      ignoreFile: garden.dotIgnoreFile,
-      cache: new TreeCache(),
-    })
-    const treeVersion = await vcsHandler.getTreeVersion({ log: garden.log, projectName: garden.projectName, config })
+      const vcsHandler = new GitHandler({
+        garden,
+        projectRoot: STATIC_DIR,
+        gardenDirPath: garden.gardenDirPath,
+        ignoreFile: garden.dotIgnoreFile,
+        cache: new TreeCache(),
+      })
+      const treeVersion = await vcsHandler.getTreeVersion({ log: garden.log, projectName: garden.projectName, config })
 
-    // eslint-disable-next-line no-console
-    console.log(`${config.name} -> ${relative(STATIC_DIR, versionFilePath)}`)
+      // eslint-disable-next-line no-console
+      console.log(`${config.name} -> ${relative(STATIC_DIR, versionFilePath)}`)
 
-    return writeTreeVersionFile(path, treeVersion)
-  })
+      return writeTreeVersionFile(path, treeVersion)
+    })
+  )
 }
 
 if (require.main === module) {

diff --git a/cli/src/build-pkg.ts b/cli/src/build-pkg.ts
@@ -9,7 +9,6 @@
 import chalk from "chalk"
 import { getAbi } from "node-abi"
 import { resolve, relative, join } from "path"
-import Bluebird from "bluebird"
 import { STATIC_DIR, GARDEN_CLI_ROOT, GARDEN_CORE_ROOT } from "@garden-io/core/build/src/constants"
 import { remove, mkdirp, copy, writeFile } from "fs-extra"
 import { exec, getPackageVersion, sleep } from "@garden-io/core/build/src/util/util"
@@ -29,7 +28,7 @@ const pkgPath = resolve(repoRoot, "cli", "node_modules", ".bin", "pkg")
 const distPath = resolve(repoRoot, "dist")
 
 // Allow larger heap size than default
-const nodeOptions = ["max-old-space-size=4096"]
+const nodeOptions = ["max-old-space-size=4096", "max-semi-space-size=64"]
 
 /* eslint-disable no-console */
 
@@ -95,48 +94,52 @@ async function buildBinaries(args: string[]) {
   const workspaces = JSON.parse(JSON.parse(res).data)
 
   console.log(chalk.cyan("Copying packages"))
-  await Bluebird.map(Object.entries(workspaces), async ([name, info]: [string, any]) => {
-    const sourcePath = resolve(repoRoot, info.location)
-    const targetPath = resolve(tmpDirPath, info.location)
-    await remove(targetPath)
-    await mkdirp(targetPath)
-    await exec("rsync", [
-      "-r",
-      "-L",
-      "--exclude=node_modules",
-      "--exclude=tmp",
-      "--exclude=test",
-      sourcePath,
-      resolve(targetPath, ".."),
-    ])
-
-    console.log(chalk.green(" ✓ " + name))
-  })
+  await Promise.all(
+    Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
+      const sourcePath = resolve(repoRoot, info.location)
+      const targetPath = resolve(tmpDirPath, info.location)
+      await remove(targetPath)
+      await mkdirp(targetPath)
+      await exec("rsync", [
+        "-r",
+        "-L",
+        "--exclude=node_modules",
+        "--exclude=tmp",
+        "--exclude=test",
+        sourcePath,
+        resolve(targetPath, ".."),
+      ])
+
+      console.log(chalk.green(" ✓ " + name))
+    })
+  )
 
   // Edit all the packages to have them directly link any internal dependencies
   console.log(chalk.cyan("Modifying package.json files for direct installation"))
-  await Bluebird.map(Object.entries(workspaces), async ([name, info]: [string, any]) => {
-    const packageRoot = resolve(tmpDirPath, info.location)
-    const packageJsonPath = resolve(packageRoot, "package.json")
-    const packageJson = require(packageJsonPath)
-
-    for (const depName of info.workspaceDependencies) {
-      const depInfo = workspaces[depName]
-      const targetRoot = resolve(tmpDirPath, depInfo.location)
-      const relPath = relative(packageRoot, targetRoot)
-      packageJson.dependencies[depName] = "file:" + relPath
-    }
-
-    if (version === "edge" || version.startsWith("edge-")) {
-      const gitHash = await exec("git", ["rev-parse", "--short", "HEAD"])
-      packageJson.version = `${packageJson.version}-${version}-${gitHash.stdout}`
-      console.log("Set package version to " + packageJson.version)
-    }
-
-    await writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))
-
-    console.log(chalk.green(" ✓ " + name))
-  })
+  await Promise.all(
+    Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
+      const packageRoot = resolve(tmpDirPath, info.location)
+      const packageJsonPath = resolve(packageRoot, "package.json")
+      const packageJson = require(packageJsonPath)
+
+      for (const depName of info.workspaceDependencies) {
+        const depInfo = workspaces[depName]
+        const targetRoot = resolve(tmpDirPath, depInfo.location)
+        const relPath = relative(packageRoot, targetRoot)
+        packageJson.dependencies[depName] = "file:" + relPath
+      }
+
+      if (version === "edge" || version.startsWith("edge-")) {
+        const gitHash = await exec("git", ["rev-parse", "--short", "HEAD"])
+        packageJson.version = `${packageJson.version}-${version}-${gitHash.stdout}`
+        console.log("Set package version to " + packageJson.version)
+      }
+
+      await writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))
+
+      console.log(chalk.green(" ✓ " + name))
+    })
+  )
 
   // Run yarn install in the cli package
   console.log(chalk.cyan("Installing packages in @garden-io/cli package"))
@@ -146,11 +149,13 @@ async function buildBinaries(args: string[]) {
   // Run pkg and pack up each platform binary
   console.log(chalk.cyan("Packaging garden binaries"))
 
-  await Bluebird.map(Object.entries(selected), async ([targetName, spec]) => {
-    await spec.handler({ targetName, sourcePath: cliPath, pkgType: spec.pkgType, version })
-    await sleep(5000) // Work around concurrency bug in pkg...
-    console.log(chalk.green(" ✓ " + targetName))
-  })
+  await Promise.all(
+    Object.entries(selected).map(async ([targetName, spec]) => {
+      await spec.handler({ targetName, sourcePath: cliPath, pkgType: spec.pkgType, version })
+      await sleep(5000) // Work around concurrency bug in pkg...
+      console.log(chalk.green(" ✓ " + targetName))
+    })
+  )
 
   console.log(chalk.green.bold("Done!"))
 }

diff --git a/core/package.json b/core/package.json
@@ -42,7 +42,6 @@
     "async": "^3.2.4",
     "@scg82/exit-hook": "^3.4.1",
     "async-lock": "^1.4.0",
-    "bluebird": "^3.7.2",
     "certpem": "^1.1.3",
     "chalk": "^4.1.2",
     "chokidar": "^3.5.3",
@@ -113,8 +112,12 @@
     "normalize-path": "^3.0.0",
     "normalize-url": "^5.3.1",
     "open": "^8.4.2",
+    "p-filter": "^2.1.0",
     "p-queue": "^6.0.0",
     "p-retry": "^4.6.2",
+    "p-limit": "^3.1.0",
+    "p-map": "^4.0.0",
+    "p-props": "^4.0.0",
     "parse-duration": "^1.0.2",
     "parse-git-config": "^3.0.0",
     "path-is-inside": "^1.0.2",
@@ -166,7 +169,6 @@
     "@types/analytics-node": "^3.1.10",
     "@types/async": "^3.2.18",
     "@types/async-lock": "^1.4.0",
-    "@types/bluebird": "^3.5.38",
     "@types/chai": "^4.3.4",
     "@types/circular-json": "^0.4.0",
     "@types/cross-spawn": "^6.0.2",
@@ -278,4 +280,4 @@
     ]
   },
   "gitHead": "b0647221a4d2ff06952bae58000b104215aed922"
-}
+}
diff --git a/core/src/actions/helpers.ts b/core/src/actions/helpers.ts
@@ -6,7 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import Bluebird from "bluebird"
 import chalk from "chalk"
 import { fromPairs, memoize } from "lodash"
 import { joi } from "../config/common"
@@ -158,22 +157,24 @@ export async function getDeployStatusPayloads({
   const actions = graph.getDeploys()
 
   return fromPairs(
-    await Bluebird.map(actions, async (action) => {
-      const startedAt = new Date().toISOString()
-      const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
-      const { result } = await router.deploy.getStatus({ action, log: actionLog, graph })
+    await Promise.all(
+      actions.map(async (action) => {
+        const startedAt = new Date().toISOString()
+        const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
+        const { result } = await router.deploy.getStatus({ action, log: actionLog, graph })
 
-      const payload = makeActionCompletePayload({
-        result,
-        operation: "getStatus",
-        startedAt,
-        force: false,
-        action,
-        sessionId,
-      }) as ActionStatusPayload<DeployStatusForEventPayload>
+        const payload = makeActionCompletePayload({
+          result,
+          operation: "getStatus",
+          startedAt,
+          force: false,
+          action,
+          sessionId,
+        }) as ActionStatusPayload<DeployStatusForEventPayload>
 
-      return [action.name, payload]
-    })
+        return [action.name, payload]
+      })
+    )
   )
 }
 
@@ -191,22 +192,24 @@ export async function getBuildStatusPayloads({
   const actions = graph.getBuilds()
 
   return fromPairs(
-    await Bluebird.map(actions, async (action) => {
-      const startedAt = new Date().toISOString()
-      const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
-      const { result } = await router.build.getStatus({ action, log: actionLog, graph })
+    await Promise.all(
+      actions.map(async (action) => {
+        const startedAt = new Date().toISOString()
+        const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
+        const { result } = await router.build.getStatus({ action, log: actionLog, graph })
 
-      const payload = makeActionCompletePayload({
-        result,
-        operation: "getStatus",
-        startedAt,
-        force: false,
-        action,
-        sessionId,
-      }) as ActionStatusPayload<BuildStatusForEventPayload>
+        const payload = makeActionCompletePayload({
+          result,
+          operation: "getStatus",
+          startedAt,
+          force: false,
+          action,
+          sessionId,
+        }) as ActionStatusPayload<BuildStatusForEventPayload>
 
-      return [action.name, payload]
-    })
+        return [action.name, payload]
+      })
+    )
   )
 }
 
@@ -224,20 +227,22 @@ export async function getTestStatusPayloads({
   const actions = graph.getTests()
 
   return fromPairs(
-    await Bluebird.map(actions, async (action) => {
-      const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
-      const startedAt = new Date().toISOString()
-      const { result } = await router.test.getResult({ action, log: actionLog, graph })
-      const payload = makeActionCompletePayload({
-        result,
-        operation: "getStatus",
-        startedAt,
-        force: false,
-        action,
-        sessionId,
-      }) as ActionStatusPayload<RunStatusForEventPayload>
-      return [action.name, payload]
-    })
+    await Promise.all(
+      actions.map(async (action) => {
+        const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
+        const startedAt = new Date().toISOString()
+        const { result } = await router.test.getResult({ action, log: actionLog, graph })
+        const payload = makeActionCompletePayload({
+          result,
+          operation: "getStatus",
+          startedAt,
+          force: false,
+          action,
+          sessionId,
+        }) as ActionStatusPayload<RunStatusForEventPayload>
+        return [action.name, payload]
+      })
+    )
   )
 }
 
@@ -255,21 +260,23 @@ export async function getRunStatusPayloads({
   const actions = graph.getRuns()
 
   return fromPairs(
-    await Bluebird.map(actions, async (action) => {
-      const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
-      const startedAt = new Date().toISOString()
-      const { result } = await router.run.getResult({ action, log: actionLog, graph })
+    await Promise.all(
+      actions.map(async (action) => {
+        const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
+        const startedAt = new Date().toISOString()
+        const { result } = await router.run.getResult({ action, log: actionLog, graph })
 
-      const payload = makeActionCompletePayload({
-        result,
-        operation: "getStatus",
-        startedAt,
-        force: false,
-        action,
-        sessionId,
-      }) as ActionStatusPayload<RunStatusForEventPayload>
+        const payload = makeActionCompletePayload({
+          result,
+          operation: "getStatus",
+          startedAt,
+          force: false,
+          action,
+          sessionId,
+        }) as ActionStatusPayload<RunStatusForEventPayload>
 
-      return [action.name, payload]
-    })
+        return [action.name, payload]
+      })
+    )
   )
 }