feat: async-ify all the depgraph builder functions

snyk · Apr 17, 2023 · 6f7da8a · 6f7da8a
1 parent 5a67037
commit 6f7da8a
Show file tree

Hide file tree

Showing 13 changed files with 115 additions and 58 deletions.
diff --git a/lib/c-map.ts b/lib/c-map.ts
@@ -0,0 +1,27 @@
+import * as pMap from 'p-map';
+import { eventLoopSpinner } from 'event-loop-spinner';
+
+/** Run a function each element in an array, with some level of concurrency.
+ *
+ * Defaults to a small, finite concurrency, so as to not to "overload" the database
+ * connection pool or the http client, both of which have small, internal limits.
+ *
+ * Can be used with async functions that don't yield; will yield for you, if necessary.
+ */
+export async function cMap<F, T>(
+  input: Iterable<F>,
+  mapper: (from: F) => Promise<T>,
+  options?: { concurrency: number },
+): Promise<T[]> {
+  const concurrency = options?.concurrency ?? 6;
+  return await pMap(
+    input,
+    async (from) => {
+      if (eventLoopSpinner.isStarving()) {
+        await eventLoopSpinner.spin();
+      }
+      return await mapper(from);
+    },
+    { concurrency },
+  );
+}
diff --git a/lib/dep-graph-builders/npm-lock-v2/index.ts b/lib/dep-graph-builders/npm-lock-v2/index.ts
@@ -5,7 +5,7 @@ import {
 } from '../types';
 import { extractPkgsFromNpmLockV2 } from './extract-npm-lock-v2-pkgs';
 import type { NpmLockPkg } from './extract-npm-lock-v2-pkgs';
-import { DepGraphBuilder } from '@snyk/dep-graph';
+import { DepGraph, DepGraphBuilder } from '@snyk/dep-graph';
 import {
   addPkgNodeToGraph,
   getGraphDependencies,
@@ -19,20 +19,21 @@ import { LockfileType } from '../../parsers';
 import * as semver from 'semver';
 import * as micromatch from 'micromatch';
 import * as pathUtil from 'path';
+import { eventLoopSpinner } from 'event-loop-spinner';
 
 export { extractPkgsFromNpmLockV2 };
 
-export const parseNpmLockV2Project = (
+export const parseNpmLockV2Project = async (
   pkgJsonContent: string,
   pkgLockContent: string,
   options: ProjectParseOptions,
-) => {
+): Promise<DepGraph> => {
   const { includeDevDeps, strictOutOfSync, includeOptionalDeps } = options;
 
   const pkgJson: PackageJsonBase = parsePkgJson(pkgJsonContent);
   const pkgs = extractPkgsFromNpmLockV2(pkgLockContent);
 
-  const depgraph = buildDepGraphNpmLockV2(pkgs, pkgJson, {
+  const depgraph = await buildDepGraphNpmLockV2(pkgs, pkgJson, {
     includeDevDeps,
     includeOptionalDeps,
     strictOutOfSync,
@@ -41,7 +42,7 @@ export const parseNpmLockV2Project = (
   return depgraph;
 };
 
-export const buildDepGraphNpmLockV2 = (
+export const buildDepGraphNpmLockV2 = async (
   npmLockPkgs: Record<string, NpmLockPkg>,
   pkgJson: PackageJsonBase,
   options: DepGraphBuildOptions,
@@ -88,7 +89,7 @@ export const buildDepGraphNpmLockV2 = (
   );
 
   const visitedMap: Set<string> = new Set();
-  dfsVisit(
+  await dfsVisit(
     depGraphBuilder,
     rootNode,
     visitedMap,
@@ -102,7 +103,7 @@ export const buildDepGraphNpmLockV2 = (
   return depGraphBuilder.build();
 };
 
-const dfsVisit = (
+const dfsVisit = async (
   depGraphBuilder: DepGraphBuilder,
   node: PkgNode,
   visitedMap: Set<string>,
@@ -112,10 +113,14 @@ const dfsVisit = (
   includeOptionalDeps: boolean,
   ancestry: { name: string; key: string; inBundle: boolean }[],
   pkgKeysByName: Map<string, string[]>,
-): void => {
+): Promise<void> => {
   visitedMap.add(node.id);
 
   for (const [name, depInfo] of Object.entries(node.dependencies || {})) {
+    if (eventLoopSpinner.isStarving()) {
+      await eventLoopSpinner.spin();
+    }
+
     const childNode = getChildNode(
       name,
       depInfo,
@@ -136,7 +141,7 @@ const dfsVisit = (
 
     if (!visitedMap.has(childNode.id)) {
       addPkgNodeToGraph(depGraphBuilder, childNode, {});
-      dfsVisit(
+      await dfsVisit(
         depGraphBuilder,
         childNode,
         visitedMap,

diff --git a/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-simple-pruned.ts b/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-simple-pruned.ts
@@ -7,13 +7,14 @@ import {
 } from '../util';
 import type { NormalisedPkgs, PackageJsonBase } from '../types';
 import type { DepGraphBuildOptions } from '../types';
+import { eventLoopSpinner } from 'event-loop-spinner';
 
 enum Color {
   GRAY,
   BLACK,
 }
 
-export const buildDepGraphYarnLockV1SimpleCyclesPruned = (
+export const buildDepGraphYarnLockV1SimpleCyclesPruned = async (
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   pkgJson: PackageJsonBase,
   options: DepGraphBuildOptions,
@@ -37,7 +38,7 @@ export const buildDepGraphYarnLockV1SimpleCyclesPruned = (
     isDev: false,
   };
 
-  dfsVisit(
+  await dfsVisit(
     depGraphBuilder,
     rootNode,
     colorMap,
@@ -58,17 +59,21 @@ export const buildDepGraphYarnLockV1SimpleCyclesPruned = (
  *  - When first exploring an edge, if it points to a GRAY node, a cycle is found and the GRAY node is pruned.
  *     - A pruned node has id `${originalId}|1`
  */
-const dfsVisit = (
+const dfsVisit = async (
   depGraphBuilder: DepGraphBuilder,
   node: PkgNode,
   colorMap: Record<string, Color>,
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   strictOutOfSync: boolean,
   includeOptionalDeps: boolean,
-): void => {
+): Promise<void> => {
   colorMap[node.id] = Color.GRAY;
 
   for (const [name, depInfo] of Object.entries(node.dependencies || {})) {
+    if (eventLoopSpinner.isStarving()) {
+      await eventLoopSpinner.spin();
+    }
+
     const childNode = getChildNode(
       name,
       depInfo,
@@ -80,7 +85,7 @@ const dfsVisit = (
 
     if (!colorMap.hasOwnProperty(childNode.id)) {
       addPkgNodeToGraph(depGraphBuilder, childNode, {});
-      dfsVisit(
+      await dfsVisit(
         depGraphBuilder,
         childNode,
         colorMap,

diff --git a/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-simple.ts b/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-simple.ts
@@ -7,8 +7,9 @@ import {
 } from '../util';
 import type { DepGraphBuildOptions } from '../types';
 import type { NormalisedPkgs, PackageJsonBase } from '../types';
+import { eventLoopSpinner } from 'event-loop-spinner';
 
-export const buildDepGraphYarnLockV1Simple = (
+export const buildDepGraphYarnLockV1Simple = async (
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   pkgJson: PackageJsonBase,
   options: DepGraphBuildOptions,
@@ -32,7 +33,7 @@ export const buildDepGraphYarnLockV1Simple = (
     isDev: false,
   };
 
-  dfsVisit(
+  await dfsVisit(
     depGraphBuilder,
     rootNode,
     visitedMap,
@@ -50,17 +51,20 @@ export const buildDepGraphYarnLockV1Simple = (
  *  - If a node doesn't exist in the map, it means it hasn't been visited.
  *  - If a node is already visited, simply connect the new node with this node.
  */
-const dfsVisit = (
+const dfsVisit = async (
   depGraphBuilder: DepGraphBuilder,
   node: PkgNode,
   visitedMap: Set<string>,
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   strictOutOfSync: boolean,
   includeOptionalDeps: boolean,
-): void => {
+): Promise<void> => {
   visitedMap.add(node.id);
 
   for (const [name, depInfo] of Object.entries(node.dependencies || {})) {
+    if (eventLoopSpinner.isStarving()) {
+      await eventLoopSpinner.spin();
+    }
     const childNode = getChildNode(
       name,
       depInfo,
@@ -71,7 +75,7 @@ const dfsVisit = (
 
     if (!visitedMap.has(childNode.id)) {
       addPkgNodeToGraph(depGraphBuilder, childNode, {});
-      dfsVisit(
+      await dfsVisit(
         depGraphBuilder,
         childNode,
         visitedMap,

diff --git a/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-workspace-package-pruned.ts b/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-workspace-package-pruned.ts
@@ -4,6 +4,7 @@ import { addPkgNodeToGraph, getTopLevelDeps, PkgNode } from '../util';
 import type { NormalisedPkgs, PackageJsonBase } from '../types';
 import type { DepGraphBuildOptions } from '../types';
 import { getChildNodeYarnLockV1Workspace } from './util';
+import { eventLoopSpinner } from 'event-loop-spinner';
 
 enum Color {
   GRAY,
@@ -12,7 +13,7 @@ enum Color {
 
 // Parse a single workspace package using yarn.lock v1
 // workspaces feature
-export const buildDepGraphYarnLockV1WorkspaceCyclesPruned = (
+export const buildDepGraphYarnLockV1WorkspaceCyclesPruned = async (
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   pkgJson: PackageJsonBase,
   workspacePkgNameToVersion: Record<string, string>,
@@ -37,7 +38,7 @@ export const buildDepGraphYarnLockV1WorkspaceCyclesPruned = (
     isDev: false,
   };
 
-  dfsVisit(
+  await dfsVisit(
     depGraphBuilder,
     rootNode,
     colorMap,
@@ -60,18 +61,21 @@ export const buildDepGraphYarnLockV1WorkspaceCyclesPruned = (
  *     - A pruned node has id `${originalId}|1`
  * When coming across another workspace package as child node, simply add the node and edge to the graph and mark it as BLACK.
  */
-const dfsVisit = (
+const dfsVisit = async (
   depGraphBuilder: DepGraphBuilder,
   node: PkgNode,
   colorMap: Record<string, Color>,
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   workspacePkgNameToVersion: Record<string, string>,
   strictOutOfSync: boolean,
   includeOptionalDeps: boolean,
-): void => {
+): Promise<void> => {
   colorMap[node.id] = Color.GRAY;
 
   for (const [name, depInfo] of Object.entries(node.dependencies || {})) {
+    if (eventLoopSpinner.isStarving()) {
+      await eventLoopSpinner.spin();
+    }
     const isWorkspacePkg = !!workspacePkgNameToVersion[name];
 
     const childNode = getChildNodeYarnLockV1Workspace(
@@ -89,7 +93,7 @@ const dfsVisit = (
         isWorkspacePkg,
       });
       if (!isWorkspacePkg) {
-        dfsVisit(
+        await dfsVisit(
           depGraphBuilder,
           childNode,
           colorMap,

diff --git a/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-workspace-package.ts b/lib/dep-graph-builders/yarn-lock-v1/build-depgraph-workspace-package.ts
@@ -3,8 +3,9 @@ import { addPkgNodeToGraph, getTopLevelDeps, PkgNode } from '../util';
 import type { NormalisedPkgs, PackageJsonBase } from '../types';
 import type { DepGraphBuildOptions } from '../types';
 import { getChildNodeYarnLockV1Workspace } from './util';
+import { eventLoopSpinner } from 'event-loop-spinner';
 
-export const buildDepGraphYarnLockV1Workspace = (
+export const buildDepGraphYarnLockV1Workspace = async (
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   pkgJson: PackageJsonBase,
   workspacePkgNameToVersion: Record<string, string>,
@@ -29,7 +30,7 @@ export const buildDepGraphYarnLockV1Workspace = (
     isDev: false,
   };
 
-  dfsVisit(
+  await dfsVisit(
     depGraphBuilder,
     rootNode,
     visitedMap,
@@ -52,18 +53,21 @@ export const buildDepGraphYarnLockV1Workspace = (
  *     - A pruned node has id `${originalId}|1`
  * When coming across another workspace package as child node, simply add the node and edge to the graph and mark it as BLACK.
  */
-const dfsVisit = (
+const dfsVisit = async (
   depGraphBuilder: DepGraphBuilder,
   node: PkgNode,
   visitedMap: Set<string>,
   extractedYarnLockV1Pkgs: NormalisedPkgs,
   workspacePkgNameToVersion: Record<string, string>,
   strictOutOfSync: boolean,
   includeOptionalDeps: boolean,
-): void => {
+): Promise<void> => {
   visitedMap.add(node.id);
 
   for (const [name, depInfo] of Object.entries(node.dependencies || {})) {
+    if (eventLoopSpinner.isStarving()) {
+      await eventLoopSpinner.spin();
+    }
     const isWorkspacePkg = !!workspacePkgNameToVersion[name];
 
     const childNode = getChildNodeYarnLockV1Workspace(
@@ -81,7 +85,7 @@ const dfsVisit = (
         isWorkspacePkg,
       });
       if (!isWorkspacePkg) {
-        dfsVisit(
+        await dfsVisit(
           depGraphBuilder,
           childNode,
           visitedMap,

diff --git a/lib/dep-graph-builders/yarn-lock-v1/simple.ts b/lib/dep-graph-builders/yarn-lock-v1/simple.ts
@@ -18,12 +18,12 @@ export const parseYarnLockV1Project = async (
   const pkgJson: PackageJsonBase = parsePkgJson(pkgJsonContent);
 
   const depGraph = pruneCycles
-    ? buildDepGraphYarnLockV1SimpleCyclesPruned(pkgs, pkgJson, {
+    ? await buildDepGraphYarnLockV1SimpleCyclesPruned(pkgs, pkgJson, {
         includeDevDeps,
         strictOutOfSync,
         includeOptionalDeps,
       })
-    : buildDepGraphYarnLockV1Simple(pkgs, pkgJson, {
+    : await buildDepGraphYarnLockV1Simple(pkgs, pkgJson, {
         includeDevDeps,
         strictOutOfSync,
         includeOptionalDeps,

diff --git a/lib/dep-graph-builders/yarn-lock-v1/workspaces.ts b/lib/dep-graph-builders/yarn-lock-v1/workspaces.ts
@@ -5,6 +5,7 @@ import { buildDepGraphYarnLockV1Workspace } from './build-depgraph-workspace-pac
 import { extractPkgsFromYarnLockV1 } from './extract-yarnlock-v1-pkgs';
 import { parsePkgJson } from '../util';
 import { ProjectParseOptions } from '../types';
+import { cMap } from '../../c-map';
 
 export const parseYarnLockV1WorkspaceProject = async (
   yarnLockContent: string,
@@ -26,9 +27,9 @@ export const parseYarnLockV1WorkspaceProject = async (
     },
   );
 
-  const depGraphs = parsedWorkspacePkgJsons.map((parsedPkgJson) => {
+  const depGraphs = cMap(parsedWorkspacePkgJsons, async (parsedPkgJson) => {
     return pruneCycles
-      ? buildDepGraphYarnLockV1WorkspaceCyclesPruned(
+      ? await buildDepGraphYarnLockV1WorkspaceCyclesPruned(
           extractedYarnLockV1Pkgs,
           parsedPkgJson,
           workspacePkgNameToVersion,
@@ -38,7 +39,7 @@ export const parseYarnLockV1WorkspaceProject = async (
             includeOptionalDeps,
           },
         )
-      : buildDepGraphYarnLockV1Workspace(
+      : await buildDepGraphYarnLockV1Workspace(
           extractedYarnLockV1Pkgs,
           parsedPkgJson,
           workspacePkgNameToVersion,