refactor: migrate npm walker to tree-walker workspace

chore: continue to improve interfaces

chore: continue to improve types

chore: update combine-async-iterators

index.d.ts

@@ -9,4 +9,4 @@ export {
   Logger,
   LoggerEventData,
   depWalker
-}
+};

package.json

@@ -8,7 +8,8 @@
   },
   "type": "module",
   "workspaces": [
-    "workspaces/tarball"
+    "workspaces/tarball",
+    "workspaces/tree-walker"
   ],
   "scripts": {
     "build": "tsc -b",
@@ -59,6 +60,7 @@
     "@slimio/is": "^2.0.0",
     "@types/node": "^20.10.0",
     "@types/pacote": "^11.1.8",
+    "@types/sinon": "^17.0.3",
     "c8": "^9.1.0",
     "dotenv": "^16.3.1",
     "eslint": "8.57.0",
@@ -81,11 +83,8 @@
     "@nodesecure/utils": "^2.0.1",
     "@nodesecure/vuln": "^1.7.0",
     "@npm/types": "^1.0.2",
-    "@npmcli/arborist": "^7.2.1",
     "@slimio/lock": "^1.0.0",
     "builtins": "^5.0.1",
-    "combine-async-iterators": "^2.1.0",
-    "itertools": "^2.1.2",
     "lodash.difference": "^4.5.0",
     "pacote": "^17.0.4",
     "semver": "^7.5.4"

src/depWalker.js

@@ -5,24 +5,19 @@ import timers from "node:timers/promises";
 import os from "node:os";
 
 // Import Third-party Dependencies
-import combineAsyncIterators from "combine-async-iterators";
-import * as iter from "itertools";
-import pacote from "pacote";
-import Arborist from "@npmcli/arborist";
 import Lock from "@slimio/lock";
 import * as vuln from "@nodesecure/vuln";
 import { ScannerLoggerEvents } from "./constants.js";
 
 // Import Workspaces Dependencies
 import { scanDirOrArchive } from "@nodesecure/tarball";
+import * as treeWalker from "@nodesecure/tree-walker";
 
 // Import Internal Dependencies
 import {
-  mergeDependencies, getCleanDependencyName, getDependenciesWarnings, addMissingVersionFlags, isGitDependency,
-  NPM_TOKEN
+  getDependenciesWarnings, addMissingVersionFlags
 } from "./utils/index.js";
 import { packageMetadata, manifestMetadata } from "./npmRegistry.js";
-import Dependency from "./class/dependency.class.js";
 import Logger from "./class/logger.class.js";
 
 const { version: packageVersion } = JSON.parse(
@@ -31,193 +26,6 @@ const { version: packageVersion } = JSON.parse(
   )
 );
 
-export async function* searchDeepDependencies(packageName, gitURL, options) {
-  const { exclude, currDepth = 0, parent, maxDepth, registry } = options;
-
-  const { name, version, deprecated, ...pkg } = await pacote.manifest(gitURL ?? packageName, {
-    ...NPM_TOKEN,
-    registry,
-    cache: `${os.homedir()}/.npm`
-  });
-  const { dependencies, customResolvers, alias } = mergeDependencies(pkg);
-
-  const current = new Dependency(name, version, parent);
-  current.alias = Object.fromEntries(alias);
-
-  if (gitURL !== null) {
-    current.isGit(gitURL);
-    try {
-      await pacote.manifest(`${name}@${version}`, {
-        ...NPM_TOKEN,
-        registry,
-        cache: `${os.homedir()}/.npm`
-      });
-    }
-    catch {
-      current.existOnRemoteRegistry = false;
-    }
-  }
-  current.addFlag("isDeprecated", deprecated === true);
-  current.addFlag("hasCustomResolver", customResolvers.size > 0);
-  current.addFlag("hasDependencies", dependencies.size > 0);
-
-  if (currDepth !== maxDepth) {
-    const config = {
-      exclude, currDepth: currDepth + 1, parent: current, maxDepth, registry
-    };
-
-    const gitDependencies = iter.filter(customResolvers.entries(), ([, valueStr]) => isGitDependency(valueStr));
-    for (const [depName, valueStr] of gitDependencies) {
-      yield* searchDeepDependencies(depName, valueStr, config);
-    }
-
-    const depsNames = await Promise.all(iter.map(dependencies.entries(), getCleanDependencyName));
-    for (const [fullName, cleanName, isLatest] of depsNames) {
-      if (!isLatest) {
-        current.addFlag("hasOutdatedDependency");
-      }
-
-      if (exclude.has(cleanName)) {
-        current.addChildren();
-        exclude.get(cleanName).add(current.fullName);
-      }
-      else {
-        exclude.set(cleanName, new Set([current.fullName]));
-        yield* searchDeepDependencies(fullName, null, config);
-      }
-    }
-  }
-
-  yield current;
-}
-
-export async function* deepReadEdges(currentPackageName, options) {
-  const { to, parent, exclude, fullLockMode, includeDevDeps, registry } = options;
-  const { version, integrity = to.integrity } = to.package;
-
-  const updatedVersion = version === "*" || typeof version === "undefined" ? "latest" : version;
-  const current = new Dependency(currentPackageName, updatedVersion, parent);
-  current.dev = to.dev;
-
-  if (fullLockMode && !includeDevDeps) {
-    const { _integrity, ...pkg } = await pacote.manifest(`${currentPackageName}@${updatedVersion}`, {
-      ...NPM_TOKEN,
-      registry,
-      cache: `${os.homedir()}/.npm`
-    });
-    const { customResolvers, alias } = mergeDependencies(pkg);
-
-    current.alias = Object.fromEntries(alias);
-    current.addFlag("hasValidIntegrity", _integrity === integrity);
-    current.addFlag("isDeprecated");
-    current.addFlag("hasCustomResolver", customResolvers.size > 0);
-
-    if (isGitDependency(to.resolved)) {
-      current.isGit(to.resolved);
-    }
-  }
-  current.addFlag("hasDependencies", to.edgesOut.size > 0);
-
-  for (const [packageName, { to: toNode }] of to.edgesOut) {
-    if (toNode === null || (!includeDevDeps && toNode.dev)) {
-      continue;
-    }
-    const cleanName = `${packageName}@${toNode.package.version}`;
-
-    if (exclude.has(cleanName)) {
-      current.addChildren();
-      exclude.get(cleanName).add(current.fullName);
-    }
-    else {
-      exclude.set(cleanName, new Set([current.fullName]));
-      yield* deepReadEdges(packageName, { parent: current, to: toNode, exclude, registry });
-    }
-  }
-  yield current;
-}
-
-export async function* getRootDependencies(manifest, options) {
-  const {
-    maxDepth = 4, exclude,
-    usePackageLock, fullLockMode, includeDevDeps,
-    location,
-    registry
-  } = options;
-
-  const { dependencies, customResolvers, alias } = mergeDependencies(manifest, void 0);
-  const parent = new Dependency(manifest.name, manifest.version);
-  parent.alias = Object.fromEntries(alias);
-
-  try {
-    await pacote.manifest(`${manifest.name}@${manifest.version}`, {
-      ...NPM_TOKEN,
-      registry,
-      cache: `${os.homedir()}/.npm`
-    });
-  }
-  catch {
-    parent.existOnRemoteRegistry = false;
-  }
-  parent.addFlag("hasCustomResolver", customResolvers.size > 0);
-  parent.addFlag("hasDependencies", dependencies.size > 0);
-
-  let iterators;
-  if (usePackageLock) {
-    const arb = new Arborist({
-      ...NPM_TOKEN,
-      path: location,
-      registry
-    });
-    let tree;
-    try {
-      await fs.access(path.join(location, "node_modules"));
-      tree = await arb.loadActual();
-    }
-    catch {
-      tree = await arb.loadVirtual();
-    }
-
-    iterators = [
-      ...iter
-        .filter(tree.edgesOut.entries(), ([, { to }]) => to !== null && (includeDevDeps ? true : (!to.dev || to.isWorkspace)))
-        .map(([packageName, { to }]) => [packageName, to.isWorkspace ? to.target : to])
-        .map(([packageName, to]) => deepReadEdges(packageName, {
-          to,
-          parent,
-          fullLockMode,
-          includeDevDeps,
-          exclude,
-          registry
-        }))
-    ];
-  }
-  else {
-    const configRef = { exclude, maxDepth, parent, registry };
-    iterators = [
-      ...iter.filter(customResolvers.entries(), ([, valueStr]) => isGitDependency(valueStr))
-        .map(([depName, valueStr]) => searchDeepDependencies(depName, valueStr, configRef)),
-      ...iter.map(dependencies.entries(), ([name, ver]) => searchDeepDependencies(`${name}@${ver}`, null, configRef))
-    ];
-  }
-  for await (const dep of combineAsyncIterators({}, ...iterators)) {
-    yield dep;
-  }
-
-  // Add root dependencies to the exclude Map (because the parent is not handled by searchDeepDependencies)
-  // if we skip this the code will fail to re-link properly dependencies in the following steps
-  const depsName = await Promise.all(iter.map(dependencies.entries(), getCleanDependencyName));
-  for (const [, fullRange, isLatest] of depsName) {
-    if (!isLatest) {
-      parent.addFlag("hasOutdatedDependency");
-    }
-    if (exclude.has(fullRange)) {
-      exclude.get(fullRange).add(parent.fullName);
-    }
-  }
-
-  yield parent;
-}
-
 /**
  * @param {*} manifest
  * @param {*} options
@@ -262,7 +70,7 @@ export async function depWalker(manifest, options = {}, logger = new Logger()) {
     tarballLocker.on("freeOne", () => logger.tick(ScannerLoggerEvents.analysis.tarball));
 
     const rootDepsOptions = { maxDepth, exclude, usePackageLock, fullLockMode, includeDevDeps, location, registry };
-    for await (const currentDep of getRootDependencies(manifest, rootDepsOptions)) {
+    for await (const currentDep of treeWalker.npm.walk(manifest, rootDepsOptions)) {
       const { name, version, dev } = currentDep;
 
       const current = currentDep.exportAsPlainObject(name === manifest.name ? 0 : void 0);

src/utils/index.js

@@ -1,6 +1,3 @@
-export * from "./isGitDependency.js";
-export * from "./mergeDependencies.js";
-export * from "./semver.js";
 export * from "./dirname.js";
 export * from "./warnings.js";
 export * from "./addMissingVersionFlags.js";

test/class/logger.spec.js → test/logger.spec.js

@@ -1,13 +1,13 @@
 // Import Node.js Dependencies
-import EventEmitter, { once } from "events";
+import EventEmitter, { once } from "node:events";
 import { test } from "node:test";
 import assert from "node:assert";
 
 // Import Third-party Dependencies
 import is from "@slimio/is";
 
 // Import Internal Dependencies
-import Logger from "../../src/class/logger.class.js";
+import Logger from "../src/class/logger.class.js";
 
 test("Logger: Creating a new class instance and assert all properties", () => {
   assert.ok(is.classObject(Logger));