feat: support npm project scan without lock files UNIFY-68

.circleci/config.yml

@@ -8,7 +8,7 @@ orbs:
   snyk: snyk/snyk@1
 
 defaults: &defaults
-  resource_class: small
+  resource_class: large
   docker:
     - image: cimg/node:14.18
   working_directory: ~/snyk-docker-plugin

lib/analyzer/applications/node-modules-utils.ts

@@ -0,0 +1,107 @@
+import * as Debug from "debug";
+import * as fs from "fs";
+import * as path from "path";
+import * as tmp from "tmp";
+import { FilePathToContent, FilesByDir } from "./types";
+const debug = Debug("snyk");
+
+export { persistAppNodeModules, cleanupAppNodeModules, groupFilesByDirectory };
+
+async function persistAppNodeModules(
+  filePathToContent: FilePathToContent,
+  fileNamesGroupedByDirectory: FilesByDir,
+): Promise<string[]> {
+  const directoryTree = Object.keys(fileNamesGroupedByDirectory).sort();
+  if (directoryTree.length === 0) {
+    debug(`Empty application directory tree.`);
+    return ["", ""];
+  }
+
+  const tempAppRootPath = tmp.dirSync().name;
+  const appRootDirNameLength = directoryTree[0].indexOf("node_modules");
+  const appRootDir =
+    appRootDirNameLength === -1
+      ? directoryTree[0]
+      : directoryTree[0].substring(0, appRootDirNameLength);
+  const appRootDirPath = path.join(tempAppRootPath, appRootDir);
+
+  try {
+    fs.mkdirSync(appRootDirPath, { recursive: true });
+  } catch (error) {
+    debug(
+      `Failed to create the temporary directory structure of the application: ${error.message}`,
+    );
+  }
+  const appRootPackageJsonPath = path.join(appRootDirPath, "package.json");
+
+  for (const directoryPath of directoryTree) {
+    const filesInDirectory = fileNamesGroupedByDirectory[directoryPath];
+    if (filesInDirectory.length < 1) {
+      continue;
+    }
+    const moduleJsonFilePath = path.join(directoryPath, filesInDirectory[0]);
+    const moduleJsonFileContent = filePathToContent[moduleJsonFilePath];
+    const tempModuleJsonFilePath = path.join(
+      tempAppRootPath,
+      moduleJsonFilePath,
+    );
+    try {
+      await createFile(tempModuleJsonFilePath, moduleJsonFileContent);
+    } catch (error) {
+      debug(
+        `Failed to create the temporary directory structure of the application: ${error.message}`,
+      );
+    }
+  }
+
+  try {
+    // Check if the package.json exists in the app root dir,
+    // if it doesn't, create an empty "package.json" required by resolve-deps
+    fs.statSync(appRootPackageJsonPath);
+  } catch (error) {
+    debug(
+      `Creating an empty ${appRootPackageJsonPath} required by resolveDeps`,
+    );
+    fs.writeFileSync(appRootPackageJsonPath, "{}", "utf-8");
+  }
+
+  return [tempAppRootPath, appRootDirPath];
+}
+
+// Function to create file with content asynchronously
+async function createFile(filePath, fileContent) {
+  try {
+    const fileDir = path.dirname(filePath);
+    // Ensure directory existence before writing the file
+    fs.mkdirSync(fileDir, { recursive: true });
+
+    const fileContentJson = JSON.parse(fileContent);
+    // Write content to the file
+    fs.writeFileSync(filePath, JSON.stringify(fileContentJson), "utf-8");
+  } catch (error) {
+    debug(`Error while creating ${filePath} : ${error.message}`);
+  }
+}
+
+function groupFilesByDirectory(
+  filePathToContent: FilePathToContent,
+): FilesByDir {
+  const fileNamesGrouped: FilesByDir = {};
+  for (const filePath of Object.keys(filePathToContent)) {
+    const directory = path.dirname(filePath);
+    const fileName = path.basename(filePath);
+    if (!fileNamesGrouped[directory]) {
+      fileNamesGrouped[directory] = [];
+    }
+    fileNamesGrouped[directory].push(fileName);
+  }
+  return fileNamesGrouped;
+}
+
+async function cleanupAppNodeModules(appRootDir: string) {
+  try {
+    fs.rmSync(appRootDir, { recursive: true, force: true });
+  } catch (error) {
+    debug(`Error while removing ${appRootDir} : ${error.message}`);
+  }
+}

lib/analyzer/applications/node.ts

@@ -1,9 +1,22 @@
 import { legacy } from "@snyk/dep-graph";
+import * as Debug from "debug";
 import * as path from "path";
 import * as lockFileParser from "snyk-nodejs-lockfile-parser";
+import * as resolveDeps from "snyk-resolve-deps";
 import { DepGraphFact, TestedFilesFact } from "../../facts";
 
-import { AppDepsScanResultWithoutTarget, FilePathToContent } from "./types";
+const debug = Debug("snyk");
+
+import {
+  cleanupAppNodeModules,
+  groupFilesByDirectory,
+  persistAppNodeModules,
+} from "./node-modules-utils";
+import {
+  AppDepsScanResultWithoutTarget,
+  FilePathToContent,
+  FilesByDir,
+} from "./types";
 
 interface ManifestLockPathPair {
   manifest: string;
@@ -14,8 +27,6 @@ interface ManifestLockPathPair {
 export async function nodeFilesToScannedProjects(
   filePathToContent: FilePathToContent,
 ): Promise<AppDepsScanResultWithoutTarget[]> {
-  const scanResults: AppDepsScanResultWithoutTarget[] = [];
-
   /**
    * TODO: Add support for Yarn workspaces!
    * https://github.com/snyk/nodejs-lockfile-parser/blob/af8ba81930e950156b539281ecf41c1bc63dacf4/test/lib/yarn-workflows.test.ts#L7-L17
@@ -26,12 +37,81 @@ export async function nodeFilesToScannedProjects(
    * };
    */
 
-  const filePairs = findManifestLockPairsInSameDirectory(filePathToContent);
+  if (Object.keys(filePathToContent).length === 0) {
+    return [];
+  }
+
+  const fileNamesGroupedByDirectory = groupFilesByDirectory(filePathToContent);
+  const manifestFilePairs = findManifestLockPairsInSameDirectory(
+    fileNamesGroupedByDirectory,
+  );
+
+  if (manifestFilePairs.length === 0) {
+    debug(
+      "No manifest lock pairs found, computing the depGraph from node modules",
+    );
+    return depGraphFromNodeModules(
+      filePathToContent,
+      fileNamesGroupedByDirectory,
+    );
+  } else {
+    return depGraphFromManifestFiles(filePathToContent, manifestFilePairs);
+  }
+  return [];
+}
+
+async function depGraphFromNodeModules(
+  filePathToContent: FilePathToContent,
+  fileNamesGroupedByDirectory: FilesByDir,
+): Promise<AppDepsScanResultWithoutTarget[]> {
+  const scanResults: AppDepsScanResultWithoutTarget[] = [];
+
+  const [appRootPath, appRootDir] = await persistAppNodeModules(
+    filePathToContent,
+    fileNamesGroupedByDirectory,
+  );
+
+  try {
+    const depRes: lockFileParser.PkgTree = await resolveDeps(appRootDir, {
+      dev: false,
+      noFromArrays: true,
+    });
+
+    const depGraph = await legacy.depTreeToGraph(depRes, "npm");
+    const depGraphFact: DepGraphFact = {
+      type: "depGraph",
+      data: depGraph,
+    };
+    const testedFilesFact: TestedFilesFact = {
+      type: "testedFiles",
+      data: Object.keys(filePathToContent),
+    };
+    scanResults.push({
+      facts: [depGraphFact, testedFilesFact],
+      identity: {
+        type: depGraph.pkgManager ? depGraph.pkgManager.name : "npm",
+        targetFile: "package.json",
+      },
+    });
+  } catch (error) {
+    debug(
+      `An error occurred while analysing node_modules dir: ${error.message}`,
+    );
+  }
+
+  await cleanupAppNodeModules(appRootPath);
+  return scanResults;
+}
 
+async function depGraphFromManifestFiles(
+  filePathToContent: FilePathToContent,
+  manifestFilePairs: ManifestLockPathPair[],
+): Promise<AppDepsScanResultWithoutTarget[]> {
+  const scanResults: AppDepsScanResultWithoutTarget[] = [];
   const shouldIncludeDevDependencies = false;
   const shouldBeStrictForManifestAndLockfileOutOfSync = false;
 
-  for (const pathPair of filePairs) {
+  for (const pathPair of manifestFilePairs) {
     // TODO: initially generate as DepGraph
     const parserResult = await lockFileParser.buildDepTree(
       filePathToContent[pathPair.manifest],
@@ -64,17 +144,18 @@ export async function nodeFilesToScannedProjects(
       },
     });
   }
-
   return scanResults;
 }
 
 function findManifestLockPairsInSameDirectory(
-  filePathToContent: FilePathToContent,
+  fileNamesGroupedByDirectory: FilesByDir,
 ): ManifestLockPathPair[] {
-  const fileNamesGroupedByDirectory = groupFilesByDirectory(filePathToContent);
   const manifestLockPathPairs: ManifestLockPathPair[] = [];
 
   for (const directoryPath of Object.keys(fileNamesGroupedByDirectory)) {
+    if (directoryPath.includes("node_modules")) {
+      continue;
+    }
     const filesInDirectory = fileNamesGroupedByDirectory[directoryPath];
     if (filesInDirectory.length !== 2) {
       // either a missing file or too many files, ignore
@@ -107,22 +188,6 @@ function findManifestLockPairsInSameDirectory(
   return manifestLockPathPairs;
 }
 
-// assumption: we only care about manifest+lock files if they are in the same directory
-function groupFilesByDirectory(filePathToContent: FilePathToContent): {
-  [directoryName: string]: string[];
-} {
-  const fileNamesGroupedByDirectory: { [directoryName: string]: string[] } = {};
-  for (const filePath of Object.keys(filePathToContent)) {
-    const directory = path.dirname(filePath);
-    const fileName = path.basename(filePath);
-    if (!fileNamesGroupedByDirectory[directory]) {
-      fileNamesGroupedByDirectory[directory] = [];
-    }
-    fileNamesGroupedByDirectory[directory].push(fileName);
-  }
-  return fileNamesGroupedByDirectory;
-}
-
 function stripUndefinedLabels(
   parserResult: lockFileParser.PkgTree,
 ): lockFileParser.PkgTree {

lib/analyzer/applications/types.ts

@@ -31,3 +31,7 @@ export interface FilePathToElfContent {
 export interface AggregatedJars {
   [path: string]: JarBuffer[];
 }
+
+export interface FilesByDir {
+  [directoryName: string]: string[];
+}

lib/analyzer/static-analyzer.ts

@@ -103,7 +103,6 @@ export async function analyze(
   }
 
   const appScan = !isTrue(options["exclude-app-vulns"]);
-
   if (appScan) {
     staticAnalysisActions.push(
       ...[

lib/inputs/node/static.ts

@@ -1,15 +1,19 @@
 import { basename } from "path";
 
+import * as path from "path";
 import { ExtractAction } from "../../extractor/types";
 import { streamToString } from "../../stream-utils";
 
+const ignoredPaths = ["/usr", "/bin", "/tmp", "/opt", "C:\\"];
 const nodeAppFiles = ["package.json", "package-lock.json", "yarn.lock"];
 const deletedAppFiles = nodeAppFiles.map((file) => ".wh." + file);
 
 function filePathMatches(filePath: string): boolean {
   const fileName = basename(filePath);
+  const dirName = path.dirname(filePath);
+
   return (
-    filePath.indexOf("node_modules") === -1 &&
+    !ignoredPaths.some((ignorePath) => dirName.includes(ignorePath)) &&
     (nodeAppFiles.includes(fileName) || deletedAppFiles.includes(fileName))
   );
 }