Merge pull request #578 from snyk/feat/npm-scan-without-lockfiles

Feat/npm support scan of npm projects without lockfiles

lib/analyzer/applications/node-modules-utils.ts

@@ -0,0 +1,150 @@
+import * as Debug from "debug";
+import { mkdir, mkdtemp, rm, stat, writeFile } from "fs/promises";
+import * as path from "path";
+
+import { FilePathToContent, FilesByDir } from "./types";
+const debug = Debug("snyk");
+
+export { persistAppNodeModules, cleanupAppNodeModules, groupFilesByDirectory };
+
+interface ScanPaths {
+  tempDir: string;
+  tempApplicationPath: string;
+  manifestPath?: string;
+}
+
+async function createTempAppDir(appParentDir: string): Promise<string[]> {
+  const tmpDir = await mkdtemp("snyk");
+
+  const appRootDir = appParentDir.includes("node_modules")
+    ? appParentDir.substring(0, appParentDir.indexOf("node_modules"))
+    : appParentDir;
+
+  const tempAppRootDirPath = path.join(tmpDir, appRootDir);
+
+  await mkdir(tempAppRootDirPath, { recursive: true });
+
+  return [tmpDir, tempAppRootDirPath];
+}
+
+const manifestName: string = "package.json";
+
+async function fileExists(path: string): Promise<boolean> {
+  return await stat(path)
+    .then(() => true)
+    .catch(() => false);
+}
+
+async function createAppSyntheticManifest(
+  tempRootManifestDir: string,
+): Promise<void> {
+  const tempRootManifestPath = path.join(tempRootManifestDir, manifestName);
+  debug(`Creating an empty synthetic manifest file: ${tempRootManifestPath}`);
+  await writeFile(tempRootManifestPath, "{}", "utf-8");
+}
+
+async function copyAppModulesManifestFiles(
+  appDirs: string[],
+  tempAppRootDirPath: string,
+  fileNamesGroupedByDirectory: FilesByDir,
+  filePathToContent: FilePathToContent,
+) {
+  for (const dependencyPath of appDirs) {
+    const filesInDirectory = fileNamesGroupedByDirectory[dependencyPath];
+    if (filesInDirectory.length === 0) {
+      continue;
+    }
+
+    const manifestPath = path.join(dependencyPath, "package.json");
+    const manifestContent = filePathToContent[manifestPath];
+
+    await createFile(
+      path.join(tempAppRootDirPath, manifestPath),
+      manifestContent,
+    );
+  }
+}
+
+async function persistAppNodeModules(
+  filePathToContent: FilePathToContent,
+  fileNamesGroupedByDirectory: FilesByDir,
+): Promise<ScanPaths> {
+  const appDirs = Object.keys(fileNamesGroupedByDirectory);
+  let tmpDir: string = "";
+  let tempAppRootDirPath: string = "";
+
+  if (appDirs.length === 0) {
+    debug(`Empty application directory tree.`);
+
+    return {
+      tempDir: tmpDir,
+      tempApplicationPath: tempAppRootDirPath,
+    };
+  }
+
+  try {
+    [tmpDir, tempAppRootDirPath] = await createTempAppDir(appDirs.sort()[0]);
+
+    await copyAppModulesManifestFiles(
+      appDirs,
+      tmpDir,
+      fileNamesGroupedByDirectory,
+      filePathToContent,
+    );
+
+    const result: ScanPaths = {
+      tempDir: tmpDir,
+      tempApplicationPath: tempAppRootDirPath,
+      manifestPath: path.join(
+        tempAppRootDirPath.substring(tmpDir.length),
+        manifestName,
+      ),
+    };
+
+    const manifestFileExists = await fileExists(
+      path.join(tempAppRootDirPath, manifestName),
+    );
+
+    if (!manifestFileExists) {
+      await createAppSyntheticManifest(tempAppRootDirPath);
+      delete result.manifestPath;
+    }
+    return result;
+  } catch (error) {
+    debug(
+      `Failed to copy the application manifest files locally: ${error.message}`,
+    );
+    return {
+      tempDir: tmpDir,
+      tempApplicationPath: tempAppRootDirPath,
+    };
+  }
+}
+
+async function createFile(filePath, fileContent) {
+  await mkdir(path.dirname(filePath), { recursive: true });
+  await writeFile(filePath, fileContent, "utf-8");
+}
+
+function groupFilesByDirectory(
+  filePathToContent: FilePathToContent,
+): FilesByDir {
+  const fileNamesGrouped: FilesByDir = {};
+  for (const filePath of Object.keys(filePathToContent)) {
+    const directory = path.dirname(filePath);
+    const fileName = path.basename(filePath);
+    if (!fileNamesGrouped[directory]) {
+      fileNamesGrouped[directory] = [];
+    }
+    fileNamesGrouped[directory].push(fileName);
+  }
+  return fileNamesGrouped;
+}
+
+async function cleanupAppNodeModules(appRootDir: string) {
+  try {
+    rm(appRootDir, { recursive: true });
+  } catch (error) {
+    debug(`Error while removing ${appRootDir} : ${error.message}`);
+  }
+}

lib/analyzer/applications/node.ts

@@ -1,9 +1,22 @@
 import { legacy } from "@snyk/dep-graph";
+import * as Debug from "debug";
 import * as path from "path";
 import * as lockFileParser from "snyk-nodejs-lockfile-parser";
+import * as resolveDeps from "snyk-resolve-deps";
 import { DepGraphFact, TestedFilesFact } from "../../facts";
 
-import { AppDepsScanResultWithoutTarget, FilePathToContent } from "./types";
+const debug = Debug("snyk");
+
+import {
+  cleanupAppNodeModules,
+  groupFilesByDirectory,
+  persistAppNodeModules,
+} from "./node-modules-utils";
+import {
+  AppDepsScanResultWithoutTarget,
+  FilePathToContent,
+  FilesByDir,
+} from "./types";
 
 interface ManifestLockPathPair {
   manifest: string;
@@ -14,8 +27,6 @@ interface ManifestLockPathPair {
 export async function nodeFilesToScannedProjects(
   filePathToContent: FilePathToContent,
 ): Promise<AppDepsScanResultWithoutTarget[]> {
-  const scanResults: AppDepsScanResultWithoutTarget[] = [];
-
   /**
    * TODO: Add support for Yarn workspaces!
    * https://github.com/snyk/nodejs-lockfile-parser/blob/af8ba81930e950156b539281ecf41c1bc63dacf4/test/lib/yarn-workflows.test.ts#L7-L17
@@ -26,12 +37,82 @@ export async function nodeFilesToScannedProjects(
    * };
    */
 
-  const filePairs = findManifestLockPairsInSameDirectory(filePathToContent);
+  if (Object.keys(filePathToContent).length === 0) {
+    return [];
+  }
+
+  const fileNamesGroupedByDirectory = groupFilesByDirectory(filePathToContent);
+  const manifestFilePairs = findManifestLockPairsInSameDirectory(
+    fileNamesGroupedByDirectory,
+  );
+
+  return manifestFilePairs.length === 0
+    ? depGraphFromNodeModules(filePathToContent, fileNamesGroupedByDirectory)
+    : depGraphFromManifestFiles(filePathToContent, manifestFilePairs);
+}
+
+async function depGraphFromNodeModules(
+  filePathToContent: FilePathToContent,
+  fileNamesGroupedByDirectory: FilesByDir,
+): Promise<AppDepsScanResultWithoutTarget[]> {
+  const { tempDir, tempApplicationPath, manifestPath } =
+    await persistAppNodeModules(filePathToContent, fileNamesGroupedByDirectory);
+
+  if (!tempApplicationPath) {
+    return [];
+  }
+
+  const scanResults: AppDepsScanResultWithoutTarget[] = [];
 
+  try {
+    const pkgTree: lockFileParser.PkgTree = await resolveDeps(
+      tempApplicationPath,
+      {
+        dev: false,
+        noFromArrays: true,
+      },
+    );
+
+    const depGraph = await legacy.depTreeToGraph(
+      pkgTree,
+      pkgTree.type || "npm",
+    );
+    scanResults.push({
+      facts: [
+        {
+          type: "depGraph",
+          data: depGraph,
+        },
+        {
+          type: "testedFiles",
+          data: Object.keys(filePathToContent),
+        },
+      ],
+      identity: {
+        type: depGraph.pkgManager.name,
+        targetFile: manifestPath,
+      },
+    });
+  } catch (error) {
+    debug(
+      `An error occurred while analysing node_modules dir: ${error.message}`,
+    );
+  } finally {
+    await cleanupAppNodeModules(tempDir);
+  }
+
+  return scanResults;
+}
+
+async function depGraphFromManifestFiles(
+  filePathToContent: FilePathToContent,
+  manifestFilePairs: ManifestLockPathPair[],
+): Promise<AppDepsScanResultWithoutTarget[]> {
+  const scanResults: AppDepsScanResultWithoutTarget[] = [];
   const shouldIncludeDevDependencies = false;
   const shouldBeStrictForManifestAndLockfileOutOfSync = false;
 
-  for (const pathPair of filePairs) {
+  for (const pathPair of manifestFilePairs) {
     // TODO: initially generate as DepGraph
     const parserResult = await lockFileParser.buildDepTree(
       filePathToContent[pathPair.manifest],
@@ -64,17 +145,18 @@ export async function nodeFilesToScannedProjects(
       },
     });
   }
-
   return scanResults;
 }
 
 function findManifestLockPairsInSameDirectory(
-  filePathToContent: FilePathToContent,
+  fileNamesGroupedByDirectory: FilesByDir,
 ): ManifestLockPathPair[] {
-  const fileNamesGroupedByDirectory = groupFilesByDirectory(filePathToContent);
   const manifestLockPathPairs: ManifestLockPathPair[] = [];
 
   for (const directoryPath of Object.keys(fileNamesGroupedByDirectory)) {
+    if (directoryPath.includes("node_modules")) {
+      continue;
+    }
     const filesInDirectory = fileNamesGroupedByDirectory[directoryPath];
     if (filesInDirectory.length !== 2) {
       // either a missing file or too many files, ignore
@@ -107,22 +189,6 @@ function findManifestLockPairsInSameDirectory(
   return manifestLockPathPairs;
 }
 
-// assumption: we only care about manifest+lock files if they are in the same directory
-function groupFilesByDirectory(filePathToContent: FilePathToContent): {
-  [directoryName: string]: string[];
-} {
-  const fileNamesGroupedByDirectory: { [directoryName: string]: string[] } = {};
-  for (const filePath of Object.keys(filePathToContent)) {
-    const directory = path.dirname(filePath);
-    const fileName = path.basename(filePath);
-    if (!fileNamesGroupedByDirectory[directory]) {
-      fileNamesGroupedByDirectory[directory] = [];
-    }
-    fileNamesGroupedByDirectory[directory].push(fileName);
-  }
-  return fileNamesGroupedByDirectory;
-}
-
 function stripUndefinedLabels(
   parserResult: lockFileParser.PkgTree,
 ): lockFileParser.PkgTree {

lib/analyzer/applications/types.ts

@@ -31,3 +31,7 @@ export interface FilePathToElfContent {
 export interface AggregatedJars {
   [path: string]: JarBuffer[];
 }
+
+export interface FilesByDir {
+  [directoryName: string]: string[];
+}

lib/inputs/node/static.ts

@@ -1,15 +1,19 @@
 import { basename } from "path";
 
+import * as path from "path";
 import { ExtractAction } from "../../extractor/types";
 import { streamToString } from "../../stream-utils";
 
+const ignoredPaths = ["/usr", "/bin", "/opt", "C:\\"];
 const nodeAppFiles = ["package.json", "package-lock.json", "yarn.lock"];
 const deletedAppFiles = nodeAppFiles.map((file) => ".wh." + file);
 
 function filePathMatches(filePath: string): boolean {
   const fileName = basename(filePath);
+  const dirName = path.dirname(filePath);
+
   return (
-    filePath.indexOf("node_modules") === -1 &&
+    !ignoredPaths.some((ignorePath) => dirName.includes(ignorePath)) &&
     (nodeAppFiles.includes(fileName) || deletedAppFiles.includes(fileName))
   );
 }