Merge pull request #884 from snyk/feat/parallelize-monitor

feat: asynchronously process monitor args
snyk · Nov 28, 2019 · 631b0fa · 631b0fa
2 parents f92319f + 22629c9
commit 631b0fa
Show file tree

Hide file tree

Showing 3 changed files with 160 additions and 115 deletions.
diff --git a/src/cli/commands/monitor/index.ts b/src/cli/commands/monitor/index.ts
@@ -54,12 +54,135 @@ async function promiseOrCleanup<T>(
   });
 }
 
+// detects package manager
+// loads appropriate plugin and inspects for dependencies
+// finally posts dependencies to registry
+async function detectInspectMonitor(
+  path: string,
+  options: MonitorOptions = {},
+): Promise<GoodResult | BadResult> {
+  try {
+    await validateMonitorPath(path, options.docker);
+
+    let packageManager = detect.detectPackageManager(path, options);
+
+    const targetFile =
+      options.docker && !options.file // snyk monitor --docker (without --file)
+        ? undefined
+        : options.file || detect.detectPackageFile(path);
+
+    const plugin = plugins.loadPlugin(packageManager, options);
+
+    const moduleInfo = ModuleInfo(plugin, options.policy);
+
+    const displayPath = pathUtil.relative(
+      '.',
+      pathUtil.join(path, targetFile || ''),
+    );
+
+    const analysisType = options.docker ? 'docker' : packageManager;
+
+    const analyzingDepsSpinnerLabel =
+      'Analyzing ' + analysisType + ' dependencies for ' + displayPath;
+
+    const postingMonitorSpinnerLabel =
+      'Posting monitor snapshot for ' + displayPath + ' ...';
+
+    await spinner(analyzingDepsSpinnerLabel);
+
+    // Scan the project dependencies via a plugin
+
+    analytics.add('packageManager', packageManager);
+    analytics.add('pluginOptions', options);
+
+    // TODO: the type should depend on allSubProjects flag
+    const inspectResult: pluginApi.InspectResult = await promiseOrCleanup(
+      moduleInfo.inspect(path, targetFile, { ...options }),
+      spinner.clear(analyzingDepsSpinnerLabel),
+    );
+
+    analytics.add('pluginName', inspectResult.plugin.name);
+
+    await spinner.clear(analyzingDepsSpinnerLabel)(inspectResult);
+
+    await spinner(postingMonitorSpinnerLabel);
+    if (inspectResult.plugin.packageManager) {
+      packageManager = inspectResult.plugin.packageManager;
+    }
+    const meta: MonitorMeta = {
+      method: 'cli',
+      packageManager,
+      'policy-path': options['policy-path'],
+      'project-name': options['project-name'] || config.PROJECT_NAME,
+      isDocker: !!options.docker,
+      prune: !!options['prune-repeated-subdependencies'],
+      'experimental-dep-graph': !!options['experimental-dep-graph'],
+      'remote-repo-url': options['remote-repo-url'],
+    };
+
+    // We send results from "all-sub-projects" scanning as different Monitor objects
+
+    // SinglePackageResult is a legacy format understood by Registry, so we have to convert
+    // a MultiProjectResult to an array of these.
+
+    let perProjectResult: pluginApi.SinglePackageResult[] = [];
+    let foundProjectCount;
+    if (pluginApi.isMultiResult(inspectResult)) {
+      perProjectResult = convertMultiPluginResultToSingle(inspectResult);
+    } else {
+      foundProjectCount = getSubProjectCount(inspectResult);
+      perProjectResult = [inspectResult];
+    }
+
+    // Post the project dependencies to the Registry
+    for (const projectDeps of perProjectResult) {
+      maybePrintDeps(options, projectDeps.package);
+
+      const res = await promiseOrCleanup(
+        snykMonitor(path, meta, projectDeps, targetFile),
+        spinner.clear(postingMonitorSpinnerLabel),
+      );
+
+      await spinner.clear(postingMonitorSpinnerLabel)(res);
+
+      res.path = path;
+      const endpoint = url.parse(config.API);
+      let leader = '';
+      if (res.org) {
+        leader = '/org/' + res.org;
+      }
+      endpoint.pathname = leader + '/manage';
+      const manageUrl = url.format(endpoint);
+
+      endpoint.pathname = leader + '/monitor/' + res.id;
+      const projectName = pluginApi.isMultiResult(inspectResult)
+        ? projectDeps.package.name
+        : undefined;
+      const monOutput = formatMonitorOutput(
+        packageManager,
+        res,
+        manageUrl,
+        options,
+        projectName,
+        foundProjectCount,
+      );
+      return { ok: true, data: monOutput, path, projectName };
+    }
+  } catch (err) {
+    return { ok: false, data: err, path };
+  }
+  return {
+    ok: false,
+    data: new MonitorError(500, 'Failed to monitor path.'),
+    path,
+  };
+}
+
 // Returns an array of Registry responses (one per every sub-project scanned), a single response,
 // or an error message.
 async function monitor(...args0: MethodArgs): Promise<any> {
   let args = [...args0];
   let options: MonitorOptions = {};
-  const results: Array<GoodResult | BadResult> = [];
   if (typeof args[args.length - 1] === 'object') {
     options = (args.pop() as ArgsOptions) as MonitorOptions;
   }
@@ -104,121 +227,11 @@ async function monitor(...args0: MethodArgs): Promise<any> {
     }
   }
 
-  // Part 1: every argument is a scan target; process them sequentially
-  for (const path of args as string[]) {
-    try {
-      await validateMonitorPath(path, options.docker);
-
-      let packageManager = detect.detectPackageManager(path, options);
+  // Part 1: every argument is a scan target; process them asynchronously
+  const results = await Promise.all(
+    (args as string[]).map((arg) => detectInspectMonitor(arg, options)),
+  );
 
-      const targetFile =
-        options.docker && !options.file // snyk monitor --docker (without --file)
-          ? undefined
-          : options.file || detect.detectPackageFile(path);
-
-      const plugin = plugins.loadPlugin(packageManager, options);
-
-      const moduleInfo = ModuleInfo(plugin, options.policy);
-
-      const displayPath = pathUtil.relative(
-        '.',
-        pathUtil.join(path, targetFile || ''),
-      );
-
-      const analysisType = options.docker ? 'docker' : packageManager;
-
-      const analyzingDepsSpinnerLabel =
-        'Analyzing ' + analysisType + ' dependencies for ' + displayPath;
-
-      const postingMonitorSpinnerLabel =
-        'Posting monitor snapshot for ' + displayPath + ' ...';
-
-      await spinner(analyzingDepsSpinnerLabel);
-
-      // Scan the project dependencies via a plugin
-
-      analytics.add('packageManager', packageManager);
-      analytics.add('pluginOptions', options);
-
-      // TODO: the type should depend on allSubProjects flag
-      const inspectResult: pluginApi.InspectResult = await promiseOrCleanup(
-        moduleInfo.inspect(path, targetFile, { ...options }),
-        spinner.clear(analyzingDepsSpinnerLabel),
-      );
-
-      analytics.add('pluginName', inspectResult.plugin.name);
-
-      await spinner.clear(analyzingDepsSpinnerLabel)(inspectResult);
-
-      await spinner(postingMonitorSpinnerLabel);
-      if (inspectResult.plugin.packageManager) {
-        packageManager = inspectResult.plugin.packageManager;
-      }
-      const meta: MonitorMeta = {
-        method: 'cli',
-        packageManager: packageManager,
-        'policy-path': options['policy-path'],
-        'project-name': options['project-name'] || config.PROJECT_NAME,
-        isDocker: !!options.docker,
-        prune: !!options['prune-repeated-subdependencies'],
-        'experimental-dep-graph': !!options['experimental-dep-graph'],
-        'remote-repo-url': options['remote-repo-url'],
-      };
-
-      // We send results from "all-sub-projects" scanning as different Monitor objects
-
-      // SinglePackageResult is a legacy format understood by Registry, so we have to convert
-      // a MultiProjectResult to an array of these.
-
-      let perProjectResult: pluginApi.SinglePackageResult[] = [];
-      let foundProjectCount;
-      if (pluginApi.isMultiResult(inspectResult)) {
-        perProjectResult = convertMultiPluginResultToSingle(inspectResult);
-      } else {
-        foundProjectCount = getSubProjectCount(inspectResult);
-        perProjectResult = [inspectResult];
-      }
-
-      // Post the project dependencies to the Registry
-      for (const projectDeps of perProjectResult) {
-        maybePrintDeps(options, projectDeps.package);
-
-        const res = await promiseOrCleanup(
-          snykMonitor(path, meta, projectDeps, targetFile),
-          spinner.clear(postingMonitorSpinnerLabel),
-        );
-
-        await spinner.clear(postingMonitorSpinnerLabel)(res);
-
-        res.path = path;
-        const endpoint = url.parse(config.API);
-        let leader = '';
-        if (res.org) {
-          leader = '/org/' + res.org;
-        }
-        endpoint.pathname = leader + '/manage';
-        const manageUrl = url.format(endpoint);
-
-        endpoint.pathname = leader + '/monitor/' + res.id;
-        const projectName = pluginApi.isMultiResult(inspectResult)
-          ? projectDeps.package.name
-          : undefined;
-        const monOutput = formatMonitorOutput(
-          packageManager,
-          res,
-          manageUrl,
-          options,
-          projectName,
-          foundProjectCount,
-        );
-        results.push({ ok: true, data: monOutput, path, projectName });
-      }
-      // push a good result
-    } catch (err) {
-      // push this error, the loop continues
-      results.push({ ok: false, data: err, path });
-    }
-  }
   // Part 2: process the output from the Registry
   if (options.json) {
     let dataToSend = results.map((result) => {

diff --git a/test/acceptance/cli-monitor/cli-monitor.acceptance.test.ts b/test/acceptance/cli-monitor/cli-monitor.acceptance.test.ts
@@ -1161,6 +1161,34 @@ test('`monitor foo:latest --docker`', async (t) => {
   );
 });
 
+test('monitor with multiple paths', async (t) => {
+  chdirWorkspaces();
+  await cli.monitor('npm-package', 'yarn-app', 'ruby-app');
+  // test last three requests to fake server
+  server.popRequests(3).forEach((req) => {
+    t.equal(req.method, 'PUT', 'makes PUT request');
+    t.equal(
+      req.headers['x-snyk-cli-version'],
+      versionNumber,
+      'sends version number',
+    );
+    t.match(req.url, /\/monitor\/(npm|yarn|rubygems)/, 'puts at correct url');
+    t.match(
+      req.body,
+      {
+        meta: {
+          name: /(npm-package|yarn-app|ruby-app)/,
+        },
+        package: {
+          name: /(npm-package|yarn-app|ruby-app)/,
+        },
+      },
+      'sends name in body',
+    );
+    t.ok(req.body.package.dependencies, 'sends dependencies in body');
+  });
+});
+
 test('`monitor foo:latest --docker --file=Dockerfile`', async (t) => {
   const dockerImageId =
     'sha256:' +

diff --git a/test/acceptance/fake-server.ts b/test/acceptance/fake-server.ts
@@ -5,6 +5,7 @@ interface FakeServer extends restify.Server {
   _nextResponse?: restify.Response;
   _nextStatusCode?: number;
   popRequest: () => restify.Request;
+  popRequests: (num: number) => restify.Request[];
   setNextResponse: (r: any) => void;
   setNextStatusCodeAndResponse: (c: number, r: any) => void;
 }
@@ -18,6 +19,9 @@ export function fakeServer(root, apikey) {
   server.popRequest = function() {
     return server._reqLog.pop()!;
   };
+  server.popRequests = function(num: number) {
+    return server._reqLog.splice(server._reqLog.length - num, num);
+  };
   server.use(restify.acceptParser(server.acceptable));
   server.use(restify.queryParser());
   server.use(restify.bodyParser());