Skip to content

Commit

Permalink
feat: unify policy handling & plucking
Browse files Browse the repository at this point in the history
  • Loading branch information
lili2311 committed Jun 18, 2020
1 parent aeee8c8 commit 197f1ec
Show file tree
Hide file tree
Showing 9 changed files with 146 additions and 99 deletions.
9 changes: 6 additions & 3 deletions src/cli/commands/monitor/index.ts
Expand Up @@ -12,6 +12,8 @@ import {
MonitorMeta,
MonitorResult,
Options,
PolicyOptions,
Contributors,
} from '../../../lib/types';
import * as config from '../../../lib/config';
import * as detect from '../../../lib/detect';
Expand Down Expand Up @@ -61,17 +63,18 @@ async function promiseOrCleanup<T>(
// or an error message.
async function monitor(...args0: MethodArgs): Promise<any> {
let args = [...args0];
let options: MonitorOptions = {};
let monitorOptions = {};
const results: Array<GoodResult | BadResult> = [];
if (typeof args[args.length - 1] === 'object') {
options = (args.pop() as ArgsOptions) as MonitorOptions;
monitorOptions = args.pop() as ArgsOptions;
}
args = args.filter(Boolean);

// populate with default path (cwd) if no path given
if (args.length === 0) {
args.unshift(process.cwd());
}
const options = monitorOptions as Options & PolicyOptions & MonitorOptions;

if (options.id) {
snyk.id = options.id;
Expand All @@ -89,7 +92,7 @@ async function monitor(...args0: MethodArgs): Promise<any> {

apiTokenExists();

let contributors: { userId: string; lastCommitDate: string }[] = [];
let contributors: Contributors[] = [];
if (!options.docker && analytics.allowAnalytics()) {
try {
const repoPath = process.cwd();
Expand Down
4 changes: 2 additions & 2 deletions src/cli/commands/protect/index.ts
Expand Up @@ -12,7 +12,7 @@ import * as errors from '../../../lib/errors';
const debug = debugModule('snyk');

async function protectFunc(
options: types.ProtectOptions & types.Options & types.TestOptions,
options: types.PolicyOptions & types.Options & types.TestOptions,
) {
const protectOptions = { ...options };
protectOptions.loose = true; // replace missing policies with empty ones
Expand Down Expand Up @@ -79,7 +79,7 @@ async function protectFunc(
}
}

async function patch(options: types.ProtectOptions & types.Options) {
async function patch(options: types.PolicyOptions & types.Options) {
try {
const response = (await snyk.test(
process.cwd(),
Expand Down
11 changes: 3 additions & 8 deletions src/cli/commands/protect/wizard.ts
Expand Up @@ -43,6 +43,7 @@ import {
MonitorMeta,
MonitorResult,
WizardOptions,
PackageJson,
} from '../../../lib/types';
import { LegacyVulnApiResult } from '../../../lib/snyk-test/legacy';
import { MultiProjectResult } from '@snyk/cli-interface/legacy/plugin';
Expand Down Expand Up @@ -318,13 +319,6 @@ function calculatePkgFileIndentation(packageFile: string): number {
return pkgIndentation;
}

interface Pkg {
scripts: any;
snyk: boolean;
dependencies: any;
devDependencies: any;
}

function processAnswers(answers, policy, options) {
if (!options) {
options = {};
Expand All @@ -347,7 +341,7 @@ function processAnswers(answers, policy, options) {
targetFile.endsWith('package-lock.json') ||
targetFile.endsWith('yarn.lock');

let pkg = {} as Pkg;
let pkg = {} as PackageJson;
let pkgIndentation = 2;

sendWizardAnalyticsData(answers);
Expand Down Expand Up @@ -599,6 +593,7 @@ function processAnswers(answers, policy, options) {
cwd,
meta as MonitorMeta,
(inspectRes as MultiProjectResult).scannedProjects[0],
options,
inspectRes.plugin,
options,
);
Expand Down
87 changes: 45 additions & 42 deletions src/lib/monitor/index.ts
Expand Up @@ -8,7 +8,16 @@ import * as os from 'os';
import * as _ from '@snyk/lodash';
import { isCI } from '../is-ci';
import * as analytics from '../analytics';
import { DepTree, MonitorMeta, MonitorResult } from '../types';
import {
DepTree,
MonitorMeta,
MonitorResult,
PolicyOptions,
MonitorOptions,
PackageJson,
Options,
Contributors,
} from '../types';
import * as projectMetadata from '../project-metadata';

import {
Expand All @@ -24,7 +33,7 @@ import { countTotalDependenciesInTree } from './count-total-deps-in-tree';
import { filterOutMissingDeps } from './filter-out-missing-deps';
import { dropEmptyDeps } from './drop-empty-deps';
import { pruneTree } from './prune-dep-tree';
import { pluckPolicies } from '../policy';
import { findAndLoadPolicy } from '../policy';
import { PluginMetadata } from '@snyk/cli-interface/legacy/plugin';
import { CallGraph, ScannedProject } from '@snyk/cli-interface/legacy/common';
import { isGitTarget } from '../project-metadata/types';
Expand All @@ -49,7 +58,7 @@ interface MonitorBody {
target: {};
targetFileRelativePath: string;
targetFile: string;
contributors?: { userId: string; lastCommitDate: string }[];
contributors?: Contributors[];
}

interface Meta {
Expand All @@ -74,10 +83,10 @@ export async function monitor(
root: string,
meta: MonitorMeta,
scannedProject: ScannedProject,
options,
options: Options & MonitorOptions & PolicyOptions,
pluginMeta: PluginMetadata,
targetFileRelativePath?: string,
contributors?: { userId: string; lastCommitDate: string }[],
contributors?: Contributors[],
): Promise<MonitorResult> {
apiTokenExists();

Expand All @@ -91,6 +100,7 @@ export async function monitor(
meta,
scannedProject,
pluginMeta,
options,
targetFileRelativePath,
contributors,
);
Expand All @@ -115,6 +125,7 @@ export async function monitor(
meta,
scannedProject,
pluginMeta,
options,
targetFileRelativePath,
contributors,
);
Expand All @@ -129,6 +140,7 @@ export async function monitor(
meta,
scannedProject,
pluginMeta,
options,
targetFileRelativePath,
contributors,
);
Expand All @@ -139,8 +151,9 @@ async function monitorDepTree(
meta: MonitorMeta,
scannedProject: ScannedProject,
pluginMeta: PluginMetadata,
options: MonitorOptions & PolicyOptions,
targetFileRelativePath?: string,
contributors?: { userId: string; lastCommitDate: string }[],
contributors?: Contributors[],
): Promise<MonitorResult> {
let treeMissingDeps: string[] = [];

Expand Down Expand Up @@ -173,15 +186,12 @@ async function monitorDepTree(
treeMissingDeps = missingDeps;
}

const policyPath = meta['policy-path'] || root;
const policyLocations = [policyPath]
.concat(pluckPolicies(depTree))
.filter(Boolean);
// docker doesn't have a policy as it can be run from anywhere
if (!meta.isDocker || !policyLocations.length) {
await snyk.policy.create();
}
const policy = await snyk.policy.load(policyLocations, { loose: true });
const policy = await findAndLoadPolicy(
root,
meta.isDocker ? 'docker' : packageManager!,
options,
depTree as PackageJson, // TODO: fix this and send only a manifest
);

const target = await projectMetadata.getInfo(scannedProject, meta, depTree);

Expand Down Expand Up @@ -261,7 +271,7 @@ async function monitorDepTree(
// WARNING: be careful changing this as it affects project uniqueness
targetFile: getTargetFile(scannedProject, pluginMeta),
targetFileRelativePath,
contributors: contributors,
contributors,
} as MonitorBody,
gzip: true,
method: 'PUT',
Expand Down Expand Up @@ -299,8 +309,9 @@ export async function monitorDepGraph(
meta: MonitorMeta,
scannedProject: ScannedProject,
pluginMeta: PluginMetadata,
options: MonitorOptions & PolicyOptions,
targetFileRelativePath?: string,
contributors?: { userId: string; lastCommitDate: string }[],
contributors?: Contributors[],
): Promise<MonitorResult> {
const packageManager = meta.packageManager;
analytics.add('monitorDepGraph', true);
Expand All @@ -316,16 +327,12 @@ export async function monitorDepGraph(
);
}

const policyPath = meta['policy-path'] || root;
const policyLocations = [policyPath]
.concat(pluckPolicies(depGraph))
.filter(Boolean);

if (!policyLocations.length) {
await snyk.policy.create();
}
const policy = await findAndLoadPolicy(
root,
meta.isDocker ? 'docker' : packageManager!,
options,
);

const policy = await snyk.policy.load(policyLocations, { loose: true });
const target = await projectMetadata.getInfo(scannedProject, meta);
if (isGitTarget(target) && target.branch) {
analytics.add('targetBranch', target.branch);
Expand Down Expand Up @@ -374,7 +381,7 @@ export async function monitorDepGraph(
target,
targetFile: getTargetFile(scannedProject, pluginMeta),
targetFileRelativePath,
contributors: contributors,
contributors,
} as MonitorBody,
gzip: true,
method: 'PUT',
Expand Down Expand Up @@ -408,16 +415,17 @@ export async function monitorDepGraph(

/**
* @deprecated it will be deleted once experimentalDepGraph FF will be deleted
and npm, yarn, sbt and rubygems usage of `experimentalMonitorDepGraphFromDepTree`
will be replaced with `monitorDepGraph` method
* and npm, yarn, sbt and rubygems usage of `experimentalMonitorDepGraphFromDepTree`
* will be replaced with `monitorDepGraph` method
*/
async function experimentalMonitorDepGraphFromDepTree(
root: string,
meta: MonitorMeta,
scannedProject: ScannedProject,
pluginMeta: PluginMetadata,
options: MonitorOptions & PolicyOptions,
targetFileRelativePath?: string,
contributors?: { userId: string; lastCommitDate: string }[],
contributors?: Contributors[],
): Promise<MonitorResult> {
const packageManager = meta.packageManager;
analytics.add('experimentalMonitorDepGraphFromDepTree', true);
Expand All @@ -434,10 +442,12 @@ async function experimentalMonitorDepGraphFromDepTree(
);
}

const policyPath = meta['policy-path'] || root;
const policyLocations = [policyPath]
.concat(pluckPolicies(depTree))
.filter(Boolean);
const policy = await findAndLoadPolicy(
root,
meta.isDocker ? 'docker' : packageManager!,
options,
depTree as PackageJson, // TODO: fix this and send only a manifest
);

if (['npm', 'yarn'].includes(meta.packageManager)) {
const { filteredDepTree, missingDeps } = filterOutMissingDeps(depTree);
Expand All @@ -449,13 +459,6 @@ async function experimentalMonitorDepGraphFromDepTree(
depTree,
packageManager,
);

// docker doesn't have a policy as it can be run from anywhere
if (!meta.isDocker || !policyLocations.length) {
await snyk.policy.create();
}
const policy = await snyk.policy.load(policyLocations, { loose: true });

const target = await projectMetadata.getInfo(scannedProject, meta, depTree);

if (isGitTarget(target) && target.branch) {
Expand Down Expand Up @@ -517,7 +520,7 @@ async function experimentalMonitorDepGraphFromDepTree(
target,
targetFile: getTargetFile(scannedProject, pluginMeta),
targetFileRelativePath,
contributors: contributors,
contributors,
} as MonitorBody,
gzip: true,
method: 'PUT',
Expand Down
46 changes: 46 additions & 0 deletions src/lib/policy/find-and-load-policy.ts
@@ -0,0 +1,46 @@
import * as snykPolicyLib from 'snyk-policy';
import * as debugModule from 'debug';

import { pluckPolicies } from '.';
import { SupportedPackageManagers } from '../package-managers';
import { PackageJson, PolicyOptions } from '../types';
import * as analytics from '../analytics';

const debug = debugModule('snyk');

export async function findAndLoadPolicy(
root: string,
scanType: SupportedPackageManagers | 'docker',
options: PolicyOptions,
pkg?: PackageJson,
): Promise<string | undefined> {
const isDocker = scanType === 'docker';
const isNodeProject = ['npm', 'yarn'].includes(scanType);
// monitor
let policyLocations: string[] = [options['policy-path'] || root];
if (isDocker) {
policyLocations = policyLocations.filter((loc) => loc !== root);
} else if (isNodeProject) {
// TODO: pluckPolicies expects a package.json object to
// find and apply policies in node_modules
policyLocations = policyLocations.concat(pluckPolicies(pkg as PackageJson));
}
debug('Potential policy locations found:', policyLocations);
analytics.add('policies', policyLocations.length);
analytics.add('policyLocations', policyLocations);

if (policyLocations.length === 0) {
return;
}
let policy;
try {
policy = await snykPolicyLib.load(policyLocations, options);
} catch (err) {
// note: inline catch, to handle error from .load
// if the .snyk file wasn't found, it is fine
if (err.code !== 'ENOENT') {
throw err;
}
}
return policy;
}
1 change: 1 addition & 0 deletions src/lib/policy/index.ts
@@ -1 +1,2 @@
export { pluckPolicies } from './pluck-policies';
export { findAndLoadPolicy } from './find-and-load-policy';
13 changes: 6 additions & 7 deletions src/lib/policy/pluck-policies.ts
@@ -1,23 +1,22 @@
import * as _ from '@snyk/lodash';
import { PackageJson } from '../types';

export function pluckPolicies(pkg) {
export function pluckPolicies(pkg: PackageJson): string[] {
if (!pkg) {
return null;
return [];
}

if (pkg.snyk) {
return pkg.snyk;
return []; // why is this check here?
}

if (!pkg.dependencies) {
return null;
return [];
}

return _.flatten(
Object.keys(pkg.dependencies)
.map((name) => {
return pluckPolicies(pkg.dependencies[name]);
})
.map((name: string) => pluckPolicies(pkg.dependencies[name]))
.filter(Boolean),
);
}

0 comments on commit 197f1ec

Please sign in to comment.