-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: [OSM-1024] pnpm dep graph builder
- Loading branch information
Showing
163 changed files
with
342,041 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,147 @@ | ||
import { DepGraphBuilder } from '@snyk/dep-graph'; | ||
import { getTopLevelDeps } from '../util'; | ||
import type { | ||
Overrides, | ||
PnpmProjectParseOptions, | ||
PnpmWorkspaceArgs, | ||
} from '../types'; | ||
import type { PackageJsonBase } from '../types'; | ||
import { getPnpmChildNode } from './utils'; | ||
import { eventLoopSpinner } from 'event-loop-spinner'; | ||
import { PnpmLockfileParser } from './lockfile-parser/lockfile-parser'; | ||
import { NormalisedPnpmPkgs, PnpmNode } from './types'; | ||
|
||
export const buildDepGraphPnpm = async ( | ||
lockFileParser: PnpmLockfileParser, | ||
pkgJson: PackageJsonBase, | ||
options: PnpmProjectParseOptions, | ||
workspaceArgs?: PnpmWorkspaceArgs, | ||
) => { | ||
const { strictOutOfSync, includeOptionalDeps, pruneWithinTopLevelDeps } = | ||
options; | ||
|
||
const depGraphBuilder = new DepGraphBuilder( | ||
{ name: 'pnpm' }, | ||
{ name: pkgJson.name, version: pkgJson.version }, | ||
); | ||
|
||
const extractedPnpmPkgs: NormalisedPnpmPkgs = | ||
lockFileParser.extractedPackages; | ||
|
||
const topLevelDeps = getTopLevelDeps(pkgJson, options); | ||
|
||
const extractedTopLevelDeps = | ||
lockFileParser.extractTopLevelDependencies(options) || {}; | ||
|
||
for (const [name, _] of Object.entries(topLevelDeps)) { | ||
topLevelDeps[name].version = extractedTopLevelDeps[name].version; | ||
} | ||
|
||
const rootNode: PnpmNode = { | ||
id: 'root-node', | ||
name: pkgJson.name, | ||
version: pkgJson.version, | ||
dependencies: topLevelDeps, | ||
isDev: false, | ||
}; | ||
|
||
await dfsVisit( | ||
depGraphBuilder, | ||
rootNode, | ||
extractedPnpmPkgs, | ||
strictOutOfSync, | ||
includeOptionalDeps, | ||
// we have rootWorkspaceOverrides if this is workspace pkg with overrides | ||
// at root - therefore it should take precedent | ||
// TODO: inspect if this is needed at all, seems like pnpm resolves everything in lockfile | ||
workspaceArgs?.rootOverrides || pkgJson.pnpm?.overrides || {}, | ||
pruneWithinTopLevelDeps, | ||
lockFileParser, | ||
); | ||
|
||
return depGraphBuilder.build(); | ||
}; | ||
|
||
/** | ||
* Use DFS to add all nodes and edges to the depGraphBuilder and prune cyclic nodes. | ||
* The visitedMap keep track of which nodes have already been discovered during traversal. | ||
* - If a node doesn't exist in the map, it means it hasn't been visited. | ||
* - If a node is already visited, simply connect the new node with this node. | ||
*/ | ||
const dfsVisit = async ( | ||
depGraphBuilder: DepGraphBuilder, | ||
node: PnpmNode, | ||
extractedPnpmPkgs: NormalisedPnpmPkgs, | ||
strictOutOfSync: boolean, | ||
includeOptionalDeps: boolean, | ||
overrides: Overrides, | ||
pruneWithinTopLevel: boolean, | ||
lockFileParser: PnpmLockfileParser, | ||
visited?: Set<string>, | ||
): Promise<void> => { | ||
for (const [name, depInfo] of Object.entries(node.dependencies || {})) { | ||
if (eventLoopSpinner.isStarving()) { | ||
await eventLoopSpinner.spin(); | ||
} | ||
|
||
const localVisited = visited || new Set<string>(); | ||
|
||
const childNode: PnpmNode = getPnpmChildNode( | ||
name, | ||
depInfo, | ||
extractedPnpmPkgs, | ||
strictOutOfSync, | ||
includeOptionalDeps, | ||
lockFileParser, | ||
); | ||
|
||
if (localVisited.has(childNode.id)) { | ||
if (pruneWithinTopLevel) { | ||
const prunedId = `${childNode.id}:pruned`; | ||
depGraphBuilder.addPkgNode( | ||
{ name: childNode.name, version: childNode.version }, | ||
prunedId, | ||
{ | ||
labels: { | ||
scope: childNode.isDev ? 'dev' : 'prod', | ||
pruned: 'true', | ||
...(node.missingLockFileEntry && { | ||
missingLockFileEntry: 'true', | ||
}), | ||
}, | ||
}, | ||
); | ||
depGraphBuilder.connectDep(node.id, prunedId); | ||
} else { | ||
depGraphBuilder.connectDep(node.id, childNode.id); | ||
} | ||
continue; | ||
} | ||
|
||
depGraphBuilder.addPkgNode( | ||
{ name: childNode.name, version: childNode.version }, | ||
childNode.id, | ||
{ | ||
labels: { | ||
scope: childNode.isDev ? 'dev' : 'prod', | ||
...(node.missingLockFileEntry && { | ||
missingLockFileEntry: 'true', | ||
}), | ||
}, | ||
}, | ||
); | ||
depGraphBuilder.connectDep(node.id, childNode.id); | ||
localVisited.add(childNode.id); | ||
await dfsVisit( | ||
depGraphBuilder, | ||
childNode, | ||
extractedPnpmPkgs, | ||
strictOutOfSync, | ||
includeOptionalDeps, | ||
overrides, | ||
pruneWithinTopLevel, | ||
lockFileParser, | ||
localVisited, | ||
); | ||
} | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
import { parsePkgJson } from '../util'; | ||
import { | ||
PackageJsonBase, | ||
PnpmProjectParseOptions, | ||
PnpmWorkspaceArgs, | ||
} from '../types'; | ||
import { buildDepGraphPnpm } from './build-dep-graph-pnpm'; | ||
import { DepGraph } from '@snyk/dep-graph'; | ||
import { getPnpmLockfileParser } from './lockfile-parser/index'; | ||
import { PnpmLockfileParser } from './lockfile-parser/lockfile-parser'; | ||
import { NodeLockfileVersion } from '../../utils'; | ||
|
||
export const parsePnpmProject = async ( | ||
pkgJsonContent: string, | ||
pnpmLockContent: string, | ||
options: PnpmProjectParseOptions, | ||
lockfileVersion?: NodeLockfileVersion, | ||
workspaceArgs?: PnpmWorkspaceArgs, | ||
): Promise<DepGraph> => { | ||
const { | ||
includeDevDeps, | ||
includeOptionalDeps, | ||
strictOutOfSync, | ||
pruneWithinTopLevelDeps, | ||
} = options; | ||
|
||
const pkgJson: PackageJsonBase = parsePkgJson(pkgJsonContent); | ||
|
||
const lockFileParser: PnpmLockfileParser = getPnpmLockfileParser( | ||
pnpmLockContent, | ||
lockfileVersion, | ||
workspaceArgs, | ||
); | ||
|
||
const depgraph = await buildDepGraphPnpm( | ||
lockFileParser, | ||
pkgJson, | ||
{ | ||
includeDevDeps, | ||
strictOutOfSync, | ||
includeOptionalDeps, | ||
pruneWithinTopLevelDeps, | ||
}, | ||
workspaceArgs, | ||
); | ||
|
||
return depgraph; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
import { load, FAILSAFE_SCHEMA } from 'js-yaml'; | ||
import { PnpmLockfileParser } from './lockfile-parser'; | ||
import { LockfileV6Parser } from './lockfile-v6'; | ||
import { LockfileV5Parser } from './lockfile-v5'; | ||
import { PnpmWorkspaceArgs } from '../../types'; | ||
import { OpenSourceEcosystems } from '@snyk/error-catalog-nodejs-public'; | ||
import { NodeLockfileVersion } from '../../../utils'; | ||
|
||
export function getPnpmLockfileParser( | ||
pnpmLockContent: string, | ||
lockfileVersion?: NodeLockfileVersion, | ||
workspaceArgs?: PnpmWorkspaceArgs, | ||
): PnpmLockfileParser { | ||
const rawPnpmLock = load(pnpmLockContent, { | ||
json: true, | ||
schema: FAILSAFE_SCHEMA, | ||
}); | ||
if (!lockfileVersion) { | ||
const version = rawPnpmLock.lockfileVersion; | ||
if (version.startsWith('5')) { | ||
lockfileVersion = NodeLockfileVersion.PnpmLockV5; | ||
} else if (version.startsWith('6')) { | ||
lockfileVersion = NodeLockfileVersion.PnpmLockV6; | ||
} | ||
} | ||
switch (lockfileVersion) { | ||
case NodeLockfileVersion.PnpmLockV5: | ||
return new LockfileV5Parser(rawPnpmLock, workspaceArgs); | ||
case NodeLockfileVersion.PnpmLockV6: | ||
return new LockfileV6Parser(rawPnpmLock, workspaceArgs); | ||
} | ||
throw new OpenSourceEcosystems.UnsupportedLockfileVersionError( | ||
`The pnpm-lock.yaml lockfile version ${lockfileVersion} is not supported`, | ||
); | ||
} |
Oops, something went wrong.