From 4f93998b9784c9915b1ce2d2655e2b237df79965 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Tue, 24 Oct 2023 18:08:55 +0300 Subject: [PATCH] feat: add functionality of matching manifest versions against installed ones Jira Ticket: https://issues.redhat.com/browse/APPENG-2062 Signed-off-by: Zvi Grinberg --- src/providers/golang_gomodules.js | 95 ++++++++++++++++++++++++- src/providers/python_controller.js | 27 +++++++ test/providers/golang_gomodules.test.js | 1 - test/providers/python_pip.test.js | 4 +- 4 files changed, 123 insertions(+), 4 deletions(-) diff --git a/src/providers/golang_gomodules.js b/src/providers/golang_gomodules.js index 028df74d..6c020a20 100644 --- a/src/providers/golang_gomodules.js +++ b/src/providers/golang_gomodules.js @@ -3,7 +3,7 @@ import { execSync } from "node:child_process" import fs from 'node:fs' import os from "node:os"; import {EOL} from "os"; -import { getCustomPath } from "../tools.js"; +import {getCustom, getCustomPath} from "../tools.js"; import path from 'node:path' import Sbom from '../sbom.js' import {PackageURL} from 'packageurl-js' @@ -174,6 +174,93 @@ function enforceRemovingIgnoredDepsInCaseOfAutomaticVersionUpdate(ignoredDeps, s }) } +/** + * + * @param {[string]} lines - array of lines of go.mod manifest + * @param {string} goMod - content of go.mod manifest + * @return {[string]} all dependencies from go.mod file as array + */ +function collectAllDepsFromManifest(lines, goMod) { + let result + // collect all deps that starts with require keyword + + result = lines.filter((line) => line.trim().startsWith("require") && !line.includes("(")).map((dep) => dep.substring("require".length).trim()) + + + + // collect all deps that are inside `require` blocks + let currentSegmentOfGoMod = goMod + let requirePositionObject = decideRequireBlockIndex(currentSegmentOfGoMod) + while(requirePositionObject.index > -1) { + let depsInsideRequirementsBlock = currentSegmentOfGoMod.substring(requirePositionObject.index + requirePositionObject.startingOffeset).trim(); + let endOfBlockIndex = depsInsideRequirementsBlock.indexOf(")") + let currentIndex = 0 + while(currentIndex < endOfBlockIndex) + { + let endOfLinePosition = depsInsideRequirementsBlock.indexOf(EOL, currentIndex); + let dependency = depsInsideRequirementsBlock.substring(currentIndex, endOfLinePosition) + result.push(dependency.trim()) + currentIndex = endOfLinePosition + 1 + } + currentSegmentOfGoMod = currentSegmentOfGoMod.substring(endOfBlockIndex + 1).trim() + requirePositionObject = decideRequireBlockIndex(currentSegmentOfGoMod) + } + + function decideRequireBlockIndex(goMod) { + let object = {} + let index = goMod.indexOf("require(") + object.startingOffeset = "require(".length + if (index === -1) + { + index = goMod.indexOf("require (") + object.startingOffeset = "require (".length + if(index === -1) + { + index = goMod.indexOf("require (") + object.startingOffeset = "require (".length + } + } + object.index = index + return object + } + return result +} + +/** + * + * @param {string} rootElementName the rootElementName element of go mod graph, to compare only direct deps from go mod graph against go.mod manifest + * @param{[string]} goModGraphOutputRows the goModGraphOutputRows from go mod graph' output + * @param {string }manifest path to go.mod manifest on file system + * @private + */ +function performManifestVersionsCheck(rootElementName, goModGraphOutputRows, manifest) { + let goMod = fs.readFileSync(manifest).toString().trim() + let lines = goMod.split(EOL); + let comparisonLines = goModGraphOutputRows.filter((line)=> line.startsWith(rootElementName)).map((line)=> getChildVertexFromEdge(line)) + let manifestDeps = collectAllDepsFromManifest(lines,goMod) + try { + comparisonLines.forEach((dependency) => { + let parts = dependency.split("@") + let version = parts[1] + let depName = parts[0] + manifestDeps.forEach(dep => { + let components = dep.trim().split(" "); + let currentDepName = components[0] + let currentVersion = components[1] + if (currentDepName === depName) { + if (currentVersion !== version) { + throw new Error(`versions mismatch for dependency name ${depName}, manifest version=${currentVersion}, installed Version=${version}, if you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting - MATCH_MANIFEST_VERSIONS=false`) + } + } + }) + }) + } + catch(error) { + console.error("Can't continue with analysis") + throw error + } +} + /** * Create SBOM json string for go Module. * @param {string} manifest - path for go.mod @@ -201,6 +288,12 @@ function getSBOM(manifest, opts = {}, includeTransitive) { let sbom = new Sbom(); let rows = goGraphOutput.split(EOL); let root = getParentVertexFromEdge(rows[0]) + let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS","false"); + if(matchManifestVersions === "true") { + { + performManifestVersionsCheck(root, rows, manifest) + } + } let mainModule = toPurl(root, "@", undefined) sbom.addRoot(mainModule) diff --git a/src/providers/python_controller.js b/src/providers/python_controller.js index 423df38f..3d067188 100644 --- a/src/providers/python_controller.js +++ b/src/providers/python_controller.js @@ -2,6 +2,7 @@ import {execSync} from "node:child_process"; import fs from "node:fs"; import path from 'node:path'; import {EOL} from "os"; +import {getCustom} from "../tools.js"; /** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */ @@ -126,6 +127,7 @@ export default class Python_controller { } }).toString(); let allPipShowDeps = pipShowOutput.split( EOL +"---" + EOL); + let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS","true"); let linesOfRequirements = fs.readFileSync(this.pathToRequirements).toString().split(EOL).filter( (line) => !line.startsWith("#")).map(line => line.trim()) let CachedEnvironmentDeps = {} allPipShowDeps.forEach( (record) => { @@ -135,6 +137,31 @@ export default class Python_controller { CachedEnvironmentDeps[dependencyName.replace("_","-")] = record }) linesOfRequirements.forEach( (dep) => { + // if matchManifestVersions setting is turned on , then + if(matchManifestVersions === "true") + { + let dependencyName + let manifestVersion + let installedVersion + let doubleEqualSignPosition + if(dep.includes("==")) + { + doubleEqualSignPosition = dep.indexOf("==") + manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim() + if(manifestVersion.includes("#")) + { + let hashCharIndex = manifestVersion.indexOf("#"); + manifestVersion = manifestVersion.substring(0,hashCharIndex) + } + dependencyName = getDependencyName(dep) + installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()]) + if(manifestVersion.trim() !== installedVersion.trim()) + { + throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName}, manifest version=${manifestVersion}, installed Version=${installedVersion}, if you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting - MATCH_MANIFEST_VERSIONS=false`) + } + + } + } bringAllDependencies(dependencies,getDependencyName(dep),CachedEnvironmentDeps,includeTransitive) }) dependencies.sort((dep1,dep2) =>{ diff --git a/test/providers/golang_gomodules.test.js b/test/providers/golang_gomodules.test.js index 47f413b4..848fa612 100644 --- a/test/providers/golang_gomodules.test.js +++ b/test/providers/golang_gomodules.test.js @@ -29,7 +29,6 @@ suite('testing the golang-go-modules data provider', () => { let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/golang/${testCase}/expected_sbom_stack_analysis.json`,).toString() expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) // invoke sut stack analysis for scenario manifest - let providedDataForStack = await golangGoModules.provideStack(`test/providers/tst_manifests/golang/${testCase}/go.mod`) // new(year: number, month: number, date?: number, hours?: number, minutes?: number, seconds?: number, ms?: number): Date diff --git a/test/providers/python_pip.test.js b/test/providers/python_pip.test.js index 022d116d..3faafad0 100644 --- a/test/providers/python_pip.test.js +++ b/test/providers/python_pip.test.js @@ -67,7 +67,7 @@ suite('testing the python-pip data provider', () => { }).beforeAll(() => clock = sinon.useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(()=> clock.restore()); -suite('testing the python-pip data provider', () => { +suite('testing the python-pip data provider with virtual environment', () => { [ "pip_requirements_virtual_env_txt_no_ignore", "pip_requirements_virtual_env_with_ignore" @@ -93,7 +93,7 @@ suite('testing the python-pip data provider', () => { // content: expectedSbom // }) // these test cases takes ~2500-2700 ms each pr >10000 in CI (for the first test-case) - }).timeout(process.env.GITHUB_ACTIONS ? 30000 : 15000) + }).timeout(process.env.GITHUB_ACTIONS ? 60000 : 30000) })