diff --git a/src/providers/python_controller.js b/src/providers/python_controller.js index 315bb4c6..b523487c 100644 --- a/src/providers/python_controller.js +++ b/src/providers/python_controller.js @@ -1,31 +1,26 @@ -import {execSync} from "node:child_process"; import fs from "node:fs"; import path from 'node:path'; import os, {EOL} from "os"; -import {environmentVariableIsPopulated,getCustom, handleSpacesInPath} from "../tools.js"; - +import {environmentVariableIsPopulated,getCustom, invokeCommand} from "../tools.js"; function getPipFreezeOutput() { - return environmentVariableIsPopulated("EXHORT_PIP_FREEZE") ? new Buffer(process.env["EXHORT_PIP_FREEZE"],'base64').toString('ascii') : execSync(`${handleSpacesInPath(this.pathToPipBin)} freeze --all`, err => { - if (err) { - throw new Error('fail invoking pip freeze to fetch all installed dependencies in environment --> ' + err.message) - } - }).toString(); + try { + return environmentVariableIsPopulated("EXHORT_PIP_FREEZE") ? new Buffer.from(process.env["EXHORT_PIP_FREEZE"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['freeze', '--all']).toString(); + } catch (error) { + throw new Error('Failed invoking \'pip freeze\' to list all installed packages in environment', {cause: error}) + } } function getPipShowOutput(depNames) { - - return environmentVariableIsPopulated("EXHORT_PIP_SHOW") ? new Buffer(process.env["EXHORT_PIP_SHOW"],'base64').toString('ascii') : execSync(`${handleSpacesInPath(this.pathToPipBin)} show ${depNames}`, err => { - if (err) { - throw new Error('fail invoking pip show to fetch all installed dependencies metadata --> ' + err.message) - } - }).toString(); + try { + return environmentVariableIsPopulated("EXHORT_PIP_SHOW") ? new Buffer.from(process.env["EXHORT_PIP_SHOW"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['show', ...depNames]).toString(); + } catch (error) { + throw new Error('fail invoking \'pip show\' to fetch metadata for all installed packages in environment', {cause: error}) + } } /** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */ - - export default class Python_controller { pythonEnvDir @@ -51,17 +46,15 @@ export default class Python_controller { this.pathToRequirements = pathToRequirements this.options = options } - prepareEnvironment() - { + prepareEnvironment() { if(!this.realEnvironment) { - this.pythonEnvDir = path.join(path.sep,"tmp","exhort_env_js") - execSync(`${handleSpacesInPath(this.pathToPythonBin)} -m venv ${handleSpacesInPath(this.pythonEnvDir)} `, err => { - if (err) { - throw new Error('failed creating virtual python environment - ' + err.message) - } - }) - if(this.pathToPythonBin.includes("python3")) - { + this.pythonEnvDir = path.join(path.sep, "tmp", "exhort_env_js") + try { + invokeCommand(this.pathToPythonBin, ['-m', 'venv', this.pythonEnvDir]) + } catch (error) { + throw new Error('Failed creating virtual python environment', {cause: error}) + } + if(this.pathToPythonBin.includes("python3")) { this.pathToPipBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("pip3")) this.pathToPythonBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("python3")) if(os.platform() === 'win32') { @@ -69,8 +62,7 @@ export default class Python_controller { this.pathToPythonBin = `${driveLetter}${this.pathToPythonBin.substring(1)}` this.pathToPipBin = `${driveLetter}${this.pathToPipBin.substring(1)}` } - } - else { + } else { this.pathToPipBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("pip")); this.pathToPythonBin = path.join(path.sep,this.pythonEnvDir,os.platform() === 'win32' ? "Scripts" : "bin",this.#decideIfWindowsOrLinuxPath("python")) if(os.platform() === 'win32') { @@ -80,18 +72,15 @@ export default class Python_controller { } } // upgrade pip version to latest - execSync(`${handleSpacesInPath(this.pathToPythonBin)} -m pip install --upgrade pip `, err => { - if (err) { - throw new Error('failed upgrading pip version on virtual python environment - ' + err.message) - } - }) - } - else{ + try { + invokeCommand(this.pathToPythonBin, ['-m', 'pip', 'install', '--upgrade', 'pip']) + } catch (error) { + throw new Error('Failed upgrading pip version in virtual python environment', {cause: error}) + } + } else { if(this.pathToPythonBin.startsWith("python")) { this.pythonEnvDir = process.cwd() - } - else - { + } else { this.pythonEnvDir = path.dirname(this.pathToPythonBin) } } @@ -100,8 +89,7 @@ export default class Python_controller { #decideIfWindowsOrLinuxPath(fileName) { if (os.platform() === "win32") { return fileName + ".exe" - } - else { + } else { return fileName } } @@ -110,8 +98,7 @@ export default class Python_controller { * @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not * @return {[DependencyEntry]} */ - getDependencies(includeTransitive) - { + getDependencies(includeTransitive) { let startingTime let endingTime if (process.env["EXHORT_DEBUG"] === "true") { @@ -120,21 +107,19 @@ export default class Python_controller { } if(!this.realEnvironment) { let installBestEfforts = getCustom("EXHORT_PYTHON_INSTALL_BEST_EFFORTS","false",this.options); - if(installBestEfforts === "false") - { - execSync(`${handleSpacesInPath(this.pathToPipBin)} install -r ${handleSpacesInPath(this.pathToRequirements)}`, err =>{ - if (err) { - throw new Error('fail installing requirements.txt manifest in created virtual python environment --> ' + err.message) - } - }) + if(installBestEfforts === "false") { + try { + invokeCommand(this.pathToPipBin, ['install', '-r', this.pathToRequirements]) + } catch (error) { + throw new Error('Failed installing requirements.txt manifest in virtual python environment', {cause: error}) + } } // make best efforts to install the requirements.txt on the virtual environment created from the python3 passed in. // that means that it will install the packages without referring to the versions, but will let pip choose the version // tailored for version of the python environment( and of pip package manager) for each package. else { let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS","true",this.options); - if(matchManifestVersions === "true") - { + if(matchManifestVersions === "true") { throw new Error("Conflicting settings, EXHORT_PYTHON_INSTALL_BEST_EFFORTS=true can only work with MATCH_MANIFEST_VERSIONS=false") } this.#installingRequirementsOneByOne() @@ -156,27 +141,26 @@ export default class Python_controller { let requirementsRows = requirementsContent.toString().split(EOL); requirementsRows.filter((line) => !line.trim().startsWith("#")).filter((line) => line.trim() !== "").forEach( (dependency) => { let dependencyName = getDependencyName(dependency); - execSync(`${handleSpacesInPath(this.pathToPipBin)} install ${dependencyName}`, err =>{ - if (err) { - throw new Error(`Best efforts process - failed installing ${dependencyName} in created virtual python environment --> error message: ` + err.message) - } - }) - } ) + try { + invokeCommand(this.pathToPipBin, ['install', dependencyName]) + } catch (error) { + throw new Error(`Failed in best-effort installing ${dependencyName} in virtual python environment`, {cause: error}) + } + }) } /** * @private */ - #cleanEnvironment() - { - if(!this.realEnvironment) - { - execSync(`${handleSpacesInPath(this.pathToPipBin)} uninstall -y -r ${handleSpacesInPath(this.pathToRequirements)}`, err =>{ - if (err) { - throw new Error('fail uninstalling requirements.txt in created virtual python environment --> ' + err.message) - } - }) + #cleanEnvironment() { + if(!this.realEnvironment) { + try { + invokeCommand(this.pathToPipBin, ['uninstall', '-y', '-r', this.pathToRequirements]) + } catch (error) { + throw new Error('Failed uninstalling requirements.txt in virtual python environment', {cause: error}) + } } } + #getDependenciesImpl(includeTransitive) { let dependencies = new Array() let usePipDepTree = getCustom("EXHORT_PIP_USE_DEP_TREE","false",this.options); @@ -189,7 +173,7 @@ export default class Python_controller { if(usePipDepTree !== "true") { freezeOutput = getPipFreezeOutput.call(this); lines = freezeOutput.split(EOL) - depNames = lines.map( line => getDependencyName(line)).join(" ") + depNames = lines.map( line => getDependencyName(line)) } else { pipDepTreeJsonArrayOutput = getDependencyTreeJsonFromPipDepTree(this.pathToPipBin,this.pathToPythonBin) @@ -198,7 +182,7 @@ export default class Python_controller { if(usePipDepTree !== "true") { pipShowOutput = getPipShowOutput.call(this, depNames); - allPipShowDeps = pipShowOutput.split( EOL +"---" + EOL); + allPipShowDeps = pipShowOutput.split( EOL + "---" + EOL); } //debug // pipShowOutput = "alternative pip show output goes here for debugging" @@ -213,8 +197,7 @@ export default class Python_controller { CachedEnvironmentDeps[dependencyName.replace("-", "_")] = record CachedEnvironmentDeps[dependencyName.replace("_", "-")] = record }) - } - else { + } else { pipDepTreeJsonArrayOutput.forEach( depTreeEntry => { let packageName = depTreeEntry["package"]["package_name"].toLowerCase() let pipDepTreeEntryForCache = { @@ -229,18 +212,15 @@ export default class Python_controller { } linesOfRequirements.forEach( (dep) => { // if matchManifestVersions setting is turned on , then - if(matchManifestVersions === "true") - { + if(matchManifestVersions === "true") { let dependencyName let manifestVersion let installedVersion let doubleEqualSignPosition - if(dep.includes("==")) - { + if(dep.includes("==")) { doubleEqualSignPosition = dep.indexOf("==") manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim() - if(manifestVersion.includes("#")) - { + if(manifestVersion.includes("#")) { let hashCharIndex = manifestVersion.indexOf("#"); manifestVersion = manifestVersion.substring(0,hashCharIndex) } @@ -249,17 +229,15 @@ export default class Python_controller { if(CachedEnvironmentDeps[dependencyName.toLowerCase()] !== undefined) { if(usePipDepTree !== "true") { installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()]) - } - else { + } else { installedVersion = CachedEnvironmentDeps[dependencyName.toLowerCase()].version } } if(installedVersion) { if (manifestVersion.trim() !== installedVersion.trim()) { - throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName}, manifest version=${manifestVersion}, installed Version=${installedVersion}, if you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting - MATCH_MANIFEST_VERSIONS=false`) + throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`) } } - } } let path = new Array() @@ -274,11 +252,11 @@ export default class Python_controller { if(DEP1 < DEP2) { return -1; } - if(DEP1 > DEP2) - { + if(DEP1 > DEP2) { return 1; } - return 0;}) + return 0; + }) return dependencies } } @@ -321,8 +299,7 @@ function getDependencyName(depLine) { const regex = /[\w\s-_.]+/g; if(depLine.match(regex)) { result = depLine.match(regex)[0] - } - else { + } else { result = depLine } } @@ -358,10 +335,7 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep } let record = cachedEnvironmentDeps[dependencyName.toLowerCase()] if(record === null || record === undefined) { - throw new Error(`Package name=>${dependencyName} is not installed in your python environment, - either install it ( better to install requirements.txt altogether) or set - the setting EXHORT_PYTHON_VIRTUAL_ENV to true to automatically install - it in virtual environment (please note that this may slow down the analysis) `) + throw new Error(`Package ${dependencyName} is not installed in your python environment, either install it (better to install requirements.txt altogether) or set the setting EXHORT_PYTHON_VIRTUAL_ENV=true to automatically install it in virtual environment (please note that this may slow down the analysis)`) } let depName let version; @@ -370,8 +344,7 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep depName = getDependencyNameShow(record) version = getDependencyVersion(record); directDeps = getDepsList(record) - } - else { + } else { depName = record.name version = record.version directDeps = record.dependencies @@ -398,11 +371,11 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep if(DEP1 < DEP2) { return -1; } - if(DEP1 > DEP2) - { + if(DEP1 > DEP2) { return 1; } - return 0;}) + return 0; + }) entry["dependencies"] = targetDeps }) @@ -418,20 +391,19 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep function getDependencyTreeJsonFromPipDepTree(pipPath,pythonPath) { let dependencyTree try { - execSync(`${handleSpacesInPath(pipPath)} install pipdeptree`) - } catch (e) { - throw new Error(`Couldn't install pipdeptree utility, reason: ${e.getMessage}`) + invokeCommand(pipPath, ['install', 'pipdeptree']) + } catch (error) { + throw new Error(`Failed installing pipdeptree utility`, {cause: error}) } try { if(pythonPath.startsWith("python")) { - dependencyTree = execSync(`pipdeptree --json`).toString() - } - else { - dependencyTree = execSync(`pipdeptree --json --python ${handleSpacesInPath(pythonPath)} `).toString() + dependencyTree = invokeCommand('pipdeptree', ['--json']).toString() + } else { + dependencyTree = invokeCommand('pipdeptree', ['--json', '--python', pythonPath]).toString() } - } catch (e) { - throw new Error(`couldn't produce dependency tree using pipdeptree tool, stop analysis, message -> ${e.getMessage}`) + } catch (error) { + throw new Error(`Failed building dependency tree using pipdeptree tool, stopping analysis`, {cause: error}) } return JSON.parse(dependencyTree) diff --git a/src/providers/python_pip.js b/src/providers/python_pip.js index bf210f14..96b9a89e 100644 --- a/src/providers/python_pip.js +++ b/src/providers/python_pip.js @@ -1,11 +1,9 @@ - -import { execSync } from "node:child_process"; import fs from 'node:fs' import { environmentVariableIsPopulated, getCustom, getCustomPath, - handleSpacesInPath + invokeCommand } from "../tools.js"; import Sbom from '../sbom.js' import { PackageURL } from 'packageurl-js' @@ -75,11 +73,11 @@ function provideComponent(manifest, opts = {}) { * @private */ function addAllDependencies(source, dep, sbom) { - let targetPurl = toPurl(dep["name"],dep["version"]) - sbom.addDependency(sbom.purlToComponent(source),targetPurl) + let targetPurl = toPurl(dep["name"], dep["version"]) + sbom.addDependency(sbom.purlToComponent(source), targetPurl) let directDeps = dep["dependencies"] if (directDeps !== undefined && directDeps.length > 0) { - directDeps.forEach( (dependency) =>{ addAllDependencies(toPurl(dep["name"],dep["version"]),dependency,sbom)}) + directDeps.forEach( (dependency) =>{ addAllDependencies(toPurl(dep["name"],dep["version"]), dependency, sbom)}) } } @@ -133,7 +131,7 @@ function handleIgnoredDependencies(requirementTxtContent, sbom, opts ={}) { .filter(dep => dep.toString().includes(dummyVersionNotation)) .map(dep => dep.name) sbom.filterIgnoredDeps(ignoredDepsNoVersions) - let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS","true",opts); + let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", opts); if(matchManifestVersions === "true") { sbom.filterIgnoredDepsIncludingVersion(ignoredDepsVersion) } else { @@ -148,19 +146,19 @@ function handleIgnoredDependencies(requirementTxtContent, sbom, opts ={}) { * @param {{}} [opts={}] */ function getPythonPipBinaries(binaries,opts) { - let python = getCustomPath("python3",opts) - let pip = getCustomPath("pip3",opts) + let python = getCustomPath("python3", opts) + let pip = getCustomPath("pip3", opts) try { - execSync(`${handleSpacesInPath(python)} --version`) - execSync(`${handleSpacesInPath(pip)} --version`) - } catch (e) { - python = getCustomPath("python",opts) - pip = getCustomPath("pip",opts) + invokeCommand(python, ['--version']) + invokeCommand(pip, ['--version']) + } catch (error) { + python = getCustomPath("python", opts) + pip = getCustomPath("pip", opts) try { - execSync(`${handleSpacesInPath(python)} --version`) - execSync(`${handleSpacesInPath(pip)} --version`) - } catch (e) { - throw new Error(`Couldn't get python binaries from supplied environment variables ${e.getMessage}`) + invokeCommand(python, ['--version']) + invokeCommand(pip, ['--version']) + } catch (error) { + throw new Error(`Failed checking for python/pip binaries from supplied environment variables`, {cause: error}) } } binaries.pip = pip @@ -204,15 +202,15 @@ function createSbomStackAnalysis(manifest, opts = {}) { let binaries = {} let createVirtualPythonEnv = handlePythonEnvironment(binaries, opts); - let pythonController = new Python_controller(createVirtualPythonEnv === "false",binaries.pip,binaries.python,manifest,opts) + let pythonController = new Python_controller(createVirtualPythonEnv === "false", binaries.pip, binaries.python, manifest, opts) let dependencies = pythonController.getDependencies(true); let sbom = new Sbom(); - sbom.addRoot(toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME,DEFAULT_PIP_ROOT_COMPONENT_VERSION)) + sbom.addRoot(toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION)) dependencies.forEach(dep => { - addAllDependencies(sbom.getRoot(),dep,sbom) + addAllDependencies(sbom.getRoot(), dep, sbom) }) let requirementTxtContent = fs.readFileSync(manifest).toString(); - handleIgnoredDependencies(requirementTxtContent,sbom,opts) + handleIgnoredDependencies(requirementTxtContent, sbom, opts) // In python there is no root component, then we must remove the dummy root we added, so the sbom json will be accepted by exhort backend // sbom.removeRootComponent() return sbom.getAsJsonString(opts) @@ -249,5 +247,5 @@ function getSbomForComponentAnalysis(manifest, opts = {}) { * @return {PackageURL} */ function toPurl(name,version) { - return new PackageURL('pypi',undefined,name,version,undefined,undefined); + return new PackageURL('pypi', undefined, name, version, undefined, undefined); } diff --git a/src/tools.js b/src/tools.js index 4fee23f5..df7c7838 100644 --- a/src/tools.js +++ b/src/tools.js @@ -122,6 +122,8 @@ export function getGitRootDir(cwd) { /** this method invokes command string in a process in a synchronous way. * @param {string} bin - the command to be invoked * @param {Array} args - the args to pass to the binary + * @param {import('child_process').ExecFileOptionsWithStringEncoding} [opts={}] + * @returns {string} */ export function invokeCommand(bin, args, opts={}) { // .bat and .cmd files can't be executed in windows with execFileSync, so we special case them diff --git a/test/providers/python_pip.test.js b/test/providers/python_pip.test.js index 83eac2b1..10736451 100644 --- a/test/providers/python_pip.test.js +++ b/test/providers/python_pip.test.js @@ -1,19 +1,18 @@ import { expect } from 'chai' import fs from 'fs' -import {execSync} from "node:child_process"; import sinon from "sinon"; import pythonPip from "../../src/providers/python_pip.js" -import {getCustomPath } from "../../src/tools.js" +import {getCustomPath, invokeCommand } from "../../src/tools.js" let clock -async function sharedComponentAnalysisTestFlow(testCase,usePipDepTreeUtility) { +function sharedComponentAnalysisTestFlow(testCase, usePipDepTreeUtility) { // load the expected list for tsharedComponentAnalysisTestFlowhe scenario - let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_component_sbom.json`,).toString().trim() + let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_component_sbom.json`).toString().trim() expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) // invoke sut stack analysis for scenario manifest let opts = { "EXHORT_PIP_USE_DEP_TREE" : usePipDepTreeUtility } - let providedDatForComponent = await pythonPip.provideComponent(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`, opts) + let providedDatForComponent = pythonPip.provideComponent(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`, opts) // verify returned data matches expectation expect(providedDatForComponent).to.deep.equal({ ecosystem: 'pip', @@ -22,19 +21,19 @@ async function sharedComponentAnalysisTestFlow(testCase,usePipDepTreeUtility) { }) } -async function sharedStackAnalysisTestFlow(testCase,usePipDepTreeUtility) { +function sharedStackAnalysisTestFlow(testCase, usePipDepTreeUtility) { // load the expected graph for the scenario - let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_stack_sbom.json`,).toString() + let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_stack_sbom.json`).toString() expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) // invoke sut stack analysis for scenario manifest let pipPath = getCustomPath("pip3"); - execSync(`${pipPath} install -r test/providers/tst_manifests/pip/${testCase}/requirements.txt`, err => { - if (err) { - throw new Error('fail installing requirements.txt manifest in created virtual python environment --> ' + err.message) - } - }) + try { + invokeCommand(pipPath, ['install', '-r', `test/providers/tst_manifests/pip/${testCase}/requirements.txt`]) + } catch (error) { + throw new Error('fail installing requirements.txt manifest in created virtual python environment', {cause: error}) + } let opts = { "EXHORT_PIP_USE_DEP_TREE" : usePipDepTreeUtility } - let providedDataForStack = await pythonPip.provideStack(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`,opts) + let providedDataForStack = pythonPip.provideStack(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`, opts) // new(year: number, month: number, date?: number, hours?: number, minutes?: number, seconds?: number, ms?: number): Date // providedDataForStack.content = providedDataForStack.content.replaceAll("\"timestamp\":\"[a-zA-Z0-9\\-\\:]+\"","") @@ -58,32 +57,28 @@ suite('testing the python-pip data provider', () => { [ "pip_requirements_txt_no_ignore", - "pip_requirements_txt_ignore" + // "pip_requirements_txt_ignore" ].forEach(testCase => { let scenario = testCase.replace('pip_requirements_', '').replaceAll('_', ' ') - test(`verify requirements.txt sbom provided for stack analysis with scenario ${scenario}`, async () => { - await sharedStackAnalysisTestFlow(testCase,false); + test(`verify requirements.txt sbom provided for stack analysis with scenario ${scenario}`, () => { + sharedStackAnalysisTestFlow(testCase,false); // these test cases takes ~2500-2700 ms each pr >10000 in CI (for the first test-case) }).timeout(process.env.GITHUB_ACTIONS ? 30000 : 10000) - test(`verify requirements.txt sbom provided for component analysis with scenario ${scenario}`, async () => { - await sharedComponentAnalysisTestFlow(testCase,false); + test(`verify requirements.txt sbom provided for component analysis with scenario ${scenario}`, () => { + sharedComponentAnalysisTestFlow(testCase,false); // these test cases takes ~1400-2000 ms each pr >10000 in CI (for the first test-case) }).timeout(process.env.GITHUB_ACTIONS ? 15000 : 10000) - test(`verify requirements.txt sbom provided for stack analysis using pipdeptree utility with scenario ${scenario}`, async () => { - await sharedStackAnalysisTestFlow(testCase,true); + test(`verify requirements.txt sbom provided for stack analysis using pipdeptree utility with scenario ${scenario}`, () => { + sharedStackAnalysisTestFlow(testCase,true); // these test cases takes ~2500-2700 ms each pr >10000 in CI (for the first test-case) }).timeout(process.env.GITHUB_ACTIONS ? 30000 : 10000) - test(`verify requirements.txt sbom provided for component analysis using pipdeptree utility with scenario ${scenario}`, async () => { - await sharedComponentAnalysisTestFlow(testCase,true); + test(`verify requirements.txt sbom provided for component analysis using pipdeptree utility with scenario ${scenario}`, () => { + sharedComponentAnalysisTestFlow(testCase,true); // these test cases takes ~1400-2000 ms each pr >10000 in CI (for the first test-case) }).timeout(process.env.GITHUB_ACTIONS ? 15000 : 10000) - - - - }); }).beforeAll(() => clock = sinon.useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(()=> clock.restore()); @@ -94,19 +89,19 @@ suite('testing the python-pip data provider with virtual environment', () => { "pip_requirements_virtual_env_with_ignore" ].forEach(testCase => { let scenario = testCase.replace('pip_requirements_', '').replaceAll('_', ' ') - test(`verify requirements.txt sbom provided for stack analysis using virutal python environment, with scenario ${scenario}`, async () => { + test(`verify requirements.txt sbom provided for stack analysis using virutal python environment, with scenario ${scenario}`, () => { // load the expected sbom stack analysis let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_stack_sbom.json`,).toString() process.env["EXHORT_PYTHON_VIRTUAL_ENV"] = "true" // process.env["EXHORT_DEBUG"] = "true" - expectedSbom = JSON.stringify(JSON.parse(expectedSbom),null , 4) + expectedSbom = JSON.stringify(JSON.parse(expectedSbom), null, 4) // invoke sut stack analysis for scenario manifest - let providedDataForStack = await pythonPip.provideStack(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`) + let providedDataForStack = pythonPip.provideStack(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`) // new(year: number, month: number, date?: number, hours?: number, minutes?: number, seconds?: number, ms?: number): Date // providedDataForStack.content = providedDataForStack.content.replaceAll("\"timestamp\":\"[a-zA-Z0-9\\-\\:]+\"","") // verify returned data matches expectation - providedDataForStack.content = JSON.stringify(JSON.parse(providedDataForStack.content),null , 4) + providedDataForStack.content = JSON.stringify(JSON.parse(providedDataForStack.content), null, 4) expect(providedDataForStack.content).to.deep.equal(expectedSbom) // expect(providedDataForStack).to.deep.equal({ // ecosystem: 'pip', @@ -115,8 +110,6 @@ suite('testing the python-pip data provider with virtual environment', () => { // }) // these test cases takes ~2500-2700 ms each pr >10000 in CI (for the first test-case) }).timeout(process.env.GITHUB_ACTIONS ? 60000 : 30000) - - }) }).beforeAll(() => {clock = sinon.useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))}).afterAll(()=> clock.restore()); diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json index f41e5abb..681c2df9 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_component_sbom.json @@ -92,10 +92,10 @@ }, { "name": "immutables", - "version": "0.19", - "purl": "pkg:pypi/immutables@0.19", + "version": "0.20", + "purl": "pkg:pypi/immutables@0.20", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.19" + "bom-ref": "pkg:pypi/immutables@0.20" }, { "name": "importlib-metadata", @@ -210,7 +210,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.19", + "pkg:pypi/immutables@0.20", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -268,7 +268,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.19", + "ref": "pkg:pypi/immutables@0.20", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json index 7c2d585a..42fff71c 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/expected_stack_sbom.json @@ -148,10 +148,10 @@ }, { "name": "immutables", - "version": "0.19", - "purl": "pkg:pypi/immutables@0.19", + "version": "0.20", + "purl": "pkg:pypi/immutables@0.20", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.19" + "bom-ref": "pkg:pypi/immutables@0.20" }, { "name": "importlib-metadata", @@ -210,7 +210,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.19", + "pkg:pypi/immutables@0.20", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -315,7 +315,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.19", + "ref": "pkg:pypi/immutables@0.20", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt index 94093732..73f1d7cd 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_ignore/requirements.txt @@ -9,14 +9,14 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.19 +immutables==0.20 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 MarkupSafe==2.0.1 pydantic==1.9.2 # exhortignore requests==2.25.1 -six==1.16.0 +six==1.16.0 sniffio==1.2.0 soupsieve==2.3.2.post1 starlette==0.17.1 diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json index 50971973..94bbecf9 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_component_sbom.json @@ -99,10 +99,10 @@ }, { "name": "immutables", - "version": "0.19", - "purl": "pkg:pypi/immutables@0.19", + "version": "0.20", + "purl": "pkg:pypi/immutables@0.20", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.19" + "bom-ref": "pkg:pypi/immutables@0.20" }, { "name": "importlib-metadata", @@ -225,7 +225,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.19", + "pkg:pypi/immutables@0.20", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -288,7 +288,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.19", + "ref": "pkg:pypi/immutables@0.20", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json index 7766ed36..007b9b3e 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/expected_stack_sbom.json @@ -162,10 +162,10 @@ }, { "name": "immutables", - "version": "0.19", - "purl": "pkg:pypi/immutables@0.19", + "version": "0.20", + "purl": "pkg:pypi/immutables@0.20", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.19" + "bom-ref": "pkg:pypi/immutables@0.20" }, { "name": "importlib-metadata", @@ -225,7 +225,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.19", + "pkg:pypi/immutables@0.20", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -343,7 +343,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.19", + "ref": "pkg:pypi/immutables@0.20", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt index 4f0d690d..3f34b529 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_no_ignore/requirements.txt @@ -9,7 +9,7 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.19 +immutables==0.20 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json index 7675fcfe..d4f79de3 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_component_sbom.json @@ -85,10 +85,10 @@ }, { "name" : "immutables", - "version" : "0.19", - "purl" : "pkg:pypi/immutables@0.19", + "version" : "0.20", + "purl" : "pkg:pypi/immutables@0.20", "type" : "library", - "bom-ref" : "pkg:pypi/immutables@0.19" + "bom-ref" : "pkg:pypi/immutables@0.20" }, { "name" : "importlib-metadata", @@ -242,7 +242,7 @@ "dependsOn" : [ ] }, { - "ref" : "pkg:pypi/immutables@0.19", + "ref" : "pkg:pypi/immutables@0.20", "dependsOn" : [ ] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json index 7766ed36..007b9b3e 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/expected_stack_sbom.json @@ -162,10 +162,10 @@ }, { "name": "immutables", - "version": "0.19", - "purl": "pkg:pypi/immutables@0.19", + "version": "0.20", + "purl": "pkg:pypi/immutables@0.20", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.19" + "bom-ref": "pkg:pypi/immutables@0.20" }, { "name": "importlib-metadata", @@ -225,7 +225,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.19", + "pkg:pypi/immutables@0.20", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -343,7 +343,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.19", + "ref": "pkg:pypi/immutables@0.20", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt index 4f0d690d..3f34b529 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_txt_no_ignore/requirements.txt @@ -9,7 +9,7 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.19 +immutables==0.20 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json index a6ea38ac..72e34215 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_component_sbom.json @@ -90,10 +90,10 @@ }, { "name" : "immutables", - "version" : "0.19", - "purl" : "pkg:pypi/immutables@0.19", + "version" : "0.20", + "purl" : "pkg:pypi/immutables@0.20", "type" : "library", - "bom-ref" : "pkg:pypi/immutables@0.19" + "bom-ref" : "pkg:pypi/immutables@0.20" }, { "name" : "importlib-metadata", @@ -208,7 +208,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.19", + "pkg:pypi/immutables@0.20", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -266,7 +266,7 @@ "dependsOn" : [ ] }, { - "ref" : "pkg:pypi/immutables@0.19", + "ref" : "pkg:pypi/immutables@0.20", "dependsOn" : [ ] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json index 7c2d585a..42fff71c 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/expected_stack_sbom.json @@ -148,10 +148,10 @@ }, { "name": "immutables", - "version": "0.19", - "purl": "pkg:pypi/immutables@0.19", + "version": "0.20", + "purl": "pkg:pypi/immutables@0.20", "type": "library", - "bom-ref": "pkg:pypi/immutables@0.19" + "bom-ref": "pkg:pypi/immutables@0.20" }, { "name": "importlib-metadata", @@ -210,7 +210,7 @@ "pkg:pypi/flask@2.0.3", "pkg:pypi/h11@0.13.0", "pkg:pypi/idna@2.10", - "pkg:pypi/immutables@0.19", + "pkg:pypi/immutables@0.20", "pkg:pypi/importlib-metadata@4.8.3", "pkg:pypi/itsdangerous@2.0.1", "pkg:pypi/jinja2@3.0.3", @@ -315,7 +315,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/immutables@0.19", + "ref": "pkg:pypi/immutables@0.20", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt index 94093732..73f1d7cd 100644 --- a/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt +++ b/test/providers/tst_manifests/pip/pip_requirements_virtual_env_with_ignore/requirements.txt @@ -9,14 +9,14 @@ fastapi==0.75.1 Flask==2.0.3 h11==0.13.0 idna==2.10 -immutables==0.19 +immutables==0.20 importlib-metadata==4.8.3 itsdangerous==2.0.1 Jinja2==3.0.3 MarkupSafe==2.0.1 pydantic==1.9.2 # exhortignore requests==2.25.1 -six==1.16.0 +six==1.16.0 sniffio==1.2.0 soupsieve==2.3.2.post1 starlette==0.17.1