From 09fadb335bc468209dd50398aa94ca873481f945 Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 23 Oct 2023 12:27:53 +0300 Subject: [PATCH 1/3] chore: add time measuring log messages test: add developer test commented out stub in IT to be ready for quick checks Signed-off-by: Zvi Grinberg --- src/analysis.js | 17 +++++++++++++++++ src/providers/python_controller.js | 12 ++++++++++++ src/sbom.js | 12 ++++++++++++ test/it/end-to-end.js | 27 +++++++++++++++++++++++++++ 4 files changed, 68 insertions(+) diff --git a/src/analysis.js b/src/analysis.js index b9ff5ce4..87673bee 100644 --- a/src/analysis.js +++ b/src/analysis.js @@ -19,6 +19,11 @@ const rhdaOperationTypeHeader = "rhda-operation-type" async function requestStack(provider, manifest, url, html = false, opts = {}) { let provided = provider.provideStack(manifest, opts) // throws error if content providing failed opts[rhdaOperationTypeHeader.toUpperCase().replaceAll("-","_")] = "stack-analysis" + let startTime = new Date() + let EndTime + if (process.env["EXHORT_DEBUG"] === "true") { + console.log("Starting time of sending stack analysis request to exhort server= " + startTime) + } let resp = await fetch(`${url}/api/v3/analysis`, { method: 'POST', headers: { @@ -28,6 +33,12 @@ async function requestStack(provider, manifest, url, html = false, opts = {}) { }, body: provided.content }) + if (process.env["EXHORT_DEBUG"] === "true") { + EndTime = new Date() + console.log("Ending time of sending stack analysis request to exhort server= " + EndTime) + let time = (EndTime - startTime) / 1000 + console.log("Total Time in seconds: " + time) + } return html ? resp.text() : resp.json() } @@ -42,6 +53,9 @@ async function requestStack(provider, manifest, url, html = false, opts = {}) { async function requestComponent(provider, data, url, opts = {}) { let provided = provider.provideComponent(data, opts) // throws error if content providing failed opts[rhdaOperationTypeHeader.toUpperCase().replaceAll("-","_")] = "component-analysis" + if (process.env["EXHORT_DEBUG"] === "true") { + console.log("Starting time of sending component analysis request to exhort server= " + new Date()) + } let resp = await fetch(`${url}/api/v3/analysis`, { method: 'POST', headers: { @@ -51,6 +65,9 @@ async function requestComponent(provider, data, url, opts = {}) { }, body: provided.content }) + if (process.env["EXHORT_DEBUG"] === "true") { + console.log("Ending time of sending component analysis request to exhort server= " + new Date()) + } return resp.json() } diff --git a/src/providers/python_controller.js b/src/providers/python_controller.js index b2c16e7f..9860e7df 100644 --- a/src/providers/python_controller.js +++ b/src/providers/python_controller.js @@ -74,6 +74,12 @@ export default class Python_controller { */ getDependencies(includeTransitive) { + let startingTime + let endingTime + if (process.env["EXHORT_DEBUG"] === "true") { + startingTime = new Date() + console.log("Starting time to get requirements.txt dependency tree = " + startingTime) + } if(!this.realEnvironment) { execSync(`${this.pathToPipBin} install -r ${this.pathToRequirements}`, err =>{ if (err) { @@ -83,6 +89,12 @@ export default class Python_controller { } let dependencies = this.#getDependenciesImpl(includeTransitive) this.#cleanEnvironment() + if (process.env["EXHORT_DEBUG"] === "true") { + endingTime = new Date() + console.log("Ending time to get requirements.txt dependency tree = " + endingTime) + let time = ( endingTime - startingTime ) / 1000 + console.log("total time to get requirements.txt dependency tree = " + time) + } return dependencies } /** diff --git a/src/sbom.js b/src/sbom.js index 4f0df0b5..9a4b75fc 100644 --- a/src/sbom.js +++ b/src/sbom.js @@ -2,7 +2,13 @@ import CycloneDxSbom from "./cyclone_dx_sbom.js"; export default class Sbom { sbomModel + #startTime + #endTime constructor() { + if (process.env["EXHORT_DEBUG"] === "true") { + this.#startTime = new Date() + console.log("Starting time to create sbom = " + this.#startTime) + } this.sbomModel = new CycloneDxSbom() } @@ -51,6 +57,12 @@ export default class Sbom { * @return String sbom json in a string format */ getAsJsonString(){ + if (process.env["EXHORT_DEBUG"] === "true") { + this.#endTime = new Date() + console.log("Ending time to create sbom = " + this.#endTime) + let time = (this.#endTime - this.#startTime) / 1000 + console.log("Total time in seconds to create sbom = " + time) + } return this.sbomModel.getAsJsonString() } diff --git a/test/it/end-to-end.js b/test/it/end-to-end.js index 775de054..7d7f8a0d 100644 --- a/test/it/end-to-end.js +++ b/test/it/end-to-end.js @@ -99,3 +99,30 @@ suite('Integration Tests', () => { }); }); + +// suite('Developer Test End to End', () => { +// // let opts = { +// // EXHORT_DEV_MODE: "true", +// // EXHORT_SNYK_TOKEN: "ee64316c-a4ba-4ca0-a785-18cb05ed3f25" +// // +// // } +// +// test(`Stack Analysis json`, async () => { +// // process.env["EXHORT_DEBUG"]= "true" +// // process.env["EXHORT_DEV_MODE"]= "false" +// // process.env["EXHORT_GO_PATH"]= "/home/zgrinber/test-go/go/bin/go" +// // process.env["RHDA_TOKEN"] = "34JKLDS-4234809-66666666666" +// // process.env["RHDA_SOURCE"] = "Zvika Client" +// // let result = await index.stackAnalysis("/tmp/rajan-0410/go.mod", false, opts); +// +// +// let pomPath = `/tmp/231023/requirements.txt` +// let providedDataForStack = await index.stackAnalysis(pomPath) +// console.log(JSON.stringify(providedDataForStack.summary,null , 4)) +// expect(providedDataForStack.summary.dependencies.scanned).greaterThan(0) +// }).timeout(15000); +// +// +// +// +// }); From c708b22323c63a0ca942045588e584717e036d8a Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 23 Oct 2023 12:37:37 +0300 Subject: [PATCH 2/3] fix: potential wrong parsing of pip show with multiline string when one of the packages contains license key with multi line string value, it can contains the delimiter "---" inside the value ( that separates between the packages in pip show otuput) , hence need to refine the parsing - that is , refine the delimiter Signed-off-by: Zvi Grinberg --- src/providers/python_controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/python_controller.js b/src/providers/python_controller.js index 9860e7df..423df38f 100644 --- a/src/providers/python_controller.js +++ b/src/providers/python_controller.js @@ -125,7 +125,7 @@ export default class Python_controller { throw new Error('fail invoking pip show to fetch all installed dependencies metadata --> ' + err.message) } }).toString(); - let allPipShowDeps = pipShowOutput.split("---"); + let allPipShowDeps = pipShowOutput.split( EOL +"---" + EOL); let linesOfRequirements = fs.readFileSync(this.pathToRequirements).toString().split(EOL).filter( (line) => !line.startsWith("#")).map(line => line.trim()) let CachedEnvironmentDeps = {} allPipShowDeps.forEach( (record) => { From 45d921c49b0ff3a6a775d1fab6b7ff4737cd3f6d Mon Sep 17 00:00:00 2001 From: Zvi Grinberg Date: Mon, 23 Oct 2023 15:59:25 +0300 Subject: [PATCH 3/3] tests: update exepcted sboms for npm component analysis Signed-off-by: Zvi Grinberg --- .../component_expected_sbom.json | 24 +++++++++---------- .../component_expected_sbom.json | 10 ++++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/test/providers/tst_manifests/npm/package_json_deps_with_exhortignore_object/component_expected_sbom.json b/test/providers/tst_manifests/npm/package_json_deps_with_exhortignore_object/component_expected_sbom.json index 68534075..59f4af9d 100644 --- a/test/providers/tst_manifests/npm/package_json_deps_with_exhortignore_object/component_expected_sbom.json +++ b/test/providers/tst_manifests/npm/package_json_deps_with_exhortignore_object/component_expected_sbom.json @@ -63,19 +63,19 @@ "type": "library", "bom-ref": "pkg:npm/express@4.18.2" }, - { - "name": "jsdom", - "version": "19.0.0", - "purl": "pkg:npm/jsdom@19.0.0", - "type": "library", - "bom-ref": "pkg:npm/jsdom@19.0.0" - }, + { + "name": "jsdom", + "version": "19.0.0", + "purl": "pkg:npm/jsdom@19.0.0", + "type": "library", + "bom-ref": "pkg:npm/jsdom@19.0.0" + }, { "name": "mongoose", - "version": "5.13.20", - "purl": "pkg:npm/mongoose@5.13.20", + "version": "5.13.21", + "purl": "pkg:npm/mongoose@5.13.21", "type": "library", - "bom-ref": "pkg:npm/mongoose@5.13.20" + "bom-ref": "pkg:npm/mongoose@5.13.21" }, { "name": "nodemon", @@ -96,7 +96,7 @@ "pkg:npm/dotenv@8.6.0", "pkg:npm/express@4.18.2", "pkg:npm/jsdom@19.0.0", - "pkg:npm/mongoose@5.13.20", + "pkg:npm/mongoose@5.13.21", "pkg:npm/nodemon@2.0.22" ] }, @@ -129,7 +129,7 @@ "dependsOn": [] }, { - "ref": "pkg:npm/mongoose@5.13.20", + "ref": "pkg:npm/mongoose@5.13.21", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/npm/package_json_deps_without_exhortignore_object/component_expected_sbom.json b/test/providers/tst_manifests/npm/package_json_deps_without_exhortignore_object/component_expected_sbom.json index 6d9dbc87..195db90a 100644 --- a/test/providers/tst_manifests/npm/package_json_deps_without_exhortignore_object/component_expected_sbom.json +++ b/test/providers/tst_manifests/npm/package_json_deps_without_exhortignore_object/component_expected_sbom.json @@ -72,10 +72,10 @@ }, { "name": "mongoose", - "version": "5.13.20", - "purl": "pkg:npm/mongoose@5.13.20", + "version": "5.13.21", + "purl": "pkg:npm/mongoose@5.13.21", "type": "library", - "bom-ref": "pkg:npm/mongoose@5.13.20" + "bom-ref": "pkg:npm/mongoose@5.13.21" }, { "name": "nodemon", @@ -96,7 +96,7 @@ "pkg:npm/dotenv@8.6.0", "pkg:npm/express@4.18.2", "pkg:npm/jsonwebtoken@8.5.1", - "pkg:npm/mongoose@5.13.20", + "pkg:npm/mongoose@5.13.21", "pkg:npm/nodemon@2.0.22" ] }, @@ -129,7 +129,7 @@ "dependsOn": [] }, { - "ref": "pkg:npm/mongoose@5.13.20", + "ref": "pkg:npm/mongoose@5.13.21", "dependsOn": [] }, {