PA-12768 use core urls rather than only spidered collection (#103)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "soos-dast",
-  "version": "2.0.26",
+  "version": "2.0.27",
   "description": "SOOS DAST - The affordable no limit web vulnerability scanner",
   "main": "index.js",
   "scripts": {

src/constants.ts

@@ -1,12 +1,12 @@
 export const SOOS_DAST_CONSTANTS = {
   AuthDelayTime: 5,
   Tool: "zap",
-  ToolVersion: "2.12",
+  ToolVersion: "2.14",
   Files: {
+    DiscoveredUrlsFile: "./core_urls.txt",
     ReportScanResultFile: "/zap/wrk/report.zap.json",
     ReportScanResultFilename: "report.zap.json",
     SarifResultsFilename: "results.sarif",
-    SpideredUrlsFile: "./spidered_urls.txt",
     ZapHookFile: "src/zap_hooks/soos_zap_hook.py",
   },
   StatusCheck: {

src/utilities/ZAPReportTransformer.ts

@@ -10,16 +10,11 @@ export class ZAPReportTransformer {
   }
 
   public static addDiscoveredUrls(reportData: any): void {
-    const discoveredUrls =
-      fs.existsSync(SOOS_DAST_CONSTANTS.Files.SpideredUrlsFile) &&
-      fs.statSync(SOOS_DAST_CONSTANTS.Files.SpideredUrlsFile).isFile()
-        ? fs
-            .readFileSync(SOOS_DAST_CONSTANTS.Files.SpideredUrlsFile, "utf-8")
-            .split("\n")
-            .filter((url) => url.trim() !== "")
-        : [];
-
-    reportData["discoveredUrls"] = discoveredUrls;
+    this.addArrayPropertyToReportFromFile(
+      reportData,
+      "discoveredUrls",
+      SOOS_DAST_CONSTANTS.Files.DiscoveredUrlsFile,
+    );
   }
 
   public static obfuscateFields(reportData: any): void {
@@ -34,6 +29,22 @@ export class ZAPReportTransformer {
     }
   }
 
+  private static addArrayPropertyToReportFromFile(
+    reportData: any,
+    name: string,
+    file: string,
+  ): void {
+    const lines =
+      fs.existsSync(file) && fs.statSync(file).isFile()
+        ? fs
+            .readFileSync(file, "utf-8")
+            .split("\n")
+            .filter((line) => line.trim() !== "")
+        : [];
+
+    reportData[name] = lines;
+  }
+
   private static obfuscateBearerToken(field: string): string {
     return field.replace(/(Authorization:\s*)[^\r\n]+/, "$1****");
   }

src/zap_hooks/soos_zap_hook.py

@@ -77,12 +77,11 @@ def zap_pre_shutdown(zap):
         serialize_and_save(zap.core, 'wrk/core_data_pre_shutdown.json')
         serialize_and_save(zap.pscan, 'wrk/pscan_data_pre_shutdown.json')
         serialize_and_save(zap.context, 'wrk/context_data_pre_shutdown.json')
-    log("Overview of spidered URL's:")
-    with open('spidered_urls.txt', 'w') as f:
-        for url in zap.spider.all_urls:
+    log("URLs Discovered:")
+    with open('core_urls.txt', 'w') as f:
+        for url in zap.core.urls():
             f.write(f"{url}\n")
-            log(f"found: {url}")
-
+            log(f"-- {url}")
 
 def _all_active_scanner_rules(zap, policy_name) -> List[str]: return [scanner['id'] for scanner in zap.ascan.scanners(policy_name)]