Merge pull request #97 from secureCodeBox/zap-integration-tests

Add Integration Tests for ZAP and some other improvements to improve the stability and debuggability of the tests

.github/workflows/ci.yaml

@@ -412,6 +412,9 @@ jobs:
         run: |
           # Install dummy-ssh app
           helm -n demo-apps install dummy-ssh ./demo-apps/dummy-ssh/ --wait
+          # Install plain nginx server
+          kubectl create deployment --image nginx:alpine nginx --namespace demo-apps
+          kubectl expose deployment nginx --port 80 --namespace demo-apps
       - name: "nmap Integration Tests"
         run: |
           helm -n integration-tests install nmap ./scanners/nmap/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
@@ -432,6 +435,11 @@ jobs:
           helm -n integration-tests install ssh-scan ./scanners/ssh_scan/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
           cd tests/integration/
           npx jest --ci --color ssh-scan
+      - name: "zap Integration Tests"
+        run: |
+          helm -n integration-tests install zap ./scanners/zap/ --set="parserImage.tag=sha-$(git rev-parse --short HEAD)"
+          cd tests/integration/
+          npx jest --ci --color zap
       - name: Inspect Post Failure
         if: failure()
         run: |

scanners/amass/templates/amass-scan-type.yaml

@@ -10,7 +10,9 @@ spec:
     location: "/home/securecodebox/amass-results.jsonl"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: OnFailure

scanners/amass/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/kube-hunter/templates/kubehunter-scan-type.yaml

@@ -8,7 +8,9 @@ spec:
     location: '/home/securecodebox/kube-hunter-results.json'
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: Never

scanners/kube-hunter/values.yaml

@@ -4,4 +4,5 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}

scanners/ncrack/templates/ncrack-scan-type.yaml

@@ -8,7 +8,9 @@ spec:
     location: "/home/securecodebox/ncrack-results.xml"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       backoffLimit: 3
       template:
         spec:

scanners/ncrack/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
   extraVolumes: []
   extraVolumeMounts: []

scanners/nikto/templates/nikto-scan-type.yaml

@@ -8,7 +8,9 @@ spec:
     location: '/home/securecodebox/nikto-results.json'
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: Never

scanners/nikto/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/nmap/templates/nmap-scan-type.yaml

@@ -8,7 +8,9 @@ spec:
     location: "/home/securecodebox/nmap-results.xml"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       backoffLimit: 3
       template:
         spec:

scanners/nmap/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/ssh_scan/templates/ssh-scan-scan-type.yaml

@@ -9,7 +9,9 @@ spec:
     location: "/home/securecodebox/ssh-scan-results.json"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: OnFailure

scanners/ssh_scan/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/sslyze/templates/sslyze-scan-type.yaml

@@ -8,7 +8,9 @@ spec:
     location: '/home/securecodebox/sslyze-results.json'
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: OnFailure

scanners/sslyze/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/test-scan/templates/test-scan-scan-type.yaml

@@ -8,7 +8,9 @@ spec:
     location: "/home/securecodebox/hello-world.txt"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       backoffLimit: 3
       template:
         spec:

scanners/test-scan/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/trivy/templates/trivy-scan-type.yaml

@@ -9,7 +9,9 @@ spec:
     location: "/home/securecodebox/trivy-results.json"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: OnFailure

scanners/trivy/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/wpscan/templates/wpscan-scan-type.yaml

@@ -9,7 +9,9 @@ spec:
     location: "/home/securecodebox/wpscan-results.json"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: OnFailure

scanners/wpscan/values.yaml

@@ -4,6 +4,7 @@ parserImage:
   tag: null
 
 scannerJob:
+  ttlSecondsAfterFinished: null
   resources: {}
 # scannerJob:
 #   resources:

scanners/zap/templates/zap-scan-type.yaml

@@ -8,13 +8,15 @@ spec:
     location: "/home/securecodebox/zap-results.json"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: Never
           containers:
             - name: zap-baseline
-              image: owasp/zap2docker-stable:2.9.0
+              image: owasp/zap2docker-weekly:w2020-09-15
               command:
                 - "zap-baseline.py"
                 # Force Zap to always return a zero exit code. k8s would otherwise try to restart zap.
@@ -42,13 +44,15 @@ spec:
     location: "/home/securecodebox/zap-results.json"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: Never
           containers:
             - name: zap-api-scan
-              image: owasp/zap2docker-weekly:w2020-09-08
+              image: owasp/zap2docker-weekly:w2020-09-15
               command:
                 - "zap-api-scan.py"
                 # Force Zap to always return a zero exit code. k8s would otherwise try to restart zap.
@@ -76,13 +80,15 @@ spec:
     location: "/home/securecodebox/zap-results.json"
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 10
+      {{- if .Values.scannerJob.ttlSecondsAfterFinished }}
+      ttlSecondsAfterFinished: {{ .Values.scannerJob.ttlSecondsAfterFinished }}
+      {{- end }}
       template:
         spec:
           restartPolicy: Never
           containers:
             - name: zap-full-scan
-              image: owasp/zap2docker-weekly:w2020-09-08
+              image: owasp/zap2docker-weekly:w2020-09-15
               command:
                 - "zap-full-scan.py"
                 # Force Zap to always return a zero exit code. k8s would otherwise try to restart zap.

tests/integration/helpers.js

@@ -5,11 +5,11 @@ kc.loadFromDefault();
 
 const k8sCRDApi = kc.makeApiClient(k8s.CustomObjectsApi);
 const k8sBatchApi = kc.makeApiClient(k8s.BatchV1Api);
+const k8sPodsApi = kc.makeApiClient(k8s.CoreV1Api);
 
 const namespace = "integration-tests";
 
-const sleep = (duration) =>
-  new Promise((resolve) => setTimeout(resolve, duration * 1000));
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms * 1000));
 
 async function deleteScan(name) {
   await k8sCRDApi.deleteNamespacedCustomObject(
@@ -33,28 +33,80 @@ async function getScan(name) {
   return scan;
 }
 
+async function displayAllLogsForJob(jobName) {
+  console.log(`Listing logs for Job '${jobName}':`);
+  const {
+    body: { items: pods },
+  } = await k8sPodsApi.listNamespacedPod(
+    namespace,
+    true,
+    undefined,
+    undefined,
+    undefined,
+    `job-name=${jobName}`
+  );
+
+  if (pods.length === 0) {
+    console.log(`No Pods found for Job '${jobName}'`);
+  }
+
+  for (const pod of pods) {
+    console.log(
+      `Listing logs for Job '${jobName}' > Pod '${pod.metadata.name}':`
+    );
+
+    for (const container of pod.spec.containers) {
+      try {
+        const response = await k8sPodsApi.readNamespacedPodLog(
+          pod.metadata.name,
+          namespace,
+          container.name
+        );
+        console.log(`Container ${container.name}:`);
+        console.log(response.body);
+      } catch (exception) {
+        console.error(
+          `Failed to display logs of container ${container.name}: ${exception.body.message}`
+        );
+      }
+    }
+  }
+}
+
 async function logJobs() {
   try {
     const { body: jobs } = await k8sBatchApi.listNamespacedJob(namespace);
 
+    console.log("Logging spec & status of jobs in namespace");
+
     for (const job of jobs.items) {
       console.log(`Job: '${job.metadata.name}' Spec:`);
-      console.dir(job.spec);
+      console.log(JSON.stringify(job.spec, null, 2));
       console.log(`Job: '${job.metadata.name}' Status:`);
-      console.dir(job.status);
+      console.log(JSON.stringify(job.status, null, 2));
+
+      await displayAllLogsForJob(job.metadata.name);
     }
   } catch (error) {
-    console.info(`Failed to list Jobs'`);
+    console.error("Failed to list Jobs");
+    console.error(error);
   }
 }
 
+async function disasterRecovery(scanName) {
+  const scan = await getScan(scanName);
+  console.error("Last Scan State:");
+  console.dir(scan);
+  await logJobs();
+}
+
 /**
  *
  * @param {string} name name of the scan. Actual name will be sufixed with a random number to avoid conflicts
  * @param {string} scanType type of the scan. Must match the name of a ScanType CRD
  * @param {string[]} parameters cli argument to be passed to the scanner
  * @param {number} timeout in seconds
- * @returns {scan.findings} returns findings { categories, severities, count } 
+ * @returns {scan.findings} returns findings { categories, severities, count }
  */
 async function scan(name, scanType, parameters = [], timeout = 180) {
   const scanDefinition = {
@@ -85,22 +137,22 @@ async function scan(name, scanType, parameters = [], timeout = 180) {
     const { status } = await getScan(actualName);
 
     if (status && status.state === "Done") {
+      // Wait a couple seconds to give kubernetes more time to update the fields
+      await sleep(2);
+      const { status } = await getScan(actualName);
       await deleteScan(actualName);
       return status.findings;
     } else if (status && status.state === "Errored") {
-      await deleteScan(actualName);
+      console.error("Scan Errored");
+      await disasterRecovery(actualName);
+
       throw new Error(
         `Scan failed with description "${status.errorDescription}"`
       );
     }
   }
-
   console.error("Scan Timed out!");
-
-  const scan = await getScan(actualName);
-  console.log("Last Scan State:");
-  console.dir(scan);
-  await logJobs();
+  await disasterRecovery(actualName);
 
   throw new Error("timed out while waiting for scan results");
 }