From 75c2a7da3291d5430bb6eed9c0449998c285349b Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 11:36:19 +0000
Subject: [PATCH 01/22] Trivy scanning

---
 .../stackhpc-container-image-build.yml        | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index cacfae23e..1e97d0d3a 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -18,6 +18,11 @@ on:
         type: boolean
         required: false
         default: false
+      scan_images:
+        description: Scan images?
+        type: boolean
+        required: false
+        default: false
       distro:
         description: Container image OS distribution
         type: choice
@@ -52,6 +57,10 @@ jobs:
         run: |
           sudo dnf -y install python3-virtualenv
 
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.18.3
+
       - name: Setup networking
         run: |
           if ! ip l show breth1 >/dev/null 2>&1; then
@@ -117,6 +126,33 @@ jobs:
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > overcloud-container-images
         if: github.event.inputs.overcloud == 'true'
 
+      - name: Generate the images to scan
+        run: |
+          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" > images-to-scan.txt
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+
+      - name: Scan built overcloud container images
+        run: |
+          cat /home/images-to-scan.txt | while read i; do
+          echo $i
+          trivy image --format sarif --severity HIGH,CRITICAL $i >> trivy-results.sarif
+          done
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+
+      # - name: Upload scan output artefact
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: Trivy Scan txt
+      #     path: scan.txt
+      #     retention-days: 7
+      #   if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+
       - name: Upload overcloud-container-images artifact
         uses: actions/upload-artifact@v3
         with:

From 77771938b32d0927ac6a42bc7640dfb1fb16f4bd Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 12:32:15 +0000
Subject: [PATCH 02/22] Fix Trivy install permissions

---
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 1e97d0d3a..52621d19d 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -59,7 +59,7 @@ jobs:
 
       - name: Install Trivy
         run: |
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.18.3
+          sudo curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.18.3
 
       - name: Setup networking
         run: |

From 290b0e7c75fa3a9d1a4102fe5f4aac4f695ab873 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 12:40:40 +0000
Subject: [PATCH 03/22] remove push for testing

---
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 52621d19d..9f6cca3de 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -111,7 +111,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push -e kolla_base_distro=${{ inputs.distro }}
+          kayobe overcloud container image build ${{ github.event.inputs.regexes }} -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'

From 82bb994f630fd69ff7cb4eadfcf5786bf27c8e66 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 13:10:51 +0000
Subject: [PATCH 04/22] fix input

---
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 9f6cca3de..d3c3b9749 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -18,7 +18,7 @@ on:
         type: boolean
         required: false
         default: false
-      scan_images:
+      scan:
         description: Scan images?
         type: boolean
         required: false

From 78e0eb9f9adbe74b1866841a6abd8e122df53f75 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 13:33:10 +0000
Subject: [PATCH 05/22] fix incorrect path

---
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index d3c3b9749..d08bad343 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -133,7 +133,7 @@ jobs:
 
       - name: Scan built overcloud container images
         run: |
-          cat /home/images-to-scan.txt | while read i; do
+          cat images-to-scan.txt | while read i; do
           echo $i
           trivy image --format sarif --severity HIGH,CRITICAL $i >> trivy-results.sarif
           done

From 3fab758f0f91af73a7d4a85b64e818e37ef34469 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 15:08:35 +0000
Subject: [PATCH 06/22] missing sudo

---
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index d08bad343..2377270d8 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -135,7 +135,7 @@ jobs:
         run: |
           cat images-to-scan.txt | while read i; do
           echo $i
-          trivy image --format sarif --severity HIGH,CRITICAL $i >> trivy-results.sarif
+          sudo trivy image --format sarif --severity HIGH,CRITICAL $i >> trivy-results.sarif
           done
         if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
 

From 5fac63e4cfa802d7c79ea688be00e72989644ef2 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 15:38:46 +0000
Subject: [PATCH 07/22] Bump trivy version

---
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 2377270d8..41248df7a 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -59,7 +59,7 @@ jobs:
 
       - name: Install Trivy
         run: |
-          sudo curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.18.3
+          sudo curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.38.0
 
       - name: Setup networking
         run: |

From c20bf3534a8d93b6c494dceb659cf4457913f1cd Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 2 Mar 2023 16:48:39 +0000
Subject: [PATCH 08/22] try folder upload for sarif files

---
 .github/workflows/stackhpc-container-image-build.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 41248df7a..57ceeb18f 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -133,16 +133,17 @@ jobs:
 
       - name: Scan built overcloud container images
         run: |
+          mkdir sarif-upload
           cat images-to-scan.txt | while read i; do
-          echo $i
-          sudo trivy image --format sarif --severity HIGH,CRITICAL $i >> trivy-results.sarif
+          filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+          sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
           done
         if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v2
         with:
-          sarif_file: 'trivy-results.sarif'
+          sarif_file: 'sarif-upload/'
         if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
 
       # - name: Upload scan output artefact

From ed8284eb14850aceee18bf69692f3b9e7a533188 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Wed, 8 Mar 2023 10:08:19 +0000
Subject: [PATCH 09/22] Test

---
 .../stackhpc-container-image-build.yml        | 30 ++++++++++++++-----
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 57ceeb18f..3a2404b29 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -18,8 +18,13 @@ on:
         type: boolean
         required: false
         default: false
-      scan:
-        description: Scan images?
+      scan_artefact:
+        description: Scan images? output=artefact
+        type: boolean
+        required: false
+        default: false
+      scan_sarif:
+        description: Scan images? output=github
         type: boolean
         required: false
         default: false
@@ -133,11 +138,22 @@ jobs:
 
       - name: Scan built overcloud container images
         run: |
-          mkdir sarif-upload
-          cat images-to-scan.txt | while read i; do
-          filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-          sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
-          done
+            if ${{ toJSON(
+                  (  fromJSON(inputs.scan_artefact)
+                ) }}; then
+                mkdir sarif-upload
+                cat images-to-scan.txt | while read i; do
+                filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+                sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
+                done
+            if ${{ toJSON(
+                  (  fromJSON(inputs.sarif)
+                ) }}; then
+                mkdir artefact-upload
+                cat images-to-scan.txt | while read i; do
+                filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+                sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
+                done
         if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
 
       - name: Upload Trivy scan results to GitHub Security tab

From e55379185bf613679ff08c57b20af9b26f0cde84 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Wed, 8 Mar 2023 10:13:19 +0000
Subject: [PATCH 10/22] Test

---
 .../stackhpc-container-image-build.yml        | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 3a2404b29..2eea9dc44 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -134,41 +134,41 @@ jobs:
       - name: Generate the images to scan
         run: |
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" > images-to-scan.txt
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan_artefact == 'true') || (github.event.inputs.scan_sarif == 'true') }}
 
       - name: Scan built overcloud container images
         run: |
             if ${{ toJSON(
                   (  fromJSON(inputs.scan_artefact)
                 ) }}; then
-                mkdir sarif-upload
-                cat images-to-scan.txt | while read i; do
-                filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-                sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
+                mkdir upload
+                # cat images-to-scan.txt | while read i; do
+                # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+                sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
                 done
             if ${{ toJSON(
-                  (  fromJSON(inputs.sarif)
+                  (  fromJSON(inputs.scan_sarif)
                 ) }}; then
                 mkdir artefact-upload
                 cat images-to-scan.txt | while read i; do
                 filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
                 sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
                 done
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan_artefact == 'true') || (github.event.inputs.scan_sarif == 'true') }}
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: 'sarif-upload/'
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
-
-      # - name: Upload scan output artefact
-      #   uses: actions/upload-artifact@v3
-      #   with:
-      #     name: Trivy Scan txt
-      #     path: scan.txt
-      #     retention-days: 7
-      #   if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+        if: ${{ (github.event.inputs.overcloud == 'true') &&  (github.event.inputs.scan_sarif == 'true') }}
+
+      - name: Upload scan output artefact
+        uses: actions/upload-artifact@v3
+        with:
+          name: "Trivy Scan txt"
+          path: 'upload/scan.txt'
+          retention-days: 7
+        if: ${{ (github.event.inputs.overcloud == 'true') &&  (github.event.inputs.scan_artefact == 'true') }}
 
       - name: Upload overcloud-container-images artifact
         uses: actions/upload-artifact@v3

From 60d54d9fb2186eeff3633c18681df1cc137bf05b Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Wed, 8 Mar 2023 10:30:47 +0000
Subject: [PATCH 11/22] test

---
 .../stackhpc-container-image-build.yml        | 37 +++++++++++++------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 2eea9dc44..697d3280a 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -138,22 +138,37 @@ jobs:
 
       - name: Scan built overcloud container images
         run: |
-            if ${{ toJSON(
-                  (  fromJSON(inputs.scan_artefact)
-                ) }}; then
-                mkdir upload
-                # cat images-to-scan.txt | while read i; do
-                # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-                sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
-                done
-            if ${{ toJSON(
-                  (  fromJSON(inputs.scan_sarif)
-                ) }}; then
+            # if ${{ toJSON(
+            #       (  fromJSON(inputs.scan_artefact)
+            #     ) }}; then
+            #     mkdir upload
+            #     # cat images-to-scan.txt | while read i; do
+            #     # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+            #     sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
+            #     done
+            # if ${{ toJSON(
+            #       (  fromJSON(inputs.scan_sarif)
+            #     ) }}; then
+            #     mkdir artefact-upload
+            #     cat images-to-scan.txt | while read i; do
+            #     filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+            #     sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
+            #     done
+            if [[ ${{ github.event.inputs.scan_sarif }} == true ]]; then
                 mkdir artefact-upload
                 cat images-to-scan.txt | while read i; do
                 filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
                 sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
                 done
+            fi
+            if [[ ${{ github.event.inputs.scan_artefact }} == true ]]; then
+                mkdir upload
+                cat images-to-scan.txt | while read i; do
+                # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+                sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
+                done
+            fi
+        shell: bash
         if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan_artefact == 'true') || (github.event.inputs.scan_sarif == 'true') }}
 
       - name: Upload Trivy scan results to GitHub Security tab

From 0addd68008e6a72d2ab28a3ef5a78300389d71f7 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Wed, 8 Mar 2023 10:32:02 +0000
Subject: [PATCH 12/22] test

---
 .../stackhpc-container-image-build.yml        | 44 ++++++-------------
 1 file changed, 14 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 697d3280a..fc3200d06 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -138,36 +138,20 @@ jobs:
 
       - name: Scan built overcloud container images
         run: |
-            # if ${{ toJSON(
-            #       (  fromJSON(inputs.scan_artefact)
-            #     ) }}; then
-            #     mkdir upload
-            #     # cat images-to-scan.txt | while read i; do
-            #     # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-            #     sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
-            #     done
-            # if ${{ toJSON(
-            #       (  fromJSON(inputs.scan_sarif)
-            #     ) }}; then
-            #     mkdir artefact-upload
-            #     cat images-to-scan.txt | while read i; do
-            #     filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-            #     sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
-            #     done
-            if [[ ${{ github.event.inputs.scan_sarif }} == true ]]; then
-                mkdir artefact-upload
-                cat images-to-scan.txt | while read i; do
-                filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-                sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
-                done
-            fi
-            if [[ ${{ github.event.inputs.scan_artefact }} == true ]]; then
-                mkdir upload
-                cat images-to-scan.txt | while read i; do
-                # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-                sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
-                done
-            fi
+          if [[ ${{ github.event.inputs.scan_sarif }} == true ]]; then
+              mkdir artefact-upload
+              cat images-to-scan.txt | while read i; do
+              filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+              sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
+              done
+          fi
+          if [[ ${{ github.event.inputs.scan_artefact }} == true ]]; then
+              mkdir upload
+              cat images-to-scan.txt | while read i; do
+              # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+              sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
+              done
+          fi
         shell: bash
         if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan_artefact == 'true') || (github.event.inputs.scan_sarif == 'true') }}
 

From df003de6745a4d0c6aaf60d72648b8fd4d9aefeb Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Wed, 8 Mar 2023 11:15:23 +0000
Subject: [PATCH 13/22] test

---
 .github/workflows/stackhpc-container-image-build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index fc3200d06..327383c40 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -139,10 +139,10 @@ jobs:
       - name: Scan built overcloud container images
         run: |
           if [[ ${{ github.event.inputs.scan_sarif }} == true ]]; then
-              mkdir artefact-upload
+              mkdir upload
               cat images-to-scan.txt | while read i; do
               filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-              sudo trivy image --format sarif --severity HIGH,CRITICAL $i > sarif-upload/${filename}.sarif
+              sudo trivy image --format sarif --severity HIGH,CRITICAL $i > upload/${filename}.sarif
               done
           fi
           if [[ ${{ github.event.inputs.scan_artefact }} == true ]]; then

From 1489682031e695c289854e0831293b14db237ce8 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Thu, 9 Mar 2023 12:50:50 +0000
Subject: [PATCH 14/22] Code clean up and fixes

---
 .../stackhpc-container-image-build.yml        | 76 +++++++------------
 1 file changed, 29 insertions(+), 47 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 327383c40..98f76c5da 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -18,16 +18,11 @@ on:
         type: boolean
         required: false
         default: false
-      scan_artefact:
-        description: Scan images? output=artefact
+      scan:
+        description: Scan images?
         type: boolean
         required: false
-        default: false
-      scan_sarif:
-        description: Scan images? output=github
-        type: boolean
-        required: false
-        default: false
+        default: true
       distro:
         description: Container image OS distribution
         type: choice
@@ -64,7 +59,7 @@ jobs:
 
       - name: Install Trivy
         run: |
-          sudo curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.38.0
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.38.0
 
       - name: Setup networking
         run: |
@@ -131,44 +126,6 @@ jobs:
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > overcloud-container-images
         if: github.event.inputs.overcloud == 'true'
 
-      - name: Generate the images to scan
-        run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" > images-to-scan.txt
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan_artefact == 'true') || (github.event.inputs.scan_sarif == 'true') }}
-
-      - name: Scan built overcloud container images
-        run: |
-          if [[ ${{ github.event.inputs.scan_sarif }} == true ]]; then
-              mkdir upload
-              cat images-to-scan.txt | while read i; do
-              filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-              sudo trivy image --format sarif --severity HIGH,CRITICAL $i > upload/${filename}.sarif
-              done
-          fi
-          if [[ ${{ github.event.inputs.scan_artefact }} == true ]]; then
-              mkdir upload
-              cat images-to-scan.txt | while read i; do
-              # filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-              sudo trivy image --severity HIGH,CRITICAL $i >> upload/scan.txt
-              done
-          fi
-        shell: bash
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan_artefact == 'true') || (github.event.inputs.scan_sarif == 'true') }}
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: 'sarif-upload/'
-        if: ${{ (github.event.inputs.overcloud == 'true') &&  (github.event.inputs.scan_sarif == 'true') }}
-
-      - name: Upload scan output artefact
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Trivy Scan txt"
-          path: 'upload/scan.txt'
-          retention-days: 7
-        if: ${{ (github.event.inputs.overcloud == 'true') &&  (github.event.inputs.scan_artefact == 'true') }}
-
       - name: Upload overcloud-container-images artifact
         uses: actions/upload-artifact@v3
         with:
@@ -204,6 +161,31 @@ jobs:
           retention-days: 7
         if: github.event.inputs.seed == 'true'
 
+      - name: Generate the images to scan
+        run: |
+          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" > images-to-scan.txt
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') || (github.event.inputs.scan_html == 'true') }}
+
+      - name: Scan built overcloud container images
+        run: |
+          if [[ ${{ github.event.inputs.scan }} == true ]]; then
+              mkdir upload
+              cat images-to-scan.txt | while read i; do
+                filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
+                sudo trivy image --severity HIGH,CRITICAL $i > upload/${filename}.txt
+              done
+          fi
+        shell: bash
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+
+      - name: Upload scan output artefact
+        uses: actions/upload-artifact@v3
+        with:
+          name: "Trivy Scan txt"
+          path: 'upload/'
+          retention-days: 7
+        if: ${{ (github.event.inputs.overcloud == 'true') &&  (github.event.inputs.scan == 'true') }}
+
       - name: Prune local Kolla container images over 1 week old
         run: |
           sudo docker image prune --all --force --filter until=168h --filter="label=kolla_version"

From c7307805fd047485e08242c6bced04dcaec465f1 Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Tue, 14 Mar 2023 16:12:26 +0000
Subject: [PATCH 15/22] add push back

---
 .github/workflows/stackhpc-container-image-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 98f76c5da..0e082d195 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -111,7 +111,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud container image build ${{ github.event.inputs.regexes }} -e kolla_base_distro=${{ inputs.distro }}
+          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'

From 5f2ff767e307a7c7d98b13eb6bd16559addcc37f Mon Sep 17 00:00:00 2001
From: k-s-dean <kyle@stackhpc.com>
Date: Wed, 15 Mar 2023 14:26:38 +0000
Subject: [PATCH 16/22] Get images to scan for seed and overcloud

---
 .../workflows/stackhpc-container-image-build.yml  | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 0e082d195..c84458d6b 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -126,6 +126,11 @@ jobs:
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > overcloud-container-images
         if: github.event.inputs.overcloud == 'true'
 
+      - name: Generate the images to scan
+        run: |
+          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" >> images-to-scan.txt
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+
       - name: Upload overcloud-container-images artifact
         uses: actions/upload-artifact@v3
         with:
@@ -153,6 +158,11 @@ jobs:
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > seed-container-images
         if: github.event.inputs.seed == 'true'
 
+      - name: Generate the images to scan
+        run: |
+          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" >> images-to-scan.txt
+        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+
       - name: Upload seed-container-images artifact
         uses: actions/upload-artifact@v3
         with:
@@ -161,11 +171,6 @@ jobs:
           retention-days: 7
         if: github.event.inputs.seed == 'true'
 
-      - name: Generate the images to scan
-        run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" > images-to-scan.txt
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') || (github.event.inputs.scan_html == 'true') }}
-
       - name: Scan built overcloud container images
         run: |
           if [[ ${{ github.event.inputs.scan }} == true ]]; then

From a83e64ebdb7636e6403ba27ed24120fc4d2f3426 Mon Sep 17 00:00:00 2001
From: Matt Anson <matta@stackhpc.com>
Date: Thu, 3 Aug 2023 18:07:55 +0100
Subject: [PATCH 17/22] Separate image build and push steps

---
 .../stackhpc-container-image-build.yml        | 116 ++++++++++++++----
 etc/kayobe/ansible/docker-registry-login.yml  |  10 ++
 .../inventory/group_vars/seed/dev-tools       |   4 +
 3 files changed, 103 insertions(+), 27 deletions(-)
 create mode 100644 etc/kayobe/ansible/docker-registry-login.yml
 create mode 100644 etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index e6d66d155..9702cf58b 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -23,6 +23,11 @@ on:
         type: boolean
         required: false
         default: true
+      scan-upload:
+        description: Upload scanned images that have vulnerabilities?
+        type: boolean
+        required: false
+        default: true
       distro:
         description: Container image OS distribution
         type: choice
@@ -60,7 +65,12 @@ jobs:
 
       - name: Install Trivy
         run: |
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.38.0
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.44.0
+ 
+      - name: Install jq
+        run: |
+          curl --output /usr/local/bin/jq -sfL https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64
+          chmod +x /usr/local/bin/jq
 
       - name: Setup networking
         run: |
@@ -112,7 +122,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push -e kolla_base_distro=${{ inputs.distro }}
+          kayobe overcloud container image build ${{ github.event.inputs.regexes }} -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'
@@ -127,11 +137,6 @@ jobs:
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > overcloud-container-images
         if: github.event.inputs.overcloud == 'true'
 
-      - name: Generate the images to scan
-        run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" >> images-to-scan.txt
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
-
       - name: Upload overcloud-container-images artifact
         uses: actions/upload-artifact@v3
         with:
@@ -144,7 +149,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed container image build --push -e kolla_base_distro=${{ inputs.distro }}
+          kayobe seed container image build -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.seed == 'true'
@@ -159,11 +164,6 @@ jobs:
           sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" > seed-container-images
         if: github.event.inputs.seed == 'true'
 
-      - name: Generate the images to scan
-        run: |
-          sudo docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$(cat ~/kolla_tag)" --format "{{.Repository}}:{{.Tag}}" >> images-to-scan.txt
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
-
       - name: Upload seed-container-images artifact
         uses: actions/upload-artifact@v3
         with:
@@ -172,25 +172,87 @@ jobs:
           retention-days: 7
         if: github.event.inputs.seed == 'true'
 
-      - name: Scan built overcloud container images
+      - name: Generate list of images to scan/push
         run: |
-          if [[ ${{ github.event.inputs.scan }} == true ]]; then
-              mkdir upload
-              cat images-to-scan.txt | while read i; do
-                filename=$(echo $i | sed 's/.*stackhpc-dev\///;s/:.*//')
-                sudo trivy image --severity HIGH,CRITICAL $i > upload/${filename}.txt
-              done
-          fi
-        shell: bash
-        if: ${{ (github.event.inputs.overcloud == 'true') && (github.event.inputs.scan == 'true') }}
+          # Make a file of imagename:tag
+          grep --invert-match --no-filename ^REPOSITORY overcloud-container-images seed-container-images |\
+          sed 's/ \+/:/g' |\
+          cut -f 1,2 -d: > docker-images.txt
+        
+      - name: Scan built container images
+        run: |
+          set -euo pipefail
+          mkdir -p image-scan-output
+
+          rm -f images-to-push.txt
 
-      - name: Upload scan output artefact
+          # If Trivy detects no vulnerabilities, add the image name to images-to-push.txt.
+          # If there are vulnerabilities detected, generate a CSV summary and do not add to
+          # images-to-push.txt.
+          while read -r image; do
+            filename=$(basename $image | sed 's/:/\./g')
+            if $(trivy image \
+                    --exit-code 1 \
+                    --scanners vuln \
+                    --format json \
+                    --severity HIGH,CRITICAL \
+                    --output image-scan-output/${filename}.json \
+                    --ignore-unfixed \
+                    $image); then
+              echo "${image}" >> images-to-push.txt
+              rm image-scan-output/${filename}.json
+            else
+              if [${{github.event.input.scan-upload}} == 'true' ]; then
+                echo "${image}" >> images-to-push.txt
+              fi
+              
+              echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.csv
+              jq -r '.Results[].Vulnerabilities
+                     # Ignore packages with "kernel" in the PkgName
+                     | map(select(.PkgName | test("kernel") | not ))
+                     | group_by(.VulnerabilityID)
+                     | map(
+                            [
+                              (map(.PkgName) | unique | join(";")),
+                              (map(.PkgPath | select( . != null )) | join(";")),
+                              .[0].PkgID,
+                              .[0].VulnerabilityID,
+                              .[0].FixedVersion,
+                              .[0].PrimaryURL,
+                              .[0].Severity
+                              ]
+                          ) 
+                     | .[] 
+                     | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv
+            fi
+          done < docker-images.txt
+          mv images-to-push.txt docker-images.txt
+        shell: bash
+        if: github.event.inputs.scan == 'true'
+     
+      - name: Upload Trivy scan artefacts
         uses: actions/upload-artifact@v3
         with:
-          name: "Trivy Scan txt"
-          path: 'upload/'
+          name: "trivy-scan-output"
+          path: |
+            'image-scan-output/*.json'
+            'image-scan-output/*.summary.csv'
           retention-days: 7
-        if: ${{ (github.event.inputs.overcloud == 'true') &&  (github.event.inputs.scan == 'true') }}
+        if: github.event.inputs.scan == 'true'
+
+      - name: Push images
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
+
+          while read -r image; do
+            # Retries!
+            for i in {1..10}; do 
+              docker push ${image} && break || sleep 5
+            done
+          done < docker-images.txt
+        shell: bash
 
       - name: Prune local Kolla container images over 1 week old
         run: |
diff --git a/etc/kayobe/ansible/docker-registry-login.yml b/etc/kayobe/ansible/docker-registry-login.yml
new file mode 100644
index 000000000..a2b5a185b
--- /dev/null
+++ b/etc/kayobe/ansible/docker-registry-login.yml
@@ -0,0 +1,10 @@
+- name: Login to docker registry
+  gather_facts: false
+  hosts: container-image-builders
+  tasks:
+    - name: Login to docker registry
+      docker_login:
+        registry_url: "{{ kolla_docker_registry or omit }}"
+        username: "{{ kolla_docker_registry_username }}"
+        password: "{{ kolla_docker_registry_password }}"
+        reauthorize: yes
\ No newline at end of file
diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
new file mode 100644
index 000000000..1bc4fbeba
--- /dev/null
+++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
@@ -0,0 +1,4 @@
+---
+# Used in CI workflow
+dev_tools_packages_extra:
+  - jq
\ No newline at end of file

From 1a9ab2ecc7b0e51fa3b3da0cef00d97f3c701367 Mon Sep 17 00:00:00 2001
From: Matt Anson <matta@stackhpc.com>
Date: Fri, 4 Aug 2023 09:25:13 +0100
Subject: [PATCH 18/22] Correct workflow syntax

---
 .../stackhpc-container-image-build.yml        | 75 +++++++++++--------
 etc/kayobe/ansible/docker-registry-login.yml  |  3 +-
 .../inventory/group_vars/seed/dev-tools       |  2 +-
 3 files changed, 48 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 1f8356c72..544562b7e 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -43,8 +43,8 @@ on:
         type: boolean
         required: false
         default: true
-      scan-upload:
-        description: Upload scanned images that have vulnerabilities?
+      scan-push:
+        description: Push scanned images that have vulnerabilities?
         type: boolean
         required: false
         default: true
@@ -135,11 +135,6 @@ jobs:
         run: |
           curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.44.0
  
-      - name: Install jq
-        run: |
-          curl --output /usr/local/bin/jq -sfL https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64
-          chmod +x /usr/local/bin/jq
-
       - name: Setup networking
         run: |
           if ! ip l show breth1 >/dev/null 2>&1; then
@@ -186,7 +181,7 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
-      - name: Build and push kolla overcloud images
+      - name: Build kolla overcloud images
         run: |
           args="${{ github.event.inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
@@ -198,7 +193,7 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'
 
-      - name: Build and push kolla seed images
+      - name: Build kolla seed images
         run: |
           args="kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
@@ -222,40 +217,57 @@ jobs:
 
       - name: Generate list of images to scan/push
         run: |
+          # Clean up any stale data
+          rm -rf ${{ matrix.distro }}-docker-images.txt
+
           # Make a file of imagename:tag
           grep --invert-match --no-filename ^REPOSITORY ${{ matrix.distro }}-container-images |\
           sed 's/ \+/:/g' |\
-          cut -f 1,2 -d: > docker-images.txt
+          cut -f 1,2 -d: > ${{ matrix.distro }}-docker-images.txt
         
       - name: Scan built container images
         run: |
           set -euo pipefail
-          mkdir -p image-scan-output
 
+          # Clean any stale data
+          rm -rf ${{ matrix.distro }}-image-scan-output
           rm -f images-to-push.txt
 
+          # Make a fresh output directory
+          mkdir -p ${{ matrix.distro }}-image-scan-output
+
           # If Trivy detects no vulnerabilities, add the image name to images-to-push.txt.
           # If there are vulnerabilities detected, generate a CSV summary and do not add to
           # images-to-push.txt.
           while read -r image; do
             filename=$(basename $image | sed 's/:/\./g')
             if $(trivy image \
+                    --quiet \
                     --exit-code 1 \
                     --scanners vuln \
                     --format json \
                     --severity HIGH,CRITICAL \
-                    --output image-scan-output/${filename}.json \
+                    --output ${{ matrix.distro }}-image-scan-output/${filename}.json \
                     --ignore-unfixed \
                     $image); then
-              echo "${image}" >> images-to-push.txt
-              rm image-scan-output/${filename}.json
+              # Clean up the output file for any images with no vulnerabilities
+              rm -f ${{ matrix.distro }}-image-scan-output/${filename}.json
+
+              # Add the image to the list to push
+              echo "${image}" >> ${{ matrix.distro }}-images-to-push.txt
             else
-              if [${{github.event.input.scan-upload}} == 'true' ]; then
-                echo "${image}" >> images-to-push.txt
+              # Still add the image to the list to push if we're ignoring fails
+              if [ "${{github.event.inputs.scan-push}}" == "true" ]; then
+                echo "${image}" >> ${{ matrix.distro }}-images-to-push.txt
               fi
               
-              echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.csv
-              jq -r '.Results[].Vulnerabilities
+              # Write a header for the summary CSV
+              echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > ${{ matrix.distro }}-image-scan-output/${filename}.summary.csv
+
+              # Write the summary CSV data
+              jq -r '.Results[] 
+                     | select(.Vulnerabilities) 
+                     | .Vulnerabilities
                      # Ignore packages with "kernel" in the PkgName
                      | map(select(.PkgName | test("kernel") | not ))
                      | group_by(.VulnerabilityID)
@@ -271,37 +283,40 @@ jobs:
                               ]
                           ) 
                      | .[] 
-                     | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv
+                     | @csv' ${{ matrix.distro }}-image-scan-output/${filename}.json >> ${{ matrix.distro }}-image-scan-output/${filename}.summary.csv
             fi
-          done < docker-images.txt
-          mv images-to-push.txt docker-images.txt
+          done < ${{ matrix.distro }}-docker-images.txt
+
+          # Rename the file of vulnerability scanned images so that it can
+          # be consumed by the docker push step
+          mv ${{ matrix.distro }}-images-to-push.txt ${{ matrix.distro }}-docker-images.txt
         shell: bash
         if: github.event.inputs.scan == 'true'
      
       - name: Upload Trivy scan artefacts
         uses: actions/upload-artifact@v3
         with:
-          name: "trivy-scan-output"
-          path: |
-            'image-scan-output/*.json'
-            'image-scan-output/*.summary.csv'
+          name: ${{ matrix.distro }}-image-scan-output
+          path: ${{ matrix.distro }}-image-scan-output
           retention-days: 7
-        if: github.event.inputs.scan == 'true'
+        if: always()
 
       - name: Push images
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
+          kayobe playbook run --become ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
 
           while read -r image; do
             # Retries!
             for i in {1..10}; do 
-              docker push ${image} && break || sleep 5
+              sudo docker push ${image} && break || sleep 2
             done
-          done < docker-images.txt
+          done < ${{ matrix.distro }}-docker-images.txt
         shell: bash
-        if: ${{ inputs.push }}
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: github.event.inputs.push == 'true'
 
       - name: Prune local Kolla container images over 1 week old
         run: |
diff --git a/etc/kayobe/ansible/docker-registry-login.yml b/etc/kayobe/ansible/docker-registry-login.yml
index a2b5a185b..39ad03600 100644
--- a/etc/kayobe/ansible/docker-registry-login.yml
+++ b/etc/kayobe/ansible/docker-registry-login.yml
@@ -1,3 +1,4 @@
+---
 - name: Login to docker registry
   gather_facts: false
   hosts: container-image-builders
@@ -7,4 +8,4 @@
         registry_url: "{{ kolla_docker_registry or omit }}"
         username: "{{ kolla_docker_registry_username }}"
         password: "{{ kolla_docker_registry_password }}"
-        reauthorize: yes
\ No newline at end of file
+        reauthorize: yes
diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
index 1bc4fbeba..3c7b7f01f 100644
--- a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
+++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
@@ -1,4 +1,4 @@
 ---
 # Used in CI workflow
 dev_tools_packages_extra:
-  - jq
\ No newline at end of file
+  - jq

From 78bac79895f9f07488cdcb61e26c77730fab07f2 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 23 Feb 2024 15:27:09 +0000
Subject: [PATCH 19/22] Trivy scanning Feb 2024 refresh

---
 .../stackhpc-container-image-build.yml        | 27 +++-----
 .../environments/ci-builder/stackhpc-ci.yml   |  1 +
 tools/scan-images.sh                          | 65 +++++++++++++++++++
 3 files changed, 76 insertions(+), 17 deletions(-)
 create mode 100755 tools/scan-images.sh

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 8e111ff08..f67e4c8ff 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -119,7 +119,7 @@ jobs:
       - name: Install package dependencies
         run: |
           sudo apt update
-          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -208,27 +208,20 @@ jobs:
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
-      - name: Upload container images artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.distro }} container images
-          path: ${{ matrix.distro }}-container-images
-          retention-days: 7
-
-      - name: Generate list of images to scan/push
-        run: |
-          # Make a file of imagename:tag
-          grep --invert-match --no-filename ^REPOSITORY ${{ matrix.distro }}-container-images |\
-          sed 's/ \+/:/g' |\
-          cut -f 1,2 -d: > ${{ matrix.distro }}-docker-images.txt
-          
       - name: Scan built container images
         run: |
-          src/kayobe-config/tools/scan-images.sh
+          src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
         shell: bash
         if: github.event.inputs.scan == 'true'
 
-      - name: Upload Trivy scan artefacts
+      - name: Upload built container images artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.distro }}-container-images
+          path: ${{ matrix.distro }}-container-images
+          retention-days: 7
+
+      - name: Upload Trivy scan results artifact
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.distro }}-image-scan-output
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 868668e33..9dd88242b 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -101,6 +101,7 @@ stackhpc_include_os_minor_version_in_repo_url: true
 # Host and port of container registry.
 # Push built images to the Ark registry.
 stackhpc_docker_registry: "{{ stackhpc_release_pulp_url | regex_replace('^https?://', '') }}"
+kolla_docker_registry: "{{ stackhpc_docker_registry }}"
 
 # Username and password of container registry.
 stackhpc_docker_registry_username: "release-train-ci"
diff --git a/tools/scan-images.sh b/tools/scan-images.sh
new file mode 100755
index 000000000..75657b148
--- /dev/null
+++ b/tools/scan-images.sh
@@ -0,0 +1,65 @@
+set -euo pipefail
+
+# TODO: Check trivy installed
+
+# TODO: Check inputs - requires $1 as a distro and $2 as container iamge
+
+# Make a fresh output directory
+mkdir -p $1-image-scan-output
+
+# Get built container images
+docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/$1-*:$2" > $1-container-images
+
+# Make a file of imagename:tag
+grep --invert-match --no-filename ^REPOSITORY $1-container-images |\
+sed 's/ \+/:/g' |\
+cut -f 1,2 -d: > $1-docker-images.txt
+
+# If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
+# If there are vulnerabilities detected, generate a CSV summary and do not add
+# to clean-images.txt.
+while read -r image; do
+  filename=$(basename $image | sed 's/:/\./g')
+  if $(trivy image \
+          --quiet \
+          --exit-code 1 \
+          --scanners vuln \
+          --format json \
+          --severity HIGH,CRITICAL \
+          --output $1-image-scan-output/${filename}.json \
+          --ignore-unfixed \
+          $image); then
+    # Clean up the output file for any images with no vulnerabilities
+    rm -f $1-image-scan-output/${filename}.json
+
+    # Add the image to the clean list
+    echo "${image}" >> $1-clean-images.txt
+  else
+    # Add the image to the dirty list
+    echo "${image}" >> $1-clean-images.txt
+    
+    # Write a header for the summary CSV
+    echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > $1-image-scan-output/${filename}.summary.csv
+
+    # Write the summary CSV data
+    jq -r '.Results[] 
+            | select(.Vulnerabilities) 
+            | .Vulnerabilities
+            # Ignore packages with "kernel" in the PkgName
+            | map(select(.PkgName | test("kernel") | not ))
+            | group_by(.VulnerabilityID)
+            | map(
+                  [
+                    (map(.PkgName) | unique | join(";")),
+                    (map(.PkgPath | select( . != null )) | join(";")),
+                    .[0].PkgID,
+                    .[0].VulnerabilityID,
+                    .[0].FixedVersion,
+                    .[0].PrimaryURL,
+                    .[0].Severity
+                    ]
+                ) 
+            | .[] 
+            | @csv' $1-image-scan-output/${filename}.json >> $1-image-scan-output/${filename}.summary.csv
+  fi
+done < $1-docker-images.txt

From 06857bd4cb4dce0b0c63ef19f6e4642b74f00baa Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 23 Feb 2024 17:04:12 +0000
Subject: [PATCH 20/22] Fixup container image scanning PR

---
 .../stackhpc-container-image-build.yml        | 52 +++++++++++--------
 .../inventory/group_vars/seed/dev-tools       |  4 --
 .../environments/ci-builder/stackhpc-ci.yml   |  1 -
 ...ainer-image-scanning-e5adf2c6b540b502.yaml |  6 +++
 tools/scan-images.sh                          | 43 ++++++++-------
 5 files changed, 61 insertions(+), 45 deletions(-)
 delete mode 100644 etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
 create mode 100644 releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index f67e4c8ff..c030a2147 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -38,11 +38,6 @@ on:
         type: boolean
         required: false
         default: true
-      scan:
-        description: Scan images?
-        type: boolean
-        required: false
-        default: true
       scan-push:
         description: Push scanned images that have vulnerabilities?
         type: boolean
@@ -177,6 +172,8 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Build kolla overcloud images
+        id: build_overcloud_images
+        continue-on-error: true
         run: |
           args="${{ github.event.inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
@@ -190,6 +187,8 @@ jobs:
         if: github.event.inputs.overcloud == 'true'
 
       - name: Build kolla seed images
+        id: build_seed_images
+        continue-on-error: true
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
@@ -209,48 +208,57 @@ jobs:
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Scan built container images
-        run: |
-          src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
-        shell: bash
-        if: github.event.inputs.scan == 'true'
-
-      - name: Upload built container images artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.distro }}-container-images
-          path: ${{ matrix.distro }}-container-images
-          retention-days: 7
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
 
       - name: Upload Trivy scan results artifact
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.distro }}-image-scan-output
-          path: ${{ matrix.distro }}-image-scan-output
+          path: image-scan-output
           retention-days: 7
-        if: always()
+
+      - name: Fail if no images have passed scanning
+        run: if [ $(wc -l < image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
+        if: github.event.inputs.scan-push == 'false'
+
+      - name: Append dirty images to clean list
+        run: |
+          cat image-scan-output/dirty-images.txt >> image-scan-output/clean-images.txt
+        if: github.event.inputs.scan-push == 'true'
 
       - name: Push images
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run --become ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml
+          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
 
           while read -r image; do
             # Retries!
             for i in {1..10}; do 
-              sudo docker push ${image} && break || sleep 5
+              docker push ${image} && break || sleep 5
             done
-          done < ${{ matrix.distro }}-docker-images.txt
+          done < image-scan-output/clean-images.txt
         shell: bash
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.push == 'true'
 
+      - name: Upload pushed container images artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.distro }}-pushed-container-images
+          path: image-scan-output/clean-images.txt
+          retention-days: 7
+
+      - name: Fail when images failed to build
+        run: exit 1
+        if: steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure'
+
   sync-container-repositories:
     name: Trigger container image repository sync
     needs:
       - container-image-build
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push == 'true'
     runs-on: ubuntu-latest
     permissions: {}
     steps:
diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
deleted file mode 100644
index 3c7b7f01f..000000000
--- a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/dev-tools
+++ /dev/null
@@ -1,4 +0,0 @@
----
-# Used in CI workflow
-dev_tools_packages_extra:
-  - jq
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 9dd88242b..868668e33 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -101,7 +101,6 @@ stackhpc_include_os_minor_version_in_repo_url: true
 # Host and port of container registry.
 # Push built images to the Ark registry.
 stackhpc_docker_registry: "{{ stackhpc_release_pulp_url | regex_replace('^https?://', '') }}"
-kolla_docker_registry: "{{ stackhpc_docker_registry }}"
 
 # Username and password of container registry.
 stackhpc_docker_registry_username: "release-train-ci"
diff --git a/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml
new file mode 100644
index 000000000..67a99f9c2
--- /dev/null
+++ b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml
@@ -0,0 +1,6 @@
+---
+security:
+  - |
+    Kolla container images created using the
+    ``stackhpc-container-image-build.yml`` workflow are now automatically
+    scanned for vulnerablilities.
diff --git a/tools/scan-images.sh b/tools/scan-images.sh
index 75657b148..f40d9de20 100755
--- a/tools/scan-images.sh
+++ b/tools/scan-images.sh
@@ -1,24 +1,31 @@
-set -euo pipefail
+set -eo pipefail
 
-# TODO: Check trivy installed
+# Check correct usage
+if [[ ! $2 ]]; then
+    echo "Usage: overcloud-ubuntu-upgrade.sh <os-distribution> <image-tag>"
+    exit 2
+fi
 
-# TODO: Check inputs - requires $1 as a distro and $2 as container iamge
+set -u
+
+# Check that trivy is installed
+if ! trivy --version; then
+    echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1'
+fi
 
 # Make a fresh output directory
-mkdir -p $1-image-scan-output
+mkdir -p image-scan-output
 
 # Get built container images
-docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/$1-*:$2" > $1-container-images
+docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/$1-*:$2" > $1-scanned-container-images.txt
 
 # Make a file of imagename:tag
-grep --invert-match --no-filename ^REPOSITORY $1-container-images |\
-sed 's/ \+/:/g' |\
-cut -f 1,2 -d: > $1-docker-images.txt
+images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:)
 
 # If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
-# If there are vulnerabilities detected, generate a CSV summary and do not add
-# to clean-images.txt.
-while read -r image; do
+# If there are vulnerabilities detected, add it to dirty-images.txt and
+# generate a csv summary
+for image in $images; do
   filename=$(basename $image | sed 's/:/\./g')
   if $(trivy image \
           --quiet \
@@ -26,20 +33,20 @@ while read -r image; do
           --scanners vuln \
           --format json \
           --severity HIGH,CRITICAL \
-          --output $1-image-scan-output/${filename}.json \
+          --output image-scan-output/${filename}.json \
           --ignore-unfixed \
           $image); then
     # Clean up the output file for any images with no vulnerabilities
-    rm -f $1-image-scan-output/${filename}.json
+    rm -f image-scan-output/${filename}.json
 
     # Add the image to the clean list
-    echo "${image}" >> $1-clean-images.txt
+    echo "${image}" >> image-scan-output/clean-images.txt
   else
     # Add the image to the dirty list
-    echo "${image}" >> $1-clean-images.txt
+    echo "${image}" >> image-scan-output/dirty-images.txt
     
     # Write a header for the summary CSV
-    echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > $1-image-scan-output/${filename}.summary.csv
+    echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv
 
     # Write the summary CSV data
     jq -r '.Results[] 
@@ -60,6 +67,6 @@ while read -r image; do
                     ]
                 ) 
             | .[] 
-            | @csv' $1-image-scan-output/${filename}.json >> $1-image-scan-output/${filename}.summary.csv
+            | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv
   fi
-done < $1-docker-images.txt
+done

From c032f6898301fe50bd5c81be4dc156c2b0a63c57 Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Fri, 1 Mar 2024 17:03:17 +0000
Subject: [PATCH 21/22] Container image builds misc improvements

---
 .../stackhpc-container-image-build.yml        | 83 +++++++++++++------
 tools/scan-images.sh                          | 13 ++-
 2 files changed, 66 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index c030a2147..9f5f88fb5 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -38,10 +38,11 @@ on:
         type: boolean
         required: false
         default: true
-      scan-push:
+      push-dirty:
         description: Push scanned images that have vulnerabilities?
         type: boolean
         required: false
+        # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
         default: true
 
 env:
@@ -171,11 +172,14 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
+      - name: Create build logs output directory
+        run: mkdir image-build-logs 
+
       - name: Build kolla overcloud images
         id: build_overcloud_images
         continue-on-error: true
         run: |
-          args="${{ github.event.inputs.regexes }}"
+          args="${{ inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
@@ -184,7 +188,11 @@ jobs:
           kayobe overcloud container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: github.event.inputs.overcloud == 'true'
+        if: inputs.overcloud
+
+      - name: Copy overcloud container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
+        if: inputs.overcloud
 
       - name: Build kolla seed images
         id: build_seed_images
@@ -198,7 +206,11 @@ jobs:
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: github.event.inputs.seed == 'true'
+        if: inputs.seed
+
+      - name: Copy seed container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
+        if: inputs.seed
 
       - name: Get built container images
         run: |
@@ -208,57 +220,74 @@ jobs:
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Scan built container images
-        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
-
-      - name: Upload Trivy scan results artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.distro }}-image-scan-output
-          path: image-scan-output
-          retention-days: 7
+        run: |
+          src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
 
       - name: Fail if no images have passed scanning
         run: if [ $(wc -l < image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
-        if: github.event.inputs.scan-push == 'false'
+        if: ${{ !inputs.push-dirty }}
+
+      - name: Copy clean images to push-attempt-images list
+        run: cp image-scan-output/clean-images.txt image-scan-output/push-attempt-images.txt
 
-      - name: Append dirty images to clean list
+      - name: Append dirty images to push list
         run: |
-          cat image-scan-output/dirty-images.txt >> image-scan-output/clean-images.txt
-        if: github.event.inputs.scan-push == 'true'
+          cat image-scan-output/dirty-images.txt >> image-scan-output/push-attempt-images.txt
+        if: inputs.push-dirty
 
       - name: Push images
         run: |
+          touch image-scan-output/push-failed-images.txt
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
 
           while read -r image; do
             # Retries!
-            for i in {1..10}; do 
-              docker push ${image} && break || sleep 5
+            for i in {1..5}; do 
+              if docker push $image; then
+                echo "Pushed $image"
+                break
+              elif $i == 5; then
+                echo "Failed to push $image"
+                echo $image >> image-scan-output/push-failed-images.txt
+              else
+                echo "Failed on retry $i"
+                sleep 5
+              fi;
             done
-          done < image-scan-output/clean-images.txt
+          done < image-scan-output/push-attempt-images.txt &&
+          mv image-scan-output image-build-logs/image-scan-output
         shell: bash
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: github.event.inputs.push == 'true'
+        if: inputs.push
 
-      - name: Upload pushed container images artifact
+      - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }}-pushed-container-images
-          path: image-scan-output/clean-images.txt
+          name: ${{ matrix.distro }}-logs
+          path: image-build-logs
           retention-days: 7
+        if: ${{ !cancelled() }}
 
       - name: Fail when images failed to build
-        run: exit 1
-        if: steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure'
+        run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1
+        if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}
+
+      - name: Fail when images failed to push
+        run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
+        if: ${{ !cancelled() }}
+
+      - name: Fail when images failed scanning
+        run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
+        if: ${{ !inputs.push-dirty && !cancelled() }}
 
   sync-container-repositories:
     name: Trigger container image repository sync
     needs:
       - container-image-build
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push == 'true'
+    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
     runs-on: ubuntu-latest
     permissions: {}
     steps:
@@ -267,7 +296,7 @@ jobs:
       - name: Trigger container image repository sync
         run: |
           filter='${{ inputs.regexes }}'
-          if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then
+          if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then
             filter="$filter bifrost"
           fi
           gh workflow run \
diff --git a/tools/scan-images.sh b/tools/scan-images.sh
index f40d9de20..50a04185a 100755
--- a/tools/scan-images.sh
+++ b/tools/scan-images.sh
@@ -1,18 +1,22 @@
+#!/usr/bin/env bash
 set -eo pipefail
 
 # Check correct usage
 if [[ ! $2 ]]; then
-    echo "Usage: overcloud-ubuntu-upgrade.sh <os-distribution> <image-tag>"
-    exit 2
+  echo "Usage: scan-images.sh <os-distribution> <image-tag>"
+  exit 2
 fi
 
 set -u
 
 # Check that trivy is installed
 if ! trivy --version; then
-    echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1'
+  echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1'
 fi
 
+# Clear any previous outputs
+rm -rf image-scan-output
+
 # Make a fresh output directory
 mkdir -p image-scan-output
 
@@ -22,6 +26,9 @@ docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/$1-*:$2" > $1-
 # Make a file of imagename:tag
 images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:)
 
+# Ensure output files exist
+touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt
+
 # If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
 # If there are vulnerabilities detected, add it to dirty-images.txt and
 # generate a csv summary

From 2be1d27932048f056fa7a33968ca18b26fdc842f Mon Sep 17 00:00:00 2001
From: Alex-Welsh <alex@stackhpc.com>
Date: Wed, 6 Mar 2024 14:27:30 +0000
Subject: [PATCH 22/22] Kolla image build - improve output formatting

---
 .../stackhpc-container-image-build.yml        | 45 ++++++++++---------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml
index 9f5f88fb5..ad3097d0a 100644
--- a/.github/workflows/stackhpc-container-image-build.yml
+++ b/.github/workflows/stackhpc-container-image-build.yml
@@ -115,7 +115,15 @@ jobs:
       - name: Install package dependencies
         run: |
           sudo apt update
-          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget
+
+      - name: Install gh
+        run: |
+          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
+          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install gh -y
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -213,31 +221,33 @@ jobs:
         if: inputs.seed
 
       - name: Get built container images
-        run: |
-          docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Scan built container images
-        run: |
-          src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
+
+      - name: Move image scan logs to output artifact
+        run: mv image-scan-output image-build-logs/image-scan-output
 
       - name: Fail if no images have passed scanning
-        run: if [ $(wc -l < image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
+        run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
         if: ${{ !inputs.push-dirty }}
 
       - name: Copy clean images to push-attempt-images list
-        run: cp image-scan-output/clean-images.txt image-scan-output/push-attempt-images.txt
+        run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
+        if: inputs.push
 
       - name: Append dirty images to push list
         run: |
-          cat image-scan-output/dirty-images.txt >> image-scan-output/push-attempt-images.txt
-        if: inputs.push-dirty
+          cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+        if: ${{ inputs.push && inputs.push-dirty }}
 
       - name: Push images
         run: |
-          touch image-scan-output/push-failed-images.txt
+          touch image-build-logs/push-failed-images.txt
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
@@ -250,14 +260,13 @@ jobs:
                 break
               elif $i == 5; then
                 echo "Failed to push $image"
-                echo $image >> image-scan-output/push-failed-images.txt
+                echo $image >> image-build-logs/push-failed-images.txt
               else
                 echo "Failed on retry $i"
                 sleep 5
               fi;
             done
-          done < image-scan-output/push-attempt-images.txt &&
-          mv image-scan-output image-build-logs/image-scan-output
+          done < image-build-logs/push-attempt-images.txt
         shell: bash
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
@@ -283,14 +292,6 @@ jobs:
         run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
         if: ${{ !inputs.push-dirty && !cancelled() }}
 
-  sync-container-repositories:
-    name: Trigger container image repository sync
-    needs:
-      - container-image-build
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
-    runs-on: ubuntu-latest
-    permissions: {}
-    steps:
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.
       - name: Trigger container image repository sync
@@ -307,7 +308,9 @@ jobs:
           -f sync-new-images=false
         env:
           GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}
 
       - name: Display link to container image repository sync workflows
         run: |
           echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml"
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}