use pre-built base images

.github/actions/smoke-tests/action.yaml

@@ -5,6 +5,9 @@ inputs:
   go-md5:
     required: true
     type: string
+  base-image-md5:
+    required: true
+    type: string
   k8s-version:
     description: Kubernetes version to use
     required: false
@@ -67,18 +70,36 @@ runs:
     - name: Docker Buildx
       uses: docker/setup-buildx-action@v2
 
+    - name: Authenticate to Google Cloud
+      id: auth
+      uses: google-github-actions/auth@v2
+      with:
+        token_format: access_token
+        workload_identity_provider: ${{ inputs.gcr-workload-identity-secret }}
+        service_account: ${{ inputs.gcr-service-account-secret }}
+      if: github.event.pull_request.head.repo.full_name == github.repository
+
+    - name: Login to GCR
+      uses: docker/login-action@v3
+      with:
+        registry: gcr.io
+        username: oauth2accesstoken
+        password: ${{ steps.auth.outputs.access_token }}
+      if: github.event.pull_request.head.repo.full_name == github.repository
+
     - name: Build ${{ inputs.image }} Container
       uses: docker/build-push-action@v3
       with:
         file: build/Dockerfile
         context: "."
         cache-from: type=gha,scope=${{ inputs.image }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}${{ contains(inputs.marker, 'appprotect') && '-nap' || '' }}
-        target: goreleaser
+        target: goreleaser${{ github.event.pull_request.head.repo.full_name == github.repository && '-prebuilt' || '' }}
         tags: "docker.io/nginx/${{ steps.ingress-type.outputs.name }}:${{ steps.ingress-type.outputs.tag }}"
         load: true
         pull: true
         build-args: |
           BUILD_OS=${{ inputs.image }}
+          PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ contains(inputs.image, 'plus') && 'plus' || 'oss' }}:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ contains(inputs.marker, 'appprotect') && '-waf' || '' }}${{ contains(inputs.marker, 'dos') && '-dos' || '' }}
           IC_VERSION=CI
           ${{ steps.ingress-type.outputs.modules != '' && format('NAP_MODULES={0}', steps.ingress-type.outputs.modules) || '' }}
           ${{ contains(inputs.marker, 'appprotect') && 'DEBIAN_VERSION=buster-slim' || '' }}
@@ -105,23 +126,6 @@ runs:
         sed -i 's|server:.*|server: https://${{ steps.k8s.outputs.cluster_ip }}:6443|' ~/.kube/kind/config
       shell: bash
 
-    - name: Authenticate to Google Cloud
-      id: auth
-      uses: google-github-actions/auth@v2
-      with:
-        token_format: access_token
-        workload_identity_provider: ${{ inputs.gcr-workload-identity-secret }}
-        service_account: ${{ inputs.gcr-service-account-secret }}
-      if: github.event.pull_request.head.repo.full_name == github.repository
-
-    - name: Login to GCR
-      uses: docker/login-action@v3
-      with:
-        registry: gcr.io
-        username: oauth2accesstoken
-        password: ${{ steps.auth.outputs.access_token }}
-      if: github.event.pull_request.head.repo.full_name == github.repository
-
     - name: Build Test-Runner Container
       uses: docker/build-push-action@v3
       with:

.github/workflows/build-base-images.yml

@@ -2,6 +2,7 @@ name: Build Base Images
 
 on:
   workflow_dispatch:
+  workflow_call:
   schedule:
     - cron: "30 4 * * 1-5" # run Mon-Fri at 04:30 UTC
 
@@ -11,7 +12,7 @@ defaults:
 
 concurrency:
   group: ${{ github.ref_name }}-base-image
-  cancel-in-progress: true
+  cancel-in-progress: false
 
 permissions:
   contents: read

.github/workflows/build-oss.yml

@@ -12,6 +12,9 @@ on:
       go-md5:
         required: true
         type: string
+      base-image-md5:
+        required: false
+        type: string
       tag:
         required: false
         type: string
@@ -95,6 +98,23 @@ jobs:
           password: ${{ secrets.QUAY_ROBOT_TOKEN }}
         if: ${{ inputs.publish-image }}
 
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
+          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
+      - name: Login to GCR
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
       - name: Get short tag
         id: tag
         run: |
@@ -143,7 +163,7 @@ jobs:
           context: "."
           cache-from: type=gha,scope=${{ inputs.image }}
           cache-to: type=gha,scope=${{ inputs.image }},mode=max
-          target: goreleaser
+          target: goreleaser${{ github.event.pull_request.head.repo.full_name == github.repository && '-prebuilt' || '' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           annotations: ${{ github.event_name != 'pull_request' && steps.meta.outputs.annotations || '' }}
@@ -156,6 +176,7 @@ jobs:
           provenance: false
           build-args: |
             BUILD_OS=${{ inputs.image }}
+            PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/oss:${{ inputs.base-image-md5 }}-${{ inputs.image }}
             IC_VERSION=${{ (github.event_name == 'pull_request' || startsWith(github.ref, 'refs/heads/release-')) && 'CI' || steps.meta.outputs.version }}
 
       - name: Certify Images

.github/workflows/build-plus.yml

@@ -15,6 +15,9 @@ on:
       go-md5:
         required: true
         type: string
+      base-image-md5:
+        required: false
+        type: string
       nap_modules:
         required: false
         type: string
@@ -67,15 +70,15 @@ jobs:
           token_format: access_token
           workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
           service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
-        if: ${{ inputs.publish-image }}
+        if: ${{ inputs.publish-image || github.event.pull_request.head.repo.full_name == github.repository }}
 
       - name: Login to GCR
         uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
         with:
           registry: gcr.io
           username: oauth2accesstoken
           password: ${{ steps.auth.outputs.access_token }}
-        if: ${{ inputs.publish-image }}
+        if: ${{ inputs.publish-image || github.event.pull_request.head.repo.full_name == github.repository }}
 
       - name: Authenticate to Google Cloud Marketplace
         id: auth-mktpl
@@ -162,7 +165,7 @@ jobs:
           context: "."
           cache-from: type=gha,scope=${{ inputs.image }}${{ contains(inputs.nap_modules, 'dos') && '-dos' || '' }}${{ contains(inputs.nap_modules, 'waf') && '-nap' || '' }}
           cache-to: type=gha,scope=${{ inputs.image }}${{ contains(inputs.nap_modules, 'dos') && '-dos' || '' }}${{ contains(inputs.nap_modules, 'waf') && '-nap' || '' }},mode=max
-          target: ${{ inputs.target }}
+          target: ${{ inputs.target }}${{ github.event.pull_request.head.repo.full_name == github.repository && '-prebuilt' || '' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           annotations: ${{ inputs.publish-image && steps.meta.outputs.annotations || '' }}
@@ -175,6 +178,7 @@ jobs:
           provenance: false
           build-args: |
             BUILD_OS=${{ inputs.image }}
+            PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/plus:${{ inputs.base-image-md5 }}-${{ inputs.image }}${{ contains(inputs.nap_modules, 'waf') && '-waf' || '' }}${{ contains(inputs.nap_modules, 'dos') && '-dos' || '' }}
             IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }}
             ${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', inputs.nap_modules) || '' }}
             ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
@@ -218,31 +222,18 @@ jobs:
             Use this image instead of building your own.
         if: ${{ github.ref_type == 'tag' && contains(inputs.target, 'aws') }}
 
-      - name: Load image for Trivy
-        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
-        with:
-          file: build/Dockerfile
-          context: "."
-          cache-from: type=gha,scope=${{ inputs.image }}
-          target: ${{ inputs.target }}
-          tags: docker.io/${{ inputs.image }}:${{ steps.meta.outputs.version }}
-          load: true
-          build-args: |
-            BUILD_OS=${{ inputs.image }}
-            IC_VERSION=${{ github.ref_type == 'tag' && steps.meta.outputs.version || 'CI' }}
-            ${{ inputs.nap_modules != '' && format('NAP_MODULES={0}', inputs.nap_modules) || '' }}
-            ${{ steps.nap_modules.outputs.modules != '' && format('NAP_MODULES_AWS={0}', steps.nap_modules.outputs.modules) || '' }}
-          secrets: |
-            "nginx-repo.crt=${{ inputs.nap_modules != '' && secrets.NGINX_AP_CRT || secrets.NGINX_CRT }}"
-            "nginx-repo.key=${{ inputs.nap_modules != '' && secrets.NGINX_AP_KEY || secrets.NGINX_KEY }}"
-            ${{ inputs.nap_modules != '' && contains(inputs.image, 'ubi') && format('"rhel_license={0}"', secrets.RHEL_LICENSE) || '' }}
+      - name: Extract image name for Trivy
+        id: trivy-tag
+        run: |
+          tag=$(echo $DOCKER_METADATA_OUTPUT_JSON | jq -r '.tags[] | select(contains("f5-gcs-7899"))' )
+          echo "tag=$tag" >> $GITHUB_OUTPUT
         if: ${{ inputs.publish-image }}
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@84384bd6e777ef152729993b8145ea352e9dd3ef # 0.17.0
         continue-on-error: true
         with:
-          image-ref: docker.io/${{ inputs.image }}:${{ steps.meta.outputs.version }}
+          image-ref: ${{ steps.trivy-tag.outputs.tag }}
           format: "sarif"
           output: "trivy-results-${{ inputs.image }}.sarif"
           ignore-unfixed: "true"

.github/workflows/ci.yml

@@ -44,6 +44,7 @@ jobs:
       binary_cache_hit: ${{ steps.binary-cache.outputs.cache-hit }}
       ic_version: ${{ steps.vars.outputs.ic_version }}
       publish-images: ${{ steps.vars.outputs.publish }}
+      docker_md5: ${{ steps.vars.outputs.docker_md5 }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -90,6 +91,8 @@ jobs:
             publish=true
           fi
           echo "publish=$publish" >> $GITHUB_OUTPUT
+          docker_md5=$(find . -type f \( -name "build/Dockerfile" -o -name .github/data/version.txt \) -not -path "./docs*"  -exec md5sum {} + | LC_ALL=C sort  | md5sum | awk '{ print $1 }' )
+          echo "docker_md5=${docker_md5:0:8}" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
 
       - name: Fetch Cached Binary Artifacts
@@ -228,17 +231,63 @@ jobs:
           key: nginx-ingress-${{ needs.checks.outputs.go_code_md5 }}
         if: ${{ needs.checks.outputs.binary_cache_hit != 'true' }}
 
+  rebuild-base-images:
+    name: Rebuild NIC Base images
+    runs-on: ubuntu-22.04
+    needs: checks
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
+          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
+      - name: Login to GCR
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
+      - name: Check if base images exist
+        id: base_exists
+        run: |
+          docker manifest inspect gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/oss:${{ needs.checks.outputs.docker_md5 }}-debian
+          echo "exists=$?" >> $GITHUB_OUTPUT
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
+      - name: Rebuild base images
+        uses: ./.github/workflows/build-base-images.yml
+        if: github.event.pull_request.head.repo.full_name == github.repository && steps.base_exists.outputs.exists != 0
+
   helm-tests:
     name: Helm Tests
     runs-on: ubuntu-22.04
-    needs: [checks, binaries]
+    needs: [checks, binaries, rebuild-base-images]
     strategy:
       matrix:
         include:
           - image: debian
             type: oss
           - image: debian-plus
             type: plus
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - name: Checkout Repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -252,18 +301,36 @@ jobs:
       - name: Docker Buildx
         uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
 
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
+        with:
+          token_format: access_token
+          workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY }}
+          service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }}
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
+      - name: Login to GCR
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+        if: github.event.pull_request.head.repo.full_name == github.repository
+
       - name: Build Docker Image ${{ matrix.image }}
         uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
         with:
           file: build/Dockerfile
           context: "."
           cache-from: type=gha,scope=${{ matrix.image }}
-          target: goreleaser
+          target: goreleaser${{ github.event.pull_request.head.repo.full_name == github.repository && '-prebuilt' || '' }}
           tags: ${{ matrix.type }}:${{ github.sha }}
           pull: true
           load: true
           build-args: |
             BUILD_OS=${{ matrix.image }}
+            PREBUILT_BASE_IMG=gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/nginx-ic-base/${{ contains(matrix.image, 'plus') && 'plus' || 'oss' }}:${{ needs.checks.outputs.docker_md5 }}-${{ matrix.image }}
             IC_VERSION=CI
           secrets: |
             ${{ contains(matrix.type, 'plus') && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
@@ -316,7 +383,7 @@ jobs:
   setup-matrix:
     name: Setup Matrix for Smoke Tests
     runs-on: ubuntu-22.04
-    needs: [binaries, checks]
+    needs: [binaries, checks, rebuild-base-images]
     permissions:
       contents: read
       id-token: write
@@ -402,6 +469,7 @@ jobs:
           gcr-service-account-secret: ${{ secrets.GCR_SERVICE_ACCOUNT }}
           rhel-license: ${{ contains(matrix.images.image, 'ubi') && secrets.RHEL_LICENSE || '' }}
           go-md5: ${{ needs.checks.outputs.go_code_md5 }}
+          base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
           test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt') || 'latest' }}"
 
       - name: Upload Test Results
@@ -443,6 +511,7 @@ jobs:
       image: ${{ matrix.image }}
       go-md5: ${{ needs.checks.outputs.go_code_md5 }}
       publish-image: ${{ needs.checks.outputs.publish-images == 'true' }}
+      base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
     permissions:
       contents: read
       actions: read
@@ -470,6 +539,7 @@ jobs:
       image: ${{ matrix.image }}
       target: ${{ matrix.target }}
       go-md5: ${{ needs.checks.outputs.go_code_md5 }}
+      base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
       release-url: ${{ needs.release-notes.outputs.release-url }}
       publish-image: ${{ needs.checks.outputs.publish-images == 'true' }}
     permissions:
@@ -523,6 +593,7 @@ jobs:
       image: ${{ matrix.image }}
       target: ${{ matrix.target }}
       go-md5: ${{ needs.checks.outputs.go_code_md5 }}
+      base-image-md5: ${{ needs.checks.outputs.docker_md5 }}
       nap_modules: ${{ matrix.nap_modules }}
       release-url: ${{ needs.release-notes.outputs.release-url }}
       publish-image: ${{ needs.checks.outputs.publish-images == 'true' }}