From b741bcbfcf27d1e0ca7eb425c57c47c655946783 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 5 Sep 2025 14:52:50 +0100 Subject: [PATCH 001/101] Test funcitonal tests with ipv6 only kind cluster: --- .github/workflows/functional-ipv6-only.yml | 156 +++++++++++++++++++++ .github/workflows/kind/ipv6-only.yaml | 7 + 2 files changed, 163 insertions(+) create mode 100644 .github/workflows/functional-ipv6-only.yml create mode 100644 .github/workflows/kind/ipv6-only.yaml diff --git a/.github/workflows/functional-ipv6-only.yml b/.github/workflows/functional-ipv6-only.yml new file mode 100644 index 0000000000..9281a91d72 --- /dev/null +++ b/.github/workflows/functional-ipv6-only.yml @@ -0,0 +1,156 @@ +name: Functional Testing + +on: + workflow_call: + inputs: + image: + required: true + type: string + k8s-version: + required: true + type: string + +defaults: + run: + shell: bash + +env: + PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} + +permissions: + contents: read + +jobs: + functional-tests: + name: Run Tests + runs-on: ubuntu-24.04 + if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} + env: + DOCKER_BUILD_SUMMARY: false + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Configure GOPROXY + id: goproxy + run: | + if [[ "${{ secrets.ARTIFACTORY_USER }}" == "" ]]; then + GOPROXY_VALUE="direct" + else + GOPROXY_VALUE="https://${{ secrets.ARTIFACTORY_USER }}:${{ secrets.ARTIFACTORY_TOKEN }}@${{ secrets.ARTIFACTORY_DEV_ENDPOINT }}" + fi + echo "GOPROXY=${GOPROXY_VALUE}" >> $GITHUB_ENV + + - name: Setup Golang Environment + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + go-version: stable + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: Build binary + uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + with: + version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser + args: build --single-target --snapshot --clean + env: + TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + TELEMETRY_ENDPOINT_INSECURE: "true" + + - name: Build NGF Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile + tags: ${{ steps.ngf-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=ngf + pull: true + target: goreleaser + + - name: Build NGINX Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + tags: ${{ steps.nginx-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=${{ inputs.image }} + pull: true + build-args: | + NJS_DIR=internal/controller/nginx/modules/src + NGINX_CONF_DIR=internal/controller/nginx/conf + BUILD_AGENT=gha + + - name: Setup license file for plus + if: ${{ inputs.image == 'plus' }} + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} + run: echo "${PLUS_LICENSE}" > license.jwt + + - name: Install cloud-provider-kind + run: | + CLOUD_PROVIDER_KIND_VERSION=v0.7.0 # renovate: datasource=github-tags depName=kubernetes-sigs/cloud-provider-kind + go install sigs.k8s.io/cloud-provider-kind@${CLOUD_PROVIDER_KIND_VERSION} + + - name: Run cloud-provider-kind + run: $(go env GOPATH)/bin/cloud-provider-kind & > cloud-provider-kind.log 2>&1 + + - name: Deploy Kubernetes + id: k8s + run: | + kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} --config=./kind/ipv6-only.yaml + kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} + + - name: Run functional telemetry tests + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true + working-directory: ./tests + + - name: Run functional graceful-recovery tests + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true + working-directory: ./tests + + - name: Run functional tests + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true + working-directory: ./tests diff --git a/.github/workflows/kind/ipv6-only.yaml b/.github/workflows/kind/ipv6-only.yaml new file mode 100644 index 0000000000..d217ac2597 --- /dev/null +++ b/.github/workflows/kind/ipv6-only.yaml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: ipv6 # Explicitly set the cluster to use IPv6 + disableDefaultCNI: false # Use Kind's default CNI +nodes: +- role: control-plane \ No newline at end of file From 5859e5167ce1d39897e5147eb02d5d34534ffbd4 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 5 Sep 2025 14:53:46 +0100 Subject: [PATCH 002/101] Add functional-ipv6-only-tests to ci.yml --- .github/workflows/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3cdc851cd..3587ece3bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -285,6 +285,23 @@ jobs: permissions: contents: read + functional-ipv6-only-tests: + name: Functional IPv6 Only Tests + needs: [vars, build-oss, build-plus] + strategy: + fail-fast: false + matrix: + image: [nginx, plus] + k8s-version: + [ + "${{ needs.vars.outputs.min_k8s_version }}", + "${{ needs.vars.outputs.k8s_latest }}", + ] + uses: ./.github/workflows/functional-ipv6-only.yml + with: + image: ${{ matrix.image }} + k8s-version: ${{ matrix.k8s-version }} + conformance-tests: name: Conformance tests needs: [vars, build-oss, build-plus] From 48aed80844e3a22cb03f9624218d889c52b903f4 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 5 Sep 2025 14:56:01 +0100 Subject: [PATCH 003/101] Fix end of file --- .github/workflows/kind/ipv6-only.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kind/ipv6-only.yaml b/.github/workflows/kind/ipv6-only.yaml index d217ac2597..f5337b474a 100644 --- a/.github/workflows/kind/ipv6-only.yaml +++ b/.github/workflows/kind/ipv6-only.yaml @@ -4,4 +4,4 @@ networking: ipFamily: ipv6 # Explicitly set the cluster to use IPv6 disableDefaultCNI: false # Use Kind's default CNI nodes: -- role: control-plane \ No newline at end of file +- role: control-plane From 981658a2466573715b31ed2109572310120f4e11 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 5 Sep 2025 15:14:21 +0100 Subject: [PATCH 004/101] Move config file --- .github/workflows/functional-ipv6-only.yml | 2 +- .../kind/ipv6-only.yaml => config/cluster/kind-ipv6-only.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/kind/ipv6-only.yaml => config/cluster/kind-ipv6-only.yaml (100%) diff --git a/.github/workflows/functional-ipv6-only.yml b/.github/workflows/functional-ipv6-only.yml index 9281a91d72..5477ab7842 100644 --- a/.github/workflows/functional-ipv6-only.yml +++ b/.github/workflows/functional-ipv6-only.yml @@ -131,7 +131,7 @@ jobs: - name: Deploy Kubernetes id: k8s run: | - kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} --config=./kind/ipv6-only.yaml + kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} --config=./config/cluster/kind-ipv6-only.yaml kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} - name: Run functional telemetry tests diff --git a/.github/workflows/kind/ipv6-only.yaml b/config/cluster/kind-ipv6-only.yaml similarity index 100% rename from .github/workflows/kind/ipv6-only.yaml rename to config/cluster/kind-ipv6-only.yaml From 145c98128395b0299f6ea2c44869de6a93e2918a Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 5 Sep 2025 15:29:44 +0100 Subject: [PATCH 005/101] Add `kubectl get nodes -o wide` to check cluster node output for IPv6 --- .github/workflows/functional-ipv6-only.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/functional-ipv6-only.yml b/.github/workflows/functional-ipv6-only.yml index 5477ab7842..3374941ad9 100644 --- a/.github/workflows/functional-ipv6-only.yml +++ b/.github/workflows/functional-ipv6-only.yml @@ -133,6 +133,7 @@ jobs: run: | kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} --config=./config/cluster/kind-ipv6-only.yaml kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} + kubectl get nodes -o wide - name: Run functional telemetry tests run: | From 41f36bdd4ce99647dbd98a58afec61231768ed21 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 5 Sep 2025 16:08:41 +0100 Subject: [PATCH 006/101] Update ipv6-only tests to the same files as in PR #3792 --- .github/workflows/ci.yml | 5 +- .github/workflows/functional-ipv6-only.yml | 157 --------------- .github/workflows/ipv6-only.yml | 218 +++++++++++++++++++++ config/cluster/kind-ipv6-only.yaml | 6 +- tests/manifests/ipv6-test-app.yaml | 0 tests/manifests/ipv6-test-client.yaml | 33 ++++ 6 files changed, 256 insertions(+), 163 deletions(-) delete mode 100644 .github/workflows/functional-ipv6-only.yml create mode 100644 .github/workflows/ipv6-only.yml create mode 100644 tests/manifests/ipv6-test-app.yaml create mode 100644 tests/manifests/ipv6-test-client.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3587ece3bb..89ab3291a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -287,14 +287,13 @@ jobs: functional-ipv6-only-tests: name: Functional IPv6 Only Tests - needs: [vars, build-oss, build-plus] + needs: [vars, build-oss] strategy: fail-fast: false matrix: - image: [nginx, plus] + image: [nginx] k8s-version: [ - "${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}", ] uses: ./.github/workflows/functional-ipv6-only.yml diff --git a/.github/workflows/functional-ipv6-only.yml b/.github/workflows/functional-ipv6-only.yml deleted file mode 100644 index 3374941ad9..0000000000 --- a/.github/workflows/functional-ipv6-only.yml +++ /dev/null @@ -1,157 +0,0 @@ -name: Functional Testing - -on: - workflow_call: - inputs: - image: - required: true - type: string - k8s-version: - required: true - type: string - -defaults: - run: - shell: bash - -env: - PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} - -permissions: - contents: read - -jobs: - functional-tests: - name: Run Tests - runs-on: ubuntu-24.04 - if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} - env: - DOCKER_BUILD_SUMMARY: false - steps: - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - name: Configure GOPROXY - id: goproxy - run: | - if [[ "${{ secrets.ARTIFACTORY_USER }}" == "" ]]; then - GOPROXY_VALUE="direct" - else - GOPROXY_VALUE="https://${{ secrets.ARTIFACTORY_USER }}:${{ secrets.ARTIFACTORY_TOKEN }}@${{ secrets.ARTIFACTORY_DEV_ENDPOINT }}" - fi - echo "GOPROXY=${GOPROXY_VALUE}" >> $GITHUB_ENV - - - name: Setup Golang Environment - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 - with: - go-version: stable - - - name: Set GOPATH - run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV - - - name: Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: NGF Docker meta - id: ngf-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric - tags: | - type=semver,pattern={{version}} - type=schedule - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: NGINX Docker meta - id: nginx-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} - tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: Build binary - uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 - with: - version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser - args: build --single-target --snapshot --clean - env: - TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 - TELEMETRY_ENDPOINT_INSECURE: "true" - - - name: Build NGF Docker Image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - file: build/Dockerfile - tags: ${{ steps.ngf-meta.outputs.tags }} - context: "." - load: true - cache-from: type=gha,scope=ngf - pull: true - target: goreleaser - - - name: Build NGINX Docker Image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} - tags: ${{ steps.nginx-meta.outputs.tags }} - context: "." - load: true - cache-from: type=gha,scope=${{ inputs.image }} - pull: true - build-args: | - NJS_DIR=internal/controller/nginx/modules/src - NGINX_CONF_DIR=internal/controller/nginx/conf - BUILD_AGENT=gha - - - name: Setup license file for plus - if: ${{ inputs.image == 'plus' }} - env: - PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} - run: echo "${PLUS_LICENSE}" > license.jwt - - - name: Install cloud-provider-kind - run: | - CLOUD_PROVIDER_KIND_VERSION=v0.7.0 # renovate: datasource=github-tags depName=kubernetes-sigs/cloud-provider-kind - go install sigs.k8s.io/cloud-provider-kind@${CLOUD_PROVIDER_KIND_VERSION} - - - name: Run cloud-provider-kind - run: $(go env GOPATH)/bin/cloud-provider-kind & > cloud-provider-kind.log 2>&1 - - - name: Deploy Kubernetes - id: k8s - run: | - kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} --config=./config/cluster/kind-ipv6-only.yaml - kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} - kubectl get nodes -o wide - - - name: Run functional telemetry tests - run: | - ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric - ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=telemetry GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true - working-directory: ./tests - - - name: Run functional graceful-recovery tests - run: | - ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric - ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GINKGO_LABEL=graceful-recovery GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true - working-directory: ./tests - - - name: Run functional tests - run: | - ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric - ngf_tag=${{ steps.ngf-meta.outputs.version }} - make test${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} GW_SERVICE_TYPE=LoadBalancer CLUSTER_NAME=${{ github.run_id }} CI=true - working-directory: ./tests diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml new file mode 100644 index 0000000000..2a237271fb --- /dev/null +++ b/.github/workflows/ipv6-only.yml @@ -0,0 +1,218 @@ +name: IPv6-Only Testing + +on: + workflow_call: + inputs: + image: + required: true + type: string + k8s-version: + required: true + type: string + +defaults: + run: + shell: bash + +env: + PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} + +permissions: + contents: read + +jobs: + ipv6-only-tests: + name: Run IPv6-Only Tests + runs-on: ubuntu-24.04 + if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} + env: + DOCKER_BUILD_SUMMARY: false + steps: + - name: Checkout Repository + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + fetch-depth: 0 + + - name: Configure GOPROXY + id: goproxy + run: | + if [[ "${{ secrets.ARTIFACTORY_USER }}" == "" ]]; then + GOPROXY_VALUE="direct" + else + GOPROXY_VALUE="https://${{ secrets.ARTIFACTORY_USER }}:${{ secrets.ARTIFACTORY_TOKEN }}@${{ secrets.ARTIFACTORY_DEV_ENDPOINT }}" + fi + echo "GOPROXY=${GOPROXY_VALUE}" >> $GITHUB_ENV + + - name: Setup Golang Environment + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + go-version: stable + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + - name: Build binary + uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + with: + version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser + args: build --single-target --snapshot --clean + env: + TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + TELEMETRY_ENDPOINT_INSECURE: "true" + + - name: Build NGF Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile + tags: ${{ steps.ngf-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=ngf-ipv6 + pull: true + target: goreleaser + + - name: Build NGINX Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + tags: ${{ steps.nginx-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=${{ inputs.image }}-ipv6 + pull: true + build-args: | + NJS_DIR=internal/controller/nginx/modules/src + NGINX_CONF_DIR=internal/controller/nginx/conf + BUILD_AGENT=gha + - name: Setup license file for plus + if: ${{ inputs.image == 'plus' }} + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} + run: echo "${PLUS_LICENSE}" > license.jwt + + - name: Deploy IPv6-Only Kubernetes + id: k8s + run: | + # Enable IPv6 and container network options + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + + # Create IPv6-only kind cluster + kind create cluster \ + --name ${{ github.run_id }}-ipv6 \ + --image=kindest/node:${{ inputs.k8s-version }} \ + --config=config/cluster/kind-ipv6-only.yaml + + # Load images into the cluster + kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 + + # Verify nodes are ipv6 only + kubectl get nodes -o wide + - name: Install NGF with IPv6 Configuration + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + # Install with IPv6-specific configuration + CLUSTER_NAME=${{ github.run_id }}-ipv6 \ + HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ + make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + working-directory: ./tests + + - name: Deploy Test Applications + run: | + kubectl apply -f tests/manifests/ipv6-test-app.yaml + + - name: Wait for NGF and Applications to be Ready + run: | + echo "Waiting for NGF to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway + echo "Waiting for test applications to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + + - name: Deploy IPv6 Test Client + run: | + kubectl apply -f tests/manifests/test-client-ipv6.yaml + kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + + - name: Get NGF IPv6 Address + id: ngf-address + run: | + # Get the NGF service IPv6 address + NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') + echo "NGF IPv6 Address: $NGF_IPV6" + echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT + + - name: Run IPv6 Connectivity Tests + run: | + echo "=== Running IPv6-Only Tests ===" + # Test 1: Basic connectivity test using test client pod + echo "Test 1: Basic IPv6 connectivity" + kubectl exec ipv6-test-client -- curl --version + kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local + # Test 2: Test NGF service directly via IPv6 + echo "Test 2: NGF Service IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" + # Test 3: Test via service DNS + echo "Test 3: Service DNS IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" + + - name: Validate IPv6-Only Configuration + run: | + echo "=== Validating IPv6-Only Configuration ===" + # Check NGF configuration + echo "NGF Pod IPv6 addresses:" + kubectl get pods -n nginx-gateway -o wide + echo "NGF Service configuration:" + kubectl get service nginx-gateway -n nginx-gateway -o yaml + echo "Gateway and HTTPRoute status:" + kubectl get gateway,httproute -A -o wide + echo "Test application service configuration:" + kubectl get service test-app-ipv6-service -o yaml + + - name: Collect Logs + if: always() + run: | + echo "=== Collecting logs for debugging ===" + echo "NGF Controller logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true + echo "NGINX logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true + echo "Test client logs:" + kubectl logs ipv6-test-client --tail=100 || true + echo "Cluster events:" + kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true + - name: Cleanup + if: always() + run: | + kind delete cluster --name ${{ github.run_id }}-ipv6 || true \ No newline at end of file diff --git a/config/cluster/kind-ipv6-only.yaml b/config/cluster/kind-ipv6-only.yaml index f5337b474a..21fb387af4 100644 --- a/config/cluster/kind-ipv6-only.yaml +++ b/config/cluster/kind-ipv6-only.yaml @@ -1,7 +1,7 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 -networking: - ipFamily: ipv6 # Explicitly set the cluster to use IPv6 - disableDefaultCNI: false # Use Kind's default CNI nodes: - role: control-plane +networking: + ipFamily: ipv6 + apiServerAddress: "::1" \ No newline at end of file diff --git a/tests/manifests/ipv6-test-app.yaml b/tests/manifests/ipv6-test-app.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/manifests/ipv6-test-client.yaml b/tests/manifests/ipv6-test-client.yaml new file mode 100644 index 0000000000..b7559cd8df --- /dev/null +++ b/tests/manifests/ipv6-test-client.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + name: ipv6-test-client + namespace: default + labels: + app: ipv6-test-client +spec: + restartPolicy: Never + containers: + - name: test-client + image: curlimages/curl:8.11.1 + imagePullPolicy: IfNotPresent + command: ["sleep", "3600"] # Keep pod alive for exec commands + resources: + limits: + cpu: "100m" + memory: "128Mi" + requests: + cpu: "50m" + memory: "64Mi" + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + capabilities: + drop: + - ALL + dnsConfig: + options: + - name: single-request-reopen + - name: ndots + value: "2" \ No newline at end of file From a0cd0cd89f1f6960894b7e98dd3073902f38f769 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 5 Sep 2025 16:29:57 +0100 Subject: [PATCH 007/101] Add new line --- .github/workflows/ipv6-only.yml | 2 +- config/cluster/kind-ipv6-only.yaml | 2 +- tests/manifests/ipv6-test-app.yaml | 61 +++++++++++++++++++++++++++ tests/manifests/ipv6-test-client.yaml | 2 +- 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 2a237271fb..2dc81187f9 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -215,4 +215,4 @@ jobs: - name: Cleanup if: always() run: | - kind delete cluster --name ${{ github.run_id }}-ipv6 || true \ No newline at end of file + kind delete cluster --name ${{ github.run_id }}-ipv6 || true diff --git a/config/cluster/kind-ipv6-only.yaml b/config/cluster/kind-ipv6-only.yaml index 21fb387af4..bb2dce8392 100644 --- a/config/cluster/kind-ipv6-only.yaml +++ b/config/cluster/kind-ipv6-only.yaml @@ -4,4 +4,4 @@ nodes: - role: control-plane networking: ipFamily: ipv6 - apiServerAddress: "::1" \ No newline at end of file + apiServerAddress: "::1" diff --git a/tests/manifests/ipv6-test-app.yaml b/tests/manifests/ipv6-test-app.yaml index e69de29bb2..ff4a506cd9 100644 --- a/tests/manifests/ipv6-test-app.yaml +++ b/tests/manifests/ipv6-test-app.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-app-ipv6 + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: test-app-ipv6 + template: + metadata: + labels: + app: test-app-ipv6 + spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + resources: + limits: + cpu: "100m" + memory: "128Mi" + requests: + cpu: "50m" + memory: "64Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: test-app-ipv6-service + namespace: default +spec: + selector: + app: test-app-ipv6 + ports: + - port: 80 + targetPort: 80 + ipFamilies: [IPv6] + ipFamilyPolicy: SingleStack +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: test-route-ipv6 + namespace: default +spec: + parentRefs: + - name: nginx-gateway + namespace: nginx-gateway + hostnames: + - "ipv6-test.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: test-app-ipv6-service + port: 80 diff --git a/tests/manifests/ipv6-test-client.yaml b/tests/manifests/ipv6-test-client.yaml index b7559cd8df..0eb7313742 100644 --- a/tests/manifests/ipv6-test-client.yaml +++ b/tests/manifests/ipv6-test-client.yaml @@ -30,4 +30,4 @@ spec: options: - name: single-request-reopen - name: ndots - value: "2" \ No newline at end of file + value: "2" From c9c6528316b883eba3609448d72e1d789214a1e9 Mon Sep 17 00:00:00 2001 From: Shaun Date: Fri, 5 Sep 2025 16:35:42 +0100 Subject: [PATCH 008/101] Potential fix for code scanning alert no. 486: Code injection Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/ipv6-only.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 2dc81187f9..dfcafba77d 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -214,5 +214,7 @@ jobs: kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true - name: Cleanup if: always() + env: + RUN_ID: ${{ github.run_id }} run: | - kind delete cluster --name ${{ github.run_id }}-ipv6 || true + kind delete cluster --name "$RUN_ID-ipv6" || true From 545ed039a46eec2cb23b28702e8bdc17561234d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 07:37:27 +0000 Subject: [PATCH 009/101] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/ipv6-only.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index dfcafba77d..5aa7aba41c 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -155,7 +155,7 @@ jobs: kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway echo "Waiting for test applications to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 - + - name: Deploy IPv6 Test Client run: | kubectl apply -f tests/manifests/test-client-ipv6.yaml @@ -199,7 +199,7 @@ jobs: kubectl get gateway,httproute -A -o wide echo "Test application service configuration:" kubectl get service test-app-ipv6-service -o yaml - + - name: Collect Logs if: always() run: | From 6361dfb1559140f5e7e26e96042f3dfacb090420 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 08:45:55 +0100 Subject: [PATCH 010/101] Remove sysctl --- .github/workflows/ipv6-only.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 5aa7aba41c..82b099eb25 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -121,8 +121,8 @@ jobs: id: k8s run: | # Enable IPv6 and container network options - sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 - sudo sysctl -w net.ipv6.conf.all.forwarding=1 + # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 + # sudo sysctl -w net.ipv6.conf.all.forwarding=1 # Create IPv6-only kind cluster kind create cluster \ From fc846cbe3cc68dc99bdf2be9b5be72b17ce247d6 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 09:12:48 +0100 Subject: [PATCH 011/101] Update ipv6-only job in `ci.yml` --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a7497cef1..3a133884a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -285,7 +285,7 @@ jobs: permissions: contents: read - functional-ipv6-only-tests: + ipv6-only-tests: name: Functional IPv6 Only Tests needs: [vars, build-oss] strategy: @@ -296,7 +296,7 @@ jobs: [ "${{ needs.vars.outputs.k8s_latest }}", ] - uses: ./.github/workflows/functional-ipv6-only.yml + uses: ./.github/workflows/ipv6-only.yml with: image: ${{ matrix.image }} k8s-version: ${{ matrix.k8s-version }} From 574fea9f2684296684f55ca2a80a7242ab072d3f Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 09:35:32 +0100 Subject: [PATCH 012/101] Add project name to goreleaser.yml --- .goreleaser.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.goreleaser.yml b/.goreleaser.yml index 31d20c536c..d6bcb68432 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,4 +1,5 @@ version: 2 +project_name: nginx-gateway-fabric env: - CGO_ENABLED=0 From 9cbac69274a5afa0f2eb894784df97d8f89fea4b Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 11:49:08 +0100 Subject: [PATCH 013/101] Remove binary build stage --- .github/workflows/ipv6-only.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 82b099eb25..e986dddf8a 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -78,14 +78,14 @@ jobs: type=schedule type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - name: Build binary - uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 - with: - version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser - args: build --single-target --snapshot --clean - env: - TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 - TELEMETRY_ENDPOINT_INSECURE: "true" + # - name: Build binary + # uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + # with: + # version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser + # args: build --single-target --snapshot --clean + # env: + # TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + # TELEMETRY_ENDPOINT_INSECURE: "true" - name: Build NGF Docker Image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 @@ -94,7 +94,7 @@ jobs: tags: ${{ steps.ngf-meta.outputs.tags }} context: "." load: true - cache-from: type=gha,scope=ngf-ipv6 + cache-from: type=gha,scope=ngf pull: true target: goreleaser From 5ded3a84e1d40bf371f8ea278757b3c2188a68f1 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 14:37:43 +0100 Subject: [PATCH 014/101] Re-add goreleaser step --- .github/workflows/ipv6-only.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index e986dddf8a..a23e5064dd 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -78,14 +78,14 @@ jobs: type=schedule type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - # - name: Build binary - # uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 - # with: - # version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser - # args: build --single-target --snapshot --clean - # env: - # TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 - # TELEMETRY_ENDPOINT_INSECURE: "true" + - name: Build binary + uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + with: + version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser + args: build --single-target --snapshot --clean + env: + TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + TELEMETRY_ENDPOINT_INSECURE: "true" - name: Build NGF Docker Image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 From 85c9ea2b69255e527f640017ba7ae676b82f092b Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 14:55:13 +0100 Subject: [PATCH 015/101] Add additional args to goreleaser --- .github/workflows/ipv6-only.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index a23e5064dd..0ecb3700e0 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -82,7 +82,8 @@ jobs: uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 with: version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser - args: build --single-target --snapshot --clean + args: + build --single-target --snapshot --clean release --config=./goreleaser.yml --rm-dist env: TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 TELEMETRY_ENDPOINT_INSECURE: "true" From c9a04d37ef296252ed796a3eeae9891844df78be Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 15:19:42 +0100 Subject: [PATCH 016/101] Explicitly specifiy goreleaser config path --- .github/workflows/ipv6-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 0ecb3700e0..36e73a230f 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -83,7 +83,7 @@ jobs: with: version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser args: - build --single-target --snapshot --clean release --config=./goreleaser.yml --rm-dist + build --single-target --snapshot --clean --config=./goreleaser.yml --rm-dist env: TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 TELEMETRY_ENDPOINT_INSECURE: "true" From 031f11ecbc61f5e7cfed68263a2b62931ff7ff17 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 8 Sep 2025 16:54:11 +0100 Subject: [PATCH 017/101] Remove --rm-dist flag --- .github/workflows/ipv6-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 36e73a230f..ae7fb088e3 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -83,7 +83,7 @@ jobs: with: version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser args: - build --single-target --snapshot --clean --config=./goreleaser.yml --rm-dist + build --single-target --snapshot --clean --config=./goreleaser.yml env: TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 TELEMETRY_ENDPOINT_INSECURE: "true" From c5198bcf9117e5591287fed9ad6a26396bcf68f3 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 10:08:23 +0100 Subject: [PATCH 018/101] Update go releaser config path --- .github/workflows/ipv6-only.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index ae7fb088e3..44722fdf2a 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -83,7 +83,7 @@ jobs: with: version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser args: - build --single-target --snapshot --clean --config=./goreleaser.yml + build --single-target --snapshot --clean --config=goreleaser.yml env: TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 TELEMETRY_ENDPOINT_INSECURE: "true" From d01246e6369e35030807f4555b4e48567f86923d Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 11:32:22 +0100 Subject: [PATCH 019/101] Add workflow file to test ipv6 only in GKE --- .github/workflows/ci.yml | 16 ++ .github/workflows/gke-ipv6-only.yml | 269 ++++++++++++++++++++++++++++ .github/workflows/ipv6-only.yml | 6 + tests/Makefile | 3 +- tests/scripts/create-gke-cluster.sh | 14 +- 5 files changed, 306 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/gke-ipv6-only.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a133884a0..bbd1adc7bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -301,6 +301,22 @@ jobs: image: ${{ matrix.image }} k8s-version: ${{ matrix.k8s-version }} + gke-ipv6-only-tests: + name: GKE IPv6 Only Tests + needs: [vars, build-oss] + strategy: + fail-fast: false + matrix: + image: [nginx] + k8s-version: + [ + "${{ needs.vars.outputs.k8s_latest }}", + ] + uses: ./.github/workflows/gke-ipv6-only.yml + with: + image: ${{ matrix.image }} + k8s-version: ${{ matrix.k8s-version }} + conformance-tests: name: Conformance tests needs: [vars, build-oss, build-plus] diff --git a/.github/workflows/gke-ipv6-only.yml b/.github/workflows/gke-ipv6-only.yml new file mode 100644 index 0000000000..c5a0c63241 --- /dev/null +++ b/.github/workflows/gke-ipv6-only.yml @@ -0,0 +1,269 @@ +name: GKE IPv6-Only Testing + +on: + workflow_call: + inputs: + image: + required: true + type: string + k8s-version: + required: true + type: string + +defaults: + run: + shell: bash + +env: + PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} + +permissions: + contents: read + +jobs: + ipv6-only-tests: + name: Run IPv6-Only Tests + runs-on: ubuntu-24.04 + if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} + env: + DOCKER_BUILD_SUMMARY: false + steps: + - name: Checkout Repository + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + fetch-depth: 0 + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + token_format: access_token + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + + - name: Login to GAR + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + install_components: kubectl + + - name: Create GKE cluster + working-directory: ./tests + run: make create-gke-cluster CI=true IPv6_ONLY=true + + - name: Create and setup VM + working-directory: ./tests + run: make create-and-setup-vm + + - name: Create and setup Router + working-directory: ./tests + run: make create-gke-router || true + + - name: Configure GOPROXY + id: goproxy + run: | + if [[ "${{ secrets.ARTIFACTORY_USER }}" == "" ]]; then + GOPROXY_VALUE="direct" + else + GOPROXY_VALUE="https://${{ secrets.ARTIFACTORY_USER }}:${{ secrets.ARTIFACTORY_TOKEN }}@${{ secrets.ARTIFACTORY_DEV_ENDPOINT }}" + fi + echo "GOPROXY=${GOPROXY_VALUE}" >> $GITHUB_ENV + + - name: Setup Golang Environment + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 + with: + go-version: stable + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: Build binary + uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + with: + version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser + args: + build --single-target --snapshot --clean --config=goreleaser.yml + env: + TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + TELEMETRY_ENDPOINT_INSECURE: "true" + + - name: Build NGF Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile + tags: ${{ steps.ngf-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=ngf + pull: true + target: goreleaser + + - name: Build NGINX Docker Image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + tags: ${{ steps.nginx-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=${{ inputs.image }}-ipv6 + pull: true + build-args: | + NJS_DIR=internal/controller/nginx/modules/src + NGINX_CONF_DIR=internal/controller/nginx/conf + BUILD_AGENT=gha + + # - name: Setup license file for plus + # if: ${{ inputs.image == 'plus' }} + # env: + # PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} + # run: echo "${PLUS_LICENSE}" > license.jwt + + # - name: Deploy IPv6-Only Kubernetes + # id: k8s + # run: | + # # Enable IPv6 and container network options + # # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 + # # sudo sysctl -w net.ipv6.conf.all.forwarding=1 + + # # Create IPv6-only kind cluster + # kind create cluster \ + # --name ${{ github.run_id }}-ipv6 \ + # --image=kindest/node:${{ inputs.k8s-version }} \ + # --config=config/cluster/kind-ipv6-only.yaml + + # # Load images into the cluster + # kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 + + # # Verify nodes are ipv6 only + # kubectl get nodes -o wide + + - name: Push Docker Images to GAR + # if: ${{ github.event_name != 'pull_request' }} + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: "." + push: true + tags: | + ${{ steps.ngf-meta.outputs.tags }} + ${{ steps.nginx-meta.outputs.tags }} + + - name: Install NGF with IPv6 Configuration + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + # Install with IPv6-specific configuration + CLUSTER_NAME=${{ github.run_id }}-ipv6 \ + HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ + make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + working-directory: ./tests + + - name: Deploy Test Applications + run: | + kubectl apply -f tests/manifests/ipv6-test-app.yaml + + - name: Wait for NGF and Applications to be Ready + run: | + echo "Waiting for NGF to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway + echo "Waiting for test applications to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + + - name: Deploy IPv6 Test Client + run: | + kubectl apply -f tests/manifests/test-client-ipv6.yaml + kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + + - name: Get NGF IPv6 Address + id: ngf-address + run: | + # Get the NGF service IPv6 address + NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') + echo "NGF IPv6 Address: $NGF_IPV6" + echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT + + - name: Run IPv6 Connectivity Tests + run: | + echo "=== Running IPv6-Only Tests ===" + # Test 1: Basic connectivity test using test client pod + echo "Test 1: Basic IPv6 connectivity" + kubectl exec ipv6-test-client -- curl --version + kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local + # Test 2: Test NGF service directly via IPv6 + echo "Test 2: NGF Service IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" + # Test 3: Test via service DNS + echo "Test 3: Service DNS IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" + + - name: Validate IPv6-Only Configuration + run: | + echo "=== Validating IPv6-Only Configuration ===" + # Check NGF configuration + echo "NGF Pod IPv6 addresses:" + kubectl get pods -n nginx-gateway -o wide + echo "NGF Service configuration:" + kubectl get service nginx-gateway -n nginx-gateway -o yaml + echo "Gateway and HTTPRoute status:" + kubectl get gateway,httproute -A -o wide + echo "Test application service configuration:" + kubectl get service test-app-ipv6-service -o yaml + + - name: Collect Logs + if: always() + run: | + echo "=== Collecting logs for debugging ===" + echo "NGF Controller logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true + echo "NGINX logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true + echo "Test client logs:" + kubectl logs ipv6-test-client --tail=100 || true + echo "Cluster events:" + kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true + + - name: Cleanup + if: always() + env: + RUN_ID: ${{ github.run_id }} + run: | + kind delete cluster --name "$RUN_ID-ipv6" || true diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 44722fdf2a..6b1006e0bb 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -66,6 +66,7 @@ jobs: type=edge type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + - name: NGINX Docker meta id: nginx-meta uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 @@ -78,6 +79,7 @@ jobs: type=schedule type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + - name: Build binary uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 with: @@ -112,6 +114,7 @@ jobs: NJS_DIR=internal/controller/nginx/modules/src NGINX_CONF_DIR=internal/controller/nginx/conf BUILD_AGENT=gha + - name: Setup license file for plus if: ${{ inputs.image == 'plus' }} env: @@ -136,6 +139,7 @@ jobs: # Verify nodes are ipv6 only kubectl get nodes -o wide + - name: Install NGF with IPv6 Configuration run: | ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric @@ -144,6 +148,7 @@ jobs: CLUSTER_NAME=${{ github.run_id }}-ipv6 \ HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + working-directory: ./tests - name: Deploy Test Applications @@ -213,6 +218,7 @@ jobs: kubectl logs ipv6-test-client --tail=100 || true echo "Cluster events:" kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true + - name: Cleanup if: always() env: diff --git a/tests/Makefile b/tests/Makefile index f1ccfdd3c5..b8de189533 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,5 @@ CI ?= false +IPv6_ONLY ?= false CLUSTER_NAME ?= kind CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image CONFORMANCE_TAG = latest## Tag for the conformance test runner image @@ -90,7 +91,7 @@ setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## C .PHONY: create-gke-cluster create-gke-cluster: ## Create a GKE cluster - ./scripts/create-gke-cluster.sh $(CI) + ./scripts/create-gke-cluster.sh $(CI) $(IPv6_ONLY) .PHONY: create-and-setup-vm create-and-setup-vm: ## Create and setup a GCP VM for tests diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index f0ccbfcc2b..fbea5c04bf 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -8,6 +8,17 @@ ip_random_digit=$((1 + RANDOM % 250)) IS_CI=${1:-false} +IPV6_ENABLE=${2:-${IPV6_ENABLE:-false}} + +IPV6_FLAGS="" +if [ "$IPV6_ENABLE" = "true" ]; then + IPV6_FLAGS="\ + --enable-ipv6 \ + --cluster-ipv6-cidr=fd00:1234::/56 \ + --services-ipv6-cidr=fd00:4321::/112 \ + --create-subnetwork name=\"${GKE_CLUSTER_NAME}-subnet\",range=10.0.0.0/16,fd00:abcd::/64,stack-type=IPV4_IPV6" +fi + if [ -z "$GKE_MACHINE_TYPE" ]; then # If the environment variable is not set, use a default value GKE_MACHINE_TYPE="e2-medium" @@ -31,7 +42,8 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port + --no-enable-insecure-kubelet-readonly-port \ + $IPV6_FLAGS # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From bdc569d7b2f89d14b400f3ea35ab9c566f0f5b88 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 11:33:22 +0100 Subject: [PATCH 020/101] Add cleanup step --- .github/workflows/gke-ipv6-only.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gke-ipv6-only.yml b/.github/workflows/gke-ipv6-only.yml index c5a0c63241..174019f979 100644 --- a/.github/workflows/gke-ipv6-only.yml +++ b/.github/workflows/gke-ipv6-only.yml @@ -260,10 +260,12 @@ jobs: kubectl logs ipv6-test-client --tail=100 || true echo "Cluster events:" kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true - + - name: Cleanup + working-directory: ./tests if: always() - env: - RUN_ID: ${{ github.run_id }} run: | - kind delete cluster --name "$RUN_ID-ipv6" || true + bash scripts/cleanup-vm.sh true + bash scripts/cleanup-router.sh true + make delete-gke-cluster + rm -rf scripts/vars.env From ea8ddde5d048c4161036579d8be511ff55fda87f Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 11:46:58 +0100 Subject: [PATCH 021/101] Fix pre-commit errors --- .github/workflows/gke-ipv6-only.yml | 6 ++-- .github/workflows/ipv6-only.yml | 4 +-- tests/scripts/create-gke-cluster.sh | 2 +- .../snippets-filter/invalid-duplicate-sf.yaml | 32 +++++++++---------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/gke-ipv6-only.yml b/.github/workflows/gke-ipv6-only.yml index 174019f979..77e74308c3 100644 --- a/.github/workflows/gke-ipv6-only.yml +++ b/.github/workflows/gke-ipv6-only.yml @@ -65,7 +65,7 @@ jobs: - name: Create and setup Router working-directory: ./tests run: make create-gke-router || true - + - name: Configure GOPROXY id: goproxy run: | @@ -99,7 +99,7 @@ jobs: type=edge type=ref,event=pr type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - + - name: NGINX Docker meta id: nginx-meta uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 @@ -117,7 +117,7 @@ jobs: uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 with: version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser - args: + args: build --single-target --snapshot --clean --config=goreleaser.yml env: TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 diff --git a/.github/workflows/ipv6-only.yml b/.github/workflows/ipv6-only.yml index 6b1006e0bb..c217a41242 100644 --- a/.github/workflows/ipv6-only.yml +++ b/.github/workflows/ipv6-only.yml @@ -84,7 +84,7 @@ jobs: uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 with: version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser - args: + args: build --single-target --snapshot --clean --config=goreleaser.yml env: TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 @@ -218,7 +218,7 @@ jobs: kubectl logs ipv6-test-client --tail=100 || true echo "Cluster events:" kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true - + - name: Cleanup if: always() env: diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index fbea5c04bf..cf52d27f15 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -12,7 +12,7 @@ IPV6_ENABLE=${2:-${IPV6_ENABLE:-false}} IPV6_FLAGS="" if [ "$IPV6_ENABLE" = "true" ]; then - IPV6_FLAGS="\ + IPV6_FLAGS="\ --enable-ipv6 \ --cluster-ipv6-cidr=fd00:1234::/56 \ --services-ipv6-cidr=fd00:4321::/112 \ diff --git a/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml b/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml index ba7e5b6c55..137771ed4e 100644 --- a/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml +++ b/tests/suite/manifests/snippets-filter/invalid-duplicate-sf.yaml @@ -19,21 +19,21 @@ metadata: name: tea spec: parentRefs: - - name: gateway - sectionName: http + - name: gateway + sectionName: http hostnames: - - "cafe.example.com" + - "cafe.example.com" rules: - - matches: - - path: - type: Exact - value: /tea - filters: - - type: ExtensionRef - extensionRef: - group: gateway.nginx.org - kind: SnippetsFilter - name: duplicate-directive - backendRefs: - - name: tea - port: 80 + - matches: + - path: + type: Exact + value: /tea + filters: + - type: ExtensionRef + extensionRef: + group: gateway.nginx.org + kind: SnippetsFilter + name: duplicate-directive + backendRefs: + - name: tea + port: 80 From 2cad23544dd827f4c16886830eb0b1dd4dd1feaf Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 13:09:03 +0100 Subject: [PATCH 022/101] Enable NFR tests to run in an IPv6 enabled environment --- .github/workflows/ci.yml | 16 -- .github/workflows/gke-ipv6-only.yml | 271 ---------------------------- .github/workflows/nfr.yml | 7 +- 3 files changed, 6 insertions(+), 288 deletions(-) delete mode 100644 .github/workflows/gke-ipv6-only.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbd1adc7bb..3a133884a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -301,22 +301,6 @@ jobs: image: ${{ matrix.image }} k8s-version: ${{ matrix.k8s-version }} - gke-ipv6-only-tests: - name: GKE IPv6 Only Tests - needs: [vars, build-oss] - strategy: - fail-fast: false - matrix: - image: [nginx] - k8s-version: - [ - "${{ needs.vars.outputs.k8s_latest }}", - ] - uses: ./.github/workflows/gke-ipv6-only.yml - with: - image: ${{ matrix.image }} - k8s-version: ${{ matrix.k8s-version }} - conformance-tests: name: Conformance tests needs: [vars, build-oss, build-plus] diff --git a/.github/workflows/gke-ipv6-only.yml b/.github/workflows/gke-ipv6-only.yml deleted file mode 100644 index 77e74308c3..0000000000 --- a/.github/workflows/gke-ipv6-only.yml +++ /dev/null @@ -1,271 +0,0 @@ -name: GKE IPv6-Only Testing - -on: - workflow_call: - inputs: - image: - required: true - type: string - k8s-version: - required: true - type: string - -defaults: - run: - shell: bash - -env: - PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} - -permissions: - contents: read - -jobs: - ipv6-only-tests: - name: Run IPv6-Only Tests - runs-on: ubuntu-24.04 - if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} - env: - DOCKER_BUILD_SUMMARY: false - steps: - - name: Checkout Repository - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 - with: - fetch-depth: 0 - - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 - with: - token_format: access_token - workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }} - service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} - - - name: Login to GAR - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 - with: - registry: us-docker.pkg.dev - username: oauth2accesstoken - password: ${{ steps.auth.outputs.access_token }} - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - install_components: kubectl - - - name: Create GKE cluster - working-directory: ./tests - run: make create-gke-cluster CI=true IPv6_ONLY=true - - - name: Create and setup VM - working-directory: ./tests - run: make create-and-setup-vm - - - name: Create and setup Router - working-directory: ./tests - run: make create-gke-router || true - - - name: Configure GOPROXY - id: goproxy - run: | - if [[ "${{ secrets.ARTIFACTORY_USER }}" == "" ]]; then - GOPROXY_VALUE="direct" - else - GOPROXY_VALUE="https://${{ secrets.ARTIFACTORY_USER }}:${{ secrets.ARTIFACTORY_TOKEN }}@${{ secrets.ARTIFACTORY_DEV_ENDPOINT }}" - fi - echo "GOPROXY=${GOPROXY_VALUE}" >> $GITHUB_ENV - - - name: Setup Golang Environment - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 - with: - go-version: stable - - - name: Set GOPATH - run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV - - - name: Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: NGF Docker meta - id: ngf-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric - tags: | - type=semver,pattern={{version}} - type=schedule - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: NGINX Docker meta - id: nginx-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} - tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: Build binary - uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 - with: - version: v2.12.0 # renovate: datasource=github-tags depName=goreleaser/goreleaser - args: - build --single-target --snapshot --clean --config=goreleaser.yml - env: - TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 - TELEMETRY_ENDPOINT_INSECURE: "true" - - - name: Build NGF Docker Image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - file: build/Dockerfile - tags: ${{ steps.ngf-meta.outputs.tags }} - context: "." - load: true - cache-from: type=gha,scope=ngf - pull: true - target: goreleaser - - - name: Build NGINX Docker Image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} - tags: ${{ steps.nginx-meta.outputs.tags }} - context: "." - load: true - cache-from: type=gha,scope=${{ inputs.image }}-ipv6 - pull: true - build-args: | - NJS_DIR=internal/controller/nginx/modules/src - NGINX_CONF_DIR=internal/controller/nginx/conf - BUILD_AGENT=gha - - # - name: Setup license file for plus - # if: ${{ inputs.image == 'plus' }} - # env: - # PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} - # run: echo "${PLUS_LICENSE}" > license.jwt - - # - name: Deploy IPv6-Only Kubernetes - # id: k8s - # run: | - # # Enable IPv6 and container network options - # # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 - # # sudo sysctl -w net.ipv6.conf.all.forwarding=1 - - # # Create IPv6-only kind cluster - # kind create cluster \ - # --name ${{ github.run_id }}-ipv6 \ - # --image=kindest/node:${{ inputs.k8s-version }} \ - # --config=config/cluster/kind-ipv6-only.yaml - - # # Load images into the cluster - # kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 - - # # Verify nodes are ipv6 only - # kubectl get nodes -o wide - - - name: Push Docker Images to GAR - # if: ${{ github.event_name != 'pull_request' }} - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - with: - context: "." - push: true - tags: | - ${{ steps.ngf-meta.outputs.tags }} - ${{ steps.nginx-meta.outputs.tags }} - - - name: Install NGF with IPv6 Configuration - run: | - ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric - ngf_tag=${{ steps.ngf-meta.outputs.version }} - # Install with IPv6-specific configuration - CLUSTER_NAME=${{ github.run_id }}-ipv6 \ - HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ - make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} - working-directory: ./tests - - - name: Deploy Test Applications - run: | - kubectl apply -f tests/manifests/ipv6-test-app.yaml - - - name: Wait for NGF and Applications to be Ready - run: | - echo "Waiting for NGF to be ready..." - kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway - echo "Waiting for test applications to be ready..." - kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 - - - name: Deploy IPv6 Test Client - run: | - kubectl apply -f tests/manifests/test-client-ipv6.yaml - kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client - - - name: Get NGF IPv6 Address - id: ngf-address - run: | - # Get the NGF service IPv6 address - NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') - echo "NGF IPv6 Address: $NGF_IPV6" - echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT - - - name: Run IPv6 Connectivity Tests - run: | - echo "=== Running IPv6-Only Tests ===" - # Test 1: Basic connectivity test using test client pod - echo "Test 1: Basic IPv6 connectivity" - kubectl exec ipv6-test-client -- curl --version - kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local - # Test 2: Test NGF service directly via IPv6 - echo "Test 2: NGF Service IPv6 connectivity" - kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ - -H "Host: ipv6-test.example.com" \ - "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" - # Test 3: Test via service DNS - echo "Test 3: Service DNS IPv6 connectivity" - kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ - -H "Host: ipv6-test.example.com" \ - "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" - - - name: Validate IPv6-Only Configuration - run: | - echo "=== Validating IPv6-Only Configuration ===" - # Check NGF configuration - echo "NGF Pod IPv6 addresses:" - kubectl get pods -n nginx-gateway -o wide - echo "NGF Service configuration:" - kubectl get service nginx-gateway -n nginx-gateway -o yaml - echo "Gateway and HTTPRoute status:" - kubectl get gateway,httproute -A -o wide - echo "Test application service configuration:" - kubectl get service test-app-ipv6-service -o yaml - - - name: Collect Logs - if: always() - run: | - echo "=== Collecting logs for debugging ===" - echo "NGF Controller logs:" - kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true - echo "NGINX logs:" - kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true - echo "Test client logs:" - kubectl logs ipv6-test-client --tail=100 || true - echo "Cluster events:" - kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true - - - name: Cleanup - working-directory: ./tests - if: always() - run: | - bash scripts/cleanup-vm.sh true - bash scripts/cleanup-router.sh true - make delete-gke-cluster - rm -rf scripts/vars.env diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index da4876fba6..280cc07b4a 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -31,6 +31,11 @@ on: default: both type: choice options: [oss, plus, both] + ipv6_enabled: + description: Whether to run the tests in an IPv6 enabled environment + required: false + default: false + type: boolean schedule: - cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC @@ -138,7 +143,7 @@ jobs: - name: Create GKE cluster working-directory: ./tests - run: make create-gke-cluster CI=true + run: make create-gke-cluster CI=true IPv6_ONLY=${{ needs.vars.outputs.ipv6_enabled }} - name: Create and setup VM working-directory: ./tests From fc0295f9dd7ec05bcba2c003788043944f6355e3 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 13:20:31 +0100 Subject: [PATCH 023/101] Enable NFR tests to run in IPv6 environments --- .github/workflows/nfr.yml | 1 + tests/scripts/remote-scripts/run-nfr-tests.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 280cc07b4a..f593b69340 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -134,6 +134,7 @@ jobs: echo "GKE_NUM_NODES=12" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env + echo "HELM_PARAMETERS="${{ if inputs.ipv6_enabled }}--set nginx.config.ipFamily=ipv6 >> vars.env" - name: Setup license file for plus if: matrix.type == 'plus' diff --git a/tests/scripts/remote-scripts/run-nfr-tests.sh b/tests/scripts/remote-scripts/run-nfr-tests.sh index baea40ab52..eb941ed494 100755 --- a/tests/scripts/remote-scripts/run-nfr-tests.sh +++ b/tests/scripts/remote-scripts/run-nfr-tests.sh @@ -10,7 +10,7 @@ elif [ "${STOP_LONGEVITY}" == "true" ]; then GINKGO_LABEL="longevity-teardown" fi -cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}" GKE_PROJECT="${GKE_PROJECT}" +cd nginx-gateway-fabric/tests && make .vm-nfr-test CI=${CI} TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer NGF_VERSION="${NGF_VERSION}" PLUS_USAGE_ENDPOINT="${PLUS_USAGE_ENDPOINT}" GKE_PROJECT="${GKE_PROJECT}" HELM_PARAMETERS="${HELM_PARAMETERS}" if [ "${START_LONGEVITY}" == "true" ]; then suite/scripts/longevity-wrk.sh From fa04cc48408e65e45d5f958a5fbdb614ca3c1a81 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 13:21:35 +0100 Subject: [PATCH 024/101] Remove IPv6 functional tests from ci --- .github/workflows/ci.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a133884a0..ab8fbe1e87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -285,22 +285,6 @@ jobs: permissions: contents: read - ipv6-only-tests: - name: Functional IPv6 Only Tests - needs: [vars, build-oss] - strategy: - fail-fast: false - matrix: - image: [nginx] - k8s-version: - [ - "${{ needs.vars.outputs.k8s_latest }}", - ] - uses: ./.github/workflows/ipv6-only.yml - with: - image: ${{ matrix.image }} - k8s-version: ${{ matrix.k8s-version }} - conformance-tests: name: Conformance tests needs: [vars, build-oss, build-plus] From 6bac8290dfd48b7176c8cb97e8ad83314646ceae Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 14:07:06 +0100 Subject: [PATCH 025/101] Update logic for ipFamily helm parameter --- .github/workflows/nfr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index f593b69340..a1d4e5c542 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -134,7 +134,7 @@ jobs: echo "GKE_NUM_NODES=12" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - echo "HELM_PARAMETERS="${{ if inputs.ipv6_enabled }}--set nginx.config.ipFamily=ipv6 >> vars.env" + echo "HELM_PARAMETERS=--set nginx.config.ipFamily=${{ inputs.ipv6_enabled ? 'ipv6' : 'ipv4' }}" >> vars.env - name: Setup license file for plus if: matrix.type == 'plus' From 012ac6ce530473a08a0a47023fcf79a1fd89ee8d Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 14:09:49 +0100 Subject: [PATCH 026/101] Update logic for helm parameters --- .github/workflows/nfr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index a1d4e5c542..bb6601df1a 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -134,8 +134,9 @@ jobs: echo "GKE_NUM_NODES=12" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - echo "HELM_PARAMETERS=--set nginx.config.ipFamily=${{ inputs.ipv6_enabled ? 'ipv6' : 'ipv4' }}" >> vars.env - + if [ "${{ inputs.ipv6_enabled }}" = "true" ]; then + echo "HELM_PARAMETERS=--set nginx.config.ipFamily=ipv6" >> vars.env + fi - name: Setup license file for plus if: matrix.type == 'plus' env: From eb44e3d1b8ac73100dc142fb1a0c68e977a334b4 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 14:26:35 +0100 Subject: [PATCH 027/101] Fix indentation --- .github/workflows/nfr.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index bb6601df1a..ba15a2608a 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -134,9 +134,9 @@ jobs: echo "GKE_NUM_NODES=12" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - if [ "${{ inputs.ipv6_enabled }}" = "true" ]; then - echo "HELM_PARAMETERS=--set nginx.config.ipFamily=ipv6" >> vars.env - fi + if [ "${{ inputs.ipv6_enabled }}" = "true" ]; then + echo "HELM_PARAMETERS=--set nginx.config.ipFamily=ipv6" >> vars.env + fi - name: Setup license file for plus if: matrix.type == 'plus' env: From ea982772e62e9919b0408301d94dd6ab3b28c318 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 14:30:58 +0100 Subject: [PATCH 028/101] Fix variable name --- .github/workflows/nfr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index ba15a2608a..cdf5cd35f0 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -145,7 +145,7 @@ jobs: - name: Create GKE cluster working-directory: ./tests - run: make create-gke-cluster CI=true IPv6_ONLY=${{ needs.vars.outputs.ipv6_enabled }} + run: make create-gke-cluster CI=true IPV6_ENABLE=${{ needs.vars.outputs.ipv6_enabled }} - name: Create and setup VM working-directory: ./tests From 8cc7e0aa869b44af96f32f9392f1195850275697 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 14:36:06 +0100 Subject: [PATCH 029/101] Fix helm parameters --- .github/workflows/nfr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index cdf5cd35f0..94c7e98833 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -135,7 +135,7 @@ jobs: echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env if [ "${{ inputs.ipv6_enabled }}" = "true" ]; then - echo "HELM_PARAMETERS=--set nginx.config.ipFamily=ipv6" >> vars.env + echo "HELM_PARAMETERS=\"--set nginx.config.ipFamily=ipv6\"" >> vars.env fi - name: Setup license file for plus if: matrix.type == 'plus' From 780e2af18820a38f70771cc57a3c352a41c79770 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 15:51:29 +0100 Subject: [PATCH 030/101] Add ipv6_enabled to job outputs --- .github/workflows/nfr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 94c7e98833..f286de93ad 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -62,6 +62,7 @@ jobs: version: ${{ github.event.inputs.version || 'edge' }} image_tag: ${{ github.event.inputs.image_tag || 'edge' }} types: ${{ steps.var.outputs.types }} + ipv6_enabled: ${{ github.event.inputs.ipv6_enabled || 'false' }} permissions: contents: read steps: @@ -145,7 +146,7 @@ jobs: - name: Create GKE cluster working-directory: ./tests - run: make create-gke-cluster CI=true IPV6_ENABLE=${{ needs.vars.outputs.ipv6_enabled }} + run: make create-gke-cluster CI=true IPV6_ENABLE=${{ vars.outputs.ipv6_enabled }} - name: Create and setup VM working-directory: ./tests From e2bc403b1509874b5d6310889a8afa476f9d6cab Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 15:58:22 +0100 Subject: [PATCH 031/101] Update IPv6 fariable name in tests/Makefile --- .github/workflows/nfr.yml | 2 +- tests/Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index f286de93ad..5af78a1c8a 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -136,7 +136,7 @@ jobs: echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env if [ "${{ inputs.ipv6_enabled }}" = "true" ]; then - echo "HELM_PARAMETERS=\"--set nginx.config.ipFamily=ipv6\"" >> vars.env + echo "HELM_PARAMETERS=\"--set nginx.config.ipFamily=ipv6\"" >> vars.env fi - name: Setup license file for plus if: matrix.type == 'plus' diff --git a/tests/Makefile b/tests/Makefile index b8de189533..c4c9c37982 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,5 +1,5 @@ CI ?= false -IPv6_ONLY ?= false +IPV6_ENABLE ?= false CLUSTER_NAME ?= kind CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image CONFORMANCE_TAG = latest## Tag for the conformance test runner image @@ -91,7 +91,7 @@ setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## C .PHONY: create-gke-cluster create-gke-cluster: ## Create a GKE cluster - ./scripts/create-gke-cluster.sh $(CI) $(IPv6_ONLY) + ./scripts/create-gke-cluster.sh $(CI) $(IPV6_ENABLE) .PHONY: create-and-setup-vm create-and-setup-vm: ## Create and setup a GCP VM for tests From a4828f94e8bbe66e3afa311f781eefdf49cd6b28 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 16:06:22 +0100 Subject: [PATCH 032/101] Update vars to `vars.inputs.ipv6_enabled` --- .github/workflows/nfr.yml | 5 ++--- tests/scripts/create-gke-cluster.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 5af78a1c8a..b58723ed47 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -62,7 +62,6 @@ jobs: version: ${{ github.event.inputs.version || 'edge' }} image_tag: ${{ github.event.inputs.image_tag || 'edge' }} types: ${{ steps.var.outputs.types }} - ipv6_enabled: ${{ github.event.inputs.ipv6_enabled || 'false' }} permissions: contents: read steps: @@ -135,7 +134,7 @@ jobs: echo "GKE_NUM_NODES=12" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - if [ "${{ inputs.ipv6_enabled }}" = "true" ]; then + if [ "${{ needs.vars.inputs.ipv6_enabled }}" = "true" ]; then echo "HELM_PARAMETERS=\"--set nginx.config.ipFamily=ipv6\"" >> vars.env fi - name: Setup license file for plus @@ -146,7 +145,7 @@ jobs: - name: Create GKE cluster working-directory: ./tests - run: make create-gke-cluster CI=true IPV6_ENABLE=${{ vars.outputs.ipv6_enabled }} + run: make create-gke-cluster CI=true IPV6_ENABLE=${{ needs.vars.inputs.ipv6_enabled }} - name: Create and setup VM working-directory: ./tests diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index cf52d27f15..5030c33bbd 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -8,7 +8,7 @@ ip_random_digit=$((1 + RANDOM % 250)) IS_CI=${1:-false} -IPV6_ENABLE=${2:-${IPV6_ENABLE:-false}} +IPV6_ENABLE=${2:-false} IPV6_FLAGS="" if [ "$IPV6_ENABLE" = "true" ]; then From 7843b32e0b3b09adfec4dc41a13bb5daae11ae9e Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 16:07:09 +0100 Subject: [PATCH 033/101] Update if statement --- tests/scripts/create-gke-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 5030c33bbd..90d8915296 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -11,7 +11,7 @@ IS_CI=${1:-false} IPV6_ENABLE=${2:-false} IPV6_FLAGS="" -if [ "$IPV6_ENABLE" = "true" ]; then +if [ "${IPV6_ENABLE}" = "true" ]; then IPV6_FLAGS="\ --enable-ipv6 \ --cluster-ipv6-cidr=fd00:1234::/56 \ From 6e4b8faa7617aa3ea86bf3eb9d18afa69bef1a95 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 16:21:29 +0100 Subject: [PATCH 034/101] Update IPv6 variable call --- .github/workflows/nfr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index b58723ed47..be403c46e6 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -61,6 +61,7 @@ jobs: test_label: ${{ github.event.inputs.test_label || 'all' }} version: ${{ github.event.inputs.version || 'edge' }} image_tag: ${{ github.event.inputs.image_tag || 'edge' }} + ipv6_enabled: ${{ github.event.inputs.ipv6_enabled || 'false' }} types: ${{ steps.var.outputs.types }} permissions: contents: read @@ -134,7 +135,7 @@ jobs: echo "GKE_NUM_NODES=12" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - if [ "${{ needs.vars.inputs.ipv6_enabled }}" = "true" ]; then + if [ "${{ needs.vars.outputs.ipv6_enabled }}" = "true" ]; then echo "HELM_PARAMETERS=\"--set nginx.config.ipFamily=ipv6\"" >> vars.env fi - name: Setup license file for plus @@ -145,7 +146,7 @@ jobs: - name: Create GKE cluster working-directory: ./tests - run: make create-gke-cluster CI=true IPV6_ENABLE=${{ needs.vars.inputs.ipv6_enabled }} + run: make create-gke-cluster CI=true IPV6_ENABLE=${{ needs.vars.outputs.ipv6_enabled }} - name: Create and setup VM working-directory: ./tests From 6de3ba4c7191855e591c575c4a9f62fa19669688 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 9 Sep 2025 16:41:30 +0100 Subject: [PATCH 035/101] Update `--create-subnetwork` flag for cluster creation --- tests/scripts/create-gke-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 90d8915296..b9c688824a 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -16,7 +16,7 @@ if [ "${IPV6_ENABLE}" = "true" ]; then --enable-ipv6 \ --cluster-ipv6-cidr=fd00:1234::/56 \ --services-ipv6-cidr=fd00:4321::/112 \ - --create-subnetwork name=\"${GKE_CLUSTER_NAME}-subnet\",range=10.0.0.0/16,fd00:abcd::/64,stack-type=IPV4_IPV6" + --create-subnetwork name="${GKE_CLUSTER_NAME}-subnet",range=10.0.0.0/16,stack-type=IPV4_IPV6,subnet-ipv6-range=fd00:abcd::/64" fi if [ -z "$GKE_MACHINE_TYPE" ]; then From 6a0ef07029e7574d8721c5a9a351e5e8dd59fd74 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 08:53:38 +0100 Subject: [PATCH 036/101] Test adding IPv6 VPC and Subnet. --- .github/workflows/nfr.yml | 44 ++++++++++++++--------------- tests/scripts/create-gke-cluster.sh | 27 ++++++++++-------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index be403c46e6..6ee60e9d72 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -148,28 +148,28 @@ jobs: working-directory: ./tests run: make create-gke-cluster CI=true IPV6_ENABLE=${{ needs.vars.outputs.ipv6_enabled }} - - name: Create and setup VM - working-directory: ./tests - run: make create-and-setup-vm - - - name: Create and setup Router - working-directory: ./tests - run: make create-gke-router || true - - - name: Run Tests - working-directory: ./tests - run: | - if ${{ needs.vars.outputs.test_label != 'all' }}; then - sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true; - else - make nfr-test CI=true; - fi - - - name: Upload Artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: results-${{ matrix.type }} - path: tests/results/**/*-${{ matrix.type }}.* + # - name: Create and setup VM + # working-directory: ./tests + # run: make create-and-setup-vm + + # - name: Create and setup Router + # working-directory: ./tests + # run: make create-gke-router || true + + # - name: Run Tests + # working-directory: ./tests + # run: | + # if ${{ needs.vars.outputs.test_label != 'all' }}; then + # sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true; + # else + # make nfr-test CI=true; + # fi + + # - name: Upload Artifacts + # uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + # with: + # name: results-${{ matrix.type }} + # path: tests/results/**/*-${{ matrix.type }}.* - name: Cleanup working-directory: ./tests diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index b9c688824a..7f049a0f66 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -10,15 +10,6 @@ IS_CI=${1:-false} IPV6_ENABLE=${2:-false} -IPV6_FLAGS="" -if [ "${IPV6_ENABLE}" = "true" ]; then - IPV6_FLAGS="\ - --enable-ipv6 \ - --cluster-ipv6-cidr=fd00:1234::/56 \ - --services-ipv6-cidr=fd00:4321::/112 \ - --create-subnetwork name="${GKE_CLUSTER_NAME}-subnet",range=10.0.0.0/16,stack-type=IPV4_IPV6,subnet-ipv6-range=fd00:abcd::/64" -fi - if [ -z "$GKE_MACHINE_TYPE" ]; then # If the environment variable is not set, use a default value GKE_MACHINE_TYPE="e2-medium" @@ -29,6 +20,21 @@ if [ -z "$GKE_NUM_NODES" ]; then GKE_NUM_NODES="3" fi +if [ "${IPV6_ENABLE}" = "true" ]; then + echo "Creating IPv6 Network interface for the GKE cluster" + gcloud compute networks create ${GKE_CLUSTER_NAME}-network --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 --stack-type=IPV4_IPV6 + gcloud compute networks subnets create ${GKE_CLUSTER_NAME}-subnet \ + --network=${GKE_CLUSTER_NAME}-network \ + --stack-type=IPV6_ONLY \ + --ipv6-access-type=INTERNAL \ + --region=${GKE_CLUSTER_REGION} + + echo "Deleting subnet ${GKE_CLUSTER_NAME}-subnet (if exists)..." + gcloud compute networks subnets delete ${GKE_CLUSTER_NAME}-subnet --region=${GKE_CLUSTER_REGION} --quiet || true + echo "Deleting network ${GKE_CLUSTER_NAME}-network (if exists)..." + gcloud compute networks delete ${GKE_CLUSTER_NAME}-network --quiet || true +fi + gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --project "${GKE_PROJECT}" \ --zone "${GKE_CLUSTER_ZONE}" \ @@ -42,8 +48,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port \ - $IPV6_FLAGS + --no-enable-insecure-kubelet-readonly-port # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From 185f5e2f01cb6577c2ee0f406a871de90d791be4 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 09:00:15 +0100 Subject: [PATCH 037/101] Remove unused flag --- .github/workflows/nfr.yml | 4 ++-- tests/scripts/create-gke-cluster.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 6ee60e9d72..b4e1ffe3f6 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -175,8 +175,8 @@ jobs: working-directory: ./tests if: always() run: | - bash scripts/cleanup-vm.sh true - bash scripts/cleanup-router.sh true + # bash scripts/cleanup-vm.sh true + # bash scripts/cleanup-router.sh true make delete-gke-cluster rm -rf scripts/vars.env diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 7f049a0f66..a3d219be45 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -22,7 +22,7 @@ fi if [ "${IPV6_ENABLE}" = "true" ]; then echo "Creating IPv6 Network interface for the GKE cluster" - gcloud compute networks create ${GKE_CLUSTER_NAME}-network --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 --stack-type=IPV4_IPV6 + gcloud compute networks create ${GKE_CLUSTER_NAME}-network --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 gcloud compute networks subnets create ${GKE_CLUSTER_NAME}-subnet \ --network=${GKE_CLUSTER_NAME}-network \ --stack-type=IPV6_ONLY \ From 8300b7eda38572f0bd893a1f6e3da01b5a13ae41 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 09:08:48 +0100 Subject: [PATCH 038/101] Update ipv6-access-type to EXTERNAL --- tests/scripts/create-gke-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index a3d219be45..dc2a3559cf 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -26,7 +26,7 @@ if [ "${IPV6_ENABLE}" = "true" ]; then gcloud compute networks subnets create ${GKE_CLUSTER_NAME}-subnet \ --network=${GKE_CLUSTER_NAME}-network \ --stack-type=IPV6_ONLY \ - --ipv6-access-type=INTERNAL \ + --ipv6-access-type=EXTERNAL \ --region=${GKE_CLUSTER_REGION} echo "Deleting subnet ${GKE_CLUSTER_NAME}-subnet (if exists)..." From 91f8e475ac5d89efd3a2c2cfa86f29e457bc61d0 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 11:24:29 +0100 Subject: [PATCH 039/101] Add firewall rules for VPC --- tests/scripts/create-gke-cluster.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index dc2a3559cf..ec8894040f 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -29,10 +29,14 @@ if [ "${IPV6_ENABLE}" = "true" ]; then --ipv6-access-type=EXTERNAL \ --region=${GKE_CLUSTER_REGION} + gcloud compute firewall-rules create ${GKE_CLUSTER_NAME}-firewall --network ${GKE_CLUSTER_NAME}-network --allow tcp:22,tcp:3389,icmp + echo "Deleting subnet ${GKE_CLUSTER_NAME}-subnet (if exists)..." gcloud compute networks subnets delete ${GKE_CLUSTER_NAME}-subnet --region=${GKE_CLUSTER_REGION} --quiet || true echo "Deleting network ${GKE_CLUSTER_NAME}-network (if exists)..." gcloud compute networks delete ${GKE_CLUSTER_NAME}-network --quiet || true + echo "Deleting firewall rule ${GKE_CLUSTER_NAME}-firewall (if exists)..." + gcloud compute firewall-rules delete ${GKE_CLUSTER_NAME}-firewall --quiet || true fi gcloud container clusters create "${GKE_CLUSTER_NAME}" \ From 9b5e541e986d73cdfda22f86e36a2a59ff8cbca6 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 12:15:05 +0100 Subject: [PATCH 040/101] Add custom subnet to cluster creation --- tests/scripts/create-gke-cluster.sh | 10 ++-------- tests/scripts/delete-gke-cluster.sh | 7 +++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index ec8894040f..01bfb7d6fc 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -30,13 +30,6 @@ if [ "${IPV6_ENABLE}" = "true" ]; then --region=${GKE_CLUSTER_REGION} gcloud compute firewall-rules create ${GKE_CLUSTER_NAME}-firewall --network ${GKE_CLUSTER_NAME}-network --allow tcp:22,tcp:3389,icmp - - echo "Deleting subnet ${GKE_CLUSTER_NAME}-subnet (if exists)..." - gcloud compute networks subnets delete ${GKE_CLUSTER_NAME}-subnet --region=${GKE_CLUSTER_REGION} --quiet || true - echo "Deleting network ${GKE_CLUSTER_NAME}-network (if exists)..." - gcloud compute networks delete ${GKE_CLUSTER_NAME}-network --quiet || true - echo "Deleting firewall rule ${GKE_CLUSTER_NAME}-firewall (if exists)..." - gcloud compute firewall-rules delete ${GKE_CLUSTER_NAME}-firewall --quiet || true fi gcloud container clusters create "${GKE_CLUSTER_NAME}" \ @@ -52,7 +45,8 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port + --no-enable-insecure-kubelet-readonly-port \ + --subnetwork=${GKE_CLUSTER_NAME}-subnet # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then diff --git a/tests/scripts/delete-gke-cluster.sh b/tests/scripts/delete-gke-cluster.sh index 2cebd452ad..ad908380b6 100755 --- a/tests/scripts/delete-gke-cluster.sh +++ b/tests/scripts/delete-gke-cluster.sh @@ -5,3 +5,10 @@ set -eo pipefail source scripts/vars.env gcloud container clusters delete "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" --project "${GKE_PROJECT}" --quiet + +echo "Deleting firewall rule ${GKE_CLUSTER_NAME}-firewall (if exists)..." +gcloud compute firewall-rules delete ${GKE_CLUSTER_NAME}-firewall --quiet || true +echo "Deleting subnet ${GKE_CLUSTER_NAME}-subnet (if exists)..." +gcloud compute networks subnets delete ${GKE_CLUSTER_NAME}-subnet --region=${GKE_CLUSTER_REGION} --quiet || true +echo "Deleting network ${GKE_CLUSTER_NAME}-network (if exists)..." +gcloud compute networks delete ${GKE_CLUSTER_NAME}-network --quiet || true From d6998a860cb5e4663831cfc104fd47e34dd8f5b5 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 12:21:08 +0100 Subject: [PATCH 041/101] Add network to cluster creation --- tests/scripts/create-gke-cluster.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 01bfb7d6fc..3f7889f417 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -46,6 +46,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ --no-enable-insecure-kubelet-readonly-port \ + --network=${GKE_CLUSTER_NAME}-network \ --subnetwork=${GKE_CLUSTER_NAME}-subnet # Add current IP to GKE master control node access, if this script is not invoked during a CI run. From caa83ae90d535df206f77a480006418c8ada36a9 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 15:03:55 +0100 Subject: [PATCH 042/101] Remove networking flags for debugging --- tests/scripts/create-gke-cluster.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 3f7889f417..bf0949aefb 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -22,7 +22,7 @@ fi if [ "${IPV6_ENABLE}" = "true" ]; then echo "Creating IPv6 Network interface for the GKE cluster" - gcloud compute networks create ${GKE_CLUSTER_NAME}-network --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 + gcloud compute networks create ${GKE_CLUSTER_NAME}-network --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 --quiet gcloud compute networks subnets create ${GKE_CLUSTER_NAME}-subnet \ --network=${GKE_CLUSTER_NAME}-network \ --stack-type=IPV6_ONLY \ @@ -45,9 +45,9 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port \ - --network=${GKE_CLUSTER_NAME}-network \ - --subnetwork=${GKE_CLUSTER_NAME}-subnet + --no-enable-insecure-kubelet-readonly-port + # --network=${GKE_CLUSTER_NAME}-network \ + # --subnetwork=${GKE_CLUSTER_NAME}-subnet # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From a9e6a629d34b3bb0bb1aa1b9e88857107e1ec8e2 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 15:48:21 +0100 Subject: [PATCH 043/101] Re add network flags --- tests/scripts/create-gke-cluster.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index bf0949aefb..ac0b544cad 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -45,9 +45,9 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port - # --network=${GKE_CLUSTER_NAME}-network \ - # --subnetwork=${GKE_CLUSTER_NAME}-subnet + --no-enable-insecure-kubelet-readonly-port \ + --network=${GKE_CLUSTER_NAME}-network \ + --subnetwork=${GKE_CLUSTER_NAME}-subnet # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From c5731c2cb7367c523aaf24b1c9dd103320e78aa4 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 10 Sep 2025 17:09:58 +0100 Subject: [PATCH 044/101] Update zone to us-west1-c --- .github/workflows/nfr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index b4e1ffe3f6..16a4223466 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -120,7 +120,7 @@ jobs: echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env - echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env + echo "GKE_CLUSTER_ZONE=us-west1-c" >> vars.env echo "GKE_CLUSTER_REGION=us-west1" >> vars.env echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env From 8bc4086ebd7314319f4dd94a0090e432fb98e16b Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 08:12:14 +0100 Subject: [PATCH 045/101] Create custom IPv6 networks and subnets for GCP VMs --- .github/workflows/nfr.yml | 19 ++++++++++--------- tests/Makefile | 3 +-- tests/scripts/cleanup-vm.sh | 7 +++++++ tests/scripts/create-and-setup-gcp-vm.sh | 21 +++++++++++++++++++-- tests/scripts/create-gke-cluster.sh | 18 +----------------- tests/scripts/delete-gke-cluster.sh | 7 ------- 6 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 16a4223466..c2c95f818a 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -136,6 +136,7 @@ jobs: echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env if [ "${{ needs.vars.outputs.ipv6_enabled }}" = "true" ]; then + echo "IPV6_ENABLED=true" >> vars.env echo "HELM_PARAMETERS=\"--set nginx.config.ipFamily=ipv6\"" >> vars.env fi - name: Setup license file for plus @@ -146,15 +147,15 @@ jobs: - name: Create GKE cluster working-directory: ./tests - run: make create-gke-cluster CI=true IPV6_ENABLE=${{ needs.vars.outputs.ipv6_enabled }} + run: make create-gke-cluster CI=true - # - name: Create and setup VM - # working-directory: ./tests - # run: make create-and-setup-vm + - name: Create and setup VM + working-directory: ./tests + run: make create-and-setup-vm - # - name: Create and setup Router - # working-directory: ./tests - # run: make create-gke-router || true + - name: Create and setup Router + working-directory: ./tests + run: make create-gke-router || true # - name: Run Tests # working-directory: ./tests @@ -175,8 +176,8 @@ jobs: working-directory: ./tests if: always() run: | - # bash scripts/cleanup-vm.sh true - # bash scripts/cleanup-router.sh true + bash scripts/cleanup-vm.sh true + bash scripts/cleanup-router.sh true make delete-gke-cluster rm -rf scripts/vars.env diff --git a/tests/Makefile b/tests/Makefile index c4c9c37982..f1ccfdd3c5 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,5 +1,4 @@ CI ?= false -IPV6_ENABLE ?= false CLUSTER_NAME ?= kind CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image CONFORMANCE_TAG = latest## Tag for the conformance test runner image @@ -91,7 +90,7 @@ setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## C .PHONY: create-gke-cluster create-gke-cluster: ## Create a GKE cluster - ./scripts/create-gke-cluster.sh $(CI) $(IPV6_ENABLE) + ./scripts/create-gke-cluster.sh $(CI) .PHONY: create-and-setup-vm create-and-setup-vm: ## Create and setup a GCP VM for tests diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index ecb0420d18..8a8ffd137c 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -14,4 +14,11 @@ if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ] && [ "${skip_gke_master_control_nod fi gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" + +# Clean up the custom network and subnet if IPv6 was enabled +if [ "${IPV6_ENABLED}" = "true" ]; then + gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet || true + gcloud compute networks delete ${RESOURCE_NAME} --quiet || true +fi + gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 5031e2e9ac..c3dc2b5e61 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -5,13 +5,28 @@ set -o pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) REPO_DIR=$(dirname $(dirname "$SCRIPT_DIR")) +NETWORK=default + source scripts/vars.env +# Create custom network and subnet if IPv6 is enabled +if [ "${IPV6_ENABLED}" = "true" ]; then + echo "Creating IPv6 Network interface for the GKE cluster" + gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 --quiet + gcloud compute networks subnets create ${RESOURCE_NAME} \ + --network=${RESOURCE_NAME} \ + --stack-type=IPV6_ONLY \ + --ipv6-access-type=EXTERNAL \ + --region=${GKE_CLUSTER_REGION} + + NETWORK=${RESOURCE_NAME} +fi + gcloud compute firewall-rules create "${RESOURCE_NAME}" \ --project="${GKE_PROJECT}" \ --direction=INGRESS \ --priority=1000 \ - --network=default \ + --network=${NETWORK} \ --action=ALLOW \ --rules=tcp:22 \ --source-ranges="${SOURCE_IP_RANGE}" \ @@ -21,7 +36,9 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- --network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default --maintenance-policy=MIGRATE \ --provisioning-model=STANDARD --service-account="${GKE_SVC_ACCOUNT}" \ --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append,https://www.googleapis.com/auth/cloud-platform \ - --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any + --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any \ + --network=${NETWORK} \ + --subnetwork=${NETWORK} # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index ac0b544cad..c41f6f12d1 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -8,8 +8,6 @@ ip_random_digit=$((1 + RANDOM % 250)) IS_CI=${1:-false} -IPV6_ENABLE=${2:-false} - if [ -z "$GKE_MACHINE_TYPE" ]; then # If the environment variable is not set, use a default value GKE_MACHINE_TYPE="e2-medium" @@ -20,18 +18,6 @@ if [ -z "$GKE_NUM_NODES" ]; then GKE_NUM_NODES="3" fi -if [ "${IPV6_ENABLE}" = "true" ]; then - echo "Creating IPv6 Network interface for the GKE cluster" - gcloud compute networks create ${GKE_CLUSTER_NAME}-network --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 --quiet - gcloud compute networks subnets create ${GKE_CLUSTER_NAME}-subnet \ - --network=${GKE_CLUSTER_NAME}-network \ - --stack-type=IPV6_ONLY \ - --ipv6-access-type=EXTERNAL \ - --region=${GKE_CLUSTER_REGION} - - gcloud compute firewall-rules create ${GKE_CLUSTER_NAME}-firewall --network ${GKE_CLUSTER_NAME}-network --allow tcp:22,tcp:3389,icmp -fi - gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --project "${GKE_PROJECT}" \ --zone "${GKE_CLUSTER_ZONE}" \ @@ -45,9 +31,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port \ - --network=${GKE_CLUSTER_NAME}-network \ - --subnetwork=${GKE_CLUSTER_NAME}-subnet + --no-enable-insecure-kubelet-readonly-port # Option is deprecated and will be removed in a future release # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then diff --git a/tests/scripts/delete-gke-cluster.sh b/tests/scripts/delete-gke-cluster.sh index ad908380b6..2cebd452ad 100755 --- a/tests/scripts/delete-gke-cluster.sh +++ b/tests/scripts/delete-gke-cluster.sh @@ -5,10 +5,3 @@ set -eo pipefail source scripts/vars.env gcloud container clusters delete "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" --project "${GKE_PROJECT}" --quiet - -echo "Deleting firewall rule ${GKE_CLUSTER_NAME}-firewall (if exists)..." -gcloud compute firewall-rules delete ${GKE_CLUSTER_NAME}-firewall --quiet || true -echo "Deleting subnet ${GKE_CLUSTER_NAME}-subnet (if exists)..." -gcloud compute networks subnets delete ${GKE_CLUSTER_NAME}-subnet --region=${GKE_CLUSTER_REGION} --quiet || true -echo "Deleting network ${GKE_CLUSTER_NAME}-network (if exists)..." -gcloud compute networks delete ${GKE_CLUSTER_NAME}-network --quiet || true From 373683bb82aebc13843cb0316add73f7d6b9a409 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 08:22:04 +0100 Subject: [PATCH 046/101] Update subnet flag for instance creation --- tests/scripts/create-and-setup-gcp-vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index c3dc2b5e61..be197126ac 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -38,7 +38,7 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append,https://www.googleapis.com/auth/cloud-platform \ --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any \ --network=${NETWORK} \ - --subnetwork=${NETWORK} + --subnetgit co=${NETWORK} # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then From 740f9c32e8fdc58535156f6c4ab9f7e4b5f20518 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 08:30:16 +0100 Subject: [PATCH 047/101] Fix subnet flag --- tests/scripts/create-and-setup-gcp-vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index be197126ac..940e7ffe6d 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -38,7 +38,7 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append,https://www.googleapis.com/auth/cloud-platform \ --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any \ --network=${NETWORK} \ - --subnetgit co=${NETWORK} + --subnet=${NETWORK} # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then From faf23f1331aad184c1f656475f04d4c0ec09e843 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 08:50:09 +0100 Subject: [PATCH 048/101] Update nwtwork settings --- tests/scripts/cleanup-vm.sh | 4 ++-- tests/scripts/create-and-setup-gcp-vm.sh | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index 8a8ffd137c..acea1bd3f0 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -15,10 +15,10 @@ fi gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" +gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" + # Clean up the custom network and subnet if IPv6 was enabled if [ "${IPV6_ENABLED}" = "true" ]; then gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet || true gcloud compute networks delete ${RESOURCE_NAME} --quiet || true fi - -gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 940e7ffe6d..c197ac9be4 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -5,7 +5,10 @@ set -o pipefail SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) REPO_DIR=$(dirname $(dirname "$SCRIPT_DIR")) +# Default network settings NETWORK=default +STACK_TYPE="IPV4_ONLY" +NETWORK_TIER="network-tier=PREMIUM" source scripts/vars.env @@ -20,6 +23,8 @@ if [ "${IPV6_ENABLED}" = "true" ]; then --region=${GKE_CLUSTER_REGION} NETWORK=${RESOURCE_NAME} + NETWORK_TIER="ipv6-network-tier=PREMIUM" + STACK_TYPE="IPV6_ONLY" fi gcloud compute firewall-rules create "${RESOURCE_NAME}" \ @@ -33,12 +38,10 @@ gcloud compute firewall-rules create "${RESOURCE_NAME}" \ --target-tags="${NETWORK_TAGS}" gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" --machine-type=n2-standard-2 \ - --network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default --maintenance-policy=MIGRATE \ + --network-interface=${NETWORK_TIER},stack-type=${STACK_TYPE},subnet=${NETWORK} --maintenance-policy=MIGRATE \ --provisioning-model=STANDARD --service-account="${GKE_SVC_ACCOUNT}" \ --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append,https://www.googleapis.com/auth/cloud-platform \ - --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any \ - --network=${NETWORK} \ - --subnet=${NETWORK} + --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then From 6dc68d77c0b2aefea87997a22299664a455206b3 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 10:46:38 +0100 Subject: [PATCH 049/101] Add echos for debugging --- tests/scripts/create-and-setup-gcp-vm.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index c197ac9be4..34e4800d0e 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -47,8 +47,10 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + echo "External IP of the VM is: ${EXTERNAL_IP}" CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') + echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" fi From 02ac018885c4c10512cde371d43d63b773f0ec4e Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 11:33:35 +0100 Subject: [PATCH 050/101] Add additional echos for debugging --- tests/scripts/create-and-setup-gcp-vm.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 34e4800d0e..88919d1e75 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -45,9 +45,17 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then + NETWORK_INTERFACES=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + --format='value(networkInterfaces)') + ACCESS_CONFIGS=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + --format='value(networkInterfaces[0].accessConfigs)') EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + + echo "Network interfaces of the VM are: ${NETWORK_INTERFACES}" + echo "Access configs of the VM are: ${ACCESS_CONFIGS}" echo "External IP of the VM is: ${EXTERNAL_IP}" + CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" From 5f5021013712984f515144299c9d02ab42d77933 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 12:00:46 +0100 Subject: [PATCH 051/101] Correctly extract external IP for IPv6 only environments --- tests/scripts/create-and-setup-gcp-vm.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 88919d1e75..53d425d0b9 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -45,17 +45,19 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then - NETWORK_INTERFACES=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ - --format='value(networkInterfaces)') - ACCESS_CONFIGS=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ - --format='value(networkInterfaces[0].accessConfigs)') - EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ - --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + + if [ "${IPV6_ENABLED}" = "true" ]; then + echo "IPv6 is enabled, fetching the external IPv6 address" + EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + --format='value(networkInterfaces[0].ipv6AccessConfigs[0].externalIpv6)') + else + echo "IPv6 is not enabled, fetching the external IPv4 address" + EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + fi - echo "Network interfaces of the VM are: ${NETWORK_INTERFACES}" - echo "Access configs of the VM are: ${ACCESS_CONFIGS}" echo "External IP of the VM is: ${EXTERNAL_IP}" - + CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" From fa7aa8bab1766dc0c8eddb687f81f90d836cd3c9 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 13:52:59 +0100 Subject: [PATCH 052/101] Debug network auth for IPv6 --- tests/scripts/create-and-setup-gcp-vm.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 53d425d0b9..63b993c2c2 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -60,7 +60,9 @@ if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') - echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" + + TMP_CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" + echo "TMP Current GKE master authorized networks: ${TMP_CURRENT_AUTH_NETWORK}" gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" fi From 1cc99c3e846319d53c68fc0a9f7c5f74e32bc821 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 14:06:00 +0100 Subject: [PATCH 053/101] Add missing parenthesis --- tests/scripts/create-and-setup-gcp-vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 63b993c2c2..87068faa93 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -61,7 +61,7 @@ if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') - TMP_CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" + TMP_CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}") echo "TMP Current GKE master authorized networks: ${TMP_CURRENT_AUTH_NETWORK}" gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" fi From 9a92d544adf593208be294b431c1950ceb5ab5a3 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 14:42:07 +0100 Subject: [PATCH 054/101] Do not update cluster networks with IPv6 --- tests/scripts/create-and-setup-gcp-vm.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 87068faa93..70fb1433cf 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -54,16 +54,16 @@ if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then echo "IPv6 is not enabled, fetching the external IPv4 address" EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ --format='value(networkInterfaces[0].accessConfigs[0].natIP)') - fi - - echo "External IP of the VM is: ${EXTERNAL_IP}" - CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ + echo "External IP of the VM is: ${EXTERNAL_IP}" + + CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') - TMP_CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}") - echo "TMP Current GKE master authorized networks: ${TMP_CURRENT_AUTH_NETWORK}" - gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" + echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" + + gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" + fi fi # Poll for SSH connectivity From 141933fac3bd4533a1e12d63e7adf23a998a57f2 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 15:38:25 +0100 Subject: [PATCH 055/101] Add temporary cleanup for conflicting subnet --- .github/workflows/nfr.yml | 5 +++++ tests/scripts/cleanup-vm.sh | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index c2c95f818a..782b3cd774 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -145,6 +145,11 @@ jobs: PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: echo "${PLUS_LICENSE}" > license.jwt + - name: TMP Cleanup conflicting subnet + working-directory: ./tests + run: | + bash scripts/cleanup-vm.sh true + - name: Create GKE cluster working-directory: ./tests run: make create-gke-cluster CI=true diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index acea1bd3f0..36fef646dd 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -4,6 +4,10 @@ set -o pipefail source scripts/vars.env +echo "tmp delete conflicting subnet" + +gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=${GKE_CLUSTER_REGION} --quiet || true + skip_gke_master_control_node_access="${1:-false}" # Remove VM IP from GKE master control node access, if required From 774c0e71c2e8a2a7893b5cc6b8fefa49fd54169d Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Thu, 11 Sep 2025 17:32:15 +0100 Subject: [PATCH 056/101] Set stack tpye to dual and IPv6 to internal only --- tests/scripts/create-and-setup-gcp-vm.sh | 28 ++++++++++-------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 70fb1433cf..437f6d6e58 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -18,13 +18,13 @@ if [ "${IPV6_ENABLED}" = "true" ]; then gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 --quiet gcloud compute networks subnets create ${RESOURCE_NAME} \ --network=${RESOURCE_NAME} \ - --stack-type=IPV6_ONLY \ - --ipv6-access-type=EXTERNAL \ + --stack-type=IPV4_IPV6 \ + --ipv6-access-type=INTERNAL \ --region=${GKE_CLUSTER_REGION} NETWORK=${RESOURCE_NAME} NETWORK_TIER="ipv6-network-tier=PREMIUM" - STACK_TYPE="IPV6_ONLY" + STACK_TYPE="IPV4_IPV6" fi gcloud compute firewall-rules create "${RESOURCE_NAME}" \ @@ -46,24 +46,18 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then - if [ "${IPV6_ENABLED}" = "true" ]; then - echo "IPv6 is enabled, fetching the external IPv6 address" - EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ - --format='value(networkInterfaces[0].ipv6AccessConfigs[0].externalIpv6)') - else - echo "IPv6 is not enabled, fetching the external IPv4 address" - EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ - --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + echo "IPv6 is not enabled, fetching the external IPv4 address" + EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + --format='value(networkInterfaces[0].accessConfigs[0].natIP)') - echo "External IP of the VM is: ${EXTERNAL_IP}" + echo "External IP of the VM is: ${EXTERNAL_IP}" - CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ - --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') + CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ + --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') - echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" + echo "Current GKE master authorized networks: ${CURRENT_AUTH_NETWORK}" - gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" - fi + gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" fi # Poll for SSH connectivity From a5eedd0bb88cda36a9bba19e4714b41a8ba3c977 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 12 Sep 2025 09:25:08 +0100 Subject: [PATCH 057/101] Add cidr range to `compute networks subnets create` --- tests/scripts/create-and-setup-gcp-vm.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 437f6d6e58..226e318c98 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -20,7 +20,8 @@ if [ "${IPV6_ENABLED}" = "true" ]; then --network=${RESOURCE_NAME} \ --stack-type=IPV4_IPV6 \ --ipv6-access-type=INTERNAL \ - --region=${GKE_CLUSTER_REGION} + --region=${GKE_CLUSTER_REGION} \ + --range=2001:db8::/64 NETWORK=${RESOURCE_NAME} NETWORK_TIER="ipv6-network-tier=PREMIUM" From 03671bff239017029e5058ddefe5ea9fb5d842cb Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 12 Sep 2025 09:42:02 +0100 Subject: [PATCH 058/101] Update sutnet mask --- tests/scripts/create-and-setup-gcp-vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 226e318c98..92285eab68 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -21,7 +21,7 @@ if [ "${IPV6_ENABLED}" = "true" ]; then --stack-type=IPV4_IPV6 \ --ipv6-access-type=INTERNAL \ --region=${GKE_CLUSTER_REGION} \ - --range=2001:db8::/64 + --range=2001:db8::/29 NETWORK=${RESOURCE_NAME} NETWORK_TIER="ipv6-network-tier=PREMIUM" From bf3515c92f3860532b0b2e93b72acb09da9b4ee9 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 12 Sep 2025 10:18:16 +0100 Subject: [PATCH 059/101] Update workflow to delete conflicting resources --- .github/workflows/nfr.yml | 41 +++++++++++++----------- tests/scripts/cleanup-vm.sh | 4 --- tests/scripts/create-and-setup-gcp-vm.sh | 2 +- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 782b3cd774..4bb0c1dcc0 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -145,22 +145,25 @@ jobs: PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: echo "${PLUS_LICENSE}" > license.jwt - - name: TMP Cleanup conflicting subnet + - name: TMP Cleanup conflicting resources working-directory: ./tests run: | - bash scripts/cleanup-vm.sh true + echo "tmp delete conflicting subnet" + gcloud compute addresses delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true + gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=us-west1 --quiet || true + rm -rf scripts/vars.env - - name: Create GKE cluster - working-directory: ./tests - run: make create-gke-cluster CI=true + # - name: Create GKE cluster + # working-directory: ./tests + # run: make create-gke-cluster CI=true - - name: Create and setup VM - working-directory: ./tests - run: make create-and-setup-vm + # - name: Create and setup VM + # working-directory: ./tests + # run: make create-and-setup-vm - - name: Create and setup Router - working-directory: ./tests - run: make create-gke-router || true + # - name: Create and setup Router + # working-directory: ./tests + # run: make create-gke-router || true # - name: Run Tests # working-directory: ./tests @@ -177,14 +180,14 @@ jobs: # name: results-${{ matrix.type }} # path: tests/results/**/*-${{ matrix.type }}.* - - name: Cleanup - working-directory: ./tests - if: always() - run: | - bash scripts/cleanup-vm.sh true - bash scripts/cleanup-router.sh true - make delete-gke-cluster - rm -rf scripts/vars.env + # - name: Cleanup + # working-directory: ./tests + # if: always() + # run: | + # # bash scripts/cleanup-vm.sh true + # # bash scripts/cleanup-router.sh true + # # make delete-gke-cluster + # # rm -rf scripts/vars.env pr-results: name: Open PR with results diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index 36fef646dd..acea1bd3f0 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -4,10 +4,6 @@ set -o pipefail source scripts/vars.env -echo "tmp delete conflicting subnet" - -gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=${GKE_CLUSTER_REGION} --quiet || true - skip_gke_master_control_node_access="${1:-false}" # Remove VM IP from GKE master control node access, if required diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 92285eab68..7c0d4e0781 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -21,7 +21,7 @@ if [ "${IPV6_ENABLED}" = "true" ]; then --stack-type=IPV4_IPV6 \ --ipv6-access-type=INTERNAL \ --region=${GKE_CLUSTER_REGION} \ - --range=2001:db8::/29 + --range=10.120.0.0/14 NETWORK=${RESOURCE_NAME} NETWORK_TIER="ipv6-network-tier=PREMIUM" From 624beedc9b3b046cddaa015975cfa7f65e16582a Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 12 Sep 2025 10:21:55 +0100 Subject: [PATCH 060/101] Delete forwarding rules --- .github/workflows/nfr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 4bb0c1dcc0..04065614a8 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -149,6 +149,7 @@ jobs: working-directory: ./tests run: | echo "tmp delete conflicting subnet" + gcloud compute forwarding-rules delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true gcloud compute addresses delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=us-west1 --quiet || true rm -rf scripts/vars.env From 745123b9c18d28a22d4f80e99375ef2981054854 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 12 Sep 2025 13:07:37 +0100 Subject: [PATCH 061/101] Change region to us-west1-b and set number of nodes to 1 --- .github/workflows/nfr.yml | 54 +++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 04065614a8..7953f3bbca 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -120,7 +120,7 @@ jobs: echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env - echo "GKE_CLUSTER_ZONE=us-west1-c" >> vars.env + echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env echo "GKE_CLUSTER_REGION=us-west1" >> vars.env echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env @@ -132,7 +132,7 @@ jobs: echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env echo "GINKGO_LABEL=" >> vars.env echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env - echo "GKE_NUM_NODES=12" >> vars.env + echo "GKE_NUM_NODES=1" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env if [ "${{ needs.vars.outputs.ipv6_enabled }}" = "true" ]; then @@ -145,26 +145,26 @@ jobs: PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: echo "${PLUS_LICENSE}" > license.jwt - - name: TMP Cleanup conflicting resources - working-directory: ./tests - run: | - echo "tmp delete conflicting subnet" - gcloud compute forwarding-rules delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true - gcloud compute addresses delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true - gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=us-west1 --quiet || true - rm -rf scripts/vars.env - - # - name: Create GKE cluster + # - name: TMP Cleanup conflicting resources # working-directory: ./tests - # run: make create-gke-cluster CI=true + # run: | + # echo "tmp delete conflicting subnet" + # gcloud compute forwarding-rules delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true + # gcloud compute addresses delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true + # gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=us-west1 --quiet || true + # rm -rf scripts/vars.env - # - name: Create and setup VM - # working-directory: ./tests - # run: make create-and-setup-vm + - name: Create GKE cluster + working-directory: ./tests + run: make create-gke-cluster CI=true - # - name: Create and setup Router - # working-directory: ./tests - # run: make create-gke-router || true + - name: Create and setup VM + working-directory: ./tests + run: make create-and-setup-vm + + - name: Create and setup Router + working-directory: ./tests + run: make create-gke-router || true # - name: Run Tests # working-directory: ./tests @@ -181,14 +181,14 @@ jobs: # name: results-${{ matrix.type }} # path: tests/results/**/*-${{ matrix.type }}.* - # - name: Cleanup - # working-directory: ./tests - # if: always() - # run: | - # # bash scripts/cleanup-vm.sh true - # # bash scripts/cleanup-router.sh true - # # make delete-gke-cluster - # # rm -rf scripts/vars.env + - name: Cleanup + working-directory: ./tests + if: always() + run: | + bash scripts/cleanup-vm.sh true + bash scripts/cleanup-router.sh true + make delete-gke-cluster + rm -rf scripts/vars.env pr-results: name: Open PR with results From b30d389c45f4cf0ee1ac82c56468ca3512f36dc2 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Fri, 12 Sep 2025 14:18:54 +0100 Subject: [PATCH 062/101] Update delete commands --- tests/scripts/cleanup-vm.sh | 4 ++-- tests/scripts/create-and-setup-gcp-vm.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index acea1bd3f0..7f9766f185 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -19,6 +19,6 @@ gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE # Clean up the custom network and subnet if IPv6 was enabled if [ "${IPV6_ENABLED}" = "true" ]; then - gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet || true - gcloud compute networks delete ${RESOURCE_NAME} --quiet || true + gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet + gcloud compute networks delete ${RESOURCE_NAME} --quiet fi diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 7c0d4e0781..434e207e17 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -47,7 +47,6 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then - echo "IPv6 is not enabled, fetching the external IPv4 address" EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ --format='value(networkInterfaces[0].accessConfigs[0].natIP)') From 0248e0dd7edccf5894bf9245f6ee5a450d29a8a5 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 08:04:39 +0100 Subject: [PATCH 063/101] Add additional comments to scripts --- tests/scripts/create-and-setup-gcp-vm.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 434e207e17..6e40ac6a2d 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -13,9 +13,10 @@ NETWORK_TIER="network-tier=PREMIUM" source scripts/vars.env # Create custom network and subnet if IPv6 is enabled +# For IPv6, we create a dual-stack subnet with internal IPv6 addresses and external IPv4 addresses if [ "${IPV6_ENABLED}" = "true" ]; then echo "Creating IPv6 Network interface for the GKE cluster" - gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --bgp-routing-mode=regional --mtu=1460 --quiet + gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --quiet # The --subnet-mode=custom flag allows us to create custom subnets gcloud compute networks subnets create ${RESOURCE_NAME} \ --network=${RESOURCE_NAME} \ --stack-type=IPV4_IPV6 \ From 3dba2fb9f4147ce2c0ba82ff4ff4c169eaf15ff6 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 08:17:49 +0100 Subject: [PATCH 064/101] Add || true --- tests/scripts/cleanup-router.sh | 4 ++-- tests/scripts/cleanup-vm.sh | 8 ++++---- tests/scripts/create-and-setup-gcp-vm.sh | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/scripts/cleanup-router.sh b/tests/scripts/cleanup-router.sh index ee3ea524b2..41152397e7 100755 --- a/tests/scripts/cleanup-router.sh +++ b/tests/scripts/cleanup-router.sh @@ -4,5 +4,5 @@ set -o pipefail source scripts/vars.env -gcloud compute routers nats delete "${RESOURCE_NAME}" --quiet --router "${RESOURCE_NAME}" --router-region "${GKE_CLUSTER_REGION}" -gcloud compute routers delete "${RESOURCE_NAME}" --quiet --region "${GKE_CLUSTER_REGION}" +gcloud compute routers nats delete "${RESOURCE_NAME}" --quiet --router "${RESOURCE_NAME}" --router-region "${GKE_CLUSTER_REGION}" || true +gcloud compute routers delete "${RESOURCE_NAME}" --quiet --region "${GKE_CLUSTER_REGION}" || true diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index 7f9766f185..8b593055a2 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -13,12 +13,12 @@ if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ] && [ "${skip_gke_master_control_nod gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${CURRENT_AUTH_NETWORK}" fi -gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" +gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" || true -gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" +gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" || true # Clean up the custom network and subnet if IPv6 was enabled if [ "${IPV6_ENABLED}" = "true" ]; then - gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet - gcloud compute networks delete ${RESOURCE_NAME} --quiet + gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet || true + gcloud compute networks delete ${RESOURCE_NAME} --quiet || true fi diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 6e40ac6a2d..1aa69ffb39 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -17,6 +17,7 @@ source scripts/vars.env if [ "${IPV6_ENABLED}" = "true" ]; then echo "Creating IPv6 Network interface for the GKE cluster" gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --quiet # The --subnet-mode=custom flag allows us to create custom subnets + gcloud compute networks subnets create ${RESOURCE_NAME} \ --network=${RESOURCE_NAME} \ --stack-type=IPV4_IPV6 \ From 95463a18eb4157a6ecc02db5d61f8fab30d8f1a0 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 08:37:51 +0100 Subject: [PATCH 065/101] Use pre-assigned dual stack network `v4-v6` --- tests/scripts/cleanup-vm.sh | 8 ++++---- tests/scripts/create-and-setup-gcp-vm.sh | 14 +++----------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index 8b593055a2..59b3eae661 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -18,7 +18,7 @@ gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJ gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" || true # Clean up the custom network and subnet if IPv6 was enabled -if [ "${IPV6_ENABLED}" = "true" ]; then - gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet || true - gcloud compute networks delete ${RESOURCE_NAME} --quiet || true -fi +# if [ "${IPV6_ENABLED}" = "true" ]; then +# gcloud compute networks subnets delete ${RESOURCE_NAME} --region=${GKE_CLUSTER_REGION} --quiet || true +# gcloud compute networks delete ${RESOURCE_NAME} --quiet || true +# fi diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index 1aa69ffb39..ec01ce651f 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -15,17 +15,9 @@ source scripts/vars.env # Create custom network and subnet if IPv6 is enabled # For IPv6, we create a dual-stack subnet with internal IPv6 addresses and external IPv4 addresses if [ "${IPV6_ENABLED}" = "true" ]; then - echo "Creating IPv6 Network interface for the GKE cluster" - gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --quiet # The --subnet-mode=custom flag allows us to create custom subnets - - gcloud compute networks subnets create ${RESOURCE_NAME} \ - --network=${RESOURCE_NAME} \ - --stack-type=IPV4_IPV6 \ - --ipv6-access-type=INTERNAL \ - --region=${GKE_CLUSTER_REGION} \ - --range=10.120.0.0/14 - - NETWORK=${RESOURCE_NAME} + echo "Using IPv6 Network interface for the GKE cluster" + + NETWORK="v4-v6" NETWORK_TIER="ipv6-network-tier=PREMIUM" STACK_TYPE="IPV4_IPV6" fi From d4082d5d498427bef85d18af69fcdd0319b7f15a Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 08:51:49 +0100 Subject: [PATCH 066/101] Change region to europe-west1 to allow use of v4-v6 subnet --- .github/workflows/nfr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 7953f3bbca..69f966d7e1 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -120,8 +120,8 @@ jobs: echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env - echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env - echo "GKE_CLUSTER_REGION=us-west1" >> vars.env + echo "GKE_CLUSTER_ZONE=europe-west1-b" >> vars.env + echo "GKE_CLUSTER_REGION=europe-west1" >> vars.env echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env From 96d62736b7c80829fd9aaf31735ff51b375d275d Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 09:11:37 +0100 Subject: [PATCH 067/101] Use custom network in us-west1 with `--enable-ula-internal-ipv6` for the VPC --- .github/workflows/nfr.yml | 4 ++-- tests/scripts/create-and-setup-gcp-vm.sh | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 69f966d7e1..7953f3bbca 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -120,8 +120,8 @@ jobs: echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env - echo "GKE_CLUSTER_ZONE=europe-west1-b" >> vars.env - echo "GKE_CLUSTER_REGION=europe-west1" >> vars.env + echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env + echo "GKE_CLUSTER_REGION=us-west1" >> vars.env echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index ec01ce651f..bc28115009 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -16,8 +16,16 @@ source scripts/vars.env # For IPv6, we create a dual-stack subnet with internal IPv6 addresses and external IPv4 addresses if [ "${IPV6_ENABLED}" = "true" ]; then echo "Using IPv6 Network interface for the GKE cluster" + # gcloud compute networks create ${RESOURCE_NAME} --subnet-mode=custom --enable-ula-internal-ipv6 --quiet # The --subnet-mode=custom flag allows us to create custom subnets - NETWORK="v4-v6" + # gcloud compute networks subnets create ${RESOURCE_NAME} \ + # --network=${RESOURCE_NAME} \ + # --stack-type=IPV4_IPV6 \ + # --ipv6-access-type=INTERNAL \ + # --region=${GKE_CLUSTER_REGION} \ + # --range=10.120.0.0/14 + + NETWORK="us-ipv4-ipv6" NETWORK_TIER="ipv6-network-tier=PREMIUM" STACK_TYPE="IPV4_IPV6" fi From a28c60a5b7a2674bbb0dc813b75d500779086480 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 11:22:54 +0100 Subject: [PATCH 068/101] Remove ipv6-network-tier --- tests/scripts/create-and-setup-gcp-vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index bc28115009..d30fe7e53a 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -26,7 +26,7 @@ if [ "${IPV6_ENABLED}" = "true" ]; then # --range=10.120.0.0/14 NETWORK="us-ipv4-ipv6" - NETWORK_TIER="ipv6-network-tier=PREMIUM" + # NETWORK_TIER="network-tier=PREMIUM" STACK_TYPE="IPV4_IPV6" fi From 4d667af9a469196722f0a7730fbfb94e9686cd0f Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 11:35:43 +0100 Subject: [PATCH 069/101] Re-enable test run --- .github/workflows/nfr.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 7953f3bbca..7f033c09db 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -166,20 +166,20 @@ jobs: working-directory: ./tests run: make create-gke-router || true - # - name: Run Tests - # working-directory: ./tests - # run: | - # if ${{ needs.vars.outputs.test_label != 'all' }}; then - # sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true; - # else - # make nfr-test CI=true; - # fi - - # - name: Upload Artifacts - # uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - # with: - # name: results-${{ matrix.type }} - # path: tests/results/**/*-${{ matrix.type }}.* + - name: Run Tests + working-directory: ./tests + run: | + if ${{ needs.vars.outputs.test_label != 'all' }}; then + sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true; + else + make nfr-test CI=true; + fi + + - name: Upload Artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: results-${{ matrix.type }} + path: tests/results/**/*-${{ matrix.type }}.* - name: Cleanup working-directory: ./tests From 9accc8bb01d12c01e1aa740f643a3e7bbe880552 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 14:43:57 +0100 Subject: [PATCH 070/101] Add nfr-ipv6-only workflow to test with sample deployment manifests --- .github/workflows/nfr-ipv6-only.yml | 292 +++++++++++++++++++++++ .github/workflows/nfr.yml | 9 - tests/scripts/create-and-setup-gcp-vm.sh | 2 +- 3 files changed, 293 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/nfr-ipv6-only.yml diff --git a/.github/workflows/nfr-ipv6-only.yml b/.github/workflows/nfr-ipv6-only.yml new file mode 100644 index 0000000000..0b62c8a649 --- /dev/null +++ b/.github/workflows/nfr-ipv6-only.yml @@ -0,0 +1,292 @@ +name: IPv6-Only Testing + +on: + workflow_dispatch: + inputs: + version: + description: Version of NGF under test + required: true + default: edge + image_tag: + description: Tag of the NGF and NGINX Docker images + required: true + default: edge + type: + description: Type of NGINX image to test + required: true + default: both + type: choice + options: [oss, plus, both] + schedule: + - cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC +defaults: + run: + shell: bash + +env: + PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} + +permissions: + contents: read + +jobs: + ipv6-only-tests: + name: Run IPv6-Only Tests + runs-on: ubuntu-24.04 + if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} + env: + DOCKER_BUILD_SUMMARY: false + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + ### Authenticate to GCP and set up gcloud, kubectl, and Docker + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + token_format: access_token + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + + - name: Login to GAR + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + with: + registry: us-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + install_components: kubectl + ### + + - name: Setup dotenv file + working-directory: ./tests/scripts + run: | + echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env + echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env + echo "PREFIX=ghcr.io/nginx/nginx-gateway-fabric" >> vars.env + echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env + echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env + echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env + echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env + echo "GKE_CLUSTER_REGION=us-west1" >> vars.env + echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env + echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env + echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env + echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env + echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env + echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env + echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env + echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env + echo "GINKGO_LABEL=" >> vars.env + echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env + echo "GKE_NUM_NODES=1" >> vars.env + echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env + echo "IPV6_ENABLED=true" >> vars.env + echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env + - name: Setup license file for plus + if: matrix.type == 'plus' + env: + PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} + run: echo "${PLUS_LICENSE}" > license.jwt + + - name: Create GKE cluster + working-directory: ./tests + run: make create-gke-cluster CI=true + + - name: Create and setup VM + working-directory: ./tests + run: make create-and-setup-vm + + - name: Create and setup Router + working-directory: ./tests + run: make create-gke-router || true + + - name: Setup Golang Environment + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: stable + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + # - name: Build binary + # uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + # with: + # version: v2.11.2 # renovate: datasource=github-tags depName=goreleaser/goreleaser + # args: build --single-target --snapshot --clean + # env: + # TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + # TELEMETRY_ENDPOINT_INSECURE: "true" + + # - name: Build NGF Docker Image + # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + # with: + # file: build/Dockerfile + # tags: ${{ steps.ngf-meta.outputs.tags }} + # context: "." + # load: true + # cache-from: type=gha,scope=ngf-ipv6 + # pull: true + # target: goreleaser + + # - name: Build NGINX Docker Image + # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + # with: + # file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + # tags: ${{ steps.nginx-meta.outputs.tags }} + # context: "." + # load: true + # cache-from: type=gha,scope=${{ inputs.image }}-ipv6 + # pull: true + # build-args: | + # NJS_DIR=internal/controller/nginx/modules/src + # NGINX_CONF_DIR=internal/controller/nginx/conf + # BUILD_AGENT=gha + + # - name: Deploy IPv6-Only Kubernetes + # id: k8s + # run: | + # # Enable IPv6 and container network options + # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 + # sudo sysctl -w net.ipv6.conf.all.forwarding=1 + + # # Create IPv6-only kind cluster + # kind create cluster \ + # --name ${{ github.run_id }}-ipv6 \ + # --image=kindest/node:${{ inputs.k8s-version }} \ + # --config=config/cluster/kind-cluster-ipv6-only.yaml + + # # Load images into the cluster + # kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 + + - name: Install NGF with IPv6 Configuration + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + + # Install with IPv6-specific configuration + HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ + make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + working-directory: ./tests + + - name: Deploy Test Applications + run: | + kubectl apply -f tests/manifests/ipv6-test-app.yaml + + - name: Wait for NGF and Applications to be Ready + run: | + echo "Waiting for NGF to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway + + echo "Waiting for test applications to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + + - name: Deploy IPv6 Test Client + run: | + kubectl apply -f tests/manifests/test-client-ipv6.yaml + kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + + - name: Get NGF IPv6 Address + id: ngf-address + run: | + # Get the NGF service IPv6 address + NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') + echo "NGF IPv6 Address: $NGF_IPV6" + echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT + + - name: Run IPv6 Connectivity Tests + run: | + echo "=== Running IPv6-Only Tests ===" + + # Test 1: Basic connectivity test using test client pod + echo "Test 1: Basic IPv6 connectivity" + kubectl exec ipv6-test-client -- curl --version + kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local + + # Test 2: Test NGF service directly via IPv6 + echo "Test 2: NGF Service IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" + + # Test 3: Test via service DNS + echo "Test 3: Service DNS IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" + + - name: Validate IPv6-Only Configuration + run: | + echo "=== Validating IPv6-Only Configuration ===" + + # Check NGF configuration + echo "NGF Pod IPv6 addresses:" + kubectl get pods -n nginx-gateway -o wide + + echo "NGF Service configuration:" + kubectl get service nginx-gateway -n nginx-gateway -o yaml + + echo "Gateway and HTTPRoute status:" + kubectl get gateway,httproute -A -o wide + + echo "Test application service configuration:" + kubectl get service test-app-ipv6-service -o yaml + + - name: Collect Logs + if: always() + run: | + echo "=== Collecting logs for debugging ===" + echo "NGF Controller logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true + + echo "NGINX logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true + + echo "Test client logs:" + kubectl logs ipv6-test-client --tail=100 || true + + echo "Cluster events:" + kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true + + - name: Cleanup + working-directory: ./tests + if: always() + run: | + bash scripts/cleanup-vm.sh true + bash scripts/cleanup-router.sh true + make delete-gke-cluster + rm -rf scripts/vars.env \ No newline at end of file diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 7f033c09db..513d7a5bbc 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -145,15 +145,6 @@ jobs: PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }} run: echo "${PLUS_LICENSE}" > license.jwt - # - name: TMP Cleanup conflicting resources - # working-directory: ./tests - # run: | - # echo "tmp delete conflicting subnet" - # gcloud compute forwarding-rules delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true - # gcloud compute addresses delete gk3-nfr-tests-17637219369-oss-b6387c3a-d2289ac6-pe --region=us-west1 --project=${{ secrets.GCP_PROJECT_ID }} --quiet || true - # gcloud compute networks subnets delete gke-nfr-tests-17637219369-oss-acf63488-pe-subnet --region=us-west1 --quiet || true - # rm -rf scripts/vars.env - - name: Create GKE cluster working-directory: ./tests run: make create-gke-cluster CI=true diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index d30fe7e53a..af592864da 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -26,7 +26,7 @@ if [ "${IPV6_ENABLED}" = "true" ]; then # --range=10.120.0.0/14 NETWORK="us-ipv4-ipv6" - # NETWORK_TIER="network-tier=PREMIUM" + # NETWORK_TIER="ipv6-network-tier=PREMIUM" # This will work only if STACK_TYPE is IPV6_ONLY STACK_TYPE="IPV4_IPV6" fi From d34085df054a552da2a73773ebe464eb90f94c82 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 14:55:05 +0100 Subject: [PATCH 071/101] Backup NFR test yml --- .github/workflows/nfr.yml | 291 ++++++++++++++++++++++++-------------- 1 file changed, 182 insertions(+), 109 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 513d7a5bbc..558ccb4e14 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -3,20 +3,6 @@ name: Non Functional Testing on: workflow_dispatch: inputs: - test_label: - description: NFR test to run. Choose between a specific test or all tests - required: true - default: all - type: choice - options: - [ - performance, - upgrade, - scale, - zero-downtime-scale, - reconfiguration, - all, - ] version: description: Version of NGF under test required: true @@ -31,14 +17,8 @@ on: default: both type: choice options: [oss, plus, both] - ipv6_enabled: - description: Whether to run the tests in an IPv6 enabled environment - required: false - default: false - type: boolean schedule: - cron: "0 16 1,15 * *" # Run on the 1st and 15th of every month at 16:00 UTC - defaults: run: shell: bash @@ -46,50 +26,21 @@ defaults: env: PLUS_USAGE_ENDPOINT: ${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }} -concurrency: - group: ${{ github.ref_name }}-nfr - cancel-in-progress: true - permissions: contents: read jobs: - vars: - name: Set up vars + ipv6-only-tests: + name: Run IPv6-Only Tests runs-on: ubuntu-24.04 - outputs: - test_label: ${{ github.event.inputs.test_label || 'all' }} - version: ${{ github.event.inputs.version || 'edge' }} - image_tag: ${{ github.event.inputs.image_tag || 'edge' }} - ipv6_enabled: ${{ github.event.inputs.ipv6_enabled || 'false' }} - types: ${{ steps.var.outputs.types }} - permissions: - contents: read - steps: - - name: Set vars - id: var - run: | - if ${{ github.event.inputs.type == 'both' || github.event_name == 'schedule' }}; then - echo 'types=["oss","plus"]' >> $GITHUB_OUTPUT - else - echo 'types=["${{ github.event.inputs.type }}"]' >> $GITHUB_OUTPUT - fi - - setup-and-run-tests: - name: Setup and Run NFR Tests - runs-on: ubuntu-24.04 - permissions: - contents: read - id-token: write # needed for authenticating to GCP - needs: vars - strategy: - fail-fast: false - matrix: - type: ${{ fromJson(needs.vars.outputs.types) }} + if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} + env: + DOCKER_BUILD_SUMMARY: false steps: - name: Checkout Repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + ### Authenticate to GCP and set up gcloud, kubectl, and Docker - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 @@ -110,6 +61,7 @@ jobs: with: project_id: ${{ secrets.GCP_PROJECT_ID }} install_components: kubectl + ### - name: Setup dotenv file working-directory: ./tests/scripts @@ -134,11 +86,8 @@ jobs: echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env echo "GKE_NUM_NODES=1" >> vars.env echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env - echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - if [ "${{ needs.vars.outputs.ipv6_enabled }}" = "true" ]; then echo "IPV6_ENABLED=true" >> vars.env - echo "HELM_PARAMETERS=\"--set nginx.config.ipFamily=ipv6\"" >> vars.env - fi + echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env - name: Setup license file for plus if: matrix.type == 'plus' env: @@ -157,20 +106,181 @@ jobs: working-directory: ./tests run: make create-gke-router || true - - name: Run Tests + - name: Setup Golang Environment + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version: stable + + - name: Set GOPATH + run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + - name: Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + # - name: Build binary + # uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 + # with: + # version: v2.11.2 # renovate: datasource=github-tags depName=goreleaser/goreleaser + # args: build --single-target --snapshot --clean + # env: + # TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 + # TELEMETRY_ENDPOINT_INSECURE: "true" + + # - name: Build NGF Docker Image + # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + # with: + # file: build/Dockerfile + # tags: ${{ steps.ngf-meta.outputs.tags }} + # context: "." + # load: true + # cache-from: type=gha,scope=ngf-ipv6 + # pull: true + # target: goreleaser + + # - name: Build NGINX Docker Image + # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + # with: + # file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + # tags: ${{ steps.nginx-meta.outputs.tags }} + # context: "." + # load: true + # cache-from: type=gha,scope=${{ inputs.image }}-ipv6 + # pull: true + # build-args: | + # NJS_DIR=internal/controller/nginx/modules/src + # NGINX_CONF_DIR=internal/controller/nginx/conf + # BUILD_AGENT=gha + + # - name: Deploy IPv6-Only Kubernetes + # id: k8s + # run: | + # # Enable IPv6 and container network options + # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 + # sudo sysctl -w net.ipv6.conf.all.forwarding=1 + + # # Create IPv6-only kind cluster + # kind create cluster \ + # --name ${{ github.run_id }}-ipv6 \ + # --image=kindest/node:${{ inputs.k8s-version }} \ + # --config=config/cluster/kind-cluster-ipv6-only.yaml + + # # Load images into the cluster + # kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 + + - name: Install NGF with IPv6 Configuration + run: | + ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric + ngf_tag=${{ steps.ngf-meta.outputs.version }} + + # Install with IPv6-specific configuration + HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ + make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} working-directory: ./tests + + - name: Deploy Test Applications run: | - if ${{ needs.vars.outputs.test_label != 'all' }}; then - sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true; - else - make nfr-test CI=true; - fi - - - name: Upload Artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: results-${{ matrix.type }} - path: tests/results/**/*-${{ matrix.type }}.* + kubectl apply -f tests/manifests/ipv6-test-app.yaml + + - name: Wait for NGF and Applications to be Ready + run: | + echo "Waiting for NGF to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway + + echo "Waiting for test applications to be ready..." + kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + + - name: Deploy IPv6 Test Client + run: | + kubectl apply -f tests/manifests/test-client-ipv6.yaml + kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + + - name: Get NGF IPv6 Address + id: ngf-address + run: | + # Get the NGF service IPv6 address + NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') + echo "NGF IPv6 Address: $NGF_IPV6" + echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT + + - name: Run IPv6 Connectivity Tests + run: | + echo "=== Running IPv6-Only Tests ===" + + # Test 1: Basic connectivity test using test client pod + echo "Test 1: Basic IPv6 connectivity" + kubectl exec ipv6-test-client -- curl --version + kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local + + # Test 2: Test NGF service directly via IPv6 + echo "Test 2: NGF Service IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" + + # Test 3: Test via service DNS + echo "Test 3: Service DNS IPv6 connectivity" + kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ + -H "Host: ipv6-test.example.com" \ + "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" + + - name: Validate IPv6-Only Configuration + run: | + echo "=== Validating IPv6-Only Configuration ===" + + # Check NGF configuration + echo "NGF Pod IPv6 addresses:" + kubectl get pods -n nginx-gateway -o wide + + echo "NGF Service configuration:" + kubectl get service nginx-gateway -n nginx-gateway -o yaml + + echo "Gateway and HTTPRoute status:" + kubectl get gateway,httproute -A -o wide + + echo "Test application service configuration:" + kubectl get service test-app-ipv6-service -o yaml + + - name: Collect Logs + if: always() + run: | + echo "=== Collecting logs for debugging ===" + echo "NGF Controller logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true + + echo "NGINX logs:" + kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true + + echo "Test client logs:" + kubectl logs ipv6-test-client --tail=100 || true + + echo "Cluster events:" + kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true - name: Cleanup working-directory: ./tests @@ -179,41 +289,4 @@ jobs: bash scripts/cleanup-vm.sh true bash scripts/cleanup-router.sh true make delete-gke-cluster - rm -rf scripts/vars.env - - pr-results: - name: Open PR with results - runs-on: ubuntu-24.04 - permissions: - contents: write # needed for opening PR with the results files - pull-requests: write # needed for opening PR with the results files - needs: [vars, setup-and-run-tests] - steps: - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Download Artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 - with: - path: tests/results/ - merge-multiple: true - - - name: Open a PR with the results - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: NFR Test Results for NGF version ${{ needs.vars.outputs.version }} - author: nginx-bot - committer: nginx-bot - branch: tests/nfr-tests-${{ needs.vars.outputs.version }} - delete-branch: true - title: NFR Test Results for NGF version ${{ needs.vars.outputs.version }} - add-paths: | - tests/results/ - body: | - Update with NFR test results for NGF version ${{ needs.vars.outputs.version }} ${{ needs.vars.outputs.types }} - - Auto-generated by the NFR tests workflow run ${{ github.run_id }} - - Tests ran using Docker image tag ${{ needs.vars.outputs.image_tag }} - - ${{ needs.vars.outputs.test_label }} test(s) ran - assignees: ${{ github.actor }} - draft: ${{ github.event_name != 'schedule' }} + rm -rf scripts/vars.env \ No newline at end of file From 04d62c240a5c2142db931c2b1b026450b796a340 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 15:00:33 +0100 Subject: [PATCH 072/101] Add token for GKE auth --- .github/workflows/nfr-ipv6-only.yml | 25 ++++++++++++++++++++++++- .github/workflows/nfr.yml | 25 ++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nfr-ipv6-only.yml b/.github/workflows/nfr-ipv6-only.yml index 0b62c8a649..5c285f6d0d 100644 --- a/.github/workflows/nfr-ipv6-only.yml +++ b/.github/workflows/nfr-ipv6-only.yml @@ -30,9 +30,32 @@ permissions: contents: read jobs: - ipv6-only-tests: + vars: + name: Set up vars + runs-on: ubuntu-24.04 + outputs: + version: ${{ github.event.inputs.version || 'edge' }} + image_tag: ${{ github.event.inputs.image_tag || 'edge' }} + types: ${{ steps.var.outputs.types }} + permissions: + contents: read + steps: + - name: Set vars + id: var + run: | + if ${{ github.event.inputs.type == 'both' || github.event_name == 'schedule' }}; then + echo 'types=["oss","plus"]' >> $GITHUB_OUTPUT + else + echo 'types=["${{ github.event.inputs.type }}"]' >> $GITHUB_OUTPUT + fi + + setup-and-run-tests: name: Run IPv6-Only Tests runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write # needed for authenticating to GCP + needs: vars if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} env: DOCKER_BUILD_SUMMARY: false diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 558ccb4e14..193e197703 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -30,9 +30,32 @@ permissions: contents: read jobs: - ipv6-only-tests: + vars: + name: Set up vars + runs-on: ubuntu-24.04 + outputs: + version: ${{ github.event.inputs.version || 'edge' }} + image_tag: ${{ github.event.inputs.image_tag || 'edge' }} + types: ${{ steps.var.outputs.types }} + permissions: + contents: read + steps: + - name: Set vars + id: var + run: | + if ${{ github.event.inputs.type == 'both' || github.event_name == 'schedule' }}; then + echo 'types=["oss","plus"]' >> $GITHUB_OUTPUT + else + echo 'types=["${{ github.event.inputs.type }}"]' >> $GITHUB_OUTPUT + fi + + setup-and-run-tests: name: Run IPv6-Only Tests runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write # needed for authenticating to GCP + needs: vars if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} env: DOCKER_BUILD_SUMMARY: false From fff6e3d968a2edf43712c13d2a2cd21e81238ef7 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 15:09:05 +0100 Subject: [PATCH 073/101] Add strategy --- .github/workflows/nfr-ipv6-only.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nfr-ipv6-only.yml b/.github/workflows/nfr-ipv6-only.yml index 5c285f6d0d..52dacfda44 100644 --- a/.github/workflows/nfr-ipv6-only.yml +++ b/.github/workflows/nfr-ipv6-only.yml @@ -56,6 +56,10 @@ jobs: contents: read id-token: write # needed for authenticating to GCP needs: vars + strategy: + fail-fast: false + matrix: + type: ${{ fromJson(needs.vars.outputs.types) }} if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} env: DOCKER_BUILD_SUMMARY: false From 4783f4ae632f5700f227222d85d3eadc9db08f01 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 15:09:38 +0100 Subject: [PATCH 074/101] Add strategy to nfr.yml --- .github/workflows/nfr.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 193e197703..2e27a6333d 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -56,6 +56,10 @@ jobs: contents: read id-token: write # needed for authenticating to GCP needs: vars + strategy: + fail-fast: false + matrix: + type: ${{ fromJson(needs.vars.outputs.types) }} if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} env: DOCKER_BUILD_SUMMARY: false From 40b1b3c2709d8e30a3bdec40093e692b8fe9f453 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 16:26:19 +0100 Subject: [PATCH 075/101] Update helm install command --- .github/workflows/nfr-ipv6-only.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nfr-ipv6-only.yml b/.github/workflows/nfr-ipv6-only.yml index 52dacfda44..87e14c9e8c 100644 --- a/.github/workflows/nfr-ipv6-only.yml +++ b/.github/workflows/nfr-ipv6-only.yml @@ -226,8 +226,9 @@ jobs: ngf_tag=${{ steps.ngf-meta.outputs.version }} # Install with IPv6-specific configuration - HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ - make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + # HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ + # make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + helm upgrade --install ngf oci://${ngf_prefix}/${ngf_tag} --create-namespace -n nginx-gateway --set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP working-directory: ./tests - name: Deploy Test Applications From 390713917062016fe13c043a2859f0dbc033e5ab Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 17:01:07 +0100 Subject: [PATCH 076/101] Add script to run IPv6 tests in GKE cluster --- .github/workflows/nfr-ipv6-only.yml | 1 - .github/workflows/nfr.yml | 170 +--------------------------- tests/scripts/run-tests-gcp-vm.sh | 85 ++++++++------ 3 files changed, 51 insertions(+), 205 deletions(-) diff --git a/.github/workflows/nfr-ipv6-only.yml b/.github/workflows/nfr-ipv6-only.yml index 87e14c9e8c..5856fa0e91 100644 --- a/.github/workflows/nfr-ipv6-only.yml +++ b/.github/workflows/nfr-ipv6-only.yml @@ -228,7 +228,6 @@ jobs: # Install with IPv6-specific configuration # HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ # make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} - helm upgrade --install ngf oci://${ngf_prefix}/${ngf_tag} --create-namespace -n nginx-gateway --set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP working-directory: ./tests - name: Deploy Test Applications diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 2e27a6333d..27b6176ceb 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -60,7 +60,6 @@ jobs: fail-fast: false matrix: type: ${{ fromJson(needs.vars.outputs.types) }} - if: ${{ !github.event.pull_request.head.repo.fork || inputs.image != 'plus' }} env: DOCKER_BUILD_SUMMARY: false steps: @@ -141,174 +140,11 @@ jobs: - name: Set GOPATH run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV - - name: Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - - - name: NGF Docker meta - id: ngf-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric - tags: | - type=semver,pattern={{version}} - type=schedule - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: NGINX Docker meta - id: nginx-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} - tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - # - name: Build binary - # uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 - # with: - # version: v2.11.2 # renovate: datasource=github-tags depName=goreleaser/goreleaser - # args: build --single-target --snapshot --clean - # env: - # TELEMETRY_ENDPOINT: otel-collector-opentelemetry-collector.collector.svc.cluster.local:4317 - # TELEMETRY_ENDPOINT_INSECURE: "true" - - # - name: Build NGF Docker Image - # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - # with: - # file: build/Dockerfile - # tags: ${{ steps.ngf-meta.outputs.tags }} - # context: "." - # load: true - # cache-from: type=gha,scope=ngf-ipv6 - # pull: true - # target: goreleaser - - # - name: Build NGINX Docker Image - # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 - # with: - # file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} - # tags: ${{ steps.nginx-meta.outputs.tags }} - # context: "." - # load: true - # cache-from: type=gha,scope=${{ inputs.image }}-ipv6 - # pull: true - # build-args: | - # NJS_DIR=internal/controller/nginx/modules/src - # NGINX_CONF_DIR=internal/controller/nginx/conf - # BUILD_AGENT=gha - - # - name: Deploy IPv6-Only Kubernetes - # id: k8s - # run: | - # # Enable IPv6 and container network options - # sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 - # sudo sysctl -w net.ipv6.conf.all.forwarding=1 - - # # Create IPv6-only kind cluster - # kind create cluster \ - # --name ${{ github.run_id }}-ipv6 \ - # --image=kindest/node:${{ inputs.k8s-version }} \ - # --config=config/cluster/kind-cluster-ipv6-only.yaml - - # # Load images into the cluster - # kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}-ipv6 - - - name: Install NGF with IPv6 Configuration + - name: Install and test NGF with IPv6 Configuration run: | - ngf_prefix=ghcr.io/nginx/nginx-gateway-fabric - ngf_tag=${{ steps.ngf-meta.outputs.version }} - - # Install with IPv6-specific configuration - HELM_PARAMETERS="--set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP" \ - make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + ./scripts/run-tests-gcp-vm.sh working-directory: ./tests - - name: Deploy Test Applications - run: | - kubectl apply -f tests/manifests/ipv6-test-app.yaml - - - name: Wait for NGF and Applications to be Ready - run: | - echo "Waiting for NGF to be ready..." - kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway - - echo "Waiting for test applications to be ready..." - kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 - - - name: Deploy IPv6 Test Client - run: | - kubectl apply -f tests/manifests/test-client-ipv6.yaml - kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client - - - name: Get NGF IPv6 Address - id: ngf-address - run: | - # Get the NGF service IPv6 address - NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') - echo "NGF IPv6 Address: $NGF_IPV6" - echo "ngf_ipv6=$NGF_IPV6" >> $GITHUB_OUTPUT - - - name: Run IPv6 Connectivity Tests - run: | - echo "=== Running IPv6-Only Tests ===" - - # Test 1: Basic connectivity test using test client pod - echo "Test 1: Basic IPv6 connectivity" - kubectl exec ipv6-test-client -- curl --version - kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local - - # Test 2: Test NGF service directly via IPv6 - echo "Test 2: NGF Service IPv6 connectivity" - kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ - -H "Host: ipv6-test.example.com" \ - "http://[${{ steps.ngf-address.outputs.ngf_ipv6 }}]:80/" || echo "Direct NGF test failed" - - # Test 3: Test via service DNS - echo "Test 3: Service DNS IPv6 connectivity" - kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ - -H "Host: ipv6-test.example.com" \ - "http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" - - - name: Validate IPv6-Only Configuration - run: | - echo "=== Validating IPv6-Only Configuration ===" - - # Check NGF configuration - echo "NGF Pod IPv6 addresses:" - kubectl get pods -n nginx-gateway -o wide - - echo "NGF Service configuration:" - kubectl get service nginx-gateway -n nginx-gateway -o yaml - - echo "Gateway and HTTPRoute status:" - kubectl get gateway,httproute -A -o wide - - echo "Test application service configuration:" - kubectl get service test-app-ipv6-service -o yaml - - - name: Collect Logs - if: always() - run: | - echo "=== Collecting logs for debugging ===" - echo "NGF Controller logs:" - kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true - - echo "NGINX logs:" - kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true - - echo "Test client logs:" - kubectl logs ipv6-test-client --tail=100 || true - - echo "Cluster events:" - kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true - - name: Cleanup working-directory: ./tests if: always() @@ -316,4 +152,4 @@ jobs: bash scripts/cleanup-vm.sh true bash scripts/cleanup-router.sh true make delete-gke-cluster - rm -rf scripts/vars.env \ No newline at end of file + rm -rf scripts/vars.env diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 3cff298694..9cd6e1944e 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -8,51 +8,62 @@ source scripts/vars.env gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~ -gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ - --command="export START_LONGEVITY=${START_LONGEVITY} &&\ - export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ - export CI=${CI} &&\ - bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-nfr-tests.sh -retcode=$? - -if [ ${retcode} -ne 0 ]; then - echo "Error running tests on VM" - exit 1 -fi +if [ "${IPV6_ENABLED}" = "true" ]; then + gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ + --command="bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-ipv6-tests.sh + retcode=$? -## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config -if [ "${CI}" = "false" ]; then - gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null - rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . + if [ ${retcode} -ne 0 ]; then + echo "Error running IPv6 tests on VM" + exit 1 + fi else - gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . -fi + gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ + --command="export START_LONGEVITY=${START_LONGEVITY} &&\ + export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ + export CI=${CI} &&\ + bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-nfr-tests.sh + retcode=$? -## If tearing down the longevity test, we need to collect logs from gcloud and add to the results -if [ "${STOP_LONGEVITY}" = "true" ]; then - version=${NGF_VERSION} - if [ "${version}" = "" ]; then - version=${TAG} + if [ ${retcode} -ne 0 ]; then + echo "Error running tests on VM" + exit 1 fi - runType=oss - if [ "${PLUS_ENABLED}" = "true" ]; then - runType=plus + ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config + if [ "${CI}" = "false" ]; then + gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null + rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . + else + gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . fi - results="${SCRIPT_DIR}/../results/longevity/$version/$version-$runType.md" - printf "\n## Error Logs\n\n" >>"${results}" + ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results + if [ "${STOP_LONGEVITY}" = "true" ]; then + version=${NGF_VERSION} + if [ "${version}" = "" ]; then + version=${TAG} + fi + + runType=oss + if [ "${PLUS_ENABLED}" = "true" ]; then + runType=plus + fi + + results="${SCRIPT_DIR}/../results/longevity/$version/$version-$runType.md" + printf "\n## Error Logs\n\n" >>"${results}" - ## ngf error logs - ngfErrText=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)") - ngfErrJSON=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)") - printf "### nginx-gateway\n%s\n%s\n\n" "${ngfErrText}" "${ngfErrJSON}" >>"${results}" + ## ngf error logs + ngfErrText=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)") + ngfErrJSON=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)") + printf "### nginx-gateway\n%s\n%s\n\n" "${ngfErrText}" "${ngfErrJSON}" >>"${results}" - ## nginx error logs - ngxErr=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)") - printf "### nginx\n%s\n\n" "${ngxErr}" >>"${results}" + ## nginx error logs + ngxErr=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)") + printf "### nginx\n%s\n\n" "${ngxErr}" >>"${results}" - ## nginx non-200 responses (also filter out 499 since wrk cancels connections) - ngxNon200=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)") - printf "%s\n\n" "${ngxNon200}" >>"${results}" + ## nginx non-200 responses (also filter out 499 since wrk cancels connections) + ngxNon200=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)") + printf "%s\n\n" "${ngxNon200}" >>"${results}" + fi fi From f2ebf45d5034ca86fcc5dbb341b7977d8804d9b2 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Mon, 15 Sep 2025 18:01:58 +0100 Subject: [PATCH 077/101] Fix script name --- tests/scripts/run-tests-gcp-vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 9cd6e1944e..4254910fa5 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -10,7 +10,7 @@ gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SC if [ "${IPV6_ENABLED}" = "true" ]; then gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ - --command="bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-ipv6-tests.sh + --command="bash -s" <"${SCRIPT_DIR}"/remote-scripts/run-ipv6-test.sh retcode=$? if [ ${retcode} -ne 0 ]; then From 47dacef8f423ad26dd66f7499237c143b92218c9 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 08:23:40 +0100 Subject: [PATCH 078/101] Add script to run IPv6 tests in GKE --- tests/scripts/remote-scripts/run-ipv6-test.sh | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/scripts/remote-scripts/run-ipv6-test.sh diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh new file mode 100644 index 0000000000..87ab9b587e --- /dev/null +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -e + +source "${HOME}"/vars.env + +source scripts/vars.env + +cd nginx-gateway-fabric/tests + +helm upgrade --install ngf ${PREFIX}/${TAG} --create-namespace -n nginx-gateway --set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP + +kubectl apply -f tests/manifests/ipv6-test-app.yaml + +echo "Waiting for NGF to be ready..." +kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway + +echo "Waiting for test applications to be ready..." +kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 + +echo "Deploying IPv6 test client" +kubectl apply -f tests/manifests/test-client-ipv6.yaml +kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client + + +echo "Getting NGF service IPv6 address" +NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') +echo "NGF IPv6 Address: $NGF_IPV6" + +echo "=== Running IPv6-Only Tests ===" + +# Test 1: Basic connectivity test using test client pod +echo "Test 1: Basic IPv6 connectivity" +kubectl exec ipv6-test-client -- curl --version +kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local + +# Test 2: Test NGF service directly via IPv6 +echo "Test 2: NGF Service IPv6 connectivity" +kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ +-H "Host: ipv6-test.example.com" \ +"http://${NGF_IPV6}:80/" || echo "Direct NGF test failed" + +# Test 3: Test via service DNS +echo "Test 3: Service DNS IPv6 connectivity" +kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ +-H "Host: ipv6-test.example.com" \ +"http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" + +echo "=== Validating IPv6-Only Configuration ===" + +# Check NGF configuration +echo "NGF Pod IPv6 addresses:" +kubectl get pods -n nginx-gateway -o wide + +echo "NGF Service configuration:" +kubectl get service nginx-gateway -n nginx-gateway -o yaml + +echo "Gateway and HTTPRoute status:" +kubectl get gateway,httproute -A -o wide + +echo "Test application service configuration:" +kubectl get service test-app-ipv6-service -o yaml + +echo "=== Collecting logs for debugging ===" +echo "NGF Controller logs:" +kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true + +echo "NGINX logs:" +kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true + +echo "Test client logs:" +kubectl logs ipv6-test-client --tail=100 || true + +echo "Cluster events:" +kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true \ No newline at end of file From e3eb3d822660f68f41838d9eb047289985281f6e Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 08:37:39 +0100 Subject: [PATCH 079/101] Remove unused file from test script --- tests/scripts/remote-scripts/run-ipv6-test.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh index 87ab9b587e..58b50e0752 100644 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -4,11 +4,12 @@ set -e source "${HOME}"/vars.env -source scripts/vars.env - cd nginx-gateway-fabric/tests -helm upgrade --install ngf ${PREFIX}/${TAG} --create-namespace -n nginx-gateway --set nginx.config.ipFamily=ipv6 --set nginx.service.type=ClusterIP +helm upgrade --install ngf ${PREFIX}/${TAG} \ + --create-namespace -n nginx-gateway \ + --set nginx.config.ipFamily=ipv6 \ + --set nginx.service.type=ClusterIP kubectl apply -f tests/manifests/ipv6-test-app.yaml From c9fb7462edc436b49e34b54b649cb57d030140ef Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 09:22:50 +0100 Subject: [PATCH 080/101] Fix nfg install commands --- .github/workflows/nfr-ipv6-only.yml | 2 +- .github/workflows/nfr.yml | 26 +++++++++++++++++++ tests/scripts/remote-scripts/run-ipv6-test.sh | 6 ++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nfr-ipv6-only.yml b/.github/workflows/nfr-ipv6-only.yml index 5856fa0e91..0f198a65f8 100644 --- a/.github/workflows/nfr-ipv6-only.yml +++ b/.github/workflows/nfr-ipv6-only.yml @@ -162,7 +162,7 @@ jobs: uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 with: images: | - name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image_tag == 'plus' && 'nginx-plus' || inputs.image_tag§ }} tags: | type=semver,pattern={{version}} type=edge diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 27b6176ceb..c868d7628c 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -140,6 +140,32 @@ jobs: - name: Set GOPATH run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=schedule + type=edge + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + with: + images: | + name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image_tag == 'plus' && 'nginx-plus' || inputs.image_tag }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + - name: Install and test NGF with IPv6 Configuration run: | ./scripts/run-tests-gcp-vm.sh diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh index 58b50e0752..f2dc653419 100644 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -6,11 +6,15 @@ source "${HOME}"/vars.env cd nginx-gateway-fabric/tests -helm upgrade --install ngf ${PREFIX}/${TAG} \ +echo "Prefix: ${PREFIX}, Tag: ${TAG}" + +echo "Installing NGF with IPv6 configuration" +helm install ngf oci://${PREFIX}/${TAG} \ --create-namespace -n nginx-gateway \ --set nginx.config.ipFamily=ipv6 \ --set nginx.service.type=ClusterIP +echo "Deploying IPv6 test application" kubectl apply -f tests/manifests/ipv6-test-app.yaml echo "Waiting for NGF to be ready..." From a978346d64d4420c546b355ad1bd1245210a9af6 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 09:41:24 +0100 Subject: [PATCH 081/101] Update helm install command --- tests/scripts/remote-scripts/run-ipv6-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh index f2dc653419..77726059d7 100644 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -9,7 +9,7 @@ cd nginx-gateway-fabric/tests echo "Prefix: ${PREFIX}, Tag: ${TAG}" echo "Installing NGF with IPv6 configuration" -helm install ngf oci://${PREFIX}/${TAG} \ +helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ --create-namespace -n nginx-gateway \ --set nginx.config.ipFamily=ipv6 \ --set nginx.service.type=ClusterIP From 5853629d714decb2e9fc8c5a1bae6572743ee999 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 09:54:05 +0100 Subject: [PATCH 082/101] Apply gateway API CRDs and NGF CRDs --- tests/scripts/remote-scripts/run-ipv6-test.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh index 77726059d7..57563da75d 100644 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -8,12 +8,21 @@ cd nginx-gateway-fabric/tests echo "Prefix: ${PREFIX}, Tag: ${TAG}" +echo "Applying Gateway API CRDs" +kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v2.1.1" | kubectl apply -f - + +echo "Applying NGF CRDs" +kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/v2.1.1/deploy/crds.yaml + echo "Installing NGF with IPv6 configuration" helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ --create-namespace -n nginx-gateway \ --set nginx.config.ipFamily=ipv6 \ --set nginx.service.type=ClusterIP +echo "Waiting for NGF deployment to be available" +kubectl wait --timeout=5m -n nginx-gateway deployment/ngf-nginx-gateway-fabric --for=condition=Available + echo "Deploying IPv6 test application" kubectl apply -f tests/manifests/ipv6-test-app.yaml From 886919a0eaa07fb5ee986ab394ee2d2be952d210 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 09:55:52 +0100 Subject: [PATCH 083/101] Remove wait command --- tests/scripts/remote-scripts/run-ipv6-test.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh index 57563da75d..1ef18478eb 100644 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -15,14 +15,11 @@ echo "Applying NGF CRDs" kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/v2.1.1/deploy/crds.yaml echo "Installing NGF with IPv6 configuration" -helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ +helm install nginx-gateway oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ --create-namespace -n nginx-gateway \ --set nginx.config.ipFamily=ipv6 \ --set nginx.service.type=ClusterIP -echo "Waiting for NGF deployment to be available" -kubectl wait --timeout=5m -n nginx-gateway deployment/ngf-nginx-gateway-fabric --for=condition=Available - echo "Deploying IPv6 test application" kubectl apply -f tests/manifests/ipv6-test-app.yaml From 219abc13ebee60e62d1618c403878bc869d64a98 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 10:15:49 +0100 Subject: [PATCH 084/101] Remove tests directory --- tests/scripts/remote-scripts/run-ipv6-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh index 1ef18478eb..9d8838fcb2 100644 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -21,7 +21,7 @@ helm install nginx-gateway oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ --set nginx.service.type=ClusterIP echo "Deploying IPv6 test application" -kubectl apply -f tests/manifests/ipv6-test-app.yaml +kubectl apply -f manifests/ipv6-test-app.yaml echo "Waiting for NGF to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway @@ -30,7 +30,7 @@ echo "Waiting for test applications to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 echo "Deploying IPv6 test client" -kubectl apply -f tests/manifests/test-client-ipv6.yaml +kubectl apply -f manifests/test-client-ipv6.yaml kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client From 69873aefc3db2209bd4a3e9be3efa4aae1c457b1 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 10:57:55 +0100 Subject: [PATCH 085/101] Set network to ipv6-only, both internal and external --- .github/workflows/nfr.yml | 3 ++- tests/scripts/create-and-setup-gcp-vm.sh | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index c868d7628c..fab05911a5 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -105,7 +105,8 @@ jobs: echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env - echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env + echo "SOURCE_IP_RANGE=$(curl -sS -6 icanhazip.com)/128" >> vars.env + # echo "SOURCE_IPv6_RANGE=$(curl -sS -6 icanhazip.com)/128" >> vars.env echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env echo "GINKGO_LABEL=" >> vars.env diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index af592864da..c799142fc4 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -25,9 +25,14 @@ if [ "${IPV6_ENABLED}" = "true" ]; then # --region=${GKE_CLUSTER_REGION} \ # --range=10.120.0.0/14 - NETWORK="us-ipv4-ipv6" - # NETWORK_TIER="ipv6-network-tier=PREMIUM" # This will work only if STACK_TYPE is IPV6_ONLY - STACK_TYPE="IPV4_IPV6" + # Dual-Stack Settings + # NETWORK="us-ipv4-ipv6" + # STACK_TYPE="IPV4_IPV6" + + # IPv6 Only Settings + NETWORK="ipv6-only" + NETWORK_TIER="ipv6-network-tier=PREMIUM" # This will work only if STACK_TYPE is IPV6_ONLY + STACK_TYPE="IPV6_ONLY" fi gcloud compute firewall-rules create "${RESOURCE_NAME}" \ @@ -49,8 +54,12 @@ gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" -- # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then + # EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + # --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + + echo "IPv6 is enabled, fetching the external IPv6 address" EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ - --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + --format='value(networkInterfaces[0].ipv6AccessConfigs[0].externalIpv6)') echo "External IP of the VM is: ${EXTERNAL_IP}" From 00ccc4f23071706f523d6b2ba5e271537049ad31 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 11:15:18 +0100 Subject: [PATCH 086/101] Update IP range --- .github/workflows/nfr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index fab05911a5..3932a68f99 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -105,7 +105,7 @@ jobs: echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env - echo "SOURCE_IP_RANGE=$(curl -sS -6 icanhazip.com)/128" >> vars.env + echo "SOURCE_IP_RANGE=$(curl -sS -6 icanhazip.com)/32" >> vars.env # echo "SOURCE_IPv6_RANGE=$(curl -sS -6 icanhazip.com)/128" >> vars.env echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env From 5120f1c65a6d75a949ad133c33a076acd471efa7 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 13:35:42 +0100 Subject: [PATCH 087/101] Set Stack type and ipv6 access type during cluster creation --- .github/workflows/nfr.yml | 97 +++++++++++++++-------------- tests/scripts/create-gke-cluster.sh | 4 +- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 3932a68f99..59784c6946 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -105,8 +105,9 @@ jobs: echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env - echo "SOURCE_IP_RANGE=$(curl -sS -6 icanhazip.com)/32" >> vars.env - # echo "SOURCE_IPv6_RANGE=$(curl -sS -6 icanhazip.com)/128" >> vars.env + echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env + echo "STACK_TYPE=IPV4_IPV6" >> vars.env + echo "IPV6_ACCESS_TYPE=EXTERNAL" >> vars.env echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env echo "GINKGO_LABEL=" >> vars.env @@ -125,52 +126,52 @@ jobs: working-directory: ./tests run: make create-gke-cluster CI=true - - name: Create and setup VM - working-directory: ./tests - run: make create-and-setup-vm - - - name: Create and setup Router - working-directory: ./tests - run: make create-gke-router || true - - - name: Setup Golang Environment - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version: stable - - - name: Set GOPATH - run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV - - - name: NGF Docker meta - id: ngf-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric - tags: | - type=semver,pattern={{version}} - type=schedule - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: NGINX Docker meta - id: nginx-meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 - with: - images: | - name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image_tag == 'plus' && 'nginx-plus' || inputs.image_tag }} - tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: Install and test NGF with IPv6 Configuration - run: | - ./scripts/run-tests-gcp-vm.sh - working-directory: ./tests + # - name: Create and setup VM + # working-directory: ./tests + # run: make create-and-setup-vm + + # - name: Create and setup Router + # working-directory: ./tests + # run: make create-gke-router || true + + # - name: Setup Golang Environment + # uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + # with: + # go-version: stable + + # - name: Set GOPATH + # run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV + + # - name: NGF Docker meta + # id: ngf-meta + # uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + # with: + # images: | + # name=ghcr.io/nginx/nginx-gateway-fabric + # tags: | + # type=semver,pattern={{version}} + # type=schedule + # type=edge + # type=ref,event=pr + # type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + # - name: NGINX Docker meta + # id: nginx-meta + # uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + # with: + # images: | + # name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image_tag == 'plus' && 'nginx-plus' || inputs.image_tag }} + # tags: | + # type=semver,pattern={{version}} + # type=edge + # type=schedule + # type=ref,event=pr + # type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + # - name: Install and test NGF with IPv6 Configuration + # run: | + # ./scripts/run-tests-gcp-vm.sh + # working-directory: ./tests - name: Cleanup working-directory: ./tests diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index c41f6f12d1..e8aeefacf2 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -31,7 +31,9 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --logging=SYSTEM,WORKLOAD \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ - --no-enable-insecure-kubelet-readonly-port # Option is deprecated and will be removed in a future release + --no-enable-insecure-kubelet-readonly-port \ + --stack-type=IPV4_IPV6 \ + --ipv6-access-type=EXTERNAL \ # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From d3d148112ba4b500b586cac65dad37e5ce05196b Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 13:41:41 +0100 Subject: [PATCH 088/101] Update stack type value --- .github/workflows/nfr.yml | 4 ++-- tests/scripts/create-gke-cluster.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 59784c6946..c5b6f6c637 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -100,14 +100,14 @@ jobs: echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env echo "GKE_CLUSTER_REGION=us-west1" >> vars.env + echo "GKE_CLUSTER_STACK_TYPE=ipv4-ipv6" >> vars.env + echo "GKE_CLUSTER_IPV6_ACCESS_TYPE=EXTERNAL" >> vars.env echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env - echo "STACK_TYPE=IPV4_IPV6" >> vars.env - echo "IPV6_ACCESS_TYPE=EXTERNAL" >> vars.env echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env echo "GINKGO_LABEL=" >> vars.env diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index e8aeefacf2..48d3d87296 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -32,8 +32,8 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --machine-type "${GKE_MACHINE_TYPE}" \ --num-nodes "${GKE_NUM_NODES}" \ --no-enable-insecure-kubelet-readonly-port \ - --stack-type=IPV4_IPV6 \ - --ipv6-access-type=EXTERNAL \ + --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ + --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From 8bf6aa3d145779af698e6d7651f3071641f33af2 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 13:48:21 +0100 Subject: [PATCH 089/101] Update access type --- .github/workflows/nfr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index c5b6f6c637..deb8f933dc 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -101,7 +101,7 @@ jobs: echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env echo "GKE_CLUSTER_REGION=us-west1" >> vars.env echo "GKE_CLUSTER_STACK_TYPE=ipv4-ipv6" >> vars.env - echo "GKE_CLUSTER_IPV6_ACCESS_TYPE=EXTERNAL" >> vars.env + echo "GKE_CLUSTER_IPV6_ACCESS_TYPE=external" >> vars.env echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env From 47d9994e5c676bdf46a8bc21eed55a5e60e2da4b Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 13:54:32 +0100 Subject: [PATCH 090/101] Add `--create-subnetwork` field as setting ipv6 access type is supported only with this option --- tests/scripts/create-gke-cluster.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 48d3d87296..8c2aeec595 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -34,6 +34,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --no-enable-insecure-kubelet-readonly-port \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ + --create-subnetwork name=${RESOURCE_NAME},range=/21 # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From 084d841e9ac41c7f031f0428238348129296bceb Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 14:06:24 +0100 Subject: [PATCH 091/101] Add `--enable-dataplane-v2` option for cluster creation as Stack_type IPV4_IPV6 is supported only with advanced datapath. --- tests/scripts/create-gke-cluster.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 8c2aeec595..a4295e543f 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -34,7 +34,8 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --no-enable-insecure-kubelet-readonly-port \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ - --create-subnetwork name=${RESOURCE_NAME},range=/21 + --create-subnetwork name=${RESOURCE_NAME},range=/21 \ + --enable-dataplane-v2 # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From 9a188de324a1227175b7ba0c089c789e2d1ebb06 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 14:16:49 +0100 Subject: [PATCH 092/101] Explicitly use pre made dual stack networks and subnets --- tests/scripts/create-gke-cluster.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index a4295e543f..3f2147d65f 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -34,8 +34,10 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --no-enable-insecure-kubelet-readonly-port \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ - --create-subnetwork name=${RESOURCE_NAME},range=/21 \ + --network=us-ipv4-ipv6 \ + --subnetwork=us-ipv4-ipv6 \ --enable-dataplane-v2 + # --create-subnetwork name=${RESOURCE_NAME},range=/21 \ ## Might need this later # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From f79a6f06829d14c1997fd6e6b6dea20ecf9d2395 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 15:04:32 +0100 Subject: [PATCH 093/101] Use `--create-subnetwork` as setting ipv6 access type is supported only with create-subnetwork option. Also explicitly use pre made `us-ipv4-ipv6` vpc as dual stack clusters are not supported on auto mode vpc networks --- tests/scripts/create-gke-cluster.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 3f2147d65f..ebff57fade 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -34,10 +34,10 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --no-enable-insecure-kubelet-readonly-port \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ - --network=us-ipv4-ipv6 \ - --subnetwork=us-ipv4-ipv6 \ + --create-subnetwork name=${RESOURCE_NAME},range=/21 \ + --network=us-ipv4-ipv6 \ --enable-dataplane-v2 - # --create-subnetwork name=${RESOURCE_NAME},range=/21 \ ## Might need this later + # Add current IP to GKE master control node access, if this script is not invoked during a CI run. if [ "${IS_CI}" = "false" ]; then From 6a3c7b345628518dc1b20ecc6b7ebf269c0174fc Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 15:09:52 +0100 Subject: [PATCH 094/101] Attempt to fix unrecognized arguments --- tests/scripts/create-gke-cluster.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index ebff57fade..d71ebee13b 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -35,8 +35,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ --create-subnetwork name=${RESOURCE_NAME},range=/21 \ - --network=us-ipv4-ipv6 \ - --enable-dataplane-v2 + --network=us-ipv4-ipv6 --enable-dataplane-v2 # Add current IP to GKE master control node access, if this script is not invoked during a CI run. From 61716d843702cfbbbfefaac5ea8ed1ab044cdbcb Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 15:16:53 +0100 Subject: [PATCH 095/101] Update `--create-subnetwork` with IPv6 CIDR range of `2600:1900:4041:625:0:1:0:0/96` --- tests/scripts/create-gke-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index d71ebee13b..b049effb3e 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -34,7 +34,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --no-enable-insecure-kubelet-readonly-port \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ - --create-subnetwork name=${RESOURCE_NAME},range=/21 \ + --create-subnetwork name=${RESOURCE_NAME},range=2600:1900:4041:625:0:1:0:0/96 \ --network=us-ipv4-ipv6 --enable-dataplane-v2 From 015c1c45bd22e7ebfa3f488f30bc27c9895339b9 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 15:24:23 +0100 Subject: [PATCH 096/101] Set ipv6-access-type=INTERNAL for create-subnetwork --- tests/scripts/create-gke-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index b049effb3e..1c9ee7e833 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -34,7 +34,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --no-enable-insecure-kubelet-readonly-port \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ - --create-subnetwork name=${RESOURCE_NAME},range=2600:1900:4041:625:0:1:0:0/96 \ + --create-subnetwork name=${RESOURCE_NAME},range=/21,ipv6-access-type=INTERNAL \ --network=us-ipv4-ipv6 --enable-dataplane-v2 From 07053b372291cb65aed59e1afeb8c58c7d469fa9 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Tue, 16 Sep 2025 15:29:08 +0100 Subject: [PATCH 097/101] Set IPv6 CIRD range --- tests/scripts/create-gke-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh index 1c9ee7e833..b049effb3e 100755 --- a/tests/scripts/create-gke-cluster.sh +++ b/tests/scripts/create-gke-cluster.sh @@ -34,7 +34,7 @@ gcloud container clusters create "${GKE_CLUSTER_NAME}" \ --no-enable-insecure-kubelet-readonly-port \ --stack-type="${GKE_CLUSTER_STACK_TYPE}" \ --ipv6-access-type="${GKE_CLUSTER_IPV6_ACCESS_TYPE}" \ - --create-subnetwork name=${RESOURCE_NAME},range=/21,ipv6-access-type=INTERNAL \ + --create-subnetwork name=${RESOURCE_NAME},range=2600:1900:4041:625:0:1:0:0/96 \ --network=us-ipv4-ipv6 --enable-dataplane-v2 From d8f6b9bfedd7c3a2aeb223b08015d1d1cc6f6402 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 17 Sep 2025 11:22:06 +0100 Subject: [PATCH 098/101] Update ipv6 test script to run on a local IPv6 kind cluster --- tests/manifests/ipv6-test-app.yaml | 5 +- tests/scripts/remote-scripts/run-ipv6-test.sh | 93 +++++++++---------- 2 files changed, 47 insertions(+), 51 deletions(-) mode change 100644 => 100755 tests/scripts/remote-scripts/run-ipv6-test.sh diff --git a/tests/manifests/ipv6-test-app.yaml b/tests/manifests/ipv6-test-app.yaml index ff4a506cd9..202759eb23 100644 --- a/tests/manifests/ipv6-test-app.yaml +++ b/tests/manifests/ipv6-test-app.yaml @@ -47,8 +47,9 @@ metadata: namespace: default spec: parentRefs: - - name: nginx-gateway - namespace: nginx-gateway + - name: gateway + sectionName: http + namespace: default hostnames: - "ipv6-test.example.com" rules: diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/scripts/remote-scripts/run-ipv6-test.sh old mode 100644 new mode 100755 index 9d8838fcb2..1755e8e308 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/scripts/remote-scripts/run-ipv6-test.sh @@ -1,86 +1,81 @@ #!/usr/bin/env bash -set -e +set -e # Exit immediately if a command exits with a non-zero status -source "${HOME}"/vars.env +RELEASE=$1 +HELM_RELEASE_NAME=${2:-ngf} +NAMESPACE=${3:-nginx-gateway} -cd nginx-gateway-fabric/tests +cleanup() { + echo "Cleaning up resources..." + kubectl delete -f tests/manifests/ipv6-test-app.yaml || true + kubectl delete -f tests/manifests/ipv6-test-client.yaml || true + kubectl delete -f tests/manifests/gateway.yaml || true + helm uninstall ${HELM_RELEASE_NAME} -n ${NAMESPACE} || true +} -echo "Prefix: ${PREFIX}, Tag: ${TAG}" +trap cleanup EXIT echo "Applying Gateway API CRDs" -kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v2.1.1" | kubectl apply -f - +kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=${RELEASE}" | kubectl apply -f - echo "Applying NGF CRDs" -kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/v2.1.1/deploy/crds.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/${RELEASE}/deploy/crds.yaml -echo "Installing NGF with IPv6 configuration" -helm install nginx-gateway oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ - --create-namespace -n nginx-gateway \ +helm upgrade --install ${HELM_RELEASE_NAME} oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ + --create-namespace -n ${NAMESPACE} \ --set nginx.config.ipFamily=ipv6 \ --set nginx.service.type=ClusterIP +# Make sure to create a Gateway! +echo "Deploying Gateway..." +kubectl apply -f tests/manifests/gateway.yaml +echo "Waiting for NGINX Gateway to be ready..." +kubectl wait --for=condition=accepted --timeout=300s gateway/gateway +POD_NAME=$(kubectl get pods -l app.kubernetes.io/instance=${HELM_RELEASE_NAME} -o jsonpath='{.items[0].metadata.name}') +kubectl wait --for=condition=ready --timeout=300s pod/${POD_NAME} + +# Might need to do local build for plus testing... + echo "Deploying IPv6 test application" -kubectl apply -f manifests/ipv6-test-app.yaml +kubectl apply -f tests/manifests/ipv6-test-app.yaml echo "Waiting for NGF to be ready..." -kubectl wait --for=condition=available --timeout=300s deployment/nginx-gateway -n nginx-gateway +kubectl wait --for=condition=available --timeout=300s deployment/${HELM_RELEASE_NAME}-nginx-gateway-fabric -n ${NAMESPACE} echo "Waiting for test applications to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 echo "Deploying IPv6 test client" -kubectl apply -f manifests/test-client-ipv6.yaml +kubectl apply -f tests/manifests/ipv6-test-client.yaml kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client echo "Getting NGF service IPv6 address" -NGF_IPV6=$(kubectl get service nginx-gateway -n nginx-gateway -o jsonpath='{.spec.clusterIP}') +NGF_IPV6=$(kubectl get service gateway-nginx -o jsonpath='{.spec.clusterIP}') echo "NGF IPv6 Address: $NGF_IPV6" echo "=== Running IPv6-Only Tests ===" -# Test 1: Basic connectivity test using test client pod echo "Test 1: Basic IPv6 connectivity" kubectl exec ipv6-test-client -- curl --version -kubectl exec ipv6-test-client -- nslookup nginx-gateway.nginx-gateway.svc.cluster.local +kubectl exec ipv6-test-client -- nslookup gateway-nginx.default.svc.cluster.local +test1_status=$? -# Test 2: Test NGF service directly via IPv6 echo "Test 2: NGF Service IPv6 connectivity" kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ --H "Host: ipv6-test.example.com" \ -"http://${NGF_IPV6}:80/" || echo "Direct NGF test failed" + -H "Host: ipv6-test.example.com" \ + "http://[${NGF_IPV6}]:80/" +test2_status=$? -# Test 3: Test via service DNS echo "Test 3: Service DNS IPv6 connectivity" kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ --H "Host: ipv6-test.example.com" \ -"http://nginx-gateway.nginx-gateway.svc.cluster.local:80/" || echo "Service DNS test failed" - -echo "=== Validating IPv6-Only Configuration ===" - -# Check NGF configuration -echo "NGF Pod IPv6 addresses:" -kubectl get pods -n nginx-gateway -o wide - -echo "NGF Service configuration:" -kubectl get service nginx-gateway -n nginx-gateway -o yaml - -echo "Gateway and HTTPRoute status:" -kubectl get gateway,httproute -A -o wide - -echo "Test application service configuration:" -kubectl get service test-app-ipv6-service -o yaml - -echo "=== Collecting logs for debugging ===" -echo "NGF Controller logs:" -kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx-gateway-controller --tail=100 || true - -echo "NGINX logs:" -kubectl logs -n nginx-gateway deployment/nginx-gateway -c nginx --tail=100 || true - -echo "Test client logs:" -kubectl logs ipv6-test-client --tail=100 || true - -echo "Cluster events:" -kubectl get events --sort-by='.lastTimestamp' --all-namespaces --tail=50 || true \ No newline at end of file + -H "Host: ipv6-test.example.com" \ + "http://gateway-nginx.default.svc.cluster.local:80/" +test3_status=$? + +if [[ $test1_status -eq 0 && $test2_status -eq 0 && $test3_status -eq 0 ]]; then + echo "All tests passed." +else + echo "One or more tests failed." +fi \ No newline at end of file From 89e3913482f52ea7d278212826be391aa7c96c08 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 17 Sep 2025 11:25:44 +0100 Subject: [PATCH 099/101] Move all relevant IPv6 test files to `tests/ipv6` --- tests/ipv6/config/kind-ipv6-only.yaml | 8 ++++++++ tests/ipv6/manifests/gateway.yaml | 11 +++++++++++ tests/{ => ipv6}/manifests/ipv6-test-app.yaml | 0 tests/{ => ipv6}/manifests/ipv6-test-client.yaml | 0 .../remote-scripts => ipv6/scripts}/run-ipv6-test.sh | 4 ++++ 5 files changed, 23 insertions(+) create mode 100644 tests/ipv6/config/kind-ipv6-only.yaml create mode 100644 tests/ipv6/manifests/gateway.yaml rename tests/{ => ipv6}/manifests/ipv6-test-app.yaml (100%) rename tests/{ => ipv6}/manifests/ipv6-test-client.yaml (100%) rename tests/{scripts/remote-scripts => ipv6/scripts}/run-ipv6-test.sh (95%) diff --git a/tests/ipv6/config/kind-ipv6-only.yaml b/tests/ipv6/config/kind-ipv6-only.yaml new file mode 100644 index 0000000000..ab17d5f307 --- /dev/null +++ b/tests/ipv6/config/kind-ipv6-only.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: ipv6 # Explicitly set the cluster to use IPv6 + apiServerAddress: "::1" + disableDefaultCNI: false # Use Kind's default CNI +nodes: +- role: control-plane diff --git a/tests/ipv6/manifests/gateway.yaml b/tests/ipv6/manifests/gateway.yaml new file mode 100644 index 0000000000..e6507f613b --- /dev/null +++ b/tests/ipv6/manifests/gateway.yaml @@ -0,0 +1,11 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: gateway +spec: + gatewayClassName: nginx + listeners: + - name: http + port: 80 + protocol: HTTP + hostname: "*.example.com" diff --git a/tests/manifests/ipv6-test-app.yaml b/tests/ipv6/manifests/ipv6-test-app.yaml similarity index 100% rename from tests/manifests/ipv6-test-app.yaml rename to tests/ipv6/manifests/ipv6-test-app.yaml diff --git a/tests/manifests/ipv6-test-client.yaml b/tests/ipv6/manifests/ipv6-test-client.yaml similarity index 100% rename from tests/manifests/ipv6-test-client.yaml rename to tests/ipv6/manifests/ipv6-test-client.yaml diff --git a/tests/scripts/remote-scripts/run-ipv6-test.sh b/tests/ipv6/scripts/run-ipv6-test.sh similarity index 95% rename from tests/scripts/remote-scripts/run-ipv6-test.sh rename to tests/ipv6/scripts/run-ipv6-test.sh index 1755e8e308..6aecd72c1c 100755 --- a/tests/scripts/remote-scripts/run-ipv6-test.sh +++ b/tests/ipv6/scripts/run-ipv6-test.sh @@ -5,6 +5,7 @@ set -e # Exit immediately if a command exits with a non-zero status RELEASE=$1 HELM_RELEASE_NAME=${2:-ngf} NAMESPACE=${3:-nginx-gateway} +CLUSTER_NAME=${4:-ipv6-only} cleanup() { echo "Cleaning up resources..." @@ -16,6 +17,9 @@ cleanup() { trap cleanup EXIT +echo "Creating IPv6 kind cluster..." +kind create cluster --name ${CLUSTER_NAME} --config kind/kind-ipv6-only.yaml + echo "Applying Gateway API CRDs" kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=${RELEASE}" | kubectl apply -f - From 14cf3aa12ff9b2ed37d9f4b1a1843232a9ada681 Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 17 Sep 2025 12:20:01 +0100 Subject: [PATCH 100/101] Move test script to `tests/ipv6` --- tests/ipv6/{scripts => }/run-ipv6-test.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) rename tests/ipv6/{scripts => }/run-ipv6-test.sh (84%) diff --git a/tests/ipv6/scripts/run-ipv6-test.sh b/tests/ipv6/run-ipv6-test.sh similarity index 84% rename from tests/ipv6/scripts/run-ipv6-test.sh rename to tests/ipv6/run-ipv6-test.sh index 6aecd72c1c..a2b94cbfcd 100755 --- a/tests/ipv6/scripts/run-ipv6-test.sh +++ b/tests/ipv6/run-ipv6-test.sh @@ -2,6 +2,10 @@ set -e # Exit immediately if a command exits with a non-zero status +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +cd "$SCRIPT_DIR" + RELEASE=$1 HELM_RELEASE_NAME=${2:-ngf} NAMESPACE=${3:-nginx-gateway} @@ -9,16 +13,16 @@ CLUSTER_NAME=${4:-ipv6-only} cleanup() { echo "Cleaning up resources..." - kubectl delete -f tests/manifests/ipv6-test-app.yaml || true - kubectl delete -f tests/manifests/ipv6-test-client.yaml || true - kubectl delete -f tests/manifests/gateway.yaml || true + kubectl delete -f manifests/ipv6-test-app.yaml || true + kubectl delete -f manifests/ipv6-test-client.yaml || true + kubectl delete -f manifests/gateway.yaml || true helm uninstall ${HELM_RELEASE_NAME} -n ${NAMESPACE} || true } trap cleanup EXIT echo "Creating IPv6 kind cluster..." -kind create cluster --name ${CLUSTER_NAME} --config kind/kind-ipv6-only.yaml +kind create cluster --name ${CLUSTER_NAME}-${RELEASE} --config config/kind-ipv6-only.yaml echo "Applying Gateway API CRDs" kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=${RELEASE}" | kubectl apply -f - @@ -33,7 +37,7 @@ helm upgrade --install ${HELM_RELEASE_NAME} oci://ghcr.io/nginx/charts/nginx-gat # Make sure to create a Gateway! echo "Deploying Gateway..." -kubectl apply -f tests/manifests/gateway.yaml +kubectl apply -f manifests/gateway.yaml echo "Waiting for NGINX Gateway to be ready..." kubectl wait --for=condition=accepted --timeout=300s gateway/gateway POD_NAME=$(kubectl get pods -l app.kubernetes.io/instance=${HELM_RELEASE_NAME} -o jsonpath='{.items[0].metadata.name}') @@ -42,7 +46,7 @@ kubectl wait --for=condition=ready --timeout=300s pod/${POD_NAME} # Might need to do local build for plus testing... echo "Deploying IPv6 test application" -kubectl apply -f tests/manifests/ipv6-test-app.yaml +kubectl apply -f manifests/ipv6-test-app.yaml echo "Waiting for NGF to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/${HELM_RELEASE_NAME}-nginx-gateway-fabric -n ${NAMESPACE} @@ -51,7 +55,7 @@ echo "Waiting for test applications to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 echo "Deploying IPv6 test client" -kubectl apply -f tests/manifests/ipv6-test-client.yaml +kubectl apply -f manifests/ipv6-test-client.yaml kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client From b9d31dafabd9a3d31272c27f12b71670bac5949d Mon Sep 17 00:00:00 2001 From: shaun-nx Date: Wed, 17 Sep 2025 15:47:28 +0100 Subject: [PATCH 101/101] Update release process with steps for IPv6 testing --- docs/developer/release-process.md | 2 + tests/Makefile | 6 +++ tests/ipv6/run-ipv6-test.sh | 74 ++++++++++++++++++++----------- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md index 7cd056c9a3..d882e2419a 100644 --- a/docs/developer/release-process.md +++ b/docs/developer/release-process.md @@ -44,6 +44,8 @@ To create a new release, follow these steps: 4. Once the release branch pipeline completes, run tests using the `release-X.X-rc` images that are pushed to Github (for example, `release-1.3-rc`). 1. Kick off the [longevity tests](https://github.com/nginx/nginx-gateway-fabric/blob/main/tests/README.md#longevity-testing) for both OSS and Plus. You'll need to create two clusters and VMs for this. Before running, update your `vars.env` file with the proper image tag and prefixes. NGF and nginx images will be available from `ghcr.io`, and nginx plus will be available in GCP (`us-docker.pkg.dev//nginx-gateway-fabric/nginx-plus`). These tests need to run for 4 days before releasing. The results should be committed to the main branch and then cherry-picked to the release branch. 2. Kick off the [NFR workflow](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/nfr.yml) in the browser. For `image_tag`, use `release-X.X-rc`, and for `version`, use the upcoming `X.Y.Z` NGF version. Run the workflow on the new release branch. This will run all of the NFR tests which are automated and open a PR with the results files when it is complete. Review this PR and make any necessary changes before merging. Once merged, be sure to cherry-pick the commit to the main branch as well (the original PR targets the release branch). + 3. Run the IPv6 tests using the `make ipv6-tests` target. This must be run from within the `tests` directory. This script need two arguments. The release version (e.g. `v2.1.0`) and the release image tag (e.g. `release-X.X-rc`). + For example, when running this script for release 2.1.0, it would look like this: `make ipv6-test RELEASE=2.1.1 RELEASE_IMAGE=release-2.1-rc` 5. Run the [Release PR](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/release-pr.yml) workflow to update the repo files for the release. Then there are a few manual steps to complete: 1. Update the [README](/README.md) to include information about the release. 2. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly. diff --git a/tests/Makefile b/tests/Makefile index 668fd879e3..355dd40420 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -18,6 +18,8 @@ EXPERIMENTAL_CONFORMANCE_PROFILES = GATEWAY-TLS CONFORMANCE_PROFILES = $(STANDARD_CONFORMANCE_PROFILES) # by default we use the standard conformance profiles. If experimental is enabled we override this and add the experimental profiles. SKIP_TESTS = CEL_TEST_TARGET = +RELEASE = ? main # e.g. v2.1.0 +RELEASE_IMAGE = ? latest # e.g. release-2.1.0-rc # Check if ENABLE_EXPERIMENTAL is true ifeq ($(ENABLE_EXPERIMENTAL),true) @@ -118,6 +120,10 @@ start-longevity-test: nfr-test ## Start the longevity test to run for 4 days in stop-longevity-test: export STOP_LONGEVITY=true stop-longevity-test: nfr-test ## Stop the longevity test and collects results +.PHONY: ipv6-tests +ipv6-tests: GOARCH=amd64 +ipv6-tests: ## Example usage: make ipv6-tests RELEASE=vX.Y.Z RELEASE_IMAGE=release-X.Y-rc + ./ipv6/run-ipv6-test.sh $(RELEASE) $(RELEASE_IMAGE) .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) diff --git a/tests/ipv6/run-ipv6-test.sh b/tests/ipv6/run-ipv6-test.sh index a2b94cbfcd..633727b228 100755 --- a/tests/ipv6/run-ipv6-test.sh +++ b/tests/ipv6/run-ipv6-test.sh @@ -2,27 +2,33 @@ set -e # Exit immediately if a command exits with a non-zero status -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +RELEASE=$1 +RELEASE_IMAGE=$2 -cd "$SCRIPT_DIR" +if [[ -z "$RELEASE" || -z "$RELEASE_IMAGE" ]]; then + echo "Usage: $0 [HELM_RELEASE_NAME] [NAMESPACE] [CLUSTER_NAME]" + echo "Error: RELEASE and RELEASE_IMAGE are required parameters. Example usage `make ipv6-test RELEASE=vX.Y.Z RELEASE_IMAGE=release-X.Y-rc`" + exit 1 +fi -RELEASE=$1 -HELM_RELEASE_NAME=${2:-ngf} -NAMESPACE=${3:-nginx-gateway} -CLUSTER_NAME=${4:-ipv6-only} +HELM_RELEASE_NAME=${3:-ngf} +NAMESPACE=${4:-nginx-gateway} +CLUSTER_NAME=${5:-ipv6-only-${RELEASE}} +RELEASE_REPO=ghcr.io/nginx/nginx-gateway-fabric cleanup() { echo "Cleaning up resources..." - kubectl delete -f manifests/ipv6-test-app.yaml || true - kubectl delete -f manifests/ipv6-test-client.yaml || true - kubectl delete -f manifests/gateway.yaml || true + kubectl delete -f ipv6/manifests/ipv6-test-app.yaml || true + kubectl delete -f ipv6/manifests/ipv6-test-client.yaml || true + kubectl delete -f ipv6/manifests/gateway.yaml || true helm uninstall ${HELM_RELEASE_NAME} -n ${NAMESPACE} || true + kind delete cluster --name ${CLUSTER_NAME} || true } trap cleanup EXIT echo "Creating IPv6 kind cluster..." -kind create cluster --name ${CLUSTER_NAME}-${RELEASE} --config config/kind-ipv6-only.yaml +kind create cluster --name ${CLUSTER_NAME} --config ipv6/config/kind-ipv6-only.yaml echo "Applying Gateway API CRDs" kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=${RELEASE}" | kubectl apply -f - @@ -30,23 +36,28 @@ kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gate echo "Applying NGF CRDs" kubectl apply --server-side -f https://raw.githubusercontent.com/nginx/nginx-gateway-fabric/${RELEASE}/deploy/crds.yaml +echo "Pulling NGF image ${RELEASE_REPO}..." +docker pull ${RELEASE_REPO}:${RELEASE_IMAGE} + +echo "Loading NGF image into kind cluster..." +docker save ${RELEASE_REPO}:${RELEASE_IMAGE} | docker exec -i ${CLUSTER_NAME}-control-plane ctr --namespace=k8s.io images import - + helm upgrade --install ${HELM_RELEASE_NAME} oci://ghcr.io/nginx/charts/nginx-gateway-fabric \ --create-namespace -n ${NAMESPACE} \ --set nginx.config.ipFamily=ipv6 \ - --set nginx.service.type=ClusterIP + --set nginx.service.type=ClusterIP \ + --set nginxGateway.image.repository=${RELEASE_REPO} \ + --set nginxGateway.image.tag=${RELEASE_IMAGE} -# Make sure to create a Gateway! echo "Deploying Gateway..." -kubectl apply -f manifests/gateway.yaml +kubectl apply -f ipv6/manifests/gateway.yaml echo "Waiting for NGINX Gateway to be ready..." kubectl wait --for=condition=accepted --timeout=300s gateway/gateway POD_NAME=$(kubectl get pods -l app.kubernetes.io/instance=${HELM_RELEASE_NAME} -o jsonpath='{.items[0].metadata.name}') kubectl wait --for=condition=ready --timeout=300s pod/${POD_NAME} -# Might need to do local build for plus testing... - echo "Deploying IPv6 test application" -kubectl apply -f manifests/ipv6-test-app.yaml +kubectl apply -f ipv6/manifests/ipv6-test-app.yaml echo "Waiting for NGF to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/${HELM_RELEASE_NAME}-nginx-gateway-fabric -n ${NAMESPACE} @@ -55,35 +66,46 @@ echo "Waiting for test applications to be ready..." kubectl wait --for=condition=available --timeout=300s deployment/test-app-ipv6 echo "Deploying IPv6 test client" -kubectl apply -f manifests/ipv6-test-client.yaml +kubectl apply -f ipv6/manifests/ipv6-test-client.yaml kubectl wait --for=condition=ready --timeout=300s pod/ipv6-test-client - echo "Getting NGF service IPv6 address" NGF_IPV6=$(kubectl get service gateway-nginx -o jsonpath='{.spec.clusterIP}') echo "NGF IPv6 Address: $NGF_IPV6" echo "=== Running IPv6-Only Tests ===" -echo "Test 1: Basic IPv6 connectivity" +echo "== Test 1: Basic IPv6 connectivity ==" kubectl exec ipv6-test-client -- curl --version -kubectl exec ipv6-test-client -- nslookup gateway-nginx.default.svc.cluster.local +kubectl exec ipv6-test-client -- nslookup gateway-nginx.default.svc.cluster.local || echo "Test 1: Basic IPv6 connectivity failed" test1_status=$? -echo "Test 2: NGF Service IPv6 connectivity" +if [[ $test1_status -eq 0 ]]; then + echo "✅ Test 1: Basic IPv6 connectivity succeeded" +fi + +echo "== Test 2: NGF Service IPv6 connectivity ==" kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ -H "Host: ipv6-test.example.com" \ - "http://[${NGF_IPV6}]:80/" + "http://[${NGF_IPV6}]:80/" || echo "Test 2: NGF Service IPv6 connectivity failed" test2_status=$? -echo "Test 3: Service DNS IPv6 connectivity" +if [[ $test2_status -eq 0 ]]; then + echo "✅ Test 2: NGF Service IPv6 connectivity succeeded" +fi + +echo "== Test 3: Service DNS IPv6 connectivity ==" kubectl exec ipv6-test-client -- curl -6 --connect-timeout 30 --max-time 60 -v \ -H "Host: ipv6-test.example.com" \ - "http://gateway-nginx.default.svc.cluster.local:80/" + "http://gateway-nginx.default.svc.cluster.local:80/" || echo "Test 3: Service DNS IPv6 connectivity failed" test3_status=$? +if [[ $test3_status -eq 0 ]]; then + echo "✅ Test 3: Service DNS IPv6 connectivity succeeded" +fi + if [[ $test1_status -eq 0 && $test2_status -eq 0 && $test3_status -eq 0 ]]; then - echo "All tests passed." + echo -e "✅ All tests passed!" else - echo "One or more tests failed." + echo -e "\033[31m One or more tests failed. \033[0m" fi \ No newline at end of file