From cedd12eeadaa49e7f75cb6bffe87dd1aece1fa20 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Wed, 13 Dec 2023 17:01:29 +0000 Subject: [PATCH 01/13] load test matrix from json file --- .github/data/matrix-regression.json | 18 ++++++++ .github/data/matrix-smoke.json | 69 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 22 +-------- 3 files changed, 89 insertions(+), 20 deletions(-) create mode 100644 .github/data/matrix-regression.json create mode 100644 .github/data/matrix-smoke.json diff --git a/.github/data/matrix-regression.json b/.github/data/matrix-regression.json new file mode 100644 index 0000000000..2dbcc1ff00 --- /dev/null +++ b/.github/data/matrix-regression.json @@ -0,0 +1,18 @@ +{ + "k8s": [ + "1.22.17", + "1.23.17", + "1.24.15", + "1.25.11", + "1.26.6", + "1.27.3" + ], + "images": [ + { + "image": "debian" + }, + { + "image": "debian-plus" + } + ] +} diff --git a/.github/data/matrix-smoke.json b/.github/data/matrix-smoke.json new file mode 100644 index 0000000000..d1206991c7 --- /dev/null +++ b/.github/data/matrix-smoke.json @@ -0,0 +1,69 @@ +{ + "images": [ + { + "image": "debian", + "marker": "ingresses" + }, + { + "image": "alpine", + "marker": "vsr" + }, + { + "image": "alpine", + "marker": "policies" + }, + { + "image": "debian", + "marker": "vs" + }, + { + "image": "ubi", + "marker": "ts" + }, + { + "image": "debian-plus", + "marker": "vs" + }, + { + "image": "debian-plus", + "marker": "ts" + }, + { + "image": "alpine-plus", + "marker": "ingresses" + }, + { + "image": "alpine-plus", + "marker": "vsr" + }, + { + "image": "ubi-plus", + "marker": "policies" + }, + { + "image": "debian-plus-nap", + "marker": "appprotect_integration" + }, + { + "image": "debian-plus-nap", + "marker": "appprotect_waf_policies_allow" + }, + { + "image": "debian-plus-nap", + "marker": "'appprotect_waf_policies and not appprotect_waf_policies_allow'" + }, + { + "image": "debian-plus-nap", + "marker": "appprotect_waf_policies_grpc" + }, + { + "image": "debian-plus-nap", + "marker": "'appprotect_watch or appprotect_batch'" + }, + { + "image": "debian-plus-nap", + "marker": "dos" + } + ], + "k8s": [] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 475d3f0c9c..9801ca110c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,27 +253,9 @@ jobs: - id: set-matrix run: | if [ "${{ github.event_name }}" != "schedule" ]; then - echo "matrix={\"images\": \ - [{\"image\": \"debian\", \"marker\": \"ingresses\"}, \ - {\"image\": \"alpine\", \"marker\":\"vsr\"}, \ - {\"image\": \"alpine\", \"marker\":\"policies\"}, \ - {\"image\": \"debian\", \"marker\": \"vs\"}, \ - {\"image\": \"ubi\", \"marker\": \"ts\"}, \ - {\"image\": \"debian-plus\", \"marker\": \"vs\"}, \ - {\"image\": \"debian-plus\", \"marker\": \"ts\"}, \ - {\"image\": \"alpine-plus\", \"marker\":\"ingresses\"}, \ - {\"image\": \"alpine-plus\", \"marker\": \"vsr\"}, \ - {\"image\": \"ubi-plus\", \"marker\": \"policies\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_integration\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_allow\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_waf_policies and not appprotect_waf_policies_allow'\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"appprotect_waf_policies_grpc\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"'appprotect_watch or appprotect_batch'\"}, \ - {\"image\": \"debian-plus-nap\", \"marker\": \"dos\"}], \ - \"k8s\": [\"${{ needs.checks.outputs.k8s_latest }}\"]}" >> $GITHUB_OUTPUT + echo "matrix=$(cat .github/data/matrix-smoke.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT else - echo "matrix={\"k8s\": [\"1.22.17\", \"1.23.17\", \"1.24.15\", \"1.25.11\", \"1.26.6\", \"1.27.3\", \"${{ needs.checks.outputs.k8s_latest }}\"], \ - \"images\": [{\"image\": \"debian\"}, {\"image\": \"debian-plus\"}]}" >> $GITHUB_OUTPUT + echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT fi - name: Checkout Repository From 52bf1533fd316dccf03ee101226f2e5a5bcc10dc Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Wed, 13 Dec 2023 17:31:41 +0000 Subject: [PATCH 02/13] reorder steps in test matrix job --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9801ca110c..44b8ca4007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,6 +250,9 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: + - name: Checkout Repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - id: set-matrix run: | if [ "${{ github.event_name }}" != "schedule" ]; then @@ -258,9 +261,6 @@ jobs: echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT fi - - name: Checkout Repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Docker Buildx uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 From dba524cd44fbf176dac107fcecf72081996f97fd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:32:35 +0000 Subject: [PATCH 03/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44b8ca4007..6ee8ebf70e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -252,7 +252,7 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - + - id: set-matrix run: | if [ "${{ github.event_name }}" != "schedule" ]; then From d2014bacc1a5bc7730dfb7b57c4782e9f7973665 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Wed, 13 Dec 2023 18:00:55 +0000 Subject: [PATCH 04/13] cleanup test output html filename --- .github/actions/smoke-tests/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/smoke-tests/action.yaml b/.github/actions/smoke-tests/action.yaml index 5497ef8788..b5d7c49945 100644 --- a/.github/actions/smoke-tests/action.yaml +++ b/.github/actions/smoke-tests/action.yaml @@ -87,7 +87,8 @@ runs: make -f tests/Makefile create-kind-cluster K8S_CLUSTER_NAME=${{ github.run_id }} K8S_CLUSTER_VERSION=${{ inputs.k8s-version }} K8S_TIMEOUT=${{ inputs.k8s-timeout }} make -f tests/Makefile image-load PREFIX=nginx/${{ steps.ingress-type.outputs.name }} TAG=${{ steps.ingress-type.outputs.tag }} K8S_CLUSTER_NAME=${{ github.run_id }} marker="${{ inputs.marker }}" - sanitized_marker="${marker// /_}" + nospaces="${marker// /_}" + sanitized_marker="${nospaces//\'/}" name="${sanitized_marker:-${{ inputs.k8s-version }}}" echo "cluster_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${{ github.run_id }}-control-plane)" >> $GITHUB_OUTPUT echo "cluster=$(echo nginx-${{ inputs.image }}-$name)" >> $GITHUB_OUTPUT From d43a7513a7c2316599076c4acd24ef19a55a2328 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Fri, 22 Dec 2023 14:46:17 +0000 Subject: [PATCH 05/13] add latest matrix updates --- .github/data/matrix-smoke.json | 42 ++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/.github/data/matrix-smoke.json b/.github/data/matrix-smoke.json index d1206991c7..deeb5f8553 100644 --- a/.github/data/matrix-smoke.json +++ b/.github/data/matrix-smoke.json @@ -10,11 +10,23 @@ }, { "image": "alpine", - "marker": "policies" + "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'" + }, + { + "image": "alpine", + "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'" + }, + { + "image": "debian", + "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'" }, { "image": "debian", - "marker": "vs" + "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'" + }, + { + "image": "debian", + "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'" }, { "image": "ubi", @@ -22,7 +34,15 @@ }, { "image": "debian-plus", - "marker": "vs" + "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager'" + }, + { + "image": "debian-plus", + "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns'" + }, + { + "image": "debian-plus", + "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'" }, { "image": "debian-plus", @@ -38,11 +58,11 @@ }, { "image": "ubi-plus", - "marker": "policies" + "marker": "'policies and not policies_ac and not policies_jwt and not policies_mtls'" }, { - "image": "debian-plus-nap", - "marker": "appprotect_integration" + "image": "ubi-plus", + "marker": "'policies_ac or policies_jwt or policies_mtls'" }, { "image": "debian-plus-nap", @@ -58,12 +78,16 @@ }, { "image": "debian-plus-nap", - "marker": "'appprotect_watch or appprotect_batch'" + "marker": "'appprotect_watch or appprotect_batch or appprotect_integration'" + }, + { + "image": "debian-plus-nap", + "marker": "'dos and not dos_learning'" }, { "image": "debian-plus-nap", - "marker": "dos" + "marker": "dos_learning" } ], "k8s": [] -} +} \ No newline at end of file From 7d64a2e9e3bfc0ea9abd432b0ceb3c25bba40c73 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 Dec 2023 14:46:36 +0000 Subject: [PATCH 06/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/data/matrix-smoke.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/data/matrix-smoke.json b/.github/data/matrix-smoke.json index deeb5f8553..d142317864 100644 --- a/.github/data/matrix-smoke.json +++ b/.github/data/matrix-smoke.json @@ -90,4 +90,4 @@ } ], "k8s": [] -} \ No newline at end of file +} From 886eaa887a29f6e071790647ee338d65f467e3b4 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 2 Jan 2024 10:38:49 +0000 Subject: [PATCH 07/13] dynamically load k8s versions --- .github/data/matrix-regression.json | 11 ++--------- .github/workflows/ci.yml | 6 ++---- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/data/matrix-regression.json b/.github/data/matrix-regression.json index 2dbcc1ff00..4d6f0045cb 100644 --- a/.github/data/matrix-regression.json +++ b/.github/data/matrix-regression.json @@ -1,12 +1,5 @@ { - "k8s": [ - "1.22.17", - "1.23.17", - "1.24.15", - "1.25.11", - "1.26.6", - "1.27.3" - ], + "k8s": [], "images": [ { "image": "debian" @@ -15,4 +8,4 @@ "image": "debian-plus" } ] -} +} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c70ecaac44..41806a1f86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,9 +66,7 @@ jobs: | head -n 7 \ | sort -V \ | sed 's/v//g' \ - | sed 's/^/\\\"/' \ - | sed 's/$/\\\",/' \ - | tr '\n' ' ' \ + | sed 's/$//' \ | sed 's/, $//') echo "latest_kindest_node_versions=$kindest_versions" >> $GITHUB_OUTPUT @@ -281,7 +279,7 @@ jobs: if [ "${{ github.event_name }}" != "schedule" ]; then echo "matrix=$(cat .github/data/matrix-smoke.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT else - echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT + echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest "$(echo ${{ needs.checks.outputs.latest_kindest_node_versions }} | jq -R -s -c 'split("\n")[:-1]')" '.k8s += [$latest]')" >> $GITHUB_OUTPUT fi - name: Docker Buildx From 728eeb75ac31310736393817e914f88ccd3bae43 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:39:22 +0000 Subject: [PATCH 08/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/data/matrix-regression.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/data/matrix-regression.json b/.github/data/matrix-regression.json index 4d6f0045cb..86751ca69e 100644 --- a/.github/data/matrix-regression.json +++ b/.github/data/matrix-regression.json @@ -8,4 +8,4 @@ "image": "debian-plus" } ] -} \ No newline at end of file +} From 21825ea23e70e26cbd02102f193360fd65678837 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 2 Jan 2024 10:45:44 +0000 Subject: [PATCH 09/13] change versions output to json --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41806a1f86..2a47c64a10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,8 @@ jobs: | sort -V \ | sed 's/v//g' \ | sed 's/$//' \ - | sed 's/, $//') + | sed 's/, $//' + | jq -R -s -c 'split("\n")[:-1]') echo "latest_kindest_node_versions=$kindest_versions" >> $GITHUB_OUTPUT - name: Check if go.mod and go.sum are up to date @@ -279,7 +280,7 @@ jobs: if [ "${{ github.event_name }}" != "schedule" ]; then echo "matrix=$(cat .github/data/matrix-smoke.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT else - echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest "$(echo ${{ needs.checks.outputs.latest_kindest_node_versions }} | jq -R -s -c 'split("\n")[:-1]')" '.k8s += [$latest]')" >> $GITHUB_OUTPUT + echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest "${{ needs.checks.outputs.latest_kindest_node_versions }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT fi - name: Docker Buildx From 6dc1aec3d85fd7a52682c78d30256e8036afb8b7 Mon Sep 17 00:00:00 2001 From: Eoin O'Shaughnessy Date: Tue, 2 Jan 2024 10:48:06 +0000 Subject: [PATCH 10/13] fix missing slash --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a47c64a10..920797309c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,7 @@ jobs: | sort -V \ | sed 's/v//g' \ | sed 's/$//' \ - | sed 's/, $//' + | sed 's/, $//' \ | jq -R -s -c 'split("\n")[:-1]') echo "latest_kindest_node_versions=$kindest_versions" >> $GITHUB_OUTPUT From 0f6b3aab9bad6b4d17fdde64f318ba017ea4c9f0 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 2 Jan 2024 11:49:26 +0000 Subject: [PATCH 11/13] try retry of regression tests option --- .github/workflows/ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 920797309c..6cce546858 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,12 @@ on: - release-* schedule: - cron: "0 4 * * *" # run every day at 04:00 UTC + workflow_dispatch: + inputs: + runRegression: + required: false + type: boolean + description: "Run the regression test suite if set to `true` otherwise will run smoke tests" defaults: run: @@ -277,10 +283,10 @@ jobs: - id: set-matrix run: | - if [ "${{ github.event_name }}" != "schedule" ]; then - echo "matrix=$(cat .github/data/matrix-smoke.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT - else + if [ "${{ github.event_name }}" == "schedule" ] || [ "${{ inputs.runRegression }}" ]; then echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest "${{ needs.checks.outputs.latest_kindest_node_versions }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT + else + echo "matrix=$(cat .github/data/matrix-smoke.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT fi - name: Docker Buildx From a86f9383e514aa8e731253badd63a0fd035b6510 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 2 Jan 2024 12:16:10 +0000 Subject: [PATCH 12/13] fix quoting of latest k8s version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cce546858..d9b141c8f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,7 +284,7 @@ jobs: - id: set-matrix run: | if [ "${{ github.event_name }}" == "schedule" ] || [ "${{ inputs.runRegression }}" ]; then - echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest "${{ needs.checks.outputs.latest_kindest_node_versions }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT + echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest '${{ needs.checks.outputs.latest_kindest_node_versions }}' '.k8s += $latest')" >> $GITHUB_OUTPUT else echo "matrix=$(cat .github/data/matrix-smoke.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT fi From 92451c07a3bcc9c0a4d6c0b350f070d0a2e55b73 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 2 Jan 2024 12:28:53 +0000 Subject: [PATCH 13/13] remove workflow dispatch workflow --- .github/workflows/ci.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9b141c8f0..2d93531e0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,12 +13,6 @@ on: - release-* schedule: - cron: "0 4 * * *" # run every day at 04:00 UTC - workflow_dispatch: - inputs: - runRegression: - required: false - type: boolean - description: "Run the regression test suite if set to `true` otherwise will run smoke tests" defaults: run: @@ -283,7 +277,7 @@ jobs: - id: set-matrix run: | - if [ "${{ github.event_name }}" == "schedule" ] || [ "${{ inputs.runRegression }}" ]; then + if [ "${{ github.event_name }}" == "schedule" ]; then echo "matrix=$(cat .github/data/matrix-regression.json | jq -c --argjson latest '${{ needs.checks.outputs.latest_kindest_node_versions }}' '.k8s += $latest')" >> $GITHUB_OUTPUT else echo "matrix=$(cat .github/data/matrix-smoke.json | jq -c --arg latest "${{ needs.checks.outputs.k8s_latest }}" '.k8s += [$latest]')" >> $GITHUB_OUTPUT