gh/workflows: Add IPsec key rotation action and use it in ci-eks / ci-ipsec-e2e

.github/actions/aws/k8s-versions-schema.yaml

@@ -4,3 +4,4 @@ includeItem:
   version: str()
   region: str()
   default: bool(required=False)
+  ipsec: bool(required=False)

.github/actions/aws/k8s-versions.yaml

@@ -7,10 +7,14 @@ include:
     region: us-west-2
   - version: "1.26"
     region: us-west-1
+    default: true
   - version: "1.27"
     region: us-east-2
     default: true
+    ipsec: true
   - version: "1.28"
     region: ca-central-1
+    ipsec: true
   - version: "1.29"
     region: us-east-1
+    ipsec: true

.github/actions/ipsec-key-rotate/action.yaml

@@ -0,0 +1,55 @@
+name: IPsec key rotation test
+description: Rotates keys and checks that established connections are not interrupted
+inputs:
+  key-type:
+    required: true
+    type: string
+    description: "gcm(aes) or cbc(aes)"
+  extra-connectivity-test-flags:
+    required: false
+    type: string
+runs:
+  using: composite
+  steps:
+    - name: Rotate IPsec Key & Test
+      uses: ./.github/actions/conn-disrupt-test
+      with:
+        job-name: conformance-ipsec-e2e-key-rotation
+        extra-connectivity-test-flags: ${{ inputs.extra-connectivity-test-flags }}
+        operation-cmd: |
+          KEYID=$(kubectl get secret -n kube-system cilium-ipsec-keys -o go-template --template={{.data.keys}} | base64 -d | cut -d' ' -f1)
+          if [[ $KEYID -ge 15 ]]; then KEYID=0; fi
+
+          if [[ "${{ inputs.key-type }}" == "gcm(aes)" ]]; then
+            key="rfc4106(gcm(aes)) $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64) 128"
+          elif [[ "${{ inputs.key-type }}" == "cbc(aes)" ]]; then
+            key="hmac(sha256) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) cbc(aes) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64)"
+          else
+            echo "Invalid key type"; exit 1
+          fi
+          data="{\"stringData\":{\"keys\":\"$((($KEYID+1))) ${key}\"}}"
+
+          echo "Updating IPsec secret with $data"
+          kubectl patch secret -n kube-system cilium-ipsec-keys -p="$data" -v=1
+
+          # Wait until key rotation starts
+          while true; do
+            keys_in_use=$(kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg encrypt status | awk '/Keys in use/ {print $NF}')
+            if [[ $keys_in_use == 2 ]]; then
+              break
+            fi
+            echo "Waiting until key rotation starts (seeing $keys_in_use keys)"
+            sleep 30s
+          done
+
+          # Wait until key rotation completes
+          # By default the key rotation cleanup delay is 5min, let's sleep 4min before actively polling
+          sleep $((4*60))
+          while true; do
+            keys_in_use=$(kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg encrypt status | awk '/Keys in use/ {print $NF}')
+            if [[ $keys_in_use == 1 ]]; then
+              break
+            fi
+            echo "Waiting until key rotation completes (seeing $keys_in_use keys)"
+            sleep 30s
+          done

.github/workflows/conformance-eks.yaml

@@ -67,7 +67,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set initial commit status
-        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1  
+        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
         with:
           sha: ${{ inputs.SHA || github.sha }}
 
@@ -113,7 +113,7 @@ jobs:
     name: Installation and Connectivity Test
     needs: generate-matrix
     runs-on: ubuntu-latest
-    timeout-minutes: 75
+    timeout-minutes: 90
     env:
       job_name: "Installation and Connectivity Test"
     strategy:
@@ -154,6 +154,9 @@ jobs:
             --helm-set tls.secretsBackend=k8s \
             --helm-set=bpf.monitorAggregation=none \
             --wait=false"
+          if [[ "${{ matrix.ipsec }}" == "true" ]]; then
+            CILIUM_INSTALL_DEFAULTS+=" --helm-set encryption.enabled=true --helm-set encryption.type=ipsec"
+          fi
 
           CONNECTIVITY_TEST_DEFAULTS="--flow-validation=disabled --hubble=false --collect-sysdump-on-failure \
             --external-target amazon.com"
@@ -198,6 +201,13 @@ jobs:
           version: ${{ matrix.version }}
           spot: ${{ github.event_name != 'schedule' }}
 
+      - name: Create IPsec key
+        if: ${{ matrix.ipsec == true }}
+        shell: bash
+        run: |
+          KEYID=15
+          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="${KEYID} rfc4106(gcm(aes)) $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64) 128"
+
       - name: Wait for images to be available
         timeout-minutes: 30
         shell: bash
@@ -248,14 +258,23 @@ jobs:
           sparse-checkout: |
             install/kubernetes/cilium
 
+      - name: Install Cilium CLI
+        uses: cilium/cilium-cli@32109b32259a487c803648a3c9463cdcafa4c0c3 # v0.15.19
+        with:
+          repository: ${{ env.CILIUM_CLI_RELEASE_REPO }}
+          release-version: ${{ env.CILIUM_CLI_VERSION }}
+          ci-version: ${{ env.cilium_cli_ci_version }}
+          binary-name: cilium-cli
+          binary-dir: ./
+
       - name: Install Cilium
         id: install-cilium
         run: |
-          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}
+          ./cilium-cli install ${{ steps.vars.outputs.cilium_install_defaults }}
 
       - name: Wait for Cilium to be ready
         run: |
-          cilium status --wait --wait-duration=10m
+          ./cilium-cli status --wait --wait-duration=10m
           kubectl get pods -n kube-system
 
       - name: Check that AWS leftover iptables chains have been removed
@@ -271,7 +290,7 @@ jobs:
 
       - name: Port forward Relay
         run: |
-          cilium hubble port-forward&
+          ./cilium-cli hubble port-forward&
           sleep 10s
           [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
 
@@ -281,48 +300,23 @@ jobs:
 
       - name: Run connectivity test (${{ join(matrix.*, ', ') }})
         run: |
-          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
+          ./cilium-cli connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
           --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}) - 1.xml" \
           --junit-property github_job_step="Run connectivity test (${{ join(matrix.*, ', ') }})"
 
-      - name: Clean up Cilium
-        run: |
-          pkill -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" || test $? -eq 1
-          cilium uninstall --wait
-
-      - name: Create custom IPsec secret
-        run: |
-          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="15 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
-
-      - name: Install Cilium with encryption
-        run: |
-          cilium install ${{ steps.vars.outputs.cilium_install_defaults }} \
-            --helm-set encryption.enabled=true \
-            --helm-set encryption.type=ipsec
-
-      - name: Wait for Cilium to be ready
-        run: |
-          cilium status --wait --wait-duration=10m
-          kubectl get pods -n kube-system
-
-      - name: Port forward Relay
-        run: |
-          cilium hubble port-forward&
-          sleep 10s
-          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
-
-      - name: Run connectivity test with IPSec (${{ join(matrix.*, ', ') }})
-        run: |
-          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} --force-deploy \
-          --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}) - 2.xml" \
-          --junit-property github_job_step="Run connectivity test with IPSec (${{ join(matrix.*, ', ') }})"
+      - name: Run IPsec key rotation tests (${{ join(matrix.*, ', ') }})
+        if: ${{ matrix.ipsec == true }}
+        uses: ./.github/actions/ipsec-key-rotate
+        with:
+          extra-connectivity-test-flags: ${{ steps.vars.outputs.connectivity_test_defaults }}
+          key-type: "gcm(aes)"
 
       - name: Post-test information gathering
         if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
         run: |
           kubectl get pods --all-namespaces -o wide
-          cilium status
-          cilium sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
+          ./cilium-cli status
+          ./cilium-cli sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
         shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
 
       - name: Upload artifacts
@@ -375,7 +369,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set final commit status
-        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1  
+        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
         with:
           sha: ${{ inputs.SHA || github.sha }}
           status: ${{ needs.installation-and-connectivity.result }}

.github/workflows/conformance-ipsec-e2e.yaml

@@ -121,7 +121,7 @@ jobs:
             key-one: 'cbc(aes)'
             key-two: 'gcm(aes)'
 
-    timeout-minutes: 70
+    timeout-minutes: 75
     steps:
       - name: Checkout context ref (trusted)
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -242,44 +242,9 @@ jobs:
             --flush-ct
 
       - name: Rotate IPsec Key & Test (${{ join(matrix.*, ', ') }})
-        uses: ./.github/actions/conn-disrupt-test
+        uses: ./.github/actions/ipsec-key-rotate
         with:
-          job-name: conformance-ipsec-e2e-key-rotation-${{ matrix.name }}
-          operation-cmd: |
-            KEYID=$(kubectl get secret -n kube-system cilium-ipsec-keys -o go-template --template={{.data.keys}} | base64 -d | cut -c 1)
-            if [[ $KEYID -ge 15 ]]; then KEYID=0; fi
-
-            if [[ "${{ matrix.key-two }}" == "gcm(aes)" ]]; then
-              key="rfc4106(gcm(aes)) $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64) 128"
-            elif [[ "${{ matrix.key-two }}" == "cbc(aes)" ]]; then
-              key="hmac(sha256) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) cbc(aes) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64)"
-            else
-              echo "Invalid key type"; exit 1
-            fi
-            data="{\"stringData\":{\"keys\":\"$((($KEYID+1))) ${key}\"}}"
-            kubectl patch secret -n kube-system cilium-ipsec-keys -p="$data" -v=1
-
-            # Wait until key rotation starts
-            while true; do
-              keys_in_use=$(kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg encrypt status | awk '/Keys in use/ {print $NF}')
-              if [[ $keys_in_use == 2 ]]; then
-                break
-              fi
-              echo "Waiting until key rotation starts (seeing $keys_in_use keys)"
-              sleep 30s
-            done
-
-            # Wait until key rotation completes
-            # By default the key rotation cleanup delay is 5min, let's sleep 4min before actively polling
-            sleep $((4*60))
-            while true; do
-              keys_in_use=$(kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg encrypt status | awk '/Keys in use/ {print $NF}')
-              if [[ $keys_in_use == 1 ]]; then
-                break
-              fi
-              echo "Waiting until key rotation completes (seeing $keys_in_use keys)"
-              sleep 30s
-            done
+          key-type: ${{ matrix.key-two }}
 
       - name: Fetch artifacts
         if: ${{ !success() }}