[v1.14] gh/workflows: IPsec key rotation improvements

.github/actions/aws/k8s-versions-schema.yaml

@@ -4,3 +4,4 @@ includeItem:
   version: str()
   region: str()
   default: bool(required=False)
+  ipsec: bool(required=False)

.github/actions/aws/k8s-versions.yaml

@@ -7,6 +7,8 @@ include:
     region: us-west-2
   - version: "1.26"
     region: us-west-1
+    default: true
   - version: "1.27"
     region: us-east-2
     default: true
+    ipsec: true

.github/actions/ipsec-key-rotate/action.yaml

@@ -0,0 +1,80 @@
+name: IPsec key rotation test
+description: Rotates keys and checks that established connections are not interrupted
+inputs:
+  key-algo:
+    required: true
+    type: string
+    description: "gcm(aes) or cbc(aes)"
+  key-type-one:
+    required: true
+    type: string
+    description: "'+' if we started with the per-tunnel key system"
+  key-type-two:
+    required: true
+    type: string
+    description: "'+' to rotate to the per-tunnel key system"
+  extra-connectivity-test-flags:
+    required: false
+    type: string
+runs:
+  using: composite
+  steps:
+    - name: Rotate IPsec Key & Test
+      uses: ./.github/actions/conn-disrupt-test
+      with:
+        job-name: conformance-ipsec-e2e-key-rotation
+        extra-connectivity-test-flags: ${{ inputs.extra-connectivity-test-flags }}
+        operation-cmd: |
+          KEYID=$(kubectl get secret -n kube-system cilium-ipsec-keys -o go-template --template={{.data.keys}} | base64 -d | grep -oP "^\d+")
+          if [[ $KEYID -ge 15 ]]; then KEYID=0; fi
+
+          if [[ "${{ inputs.key-algo }}" == "gcm(aes)" ]]; then
+            key="rfc4106(gcm(aes)) $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64) 128"
+          elif [[ "${{ inputs.key-algo }}" == "cbc(aes)" ]]; then
+            key="hmac(sha256) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64) cbc(aes) $(dd if=/dev/urandom count=32 bs=1 2> /dev/null| xxd -p -c 64)"
+          else
+            echo "Invalid key type"; exit 1
+          fi
+          data="{\"stringData\":{\"keys\":\"$((($KEYID+1)))${{ inputs.key-type-two }} ${key}\"}}"
+
+          echo "Updating IPsec secret with $data"
+          kubectl patch secret -n kube-system cilium-ipsec-keys -p="$data" -v=1
+
+          # Compute number of expected keys during key rotation depending on
+          # whether we use the single-key system (1 key) or the per-tunnel
+          # keys system (4 keys).
+          exp_nb_keys=2
+          if [[ "${{ inputs.key-type-one }}" == "+" ]]; then
+            ((exp_nb_keys+=7))
+          fi
+          if [[ "${{ inputs.key-type-two }}" == "+" ]]; then
+            ((exp_nb_keys+=7))
+          fi
+
+          # Wait until key rotation starts
+          # We expect the amount of keys in use to grow during rotation.
+          while true; do
+            keys_in_use=$(kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium encrypt status | awk '/Keys in use/ {print $NF}')
+            if [[ $keys_in_use == $exp_nb_keys ]]; then
+              break
+            fi
+            echo "Waiting until key rotation starts (seeing $keys_in_use keys, expected $exp_nb_keys)"
+            sleep 30s
+          done
+
+          exp_nb_keys=1
+          if [[ "${{ inputs.key-type-two }}" == "+" ]]; then
+            exp_nb_keys=8
+          fi
+
+          # Wait until key rotation completes
+          # By default the key rotation cleanup delay is 5min, let's sleep 4min before actively polling
+          sleep $((4*60))
+          while true; do
+            keys_in_use=$(kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium encrypt status | awk '/Keys in use/ {print $NF}')
+            if [[ $keys_in_use == $exp_nb_keys ]]; then
+              break
+            fi
+            echo "Waiting until key rotation completes (seeing $keys_in_use keys, expected $exp_nb_keys)"
+            sleep 30s
+          done

.github/workflows/conformance-eks.yaml

@@ -64,7 +64,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set initial commit status
-        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1  
+        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
         with:
           sha: ${{ inputs.SHA || github.sha }}
 
@@ -110,7 +110,7 @@ jobs:
     name: Installation and Connectivity Test
     needs: generate-matrix
     runs-on: ubuntu-latest
-    timeout-minutes: 75
+    timeout-minutes: 90
     env:
       job_name: "Installation and Connectivity Test"
     strategy:
@@ -159,6 +159,9 @@ jobs:
             --helm-set tls.secretsBackend=k8s \
             --helm-set=bpf.monitorAggregation=none \
             --wait=false"
+          if [[ "${{ matrix.ipsec }}" == "true" ]]; then
+            CILIUM_INSTALL_DEFAULTS+=" --helm-set encryption.enabled=true --helm-set encryption.type=ipsec"
+          fi
 
           CONNECTIVITY_TEST_DEFAULTS="--flow-validation=disabled --hubble=false --collect-sysdump-on-failure \
             --external-target amazon.com"
@@ -202,6 +205,13 @@ jobs:
           owner: "${{ steps.vars.outputs.owner }}"
           version: ${{ matrix.version }}
 
+      - name: Create IPsec key
+        if: ${{ matrix.ipsec == true }}
+        shell: bash
+        run: |
+          KEYID=15
+          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="${KEYID} rfc4106(gcm(aes)) $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64) 128"
+
       # This is a workaround for flake #16938.
       - name: Remove AWS-CNI
         run: |
@@ -226,13 +236,21 @@ jobs:
           sparse-checkout: |
             install/kubernetes/cilium
 
+      - name: Install Cilium CLI
+        uses: cilium/cilium-cli@de740a64da1b12dc8488cf3af2474c60d52c349e # v0.16.3
+        with:
+          release-version: ${{ env.CILIUM_CLI_VERSION }}
+          ci-version: ${{ env.cilium_cli_ci_version }}
+          binary-name: cilium-cli
+          binary-dir: ./
+
       - name: Install Cilium
         run: |
-          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}
+          ./cilium-cli install ${{ steps.vars.outputs.cilium_install_defaults }}
 
       - name: Wait for Cilium to be ready
         run: |
-          cilium status --wait
+          ./cilium-cli status --wait
           kubectl get pods -n kube-system
 
       - name: Check that AWS leftover iptables chains have been removed
@@ -248,7 +266,7 @@ jobs:
 
       - name: Port forward Relay
         run: |
-          cilium hubble port-forward&
+          ./cilium-cli hubble port-forward&
           sleep 10s
           [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
 
@@ -258,48 +276,25 @@ jobs:
 
       - name: Run connectivity test (${{ join(matrix.*, ', ') }})
         run: |
-          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
+          ./cilium-cli connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} \
           --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}) - 1.xml" \
           --junit-property github_job_step="Run connectivity test (${{ join(matrix.*, ', ') }})"
 
-      - name: Clean up Cilium
-        run: |
-          pkill -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" || test $? -eq 1
-          cilium uninstall --wait
-
-      - name: Create custom IPsec secret
-        run: |
-          kubectl create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="15 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
-
-      - name: Install Cilium with encryption
-        run: |
-          cilium install ${{ steps.vars.outputs.cilium_install_defaults }} \
-            --helm-set encryption.enabled=true \
-            --helm-set encryption.type=ipsec
-
-      - name: Wait for Cilium to be ready
-        run: |
-          cilium status --wait
-          kubectl get pods -n kube-system
-
-      - name: Port forward Relay
-        run: |
-          cilium hubble port-forward&
-          sleep 10s
-          [[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
-
-      - name: Run connectivity test with IPSec (${{ join(matrix.*, ', ') }})
-        run: |
-          cilium connectivity test ${{ steps.vars.outputs.connectivity_test_defaults }} --force-deploy \
-          --junit-file "cilium-junits/${{ env.job_name }} (${{ join(matrix.*, ', ') }}) - 2.xml" \
-          --junit-property github_job_step="Run connectivity test with IPSec (${{ join(matrix.*, ', ') }})"
+      - name: Run IPsec key rotation tests (${{ join(matrix.*, ', ') }})
+        if: ${{ matrix.ipsec == true }}
+        uses: ./.github/actions/ipsec-key-rotate
+        with:
+          extra-connectivity-test-flags: ${{ steps.vars.outputs.connectivity_test_defaults }}
+          key-algo: "gcm(aes)"
+          key-type-one: ""
+          key-type-two: ""
 
       - name: Post-test information gathering
         if: ${{ !success() }}
         run: |
           kubectl get pods --all-namespaces -o wide
-          cilium status
-          cilium sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
+          ./cilium-cli status
+          ./cilium-cli sysdump --output-filename cilium-sysdump-final-${{ join(matrix.*, '-') }}
         shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
 
       - name: Clean up EKS
@@ -358,7 +353,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set final commit status
-        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1  
+        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
         with:
           sha: ${{ inputs.SHA || github.sha }}
           status: ${{ needs.installation-and-connectivity.result }}