gha: extend conformance clustermesh to also cover external kvstores

[ upstream commit 403b3a2 ]

[ backporter's notes: in Cilium v1.14 and earlier, the clustermesh
  configuration secret is created only when the clustermesh-apiserver
  is enabled. For this reason, we need to enable it also when actually
  connecting to a remote kvstore cluster, although with zero replicas. ]

Let's extend the conformance clustermesh workflow to also cover the
external kvstores configuration in addition to plain clustermesh and
kvstoremesh. To avoid increasing the number of matrix entries, let's
convert two of the already existing ones over to this mode.

We leverage the newly introduced kvstore action to setup the etcd
containers and retrieve the appropriate parameters. Cluster Mesh
configurations are directly specified at installation time, as
'cilium clustermesh connect' does not support this scenario.

Signed-off-by: Marco Iorio <marco.iorio@isovalent.com>
Signed-off-by: Marco Iorio <marco.iorio@isovalent.com>

.github/workflows/conformance-clustermesh.yaml

@@ -129,9 +129,7 @@ jobs:
             ipfamily: 'dual'
             encryption: 'ipsec'
             kube-proxy: 'iptables'
-            mode: 'kvstoremesh'
-            cm-auth-mode-1: 'cluster'
-            cm-auth-mode-2: 'cluster'
+            mode: 'external'
 
           - name: '6'
             tunnel: 'vxlan'
@@ -200,6 +198,10 @@ jobs:
           fi
 
           # bpf.masquerade is disabled due to #23283
+          # We enable the clustermesh-apiserver (although with zero replicas)
+          # also when actually connecting to an external kvstore. This is a
+          # workaround to enable creating the clustermesh configuration secret,
+          # that is otherwise skipped in Cilium v1.14 (and earlier).
           CILIUM_INSTALL_DEFAULTS="--chart-directory=install/kubernetes/cilium \
             --helm-set=debug.enabled=true \
             --helm-set=image.repository=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/cilium-ci \
@@ -216,6 +218,7 @@ jobs:
             --helm-set=hubble.relay.image.override=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/hubble-relay-ci:${SHA} \
             --helm-set=hubble.relay.image.useDigest=false \
             --helm-set=clustermesh.useAPIServer=true \
+            --helm-set=clustermesh.apiserver.replicas=${{ matrix.mode == 'external' && '0' || '1' }} \
             --helm-set=clustermesh.apiserver.kvstoremesh.enabled=${{ matrix.mode == 'kvstoremesh' }} \
             --helm-set=clustermesh.apiserver.image.override=quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/clustermesh-apiserver-ci:${SHA} \
             --helm-set=clustermesh.apiserver.image.useDigest=false \
@@ -347,6 +350,30 @@ jobs:
           kubectl --context ${{ env.contextName1 }} patch deployment -n kube-system coredns --patch="$COREDNS_PATCH"
           kubectl --context ${{ env.contextName2 }} patch deployment -n kube-system coredns --patch="$COREDNS_PATCH"
 
+      - name: Start kvstore clusters
+        id: kvstore
+        if: matrix.mode == 'external'
+        uses: ./.github/actions/kvstore
+        with:
+          clusters: 2
+
+      - name: Create the secret containing the kvstore credentials
+        if: matrix.mode == 'external'
+        run: |
+          kubectl --context ${{ env.contextName1 }} create -n kube-system -f ${{ steps.kvstore.outputs.cilium_etcd_secrets_path }}
+          kubectl --context ${{ env.contextName2 }} create -n kube-system -f ${{ steps.kvstore.outputs.cilium_etcd_secrets_path }}
+
+      - name: Set clustermesh connection parameters
+        if: matrix.mode == 'external'
+        id: clustermesh-vars
+        run: |
+          echo "cilium_install_clustermesh= \
+            --set=clustermesh.config.enabled=true \
+            --set clustermesh.config.clusters[0].name=${{ env.clusterName1 }} \
+            --set clustermesh.config.clusters[1].name=${{ env.clusterName2 }} \
+            ${{ steps.kvstore.outputs.cilium_install_clustermesh }} \
+          " >> $GITHUB_OUTPUT
+
       - name: Wait for images to be available
         timeout-minutes: 30
         shell: bash
@@ -371,6 +398,8 @@ jobs:
           kubectl --context ${{ env.contextName2 }} create -n kube-system secret generic cilium-ipsec-keys --from-literal=keys="${SECRET}"
 
       - name: Install Cilium in cluster1
+        env:
+          KVSTORE_ID: 1
         run: |
           # Explicitly configure the NodePort to make sure that it is different in
           # each cluster, to workaround #24692
@@ -379,14 +408,18 @@ jobs:
             --helm-set cluster.name=${{ env.clusterName1 }} \
             --helm-set cluster.id=1 \
             --helm-set clustermesh.apiserver.service.nodePort=32379 \
-            --helm-set clustermesh.apiserver.tls.authMode=${{ matrix.cm-auth-mode-1 }}
+            --helm-set clustermesh.apiserver.tls.authMode=${{ matrix.mode != 'external' && matrix.cm-auth-mode-1 || 'legacy' }} \
+            ${{ steps.kvstore.outputs.cilium_install_kvstore }} \
+            ${{ steps.clustermesh-vars.outputs.cilium_install_clustermesh }}
 
       - name: Copy the Cilium CA secret to cluster2, as they must match
         run: |
           kubectl --context ${{ env.contextName1 }} get secret -n kube-system cilium-ca -o yaml |
             kubectl --context ${{ env.contextName2 }} create -f -
 
       - name: Install Cilium in cluster2
+        env:
+          KVSTORE_ID: 2
         run: |
           # Explicitly configure the NodePort to make sure that it is different in
           # each cluster, to workaround #24692
@@ -395,7 +428,9 @@ jobs:
             --helm-set cluster.name=${{ env.clusterName2 }} \
             --helm-set cluster.id=255 \
             --helm-set clustermesh.apiserver.service.nodePort=32380 \
-            --helm-set clustermesh.apiserver.tls.authMode=${{ matrix.cm-auth-mode-2 }}
+            --helm-set clustermesh.apiserver.tls.authMode=${{ matrix.mode != 'external' && matrix.cm-auth-mode-2 || 'legacy' }} \
+            ${{ steps.kvstore.outputs.cilium_install_kvstore }} \
+            ${{ steps.clustermesh-vars.outputs.cilium_install_clustermesh }}
 
       - name: Wait for cluster mesh status to be ready
         run: |
@@ -405,10 +440,12 @@ jobs:
           cilium --context ${{ env.contextName2 }} clustermesh status --wait
 
       - name: Connect clusters
+        if: matrix.mode != 'external'
         run: |
           cilium --context ${{ env.contextName1 }} clustermesh connect --destination-context ${{ env.contextName2 }}
 
       - name: Wait for cluster mesh status to be ready
+        if: matrix.mode != 'external'
         run: |
           cilium --context ${{ env.contextName1 }} status --wait
           cilium --context ${{ env.contextName2 }} status --wait
@@ -446,6 +483,14 @@ jobs:
           kubectl config use-context ${{ env.contextName2 }}
           kubectl get pods --all-namespaces -o wide
           cilium sysdump --output-filename cilium-sysdump-context2-final-${{ join(matrix.*, '-') }}
+
+          if [ "${{ matrix.mode }}" == "external" ]; then
+            for i in {1..2}; do
+              echo
+              echo "# Retrieving logs from kvstore$i docker container"
+              docker logs kvstore$i
+            done
+          fi
         shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
 
       - name: Upload artifacts