gha: additionally cover BPF masquerade in clustermesh E2E tests

[ upstream commit a1089a7 ]

Currently, BPF masquerade was always disabled in the clustermesh
E2E tests due to unintended interactions with Docker iptables
rules breaking DNS resolution [1]. Instead, let's explicitly
configure external upstream DNS servers for coredns, so that we
can also enable this feature when KPR is enabled.

While being there, let's also make the KPR setting explicit,
instead of relying on the Cilium CLI configuration (which is based
on whether the kube-proxy daemonset is present or not).

[1]: #23283

Signed-off-by: Marco Iorio <marco.iorio@isovalent.com>
Signed-off-by: Nicolas Busseneau <nicolas@isovalent.com>

.github/workflows/conformance-clustermesh.yaml

@@ -211,9 +211,9 @@ jobs:
         id: vars
         run: |
 
-          # bpf.masquerade is disabled due to #23283
           CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
-            --helm-set=bpf.masquerade=false \
+            --helm-set=kubeProxyReplacement=${{ matrix.kube-proxy == 'none' }} \
+            --helm-set=bpf.masquerade=${{ matrix.kube-proxy == 'none' }} \
             --helm-set=hubble.enabled=true \
             --helm-set=hubble.relay.enabled=true \
             --helm-set=clustermesh.useAPIServer=${{ matrix.mode != 'external' }} \
@@ -341,8 +341,8 @@ jobs:
 
       # Make sure that coredns uses IPv4-only upstream DNS servers also in case of clusters
       # with IP family dual, since IPv6 ones are not reachable and cause spurious failures.
+      # Additionally, this is also required to workaround #23283.
       - name: Configure the coredns nameservers
-        if: matrix.ipfamily == 'dual'
         run: |
           COREDNS_PATCH="
           spec:

.github/workflows/tests-clustermesh-upgrade.yaml

@@ -141,21 +141,21 @@ jobs:
           CILIUM_DOWNGRADE_VERSION=$(contrib/scripts/print-downgrade-version.sh)
           echo "downgrade_version=${CILIUM_DOWNGRADE_VERSION}" >> $GITHUB_OUTPUT
 
-          # * bpf.masquerade is disabled due to #23283
           # * Hubble is disabled to avoid the performance penalty in the testing
           #   environment due to the relatively high traffic load.
           # * We explicitly configure the sync timeout to a higher value to
           #   give enough time to the clustermesh-apiserver to restart after
           #   the upgrade/downgrade before that agents regenerate the endpoints.
           CILIUM_INSTALL_DEFAULTS=" \
             --set=debug.enabled=true \
-            --set=bpf.masquerade=false \
             --set=bpf.monitorAggregation=none \
             --set=hubble.enabled=false \
             --set=routingMode=tunnel \
             --set=tunnelProtocol=vxlan \
             --set=ipv4.enabled=true \
             --set=ipv6.enabled=true \
+            --set=kubeProxyReplacement=${{ matrix.kube-proxy == 'none' }} \
+            --set=bpf.masquerade=${{ matrix.kube-proxy == 'none' }} \
             --set=clustermesh.useAPIServer=${{ !matrix.external-kvstore }} \
             --set=clustermesh.config.enabled=true \
             --set=extraConfig.clustermesh-ip-identities-sync-timeout=10m"
@@ -228,6 +228,25 @@ jobs:
           config: ./.github/kind-config-cluster2.yaml
           wait: 0 # The control-plane never becomes ready, since no CNI is present
 
+      # Make sure that coredns uses IPv4-only upstream DNS servers also in case of clusters
+      # with IP family dual, since IPv6 ones are not reachable and cause spurious failures.
+      # Additionally, this is also required to workaround #23283.
+      - name: Configure the coredns nameservers
+        run: |
+          COREDNS_PATCH="
+          spec:
+            template:
+              spec:
+                dnsPolicy: None
+                dnsConfig:
+                  nameservers:
+                  - 8.8.4.4
+                  - 8.8.8.8
+          "
+
+          kubectl --context ${{ env.contextName1 }} patch deployment -n kube-system coredns --patch="$COREDNS_PATCH"
+          kubectl --context ${{ env.contextName2 }} patch deployment -n kube-system coredns --patch="$COREDNS_PATCH"
+
       - name: Create the IPSec secret in both clusters
         if: matrix.encryption == 'ipsec'
         run: |