fix(cilium): revert kubevirt changes, prepare cluster for BGP

kubernetes/apps/kube-system/cilium/templates/bgp.yaml

@@ -0,0 +1,35 @@
+# {{ if not .Values.skipCiliumExternals }}
+# disable it for now, keep for future
+# ---
+# apiVersion: cilium.io/v2alpha1
+# kind: CiliumLoadBalancerIPPool
+# metadata:
+#   name: pool
+# spec:
+#   cidrs:
+#     - cidr: "<path:kubernetes/data/internal/base#CIDR_LOADBALANCER>"
+# ---
+# apiVersion: cilium.io/v2alpha1
+# kind: CiliumBGPPeeringPolicy
+# metadata:
+#   name: bgp-peering-policy-worker
+# spec:
+#   virtualRouters:
+#     - localASN: <path:kubernetes/data/internal/base#ASN_CLUSTER>
+#       serviceSelector:
+#         matchExpressions:
+#           - key: "io.cilium/bgp-announce"
+#             operator: NotIn
+#             values:
+#               - ignore
+#       neighbors:
+#         - peerAddress: "<path:kubernetes/data/internal/base#IP_GATEWAY>/32"
+#           peerASN: <path:kubernetes/data/internal/base#ASN_GATEWAY>
+#           eBGPMultihopTTL: 10
+#           connectRetryTimeSeconds: 120
+#           holdTimeSeconds: 90
+#           keepAliveTimeSeconds: 30
+#           gracefulRestart:
+#             enabled: true
+#             restartTimeSeconds: 120
+# {{ end }}

kubernetes/apps/kube-system/cilium/values.yaml

@@ -6,8 +6,8 @@ cilium:
     bbr: true
   bpf:
     masquerade: false
-  bgp:
-    enabled: false
+  bgpControlPlane:
+    enabled: true
   cgroup:
     autoMount:
       enabled: false
@@ -117,3 +117,7 @@ cilium:
   socketLB:
     hostNamespaceOnly: true
   tunnelProtocol: geneve  # using vxlan here will break VPN pod-gateway
+
+  # when implementing full BGP, remove this encapsulation and try
+  # autoDirectNodeRoutes: true
+  # routingMode: native