move the vpc-nat generic configurations into one single ConfigMap (#2574

)

* move the vpc-nat generic configurations into one single ConfigMap

* make vpc-nat image name configurable

---------

Co-authored-by: 夜微澜 <qiutingjun@cmss.chinamobile.com>

dist/images/install.sh

@@ -36,6 +36,7 @@ CNI_CONF_DIR="/etc/cni/net.d"
 CNI_BIN_DIR="/opt/cni/bin"
 
 REGISTRY="kubeovn"
+VPC_NAT_IMAGE="vpc-nat-gateway"
 VERSION="v1.12.0"
 IMAGE_PULL_POLICY="IfNotPresent"
 POD_CIDR="10.16.0.0/16"                # Do NOT overlap with NODE/SVC/JOIN CIDR
@@ -3106,6 +3107,17 @@ echo "[Step 3/6] Install Kube-OVN"
 
 cat <<EOF > kube-ovn.yaml
 ---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovn-vpc-nat-config
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      kube-ovn vpc-nat common config
+data:
+  image: $REGISTRY/$VPC_NAT_IMAGE:$VERSION
+---
 kind: Deployment
 apiVersion: apps/v1
 metadata:

kubeovn-helm/templates/vpc-nat-config.yaml

@@ -0,0 +1,10 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: ovn-vpc-nat-config
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      kube-ovn vpc-nat common config
+data:
+  image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.vpcRepository }}:{{ .Values.global.images.kubeovn.tag }}

kubeovn-helm/values.yaml

@@ -9,6 +9,7 @@ global:
     kubeovn:
       repository: kube-ovn
       dpdkRepository: kube-ovn-dpdk
+      vpcRepository: vpc-nat-gateway
       tag: v1.12.0
       support_arm: true
       thirdparty: true

pkg/controller/controller.go

@@ -989,6 +989,10 @@ func (c *Controller) startWorkers(ctx context.Context) {
 		c.resyncExternalGateway()
 	}, time.Second, ctx.Done())
 
+	go wait.Until(func() {
+		c.resyncVpcNatConfig()
+	}, time.Second, ctx.Done())
+
 	go wait.Until(func() {
 		c.resyncVpcNatGwConfig()
 	}, time.Second, ctx.Done())

pkg/controller/service_lb.go

@@ -77,11 +77,6 @@ func (c *Controller) genLbSvcDeployment(svc *corev1.Service) (dp *v1.Deployment)
 		"service":   svc.Name,
 	}
 
-	image := "kubeovn/vpc-nat-gateway:v1.12.0"
-	if svc.Annotations[util.LbSvcPodImg] != "" {
-		image = svc.Annotations[util.LbSvcPodImg]
-	}
-
 	attachmentName, attachmentNs := parseAttachNetworkProvider(svc)
 	providerName := getAttachNetworkProvider(svc)
 	attachSubnetAnnotation := fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, providerName)
@@ -115,7 +110,7 @@ func (c *Controller) genLbSvcDeployment(svc *corev1.Service) (dp *v1.Deployment)
 					Containers: []corev1.Container{
 						{
 							Name:            "lb-svc",
-							Image:           image,
+							Image:           vpcNatImage,
 							Command:         []string{"bash"},
 							Args:            []string{"-c", "while true; do sleep 10000; done"},
 							ImagePullPolicy: corev1.PullIfNotPresent,

pkg/controller/vpc_dns.go

@@ -40,7 +40,6 @@ var (
 	cmVersion       = ""
 	k8sServiceHost  = ""
 	k8sServicePort  = ""
-	initRouteImage  = ""
 	enableCoredns   = false
 	hostNameservers []string
 )
@@ -49,7 +48,6 @@ const (
 	CorednsContainerName = "coredns"
 	CorednsLabelKey      = "k8s-app"
 	CorednsTemplateDep   = "coredns-template.yaml"
-	InitRouteImage       = "kubeovn/vpc-nat-gateway:v1.11.0"
 )
 
 func genVpcDnsDpName(name string) string {
@@ -460,7 +458,7 @@ func setVpcDnsRoute(dp *v1.Deployment, subnetGw string) {
 	allowPrivilegeEscalation := true
 	dp.Spec.Template.Spec.InitContainers = append(dp.Spec.Template.Spec.InitContainers, corev1.Container{
 		Name:            "init-route",
-		Image:           initRouteImage,
+		Image:           vpcNatImage,
 		Command:         []string{"sh", "-c", routeCmd},
 		ImagePullPolicy: corev1.PullIfNotPresent,
 		SecurityContext: &corev1.SecurityContext{
@@ -573,12 +571,6 @@ func (c *Controller) resyncVpcDnsConfig() {
 		}
 	}
 	enableCoredns = newEnableCoredns
-
-	if newInitRouteImage, ok := cm.Data["init-route-image"]; ok {
-		initRouteImage = newInitRouteImage
-	} else {
-		initRouteImage = InitRouteImage
-	}
 }
 
 func (c *Controller) getDefaultCoreDnsImage() (string, error) {

pkg/controller/vpc_lb.go

@@ -16,8 +16,6 @@ import (
 	"github.com/kubeovn/kube-ovn/pkg/util"
 )
 
-const vpcLbImage = "kubeovn/vpc-nat-gateway:v1.8.0"
-
 func vpcLbDeploymentName(vpc string) string {
 	return fmt.Sprintf("vpc-%s-lb", vpc)
 }
@@ -131,7 +129,7 @@ func (c *Controller) genVpcLbDeployment(vpc *kubeovnv1.Vpc) (*v1.Deployment, err
 					Containers: []corev1.Container{
 						{
 							Name:            "vpc-lb",
-							Image:           vpcLbImage,
+							Image:           vpcNatImage,
 							Command:         []string{"bash"},
 							Args:            []string{"-c", "while true; do sleep 10000; done"},
 							ImagePullPolicy: corev1.PullIfNotPresent,
@@ -154,7 +152,7 @@ func (c *Controller) genVpcLbDeployment(vpc *kubeovnv1.Vpc) (*v1.Deployment, err
 	if v4Gw != "" && v4Svc != "" {
 		deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, corev1.Container{
 			Name:            "init-ipv4-route",
-			Image:           vpcLbImage,
+			Image:           vpcNatImage,
 			Command:         []string{"ip"},
 			Args:            strings.Fields(fmt.Sprintf("-4 route add %s via %s", v4Svc, v4Gw)),
 			ImagePullPolicy: corev1.PullIfNotPresent,
@@ -164,7 +162,7 @@ func (c *Controller) genVpcLbDeployment(vpc *kubeovnv1.Vpc) (*v1.Deployment, err
 			},
 		}, corev1.Container{
 			Name:            "init-ipv4-iptables",
-			Image:           vpcLbImage,
+			Image:           vpcNatImage,
 			Command:         []string{"iptables"},
 			Args:            strings.Fields(fmt.Sprintf("-t nat -I POSTROUTING -d %s -j MASQUERADE", v4Svc)),
 			ImagePullPolicy: corev1.PullIfNotPresent,
@@ -177,7 +175,7 @@ func (c *Controller) genVpcLbDeployment(vpc *kubeovnv1.Vpc) (*v1.Deployment, err
 	if v6Gw != "" && v6Svc != "" {
 		deployment.Spec.Template.Spec.InitContainers = append(deployment.Spec.Template.Spec.InitContainers, corev1.Container{
 			Name:            "init-ipv6-route",
-			Image:           vpcLbImage,
+			Image:           vpcNatImage,
 			Command:         []string{"ip"},
 			Args:            strings.Fields(fmt.Sprintf("-6 route add %s via %s", v6Svc, v6Gw)),
 			ImagePullPolicy: corev1.PullIfNotPresent,
@@ -187,7 +185,7 @@ func (c *Controller) genVpcLbDeployment(vpc *kubeovnv1.Vpc) (*v1.Deployment, err
 			},
 		}, corev1.Container{
 			Name:            "init-ipv6-iptables",
-			Image:           vpcLbImage,
+			Image:           vpcNatImage,
 			Command:         []string{"ip6tables"},
 			Args:            strings.Fields(fmt.Sprintf("-t nat -I POSTROUTING -d %s -j MASQUERADE", v6Svc)),
 			ImagePullPolicy: corev1.PullIfNotPresent,

pkg/controller/vpc_nat.go

@@ -0,0 +1,21 @@
+package controller
+
+import (
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/klog/v2"
+
+	"github.com/kubeovn/kube-ovn/pkg/util"
+)
+
+var (
+	vpcNatImage = ""
+)
+
+func (c *Controller) resyncVpcNatConfig() {
+	cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.VpcNatConfig)
+	if err != nil && !k8serrors.IsNotFound(err) {
+		klog.Errorf("failed to get ovn-vpc-nat-config, %v", err)
+		return
+	}
+	vpcNatImage = cm.Data["image"]
+}

pkg/controller/vpc_nat_gateway.go

@@ -26,7 +26,6 @@ import (
 )
 
 var (
-	vpcNatImage       = ""
 	vpcNatEnabled     = "unknown"
 	VpcNatCmVersion   = ""
 	NAT_GW_CREATED_AT = ""
@@ -60,7 +59,7 @@ func (c *Controller) resyncVpcNatGwConfig() {
 		return
 	}
 
-	if k8serrors.IsNotFound(err) || cm.Data["enable-vpc-nat-gw"] == "false" || cm.Data["image"] == "" {
+	if k8serrors.IsNotFound(err) || cm.Data["enable-vpc-nat-gw"] == "false" {
 		if vpcNatEnabled == "false" {
 			return
 		}

pkg/util/const.go

@@ -166,6 +166,7 @@ const (
 	VpcLbNetworkAttachment = "ovn-vpc-lb"
 	VpcDnsConfig           = "vpc-dns-config"
 	VpcDnsDepTemplate      = "vpc-dns-dep"
+	VpcNatConfig           = "ovn-vpc-nat-config"
 
 	DefaultSecurityGroupName = "default-securitygroup"
 

pkg/webhook/vpc_nat.go

@@ -0,0 +1,30 @@
+package webhook
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/kubeovn/kube-ovn/pkg/util"
+)
+
+func (v *ValidatingHook) ValidateVpcNatConfig(ctx context.Context) error {
+	cm := &corev1.ConfigMap{}
+	cmKey := types.NamespacedName{Namespace: "kube-system", Name: util.VpcNatConfig}
+	if err := v.cache.Get(ctx, cmKey, cm); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return fmt.Errorf("configMap \"%s\" not configured", util.VpcNatConfig)
+		}
+		return err
+	}
+
+	if cm.Data["image"] == "" {
+		err := fmt.Errorf("parameter \"image\" in ConfigMap \"%s\" cannot be empty", util.VpcNatConfig)
+		return err
+	}
+
+	return nil
+}