feat: use the code way to install core-dns

Signed-off-by: qiuwei <qiuwei_yewu@cmss.chinamobile.com>

hack/globalnode_patch.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# define the node name prefix
+node_prefix="test-ylc-m-"
+
+# Generate the name of the globalnode that needs to change state and update the state to "reserved"
+# Change the node label you want in the for loop, such as 01-09 in this case
+for ((i=1; i<=9; i++))
+do
+    # Use printf to format strings so that the number part is always two digits long
+    node_number=$(printf "%02d" $i)
+    node_name="$node_prefix$node_number"
+
+    # Update the status of the corresponding globalnode to "free" using the kubectl patch command.
+    kubectl patch globalnode $node_name --type=json -p '[{"op":"replace", "path":"/spec/state", "value":"free"}]'
+done

pkg/kubenest/controlplane/coredns/coredns.go

@@ -0,0 +1,154 @@
+package coredns
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/dynamic"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/coredns/host"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/coredns/virtualcluster"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
+)
+
+func EnsureHostCoreDns(client clientset.Interface, name, namespace string) error {
+	//create core-dns cm for host
+	err := installCoreDnsConfigMap(client, namespace)
+	if err != nil {
+		return err
+	}
+	//create core-dns rbac
+	err = EnsureCoreDnsRBAC(client, namespace, name)
+	if err != nil {
+		return err
+	}
+	//service in host has been created in the NewVirtualClusterServiceTask
+	//create core-dns deployment
+	err = installCoreDnsDeployment(client, name, namespace)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func EnsureVirtualClusterCoreDns(dynamicClient dynamic.Interface, templateMapping map[string]interface{}) error {
+	// create core-dns endpoints for virtualcluster
+	err := installCoreDnsEndpointsInVirtualCluster(dynamicClient, templateMapping)
+	if err != nil {
+		return err
+	}
+	// create core-dns service in virtualcluster
+	err = installCoreDnsServiceInVirtualCluster(dynamicClient, templateMapping)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func installCoreDnsDeployment(client clientset.Interface, name, namespace string) error {
+	imageRepository, _ := util.GetImageMessage()
+	coreDnsDeploymentBytes, err := util.ParseTemplate(host.CoreDnsDeployment, struct {
+		Namespace, Name, ImageRepository string
+	}{
+		Namespace:       namespace,
+		Name:            name,
+		ImageRepository: imageRepository,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing core-dns deployment template: %w", err)
+	}
+	coreDnsDeployment := &appsv1.Deployment{}
+	if err := yaml.Unmarshal([]byte(coreDnsDeploymentBytes), coreDnsDeployment); err != nil {
+		return fmt.Errorf("error when decoding core-dns deployment: %w", err)
+	}
+
+	if err := util.CreateOrUpdateDeployment(client, coreDnsDeployment); err != nil {
+		return fmt.Errorf("error when creating deployment for %s, err: %w", coreDnsDeployment.Name, err)
+	}
+	return nil
+}
+
+func getCoreDnsConfigMapManifest(namespace string) (*v1.ConfigMap, error) {
+	coreDnsConfigMapBytes, err := util.ParseTemplate(host.CoreDnsCM, struct {
+		Namespace string
+	}{
+		Namespace: namespace,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error when parsing core-dns configMap template: %w", err)
+	}
+
+	config := &v1.ConfigMap{}
+	if err := yaml.Unmarshal([]byte(coreDnsConfigMapBytes), config); err != nil {
+		return nil, fmt.Errorf("err when decoding core-dns configMap: %w", err)
+	}
+
+	return config, nil
+}
+
+func installCoreDnsConfigMap(client clientset.Interface, namespace string) error {
+	config, err := getCoreDnsConfigMapManifest(namespace)
+	if err != nil {
+		return err
+	}
+
+	if err := util.CreateOrUpdateConfigMap(client, config); err != nil {
+		return fmt.Errorf("error when creating configMap for %s, err: %w", config.Name, err)
+	}
+	return nil
+}
+
+func installCoreDnsServiceInVirtualCluster(dynamicClient dynamic.Interface, templateMapping map[string]interface{}) error {
+	coreDnsServiceInVcBytes, err := util.ParseTemplate(virtualcluster.CoreDnsService, templateMapping)
+	if err != nil {
+		return fmt.Errorf("error when parsing core-dns service in virtual cluster template: %w", err)
+	}
+	var obj unstructured.Unstructured
+	if err := yaml.Unmarshal([]byte(coreDnsServiceInVcBytes), &obj); err != nil {
+		return fmt.Errorf("err when decoding core-dns service in virtual cluster: %w", err)
+	}
+
+	err = util.CreateObject(dynamicClient, obj.GetNamespace(), obj.GetName(), &obj)
+	if err != nil {
+		return fmt.Errorf("error when creating core-dns service in virtual cluster err: %w", err)
+	}
+	return nil
+}
+
+func installCoreDnsEndpointsInVirtualCluster(dynamicClient dynamic.Interface, templateMapping map[string]interface{}) error {
+	coreDnsEndpointsInVcBytes, err := util.ParseTemplate(virtualcluster.CoreDnsEndpoints, templateMapping)
+	if err != nil {
+		return fmt.Errorf("error when parsing core-dns service in virtual cluster template: %w", err)
+	}
+	var obj unstructured.Unstructured
+	if err := yaml.Unmarshal([]byte(coreDnsEndpointsInVcBytes), &obj); err != nil {
+		return fmt.Errorf("err when decoding core-dns service in virtual cluster: %w", err)
+	}
+
+	err = util.CreateObject(dynamicClient, obj.GetNamespace(), obj.GetName(), &obj)
+	if err != nil {
+		return fmt.Errorf("error when creating core-dns service in virtual cluster err: %w", err)
+	}
+	return nil
+}
+
+func DeleteCoreDnsDeployment(client clientset.Interface, name, namespace string) error {
+	// delete deployment
+	deployName := fmt.Sprintf("%s-%s", name, "coredns")
+	if err := util.DeleteDeployment(client, deployName, namespace); err != nil {
+		return errors.Wrapf(err, "Failed to delete deployment %s/%s", deployName, namespace)
+	}
+
+	// delete configmap
+	cmName := "coredns"
+	if err := util.DeleteConfigmap(client, cmName, namespace); err != nil {
+		return errors.Wrapf(err, "Failed to delete configmap %s/%s", cmName, namespace)
+	}
+
+	return nil
+}

pkg/kubenest/controlplane/coredns/rbac.go

@@ -0,0 +1,77 @@
+package coredns
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/coredns/host"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
+)
+
+func EnsureCoreDnsRBAC(client clientset.Interface, namespace string, name string) error {
+	if err := grantCoreDnsClusterSA(client, namespace); err != nil {
+		return err
+	}
+	if err := grantCoreDnsClusterRoleBinding(client, namespace, name); err != nil {
+		return err
+	}
+	if err := grantCoreDnsClusterRole(client, name); err != nil {
+		return err
+	}
+	return nil
+}
+
+func grantCoreDnsClusterSA(client clientset.Interface, namespace string) error {
+	coreDnsClusterSABytes, err := util.ParseTemplate(host.CoreDnsSA, struct {
+		Namespace string
+	}{
+		Namespace: namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing core-dns sa template: %w", err)
+	}
+	serviceAccount := &v1.ServiceAccount{}
+	if err := yaml.Unmarshal([]byte(coreDnsClusterSABytes), serviceAccount); err != nil {
+		return fmt.Errorf("err when decoding core-dns view Clusterrole: %w", err)
+	}
+	return util.CreateOrUpdateClusterSA(client, serviceAccount, namespace)
+}
+
+func grantCoreDnsClusterRoleBinding(client clientset.Interface, namespace string, name string) error {
+	coreDnsClusterRoleBindingBytes, err := util.ParseTemplate(host.CoreDnsClusterRoleBinding, struct {
+		Name      string
+		Namespace string
+	}{
+		Name:      name,
+		Namespace: namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing core-dns role binding template: %w", err)
+	}
+	viewClusterRoleBinding := &rbacv1.ClusterRoleBinding{}
+
+	if err := yaml.Unmarshal([]byte(coreDnsClusterRoleBindingBytes), viewClusterRoleBinding); err != nil {
+		return fmt.Errorf("err when decoding core-dns Clusterrole Binding: %w", err)
+	}
+	return util.CreateOrUpdateClusterRoleBinding(client, viewClusterRoleBinding)
+}
+
+func grantCoreDnsClusterRole(client clientset.Interface, name string) error {
+	viewClusterRole := &rbacv1.ClusterRole{}
+	coreDnsClusterRoleBytes, err := util.ParseTemplate(host.CoreDnsClusterRole, struct {
+		Name string
+	}{
+		Name: name,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing core-dns cluster role template: %w", err)
+	}
+	if err := yaml.Unmarshal([]byte(coreDnsClusterRoleBytes), viewClusterRole); err != nil {
+		return fmt.Errorf("err when decoding core-dns Clusterrole: %w", err)
+	}
+	return util.CreateOrUpdateClusterRole(client, viewClusterRole)
+}

pkg/kubenest/controlplane/service.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/apiserver"
+	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/coredns/host"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/manifest/controlplane/etcd"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
 )
@@ -30,6 +31,7 @@ func DeleteVirtualClusterService(client clientset.Interface, name, namespace str
 		fmt.Sprintf("%s-%s", name, "apiserver"),
 		fmt.Sprintf("%s-%s", name, "etcd"),
 		fmt.Sprintf("%s-%s", name, "etcd-client"),
+		"kube-dns",
 	}
 	for _, service := range services {
 		err := client.CoreV1().Services(namespace).Delete(context.TODO(), service, metav1.DeleteOptions{})
@@ -47,6 +49,7 @@ func DeleteVirtualClusterService(client clientset.Interface, name, namespace str
 }
 
 func createServerService(client clientset.Interface, name, namespace string, port int32) error {
+	//apiserver service
 	apiserverServiceBytes, err := util.ParseTemplate(apiserver.ApiserverService, struct {
 		ServiceName, Namespace, ServiceType string
 		ServicePort                         int32
@@ -68,6 +71,7 @@ func createServerService(client clientset.Interface, name, namespace string, por
 		return fmt.Errorf("err when creating virtual cluster apiserver service for %s, err: %w", apiserverService.Name, err)
 	}
 
+	//etcd service
 	etcdServicePeerBytes, err := util.ParseTemplate(etcd.EtcdPeerService, struct {
 		ServiceName, Namespace                   string
 		EtcdListenClientPort, EtcdListenPeerPort int32
@@ -90,6 +94,7 @@ func createServerService(client clientset.Interface, name, namespace string, por
 		return fmt.Errorf("error when creating etcd client service, err: %w", err)
 	}
 
+	//etcd-client service
 	etcdClientServiceBytes, err := util.ParseTemplate(etcd.EtcdClientService, struct {
 		ServiceName, Namespace string
 		EtcdListenClientPort   int32
@@ -111,6 +116,25 @@ func createServerService(client clientset.Interface, name, namespace string, por
 		return fmt.Errorf("err when creating etcd client service, err: %w", err)
 	}
 
+	//core-dns service
+	coreDnsServiceBytes, err := util.ParseTemplate(host.CoreDnsService, struct {
+		Namespace string
+	}{
+		Namespace: namespace,
+	})
+	if err != nil {
+		return fmt.Errorf("error when parsing core-dns serive template: %w", err)
+	}
+
+	coreDnsService := &corev1.Service{}
+	if err := yaml.Unmarshal([]byte(coreDnsServiceBytes), coreDnsService); err != nil {
+		return fmt.Errorf("err when decoding core-dns service: %w", err)
+	}
+
+	if err := createOrUpdateService(client, coreDnsService); err != nil {
+		return fmt.Errorf("err when creating core-dns service, err: %w", err)
+	}
+
 	return nil
 }
 

pkg/kubenest/manifest/controlplane/coredns/host/manifest_configmap.go

@@ -0,0 +1,34 @@
+package host
+
+const (
+	CoreDnsCM = `
+apiVersion: v1
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+           lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+           pods insecure
+           fallthrough in-addr.arpa ip6.arpa
+           ttl 30
+           kubeconfig /etc/apiserver/kubeconfig
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+           max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: {{ .Namespace }}
+`
+)