[Federation] Create configmap for the cluster kube-dns when cluster joins and remove when it unjoins

federation/pkg/kubefed/BUILD

@@ -20,6 +20,9 @@ go_library(
         "//federation/apis/federation:go_default_library",
         "//federation/pkg/kubefed/init:go_default_library",
         "//federation/pkg/kubefed/util:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/apis/extensions:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/kubectl:go_default_library",
         "//pkg/kubectl/cmd:go_default_library",
         "//pkg/kubectl/cmd/templates:go_default_library",
@@ -54,6 +57,7 @@ go_test(
         "//pkg/api:go_default_library",
         "//pkg/api/testapi:go_default_library",
         "//pkg/api/v1:go_default_library",
+        "//pkg/apis/extensions/v1beta1:go_default_library",
         "//pkg/kubectl/cmd/testing:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//vendor:k8s.io/apimachinery/pkg/api/equality",

federation/pkg/kubefed/init/init.go

@@ -66,6 +66,10 @@ const (
 	ControllerManagerCN         = "federation-controller-manager"
 	AdminCN                     = "admin"
 	HostClusterLocalDNSZoneName = "cluster.local."
+	APIServerNameSuffix         = "apiserver"
+	CMNameSuffix                = "controller-manager"
+	CredentialSuffix            = "credentials"
+	KubeconfigNameSuffix        = "kubeconfig"
 
 	// User name used by federation controller manager to make
 	// calls to federation API server.
@@ -225,16 +229,16 @@ func (i *initFederation) Complete(cmd *cobra.Command, args []string) error {
 // See the design doc in https://github.com/kubernetes/kubernetes/pull/34484
 // for details.
 func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error {
-	hostFactory := config.HostFactory(i.commonOptions.Host, i.commonOptions.Kubeconfig)
+	hostFactory := config.ClusterFactory(i.commonOptions.Host, i.commonOptions.Kubeconfig)
 	hostClientset, err := hostFactory.ClientSet()
 	if err != nil {
 		return err
 	}
 
-	serverName := fmt.Sprintf("%s-apiserver", i.commonOptions.Name)
-	serverCredName := fmt.Sprintf("%s-credentials", serverName)
-	cmName := fmt.Sprintf("%s-controller-manager", i.commonOptions.Name)
-	cmKubeconfigName := fmt.Sprintf("%s-kubeconfig", cmName)
+	serverName := fmt.Sprintf("%s-%s", i.commonOptions.Name, APIServerNameSuffix)
+	serverCredName := fmt.Sprintf("%s-%s", serverName, CredentialSuffix)
+	cmName := fmt.Sprintf("%s-%s", i.commonOptions.Name, CMNameSuffix)
+	cmKubeconfigName := fmt.Sprintf("%s-%s", cmName, KubeconfigNameSuffix)
 
 	// 1. Create a namespace for federation system components
 	_, err = createNamespace(hostClientset, i.commonOptions.FederationSystemNamespace, i.options.dryRun)
@@ -745,6 +749,16 @@ func createControllerManager(clientset *client.Clientset, namespace, name, svcNa
 			Name:      cmName,
 			Namespace: namespace,
 			Labels:    componentLabel,
+			// We additionally update the details (in annotations) about the
+			// kube-dns config map which needs to be created in the clusters
+			// registering to this federation (at kubefed join).
+			// We wont otherwise have this information available at kubefed join.
+			Annotations: map[string]string{
+				// TODO: the name/domain name pair should ideally be checked for naming convention
+				// as done in kube-dns federation flags check.
+				// https://github.com/kubernetes/dns/blob/master/pkg/dns/federation/federation.go
+				util.FedDomainMapKey: fmt.Sprintf("%s=%s", name, dnsZoneName),
+			},
 		},
 		Spec: extensions.DeploymentSpec{
 			Replicas: 1,

federation/pkg/kubefed/init/init_test.go

@@ -205,7 +205,7 @@ func TestInitFederation(t *testing.T) {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
 
-		adminConfig, err := kubefedtesting.NewFakeAdminConfig(hostFactory, tc.kubeconfigGlobal)
+		adminConfig, err := kubefedtesting.NewFakeAdminConfig(hostFactory, nil, "", tc.kubeconfigGlobal)
 		if err != nil {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
@@ -907,6 +907,9 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
 			Name:      cmName,
 			Namespace: namespaceName,
 			Labels:    componentLabel,
+			Annotations: map[string]string{
+				util.FedDomainMapKey: fmt.Sprintf("%s=%s", federationName, dnsZoneName),
+			},
 		},
 		Spec: v1beta1.DeploymentSpec{
 			Replicas: &replicas,

federation/pkg/kubefed/join.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/kubernetes/federation/pkg/kubefed/util"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/kubectl"
 	kubectlcmd "k8s.io/kubernetes/pkg/kubectl/cmd"
 	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
@@ -32,6 +33,9 @@ import (
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/api"
+	extensions "k8s.io/kubernetes/pkg/apis/extensions"
 )
 
 const (
@@ -40,6 +44,7 @@ const (
 	// details.
 	// TODO(madhusudancs): Make this value customizable.
 	defaultClientCIDR = "0.0.0.0/0"
+	CMNameSuffix      = "controller-manager"
 )
 
 var (
@@ -138,7 +143,12 @@ func (j *joinFederation) Run(f cmdutil.Factory, cmdOut io.Writer, config util.Ad
 	}
 	glog.V(2).Infof("Created cluster generator: %#v", generator)
 
-	hostFactory := config.HostFactory(j.commonOptions.Host, j.commonOptions.Kubeconfig)
+	hostFactory := config.ClusterFactory(j.commonOptions.Host, j.commonOptions.Kubeconfig)
+	hostClientset, err := hostFactory.ClientSet()
+	if err != nil {
+		glog.V(2).Infof("Failed to get the cluster client for the host cluster: %q", j.commonOptions.Host, err)
+		return err
+	}
 
 	// We are not using the `kubectl create secret` machinery through
 	// `RunCreateSubcommand` as we do to the cluster resource below
@@ -155,19 +165,33 @@ func (j *joinFederation) Run(f cmdutil.Factory, cmdOut io.Writer, config util.Ad
 	//    don't have to print the created secret in the default case.
 	// Having said that, secret generation machinery could be altered to
 	// suit our needs, but it is far less invasive and readable this way.
-	_, err = createSecret(hostFactory, clientConfig, j.commonOptions.FederationSystemNamespace, j.options.clusterContext, j.options.secretName, j.options.dryRun)
+	_, err = createSecret(hostClientset, clientConfig, j.commonOptions.FederationSystemNamespace, j.options.clusterContext, j.options.secretName, j.options.dryRun)
 	if err != nil {
 		glog.V(2).Infof("Failed creating the cluster credentials secret: %v", err)
 		return err
 	}
 	glog.V(2).Infof("Cluster credentials secret created")
 
-	return kubectlcmd.RunCreateSubcommand(f, cmd, cmdOut, &kubectlcmd.CreateSubcommandOptions{
+	err = kubectlcmd.RunCreateSubcommand(f, cmd, cmdOut, &kubectlcmd.CreateSubcommandOptions{
 		Name:                j.commonOptions.Name,
 		StructuredGenerator: generator,
 		DryRun:              j.options.dryRun,
 		OutputFormat:        cmdutil.GetFlagString(cmd, "output"),
 	})
+	if err != nil {
+		return err
+	}
+
+	// We further need to create a configmap named kube-config in the
+	// just registered cluster which will be consumed by the kube-dns
+	// of this cluster.
+	_, err = createConfigMap(hostClientset, config, j.commonOptions.FederationSystemNamespace, j.options.clusterContext, j.commonOptions.Kubeconfig, j.options.dryRun)
+	if err != nil {
+		glog.V(2).Infof("Failed creating the config map in cluster: %v", err)
+		return err
+	}
+
+	return err
 }
 
 // minifyConfig is a wrapper around `clientcmdapi.MinifyConfig()` that
@@ -189,7 +213,7 @@ func minifyConfig(clientConfig *clientcmdapi.Config, context string) (*clientcmd
 
 // createSecret extracts the kubeconfig for a given cluster and populates
 // a secret with that kubeconfig.
-func createSecret(hostFactory cmdutil.Factory, clientConfig *clientcmdapi.Config, namespace, contextName, secretName string, dryRun bool) (runtime.Object, error) {
+func createSecret(clientset *internalclientset.Clientset, clientConfig *clientcmdapi.Config, namespace, contextName, secretName string, dryRun bool) (runtime.Object, error) {
 	// Minify the kubeconfig to ensure that there is only information
 	// relevant to the cluster we are registering.
 	newClientConfig, err := minifyConfig(clientConfig, contextName)
@@ -206,14 +230,65 @@ func createSecret(hostFactory cmdutil.Factory, clientConfig *clientcmdapi.Config
 		return nil, err
 	}
 
-	// Boilerplate to create the secret in the host cluster.
-	clientset, err := hostFactory.ClientSet()
+	return util.CreateKubeconfigSecret(clientset, newClientConfig, namespace, secretName, dryRun)
+}
+
+// createConfigMap creates a configmap with name kube-dns in the joining cluster
+// which stores the information about this federation zone name.
+// If the configmap with this name already exists, its updated with this information.
+func createConfigMap(hostClientSet *internalclientset.Clientset, config util.AdminConfig, fedSystemNamespace, targetClusterContext, kubeconfigPath string, dryRun bool) (*api.ConfigMap, error) {
+	cmDep, err := getCMDeployment(hostClientSet, fedSystemNamespace)
 	if err != nil {
-		glog.V(2).Infof("Failed to serialize the kubeconfig for the given context %q: %v", contextName, err)
 		return nil, err
 	}
+	domainMap, ok := cmDep.Annotations[util.FedDomainMapKey]
+	if !ok {
+		return nil, fmt.Errorf("kube-dns config map data missing from controller manager annotations")
+	}
 
-	return util.CreateKubeconfigSecret(clientset, newClientConfig, namespace, secretName, dryRun)
+	targetFactory := config.ClusterFactory(targetClusterContext, kubeconfigPath)
+	targetClientSet, err := targetFactory.ClientSet()
+	if err != nil {
+		return nil, err
+	}
+
+	existingConfigMap, err := targetClientSet.Core().ConfigMaps(metav1.NamespaceSystem).Get(util.KubeDnsConfigmapName, metav1.GetOptions{})
+	if isNotFound(err) {
+		newConfigMap := &api.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      util.KubeDnsConfigmapName,
+				Namespace: metav1.NamespaceSystem,
+			},
+			Data: map[string]string{
+				util.FedDomainMapKey: domainMap,
+			},
+		}
+
+		if dryRun {
+			return newConfigMap, nil
+		}
+		return targetClientSet.Core().ConfigMaps(metav1.NamespaceSystem).Create(newConfigMap)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if existingConfigMap.Data == nil {
+		existingConfigMap.Data = make(map[string]string)
+	}
+	if _, ok := existingConfigMap.Data[util.FedDomainMapKey]; ok {
+		// Append this federation info
+		existingConfigMap.Data[util.FedDomainMapKey] = appendConfigMapString(existingConfigMap.Data[util.FedDomainMapKey], cmDep.Annotations[util.FedDomainMapKey])
+
+	} else {
+		// For some reason the configMap exists but this data is empty
+		existingConfigMap.Data[util.FedDomainMapKey] = cmDep.Annotations[util.FedDomainMapKey]
+	}
+
+	if dryRun {
+		return existingConfigMap, nil
+	}
+	return targetClientSet.Core().ConfigMaps(metav1.NamespaceSystem).Update(existingConfigMap)
 }
 
 // clusterGenerator extracts the cluster information from the supplied
@@ -261,3 +336,33 @@ func extractScheme(url string) string {
 	}
 	return scheme
 }
+
+func getCMDeployment(hostClientSet *internalclientset.Clientset, fedNamespace string) (*extensions.Deployment, error) {
+	depList, err := hostClientSet.Extensions().Deployments(fedNamespace).List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, dep := range depList.Items {
+		if strings.HasSuffix(dep.Name, CMNameSuffix) {
+			return &dep, nil
+		}
+	}
+	return nil, fmt.Errorf("could not find the deployment for controller manager in host cluster")
+}
+
+func appendConfigMapString(existing string, toAppend string) string {
+	if existing == "" {
+		return toAppend
+	}
+
+	values := strings.Split(existing, ",")
+	for _, v := range values {
+		// Somehow this federation string is already present,
+		// Nothing should be done
+		if v == toAppend {
+			return existing
+		}
+	}
+	return fmt.Sprintf("%s,%s", existing, toAppend)
+}

federation/pkg/kubefed/join_test.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/testapi"
 	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
 	cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing"
 	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 )
@@ -130,7 +131,16 @@ func TestJoinFederation(t *testing.T) {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
 
-		adminConfig, err := kubefedtesting.NewFakeAdminConfig(hostFactory, tc.kubeconfigGlobal)
+		targetClusterFactory, err := fakeJoinTargetClusterFactory(tc.cluster, tc.clusterCtx)
+		if err != nil {
+			t.Fatalf("[%d] unexpected error: %v", i, err)
+		}
+
+		targetClusterContext := tc.clusterCtx
+		if targetClusterContext == "" {
+			targetClusterContext = tc.cluster
+		}
+		adminConfig, err := kubefedtesting.NewFakeAdminConfig(hostFactory, targetClusterFactory, targetClusterContext, tc.kubeconfigGlobal)
 		if err != nil {
 			t.Fatalf("[%d] unexpected error: %v", i, err)
 		}
@@ -234,6 +244,7 @@ func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token stri
 	if err != nil {
 		return nil, err
 	}
+
 	secretObject := v1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Secret",
@@ -248,7 +259,31 @@ func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token stri
 		},
 	}
 
+	cmName := "controller-manager"
+	deploymentList := v1beta1.DeploymentList{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "DeploymentList",
+			APIVersion: testapi.Extensions.GroupVersion().String(),
+		},
+		Items: []v1beta1.Deployment{
+			{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Deployment",
+					APIVersion: testapi.Extensions.GroupVersion().String(),
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      cmName,
+					Namespace: util.DefaultFederationSystemNamespace,
+					Annotations: map[string]string{
+						util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, "test-dns-zone"),
+					},
+				},
+			},
+		},
+	}
+
 	f, tf, codec, _ := cmdtesting.NewAPIFactory()
+	extensionCodec := testapi.Extensions.Codec()
 	ns := dynamic.ContentConfig().NegotiatedSerializer
 	tf.ClientConfig = kubefedtesting.DefaultClientConfig()
 	tf.Client = &fake.RESTClient{
@@ -270,6 +305,53 @@ func fakeJoinHostFactory(clusterName, clusterCtx, secretName, server, token stri
 					return nil, fmt.Errorf("Unexpected secret object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, secretObject))
 				}
 				return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &secretObject)}, nil
+			case p == "/apis/extensions/v1beta1/namespaces/federation-system/deployments" && m == http.MethodGet:
+				return &http.Response{StatusCode: http.StatusOK, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(extensionCodec, &deploymentList)}, nil
+			default:
+				return nil, fmt.Errorf("unexpected request: %#v\n%#v", req.URL, req)
+			}
+		}),
+	}
+	return f, nil
+}
+
+func fakeJoinTargetClusterFactory(clusterName, clusterCtx string) (cmdutil.Factory, error) {
+	if clusterCtx == "" {
+		clusterCtx = clusterName
+	}
+
+	configmapObject := v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      util.KubeDnsConfigmapName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Data: map[string]string{
+			util.FedDomainMapKey: fmt.Sprintf("%s=%s", clusterCtx, "test-dns-zone"),
+		},
+	}
+
+	f, tf, codec, _ := cmdtesting.NewAPIFactory()
+	ns := dynamic.ContentConfig().NegotiatedSerializer
+	tf.ClientConfig = kubefedtesting.DefaultClientConfig()
+	tf.Client = &fake.RESTClient{
+		APIRegistry:          api.Registry,
+		NegotiatedSerializer: ns,
+		Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) {
+			switch p, m := req.URL.Path, req.Method; {
+			case p == "/api/v1/namespaces/kube-system/configmaps/" && m == http.MethodPost:
+				body, err := ioutil.ReadAll(req.Body)
+				if err != nil {
+					return nil, err
+				}
+				var got v1.ConfigMap
+				_, _, err = codec.Decode(body, nil, &got)
+				if err != nil {
+					return nil, err
+				}
+				if !apiequality.Semantic.DeepEqual(got, configmapObject) {
+					return nil, fmt.Errorf("Unexpected configmap object\n\tDiff: %s", diff.ObjectGoPrintDiff(got, configmapObject))
+				}
+				return &http.Response{StatusCode: http.StatusCreated, Header: kubefedtesting.DefaultHeader(), Body: kubefedtesting.ObjBody(codec, &configmapObject)}, nil
 			default:
 				return nil, fmt.Errorf("unexpected request: %#v\n%#v", req.URL, req)
 			}