Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Federation] Automate configuring nameserver in cluster-dns for CoreDNS provider #42895

Merged
merged 3 commits into from
May 17, 2017
Merged

[Federation] Automate configuring nameserver in cluster-dns for CoreDNS provider #42895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c107016
Add coredns-endpoints to CoreDNS provider config
Mar 9, 2017
2ba7c8e
Add stubDomains to kube-dns configmap for private nameserver
Mar 9, 2017
005e11b
Auto-generated bazel build file
Mar 10, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 3 additions & 2 deletions federation/pkg/dnsprovider/providers/coredns/coredns.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ const (
// Config to override defaults
type Config struct {
Global struct {
EtcdEndpoints string `gcfg:"etcd-endpoints"`
DNSZones string `gcfg:"zones"`
EtcdEndpoints string `gcfg:"etcd-endpoints"`
DNSZones string `gcfg:"zones"`
CoreDNSEndpoints string `gcfg:"coredns-endpoints"`
}
}

Expand Down
4 changes: 4 additions & 0 deletions federation/pkg/kubefed/init/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ go_library(
deps = [
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//federation/apis/federation:go_default_library",
"//federation/pkg/dnsprovider/providers/coredns:go_default_library",
"//federation/pkg/kubefed/util:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
Expand All @@ -27,6 +28,7 @@ go_library(
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
Expand All @@ -46,6 +48,7 @@ go_test(
tags = ["automanaged"],
deps = [
"//federation/apis/federation:go_default_library",
"//federation/pkg/dnsprovider/providers/coredns:go_default_library",
"//federation/pkg/kubefed/testing:go_default_library",
"//federation/pkg/kubefed/util:go_default_library",
"//pkg/api:go_default_library",
Expand All @@ -57,6 +60,7 @@ go_test(
"//pkg/apis/rbac/v1beta1:go_default_library",
"//pkg/kubectl/cmd/testing:go_default_library",
"//pkg/kubectl/cmd/util:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
Expand Down
33 changes: 27 additions & 6 deletions federation/pkg/kubefed/init/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
triple "k8s.io/client-go/util/cert/triple"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
"k8s.io/kubernetes/federation/apis/federation"
"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/coredns"
"k8s.io/kubernetes/federation/pkg/kubefed/util"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
Expand All @@ -55,6 +56,7 @@ import (
"github.com/golang/glog"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"gopkg.in/gcfg.v1"
)

const (
Expand Down Expand Up @@ -379,7 +381,7 @@ func (i *initFederation) Run(cmdOut io.Writer, config util.AdminConfig) error {

glog.V(4).Info("Creating federation controller manager deployment")

_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun)
_, err = createControllerManager(hostClientset, i.commonOptions.FederationSystemNamespace, i.commonOptions.Name, svc.Name, cmName, i.options.image, cmKubeconfigName, i.options.dnsZoneName, i.options.dnsProvider, i.options.dnsProviderConfig, sa.Name, dnsProviderSecret, i.options.controllerManagerOverrides, i.options.dryRun)
if err != nil {
return err
}
Expand Down Expand Up @@ -847,7 +849,7 @@ func createRoleBindings(clientset client.Interface, namespace, saName, federatio
return newRole, newRolebinding, err
}

func createControllerManager(clientset client.Interface, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
func createControllerManager(clientset client.Interface, namespace, name, svcName, cmName, image, kubeconfigName, dnsZoneName, dnsProvider, dnsProviderConfig, saName string, dnsProviderSecret *api.Secret, argOverrides map[string]string, dryRun bool) (*extensions.Deployment, error) {
command := []string{
"/hyperkube",
"federation-controller-manager",
Expand Down Expand Up @@ -935,12 +937,19 @@ func createControllerManager(clientset client.Interface, namespace, name, svcNam
dep.Spec.Template.Spec.ServiceAccountName = saName
}

if dryRun {
return dep, nil
}

if dnsProviderSecret != nil {
dep = addDNSProviderConfig(dep, dnsProviderSecret.Name)
if dnsProvider == util.FedDNSProviderCoreDNS {
var err error
dep, err = addCoreDNSServerAnnotation(dep, dnsZoneName, dnsProviderConfig)
if err != nil {
return nil, err
}
}
}

if dryRun {
return dep, nil
}

return clientset.Extensions().Deployments(namespace).Create(dep)
Expand Down Expand Up @@ -1154,3 +1163,15 @@ func addDNSProviderConfig(dep *extensions.Deployment, secretName string) *extens
func authFileContents(username, authSecret string) []byte {
return []byte(fmt.Sprintf("%s,%s,%s\n", authSecret, username, uuid.NewUUID()))
}

func addCoreDNSServerAnnotation(deployment *extensions.Deployment, dnsZoneName, dnsProviderConfig string) (*extensions.Deployment, error) {
var cfg coredns.Config
if err := gcfg.ReadFileInto(&cfg, dnsProviderConfig); err != nil {
return nil, err
}

deployment.Annotations[util.FedDNSZoneName] = dnsZoneName
deployment.Annotations[util.FedNameServer] = cfg.Global.CoreDNSEndpoints
deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
return deployment, nil
}
32 changes: 29 additions & 3 deletions federation/pkg/kubefed/init/init_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import (
"k8s.io/client-go/rest/fake"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/kubernetes/federation/apis/federation"
"k8s.io/kubernetes/federation/pkg/dnsprovider/providers/coredns"
kubefedtesting "k8s.io/kubernetes/federation/pkg/kubefed/testing"
"k8s.io/kubernetes/federation/pkg/kubefed/util"
"k8s.io/kubernetes/pkg/api"
Expand All @@ -55,6 +56,8 @@ import (
rbacv1beta1 "k8s.io/kubernetes/pkg/apis/rbac/v1beta1"
cmdtesting "k8s.io/kubernetes/pkg/kubectl/cmd/testing"
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"

"gopkg.in/gcfg.v1"
)

const (
Expand All @@ -74,7 +77,6 @@ const (

func TestInitFederation(t *testing.T) {
cmdErrMsg := ""
dnsProvider := "google-clouddns"
cmdutil.BehaviorOnFatal(func(str string, code int) {
cmdErrMsg = str
})
Expand All @@ -97,6 +99,7 @@ func TestInitFederation(t *testing.T) {
etcdPVCapacity string
etcdPersistence string
expectedErr string
dnsProvider string
dnsProviderConfig string
dryRun string
apiserverArgOverrides string
Expand All @@ -116,6 +119,7 @@ func TestInitFederation(t *testing.T) {
etcdPVCapacity: "5Gi",
etcdPersistence: "true",
expectedErr: "",
dnsProvider: util.FedDNSProviderCoreDNS,
dnsProviderConfig: "dns-provider.conf",
dryRun: "",
apiserverArgOverrides: "--client-ca-file=override,--log-dir=override",
Expand Down Expand Up @@ -210,6 +214,9 @@ func TestInitFederation(t *testing.T) {
tmpDirPath := ""
buf := bytes.NewBuffer([]byte{})

if tc.dnsProvider == "" {
tc.dnsProvider = "google-clouddns"
}
if tc.dnsProviderConfig != "" {
tmpfile, err := ioutil.TempFile("", tc.dnsProviderConfig)
if err != nil {
Expand All @@ -227,7 +234,7 @@ func TestInitFederation(t *testing.T) {
}
defer os.Remove(tmpDirPath)

hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.apiserverArgOverrides, tc.cmArgOverrides, tmpDirPath, tc.apiserverEnableHTTPBasicAuth, tc.apiserverEnableTokenAuth, tc.isRBACAPIAvailable)
hostFactory, err := fakeInitHostFactory(tc.apiserverServiceType, tc.federation, util.DefaultFederationSystemNamespace, tc.advertiseAddress, tc.lbIP, tc.dnsZoneName, tc.image, tc.dnsProvider, tc.dnsProviderConfig, tc.etcdPersistence, tc.etcdPVCapacity, tc.apiserverArgOverrides, tc.cmArgOverrides, tmpDirPath, tc.apiserverEnableHTTPBasicAuth, tc.apiserverEnableTokenAuth, tc.isRBACAPIAvailable)
if err != nil {
t.Fatalf("[%d] unexpected error: %v", i, err)
}
Expand All @@ -243,7 +250,7 @@ func TestInitFederation(t *testing.T) {
cmd.Flags().Set("host-cluster-context", "substrate")
cmd.Flags().Set("dns-zone-name", tc.dnsZoneName)
cmd.Flags().Set("image", tc.image)
cmd.Flags().Set("dns-provider", dnsProvider)
cmd.Flags().Set("dns-provider", tc.dnsProvider)
cmd.Flags().Set("apiserver-arg-overrides", tc.apiserverArgOverrides)
cmd.Flags().Set("controllermanager-arg-overrides", tc.cmArgOverrides)

Expand Down Expand Up @@ -1043,6 +1050,12 @@ func fakeInitHostFactory(apiserverServiceType v1.ServiceType, federationName, na
}
if dnsProviderConfig != "" {
cm = addDNSProviderConfigTest(cm, cmDNSProviderSecret.Name)
if dnsProvider == util.FedDNSProviderCoreDNS {
cm, err = addCoreDNSServerAnnotationTest(cm, dnsZoneName, dnsProviderConfig)
if err != nil {
return nil, err
}
}
}

podList := v1.PodList{}
Expand Down Expand Up @@ -1540,3 +1553,16 @@ func addDNSProviderConfigTest(dep *v1beta1.Deployment, secretName string) *v1bet

return dep
}

// TODO: Reuse the function addCoreDNSServerAnnotation once that function is converted to use versioned objects.
func addCoreDNSServerAnnotationTest(deployment *v1beta1.Deployment, dnsZoneName, dnsProviderConfig string) (*v1beta1.Deployment, error) {
var cfg coredns.Config
if err := gcfg.ReadFileInto(&cfg, dnsProviderConfig); err != nil {
return nil, err
}

deployment.Annotations[util.FedDNSZoneName] = dnsZoneName
deployment.Annotations[util.FedNameServer] = cfg.Global.CoreDNSEndpoints
deployment.Annotations[util.FedDNSProvider] = util.FedDNSProviderCoreDNS
return deployment, nil
}
13 changes: 13 additions & 0 deletions federation/pkg/kubefed/join.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ func createConfigMap(hostClientSet internalclientset.Interface, config util.Admi
util.FedDomainMapKey: domainMap,
},
}
newConfigMap = populateStubDomainsIfRequired(newConfigMap, cmDep.Annotations)

if dryRun {
return newConfigMap, nil
Expand Down Expand Up @@ -397,3 +398,15 @@ func getFederationName(hostClientSet internalclientset.Interface, fedNamespace s

return name, nil
}

func populateStubDomainsIfRequired(configMap *api.ConfigMap, annotations map[string]string) *api.ConfigMap {
dnsProvider := annotations[util.FedDNSProvider]
dnsZoneName := annotations[util.FedDNSZoneName]
nameServer := annotations[util.FedNameServer]

if dnsProvider != util.FedDNSProviderCoreDNS || dnsZoneName == "" || nameServer == "" {
return configMap
}
configMap.Data[util.KubeDnsStubDomains] = fmt.Sprintf(`{"%s":["%s"]}`, dnsZoneName, nameServer)
return configMap
}