Merge pull request #42246 from jamiehannaford/add-etcd-flags-kubeadm

Automatic merge from submit-queue

Allow configurable etcd options

**What this PR does / why we need it**:

Allows users to set the `--listen-client-urls` and `--advertise-client-urls` flags on etcd binaries for clusters set up with kubeadm.

**Which issue this PR fixes**:

As far as I can tell right now, other nodes in a cluster set up with kubeadm cannot communicate with the etcd static pod running on the master. This is needed in order to set up calico/canal SDN which needs access to a publicly addressable IPv4 _before_ the overlay network and inter-cluster subnet is created.

Addresses kubernetes/enhancements#138 and kubernetes/enhancements#11.

**Release note**:
```release-note
Users can now specify listen and advertise URLs for etcd in a kubeadm cluster 
```

cmd/kubeadm/app/apis/kubeadm/env.go

@@ -31,7 +31,6 @@ func SetEnvParams() *EnvParams {
 
 	envParams := map[string]string{
 		"kubernetes_dir":  "/etc/kubernetes",
-		"host_etcd_path":  "/var/lib/etcd",
 		"hyperkube_image": "",
 		"repo_prefix":     "gcr.io/google_containers",
 		"etcd_image":      "",
@@ -45,7 +44,6 @@ func SetEnvParams() *EnvParams {
 
 	return &EnvParams{
 		KubernetesDir:    path.Clean(envParams["kubernetes_dir"]),
-		HostEtcdPath:     path.Clean(envParams["host_etcd_path"]),
 		HyperkubeImage:   envParams["hyperkube_image"],
 		RepositoryPrefix: envParams["repo_prefix"],
 		EtcdImage:        envParams["etcd_image"],

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -36,6 +36,7 @@ func KubeadmFuzzerFuncs(t apitesting.TestingCommon) []interface{} {
 			obj.CertificatesDir = "foo"
 			obj.APIServerCertSANs = []string{}
 			obj.Token = "foo"
+			obj.Etcd.DataDir = "foo"
 		},
 		func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -24,7 +24,6 @@ import (
 
 type EnvParams struct {
 	KubernetesDir    string
-	HostEtcdPath     string
 	HyperkubeImage   string
 	RepositoryPrefix string
 	EtcdImage        string
@@ -82,6 +81,8 @@ type Etcd struct {
 	CAFile    string
 	CertFile  string
 	KeyFile   string
+	DataDir   string
+	ExtraArgs map[string]string
 }
 
 type NodeConfiguration struct {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/defaults.go

@@ -32,6 +32,7 @@ const (
 	DefaultAuthorizationMode = "RBAC"
 	DefaultCACertPath        = "/etc/kubernetes/pki/ca.crt"
 	DefaultCertificatesDir   = "/etc/kubernetes/pki"
+	DefaultEtcdDataDir       = "/var/lib/etcd"
 )
 
 func addDefaultingFuncs(scheme *runtime.Scheme) error {
@@ -66,6 +67,10 @@ func SetDefaults_MasterConfiguration(obj *MasterConfiguration) {
 	if obj.TokenTTL == 0 {
 		obj.TokenTTL = constants.DefaultTokenDuration
 	}
+
+	if obj.Etcd.DataDir == "" {
+		obj.Etcd.DataDir = DefaultEtcdDataDir
+	}
 }
 
 func SetDefaults_NodeConfiguration(obj *NodeConfiguration) {

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -70,10 +70,12 @@ type Networking struct {
 }
 
 type Etcd struct {
-	Endpoints []string `json:"endpoints"`
-	CAFile    string   `json:"caFile"`
-	CertFile  string   `json:"certFile"`
-	KeyFile   string   `json:"keyFile"`
+	Endpoints []string          `json:"endpoints"`
+	CAFile    string            `json:"caFile"`
+	CertFile  string            `json:"certFile"`
+	KeyFile   string            `json:"keyFile"`
+	DataDir   string            `json:"dataDir"`
+	ExtraArgs map[string]string `json:"extraArgs"`
 }
 
 type NodeConfiguration struct {

cmd/kubeadm/app/master/manifests.go

@@ -112,21 +112,16 @@ func WriteStaticPodManifests(cfg *kubeadmapi.MasterConfiguration) error {
 	// Add etcd static pod spec only if external etcd is not configured
 	if len(cfg.Etcd.Endpoints) == 0 {
 		etcdPod := componentPod(api.Container{
-			Name: etcd,
-			Command: []string{
-				"etcd",
-				"--listen-client-urls=http://127.0.0.1:2379",
-				"--advertise-client-urls=http://127.0.0.1:2379",
-				"--data-dir=/var/lib/etcd",
-			},
-			VolumeMounts:  []api.VolumeMount{certsVolumeMount(), etcdVolumeMount(), k8sVolumeMount()},
+			Name:          etcd,
+			Command:       getEtcdCommand(cfg),
+			VolumeMounts:  []api.VolumeMount{certsVolumeMount(), etcdVolumeMount(cfg.Etcd.DataDir), k8sVolumeMount()},
 			Image:         images.GetCoreImage(images.KubeEtcdImage, cfg, kubeadmapi.GlobalEnvParams.EtcdImage),
 			LivenessProbe: componentProbe(2379, "/health", api.URISchemeHTTP),
 		}, certsVolume(cfg), etcdVolume(cfg), k8sVolume(cfg))
 
 		etcdPod.Spec.SecurityContext = &api.PodSecurityContext{
 			SELinuxOptions: &api.SELinuxOptions{
-				// Unconfine the etcd container so it can write to /var/lib/etcd with SELinux enforcing:
+				// Unconfine the etcd container so it can write to the data dir with SELinux enforcing:
 				Type: "spc_t",
 			},
 		}
@@ -156,15 +151,15 @@ func etcdVolume(cfg *kubeadmapi.MasterConfiguration) api.Volume {
 	return api.Volume{
 		Name: "etcd",
 		VolumeSource: api.VolumeSource{
-			HostPath: &api.HostPathVolumeSource{Path: kubeadmapi.GlobalEnvParams.HostEtcdPath},
+			HostPath: &api.HostPathVolumeSource{Path: cfg.Etcd.DataDir},
 		},
 	}
 }
 
-func etcdVolumeMount() api.VolumeMount {
+func etcdVolumeMount(dataDir string) api.VolumeMount {
 	return api.VolumeMount{
 		Name:      "etcd",
-		MountPath: "/var/lib/etcd",
+		MountPath: dataDir,
 	}
 }
 
@@ -379,6 +374,21 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool, k
 	return command
 }
 
+func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string {
+	var command []string
+
+	defaultArguments := map[string]string{
+		"listen-client-urls":    "http://127.0.0.1:2379",
+		"advertise-client-urls": "http://127.0.0.1:2379",
+		"data-dir":              cfg.Etcd.DataDir,
+	}
+
+	command = append(command, "etcd")
+	command = append(command, getExtraParameters(cfg.Etcd.ExtraArgs, defaultArguments)...)
+
+	return command
+}
+
 func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, selfHosted bool) []string {
 	var command []string
 

cmd/kubeadm/app/master/manifests_test.go

@@ -32,7 +32,10 @@ import (
 	"k8s.io/kubernetes/pkg/util/version"
 )
 
-const testCertsDir = "/var/lib/certs"
+const (
+	testCertsDir = "/var/lib/certs"
+	etcdDataDir  = "/var/lib/etcd"
+)
 
 func TestWriteStaticPodManifests(t *testing.T) {
 	tmpdir, err := ioutil.TempDir("", "")
@@ -127,12 +130,13 @@ func TestEtcdVolume(t *testing.T) {
 		expected api.Volume
 	}{
 		{
-			cfg: &kubeadmapi.MasterConfiguration{},
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{DataDir: etcdDataDir},
+			},
 			expected: api.Volume{
 				Name: "etcd",
 				VolumeSource: api.VolumeSource{
-					HostPath: &api.HostPathVolumeSource{
-						Path: kubeadmapi.GlobalEnvParams.HostEtcdPath},
+					HostPath: &api.HostPathVolumeSource{Path: etcdDataDir},
 				}},
 		},
 	}
@@ -163,13 +167,13 @@ func TestEtcdVolumeMount(t *testing.T) {
 		{
 			expected: api.VolumeMount{
 				Name:      "etcd",
-				MountPath: "/var/lib/etcd",
+				MountPath: etcdDataDir,
 			},
 		},
 	}
 
 	for _, rt := range tests {
-		actual := etcdVolumeMount()
+		actual := etcdVolumeMount(etcdDataDir)
 		if actual.Name != rt.expected.Name {
 			t.Errorf(
 				"failed etcdVolumeMount:\n\texpected: %s\n\t  actual: %s",
@@ -664,6 +668,62 @@ func TestGetControllerManagerCommand(t *testing.T) {
 	}
 }
 
+func TestGetEtcdCommand(t *testing.T) {
+	var tests = []struct {
+		cfg      *kubeadmapi.MasterConfiguration
+		expected []string
+	}{
+		{
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{DataDir: "/var/lib/etcd"},
+			},
+			expected: []string{
+				"etcd",
+				"--listen-client-urls=http://127.0.0.1:2379",
+				"--advertise-client-urls=http://127.0.0.1:2379",
+				"--data-dir=/var/lib/etcd",
+			},
+		},
+		{
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{
+					DataDir: "/var/lib/etcd",
+					ExtraArgs: map[string]string{
+						"listen-client-urls":    "http://10.0.1.10:2379",
+						"advertise-client-urls": "http://10.0.1.10:2379",
+					},
+				},
+			},
+			expected: []string{
+				"etcd",
+				"--listen-client-urls=http://10.0.1.10:2379",
+				"--advertise-client-urls=http://10.0.1.10:2379",
+				"--data-dir=/var/lib/etcd",
+			},
+		},
+		{
+			cfg: &kubeadmapi.MasterConfiguration{
+				Etcd: kubeadmapi.Etcd{DataDir: "/etc/foo"},
+			},
+			expected: []string{
+				"etcd",
+				"--listen-client-urls=http://127.0.0.1:2379",
+				"--advertise-client-urls=http://127.0.0.1:2379",
+				"--data-dir=/etc/foo",
+			},
+		},
+	}
+
+	for _, rt := range tests {
+		actual := getEtcdCommand(rt.cfg)
+		sort.Strings(actual)
+		sort.Strings(rt.expected)
+		if !reflect.DeepEqual(actual, rt.expected) {
+			t.Errorf("failed getEtcdCommand:\nexpected:\n%v\nsaw:\n%v", rt.expected, actual)
+		}
+	}
+}
+
 func TestGetSchedulerCommand(t *testing.T) {
 	var tests = []struct {
 		cfg      *kubeadmapi.MasterConfiguration

cmd/kubeadm/app/preflight/checks.go

@@ -515,7 +515,7 @@ func RunInitMasterChecks(cfg *kubeadmapi.MasterConfiguration) error {
 		// Only do etcd related checks when no external endpoints were specified
 		checks = append(checks,
 			PortOpenCheck{port: 2379},
-			DirAvailableCheck{Path: "/var/lib/etcd"},
+			DirAvailableCheck{Path: cfg.Etcd.DataDir},
 		)
 	} else {
 		// Only check etcd version when external endpoints are specified