Merge pull request #57415 from stealthybox/feature/kubeadm_594-etcd_tls

Automatic merge from submit-queue (batch tested with PRs 59159, 60318, 60079, 59371, 57415). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Feature/kubeadm 594 etcd TLS on init/upgrade

**What this PR does / why we need it**:
On `kubeadm init`/`kubeadm upgrade`, this PR generates certificates for securing local etcd:
- etcd serving cert
- etcd peer cert
- apiserver etcd client cert

Flags and hostMounts are added to the etcd and apiserver static-pods to load these certs.
For connections to etcd, `https` is now used in favor of `http` and tests have been added/updated.

Etcd only listens on localhost, so the serving cert SAN defaults to `DNS:localhost,IP:127.0.0.1`.
The etcd peer cert has SANs for `<hostname>,<api-advertise-address>`, but is unused.

New kubeadm config options, `Etcd.ServerCertSANs` and `Etcd.PeerCertSANs`, are used for user additions to the default certificate SANs for the etcd server and peer certs.

This feature continues to utilize the existence of `MasterConfiguration.Etcd.Endpoints` as a feature gate for external-etcd.
If the user passes flags to configure `Etcd.{CAFile,CertFile,KeyFile}` but they omit `Endpoints`, these flags will be unused, and a warning is printed.

New phase commands:
```
kubeadm alpha phase certs etcd-server
kubeadm alpha phase certs etcd-peer
kubeadm alpha phase certs apiserver-etcd-client 
```

**Which issue(s) this PR fixes**
Fixes kubernetes/kubeadm#594

**Special notes for your reviewer**:

#### on the master
these should fail:
```bash
curl localhost:2379/v2/keys  # no output
curl --cacert /etc/kubernetes/pki/ca.crt https://localhost:2379/v2/keys  # handshake error
```
these should succeed:
```
cd /etc/kubernetes/pki
curl --cacert ca.crt --cert apiserver-etcd-client.crt --key apiserver-etcd-client.key https://localhost:2379/v2/keys
```

**Release note**:
```release-note
On cluster provision or upgrade, kubeadm now generates certs and secures all connections to the etcd static-pod with mTLS.
```

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -43,6 +43,8 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.AuthorizationModes = []string{"foo"}
 			obj.CertificatesDir = "foo"
 			obj.APIServerCertSANs = []string{"foo"}
+			obj.Etcd.ServerCertSANs = []string{"foo"}
+			obj.Etcd.PeerCertSANs = []string{"foo"}
 			obj.Token = "foo"
 			obj.CRISocket = "foo"
 			obj.Etcd.Image = "foo"

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -178,6 +178,12 @@ type Etcd struct {
 	Image string
 	// SelfHosted holds configuration for self-hosting etcd.
 	SelfHosted *SelfHostedEtcd
+	// ServerCertSANs sets extra Subject Alternative Names for the etcd server
+	// signing cert. This is currently used for the etcd static-pod.
+	ServerCertSANs []string
+	// PeerCertSANs sets extra Subject Alternative Names for the etcd peer
+	// signing cert. This is currently used for the etcd static-pod.
+	PeerCertSANs []string
 }
 
 // SelfHostedEtcd describes options required to configure self-hosted etcd.

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -170,6 +170,10 @@ type Etcd struct {
 	Image string `json:"image"`
 	// SelfHosted holds configuration for self-hosting etcd.
 	SelfHosted *SelfHostedEtcd `json:"selfHosted,omitempty"`
+	// ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert.
+	ServerCertSANs []string `json:"serverCertSANs,omitempty"`
+	// PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert.
+	PeerCertSANs []string `json:"peerCertSANs,omitempty"`
 }
 
 // SelfHostedEtcd describes options required to configure self-hosted etcd.

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -72,7 +72,9 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
 	allErrs = append(allErrs, ValidateCloudProvider(c.CloudProvider, field.NewPath("cloudprovider"))...)
 	allErrs = append(allErrs, ValidateAuthorizationModes(c.AuthorizationModes, field.NewPath("authorization-modes"))...)
 	allErrs = append(allErrs, ValidateNetworking(&c.Networking, field.NewPath("networking"))...)
-	allErrs = append(allErrs, ValidateAPIServerCertSANs(c.APIServerCertSANs, field.NewPath("cert-altnames"))...)
+	allErrs = append(allErrs, ValidateCertSANs(c.APIServerCertSANs, field.NewPath("api-server-cert-altnames"))...)
+	allErrs = append(allErrs, ValidateCertSANs(c.Etcd.ServerCertSANs, field.NewPath("etcd-server-cert-altnames"))...)
+	allErrs = append(allErrs, ValidateCertSANs(c.Etcd.PeerCertSANs, field.NewPath("etcd-peer-cert-altnames"))...)
 	allErrs = append(allErrs, ValidateAbsolutePath(c.CertificatesDir, field.NewPath("certificates-dir"))...)
 	allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("node-name"))...)
 	allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
@@ -228,8 +230,8 @@ func ValidateToken(t string, fldPath *field.Path) field.ErrorList {
 	return allErrs
 }
 
-// ValidateAPIServerCertSANs validates alternative names
-func ValidateAPIServerCertSANs(altnames []string, fldPath *field.Path) field.ErrorList {
+// ValidateCertSANs validates alternative names
+func ValidateCertSANs(altnames []string, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 	for _, altname := range altnames {
 		if len(validation.IsDNS1123Subdomain(altname)) != 0 && net.ParseIP(altname) == nil {

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -134,7 +134,7 @@ func TestValidateCloudProvider(t *testing.T) {
 	}
 }
 
-func TestValidateAPIServerCertSANs(t *testing.T) {
+func TestValidateCertSANs(t *testing.T) {
 	var tests = []struct {
 		sans     []string
 		expected bool
@@ -148,10 +148,10 @@ func TestValidateAPIServerCertSANs(t *testing.T) {
 		{[]string{"my-hostname2", "my.other.subdomain", "2001:db8::10"}, true}, // supported
 	}
 	for _, rt := range tests {
-		actual := ValidateAPIServerCertSANs(rt.sans, nil)
+		actual := ValidateCertSANs(rt.sans, nil)
 		if (len(actual) == 0) != rt.expected {
 			t.Errorf(
-				"failed ValidateAPIServerCertSANs:\n\texpected: %t\n\t  actual: %t",
+				"failed ValidateCertSANs:\n\texpected: %t\n\t  actual: %t",
 				rt.expected,
 				(len(actual) == 0),
 			)

cmd/kubeadm/app/cmd/phases/certs.go

@@ -60,8 +60,8 @@ var (
 	apiServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(`
 		Generates the API server serving certificate and key and saves them into %s and %s files.
 		
-		The certificate includes default subject alternative names and additional sans eventually provided by the user;
-		default sans are: <node-name>, <apiserver-advertise-address>, kubernetes, kubernetes.default, kubernetes.default.svc, 
+		The certificate includes default subject alternative names and additional SANs provided by the user;
+		default SANs are: <node-name>, <apiserver-advertise-address>, kubernetes, kubernetes.default, kubernetes.default.svc,
 		kubernetes.default.svc.<service-dns-domain>, <internalAPIServerVirtualIP> (that is the .10 address in <service-cidr> address space).
 
 		If both files already exist, kubeadm skips the generation step and existing files will be used.
@@ -74,6 +74,31 @@ var (
 		If both files already exist, kubeadm skips the generation step and existing files will be used.
 		`+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerKubeletClientCertName, kubeadmconstants.APIServerKubeletClientKeyName)
 
+	etcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(`
+		Generates the etcd serving certificate and key and saves them into %s and %s files.
+		
+		The certificate includes default subject alternative names and additional SANs provided by the user;
+		default SANs are: localhost, 127.0.0.1.
+
+		If both files already exist, kubeadm skips the generation step and existing files will be used.
+		`+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdServerCertName, kubeadmconstants.EtcdServerKeyName)
+
+	etcdPeerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(`
+		Generates the etcd peer certificate and key and saves them into %s and %s files.
+		
+		The certificate includes default subject alternative names and additional SANs provided by the user;
+		default SANs are: <node-name>, <apiserver-advertise-address>.
+
+		If both files already exist, kubeadm skips the generation step and existing files will be used.
+		`+cmdutil.AlphaDisclaimer), kubeadmconstants.EtcdPeerCertName, kubeadmconstants.EtcdPeerKeyName)
+
+	apiServerEtcdServerCertLongDesc = fmt.Sprintf(normalizer.LongDesc(`
+		Generates the client certificate for the API server to connect to etcd securely and the respective key,
+		and saves them into %s and %s files.
+
+		If both files already exist, kubeadm skips the generation step and existing files will be used.
+		`+cmdutil.AlphaDisclaimer), kubeadmconstants.APIServerEtcdClientCertName, kubeadmconstants.APIServerEtcdClientKeyName)
+
 	saKeyLongDesc = fmt.Sprintf(normalizer.LongDesc(`
 		Generates the private key for signing service account tokens along with its public key, and saves them into
 		%s and %s files.
@@ -157,6 +182,24 @@ func getCertsSubCommands(defaultKubernetesVersion string) []*cobra.Command {
 			long:    apiServerKubeletCertLongDesc,
 			cmdFunc: certsphase.CreateAPIServerKubeletClientCertAndKeyFiles,
 		},
+		{
+			use:     "etcd-server",
+			short:   "Generates etcd serving certificate and key",
+			long:    etcdServerCertLongDesc,
+			cmdFunc: certsphase.CreateEtcdServerCertAndKeyFiles,
+		},
+		{
+			use:     "etcd-peer",
+			short:   "Generates etcd peer certificate and key",
+			long:    etcdPeerCertLongDesc,
+			cmdFunc: certsphase.CreateEtcdPeerCertAndKeyFiles,
+		},
+		{
+			use:     "apiserver-etcd-client",
+			short:   "Generates client certificate for the API server to connect to etcd securely",
+			long:    apiServerEtcdServerCertLongDesc,
+			cmdFunc: certsphase.CreateAPIServerEtcdClientCertAndKeyFiles,
+		},
 		{
 			use:     "sa",
 			short:   "Generates a private key for signing service account tokens along with its public key",

cmd/kubeadm/app/constants/constants.go

@@ -65,6 +65,33 @@ const (
 	// APIServerKubeletClientCertCommonName defines kubelet client certificate common name (CN)
 	APIServerKubeletClientCertCommonName = "kube-apiserver-kubelet-client"
 
+	// EtcdServerCertAndKeyBaseName defines etcd's server certificate and key base name
+	EtcdServerCertAndKeyBaseName = "etcd/server"
+	// EtcdServerCertName defines etcd's server certificate name
+	EtcdServerCertName = "etcd/server.crt"
+	// EtcdServerKeyName defines etcd's server key name
+	EtcdServerKeyName = "etcd/server.key"
+	// EtcdServerCertCommonName defines etcd's server certificate common name (CN)
+	EtcdServerCertCommonName = "kube-etcd"
+
+	// EtcdPeerCertAndKeyBaseName defines etcd's peer certificate and key base name
+	EtcdPeerCertAndKeyBaseName = "etcd/peer"
+	// EtcdPeerCertName defines etcd's peer certificate name
+	EtcdPeerCertName = "etcd/peer.crt"
+	// EtcdPeerKeyName defines etcd's peer key name
+	EtcdPeerKeyName = "etcd/peer.key"
+	// EtcdPeerCertCommonName defines etcd's peer certificate common name (CN)
+	EtcdPeerCertCommonName = "kube-etcd-peer"
+
+	// APIServerEtcdClientCertAndKeyBaseName defines etcd client certificate and key base name
+	APIServerEtcdClientCertAndKeyBaseName = "apiserver-etcd-client"
+	// APIServerEtcdClientCertName defines etcd client certificate name
+	APIServerEtcdClientCertName = "apiserver-etcd-client.crt"
+	// APIServerEtcdClientKeyName defines etcd client key name
+	APIServerEtcdClientKeyName = "apiserver-etcd-client.key"
+	// APIServerEtcdClientCertCommonName defines etcd client certificate common name (CN)
+	APIServerEtcdClientCertCommonName = "kube-apiserver-etcd-client"
+
 	// ServiceAccountKeyBaseName defines SA key base name
 	ServiceAccountKeyBaseName = "sa"
 	// ServiceAccountPublicKeyName defines SA public key base name

cmd/kubeadm/app/phases/certs/BUILD

@@ -30,8 +30,6 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
-        "//pkg/registry/core/service/ipallocator:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
     ],
 )