Merge pull request #48228 from danehans/kubeadm_v6masterep

Automatic merge from submit-queue

Updates Kubeadm Master Endpoint for IPv6

**What this PR does / why we need it**:
Previously, kubeadm would use ip:port to construct a master
endpoint. This works fine for IPv4 addresses, but not for IPv6.
Per [RFC 3986](https://www.ietf.org/rfc/rfc3986.txt), IPv6 requires the ip to be encased in brackets
when being joined to a port with a colon.

This patch updates kubeadm to support wrapping a v6 address with
[] to form the master endpoint url. Since this functionality is
needed in multiple areas, a dedicated util function was created
for this purpose.

**Which issue this PR fixes**
Fixes Issue kubernetes/kubeadm#334

**Special notes for your reviewer**:
As part of a bigger effort to add IPv6 support to Kubernetes:
Issue #1443
Issue #47666

**Release note**:
```NONE
```
/area kubeadm
/area ipv6
/sig network
/sig cluster-ops

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -17,7 +17,6 @@ limitations under the License.
 package kubeadm
 
 import (
-	"fmt"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -116,7 +115,3 @@ type NodeConfiguration struct {
 	// the security of kubeadm since other nodes can impersonate the master.
 	DiscoveryTokenUnsafeSkipCAVerification bool
 }
-
-func (cfg *MasterConfiguration) GetMasterEndpoint() string {
-	return fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort)
-}

cmd/kubeadm/app/apis/kubeadm/validation/BUILD

@@ -24,6 +24,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/cmd/features:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/token:go_default_library",
         "//pkg/api/validation:go_default_library",
         "//pkg/kubeapiserver/authorizer/modes:go_default_library",

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	tokenutil "k8s.io/kubernetes/cmd/kubeadm/app/util/token"
 	apivalidation "k8s.io/kubernetes/pkg/api/validation"
 	authzmodes "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
@@ -68,6 +69,7 @@ func ValidateMasterConfiguration(c *kubeadm.MasterConfiguration) field.ErrorList
 	allErrs = append(allErrs, ValidateNodeName(c.NodeName, field.NewPath("node-name"))...)
 	allErrs = append(allErrs, ValidateToken(c.Token, field.NewPath("token"))...)
 	allErrs = append(allErrs, ValidateFeatureFlags(c.FeatureFlags, field.NewPath("feature-flags"))...)
+	allErrs = append(allErrs, ValidateAPIEndpoint(c, field.NewPath("api-endpoint"))...)
 	return allErrs
 }
 
@@ -309,3 +311,13 @@ func ValidateFeatureFlags(featureFlags map[string]bool, fldPath *field.Path) fie
 
 	return allErrs
 }
+
+func ValidateAPIEndpoint(c *kubeadm.MasterConfiguration, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	endpoint, err := kubeadmutil.GetMasterEndpoint(c)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, endpoint, "Invalid API Endpoint"))
+	}
+	return allErrs
+}

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -209,6 +209,71 @@ func TestValidateIPNetFromString(t *testing.T) {
 	}
 }
 
+func TestValidateAPIEndpoint(t *testing.T) {
+	var tests = []struct {
+		name     string
+		s        *kubeadm.MasterConfiguration
+		expected bool
+	}{
+		{
+			name:     "Missing configuration",
+			s:        &kubeadm.MasterConfiguration{},
+			expected: false,
+		},
+		{
+			name: "Valid IPv4 address and default port",
+			s: &kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "1.2.3.4",
+					BindPort:         6443,
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "Valid IPv6 address and port",
+			s: &kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "2001:db7::1",
+					BindPort:         3446,
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "Invalid IPv4 address",
+			s: &kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "1.2.34",
+					BindPort:         6443,
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "Invalid IPv6 address",
+			s: &kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "2001:db7:1",
+					BindPort:         3446,
+				},
+			},
+			expected: false,
+		},
+	}
+	for _, rt := range tests {
+		actual := ValidateAPIEndpoint(rt.s, nil)
+		if (len(actual) == 0) != rt.expected {
+			t.Errorf(
+				"%s test case failed:\n\texpected: %t\n\t  actual: %t",
+				rt.name,
+				rt.expected,
+				(len(actual) == 0),
+			)
+		}
+	}
+}
+
 func TestValidateMasterConfiguration(t *testing.T) {
 	nodename := "valid-nodename"
 	var tests = []struct {
@@ -220,6 +285,10 @@ func TestValidateMasterConfiguration(t *testing.T) {
 			&kubeadm.MasterConfiguration{}, false},
 		{"invalid missing token with IPv4 service subnet",
 			&kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "1.2.3.4",
+					BindPort:         6443,
+				},
 				AuthorizationModes: []string{"Node", "RBAC"},
 				Networking: kubeadm.Networking{
 					ServiceSubnet: "10.96.0.1/12",
@@ -230,6 +299,10 @@ func TestValidateMasterConfiguration(t *testing.T) {
 			}, false},
 		{"invalid missing token with IPv6 service subnet",
 			&kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "1.2.3.4",
+					BindPort:         6443,
+				},
 				AuthorizationModes: []string{"Node", "RBAC"},
 				Networking: kubeadm.Networking{
 					ServiceSubnet: "2001:db8::1/98",
@@ -240,6 +313,10 @@ func TestValidateMasterConfiguration(t *testing.T) {
 			}, false},
 		{"invalid missing node name",
 			&kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "1.2.3.4",
+					BindPort:         6443,
+				},
 				AuthorizationModes: []string{"Node", "RBAC"},
 				Networking: kubeadm.Networking{
 					ServiceSubnet: "10.96.0.1/12",
@@ -250,6 +327,10 @@ func TestValidateMasterConfiguration(t *testing.T) {
 			}, false},
 		{"valid master configuration with IPv4 service subnet",
 			&kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "1.2.3.4",
+					BindPort:         6443,
+				},
 				AuthorizationModes: []string{"Node", "RBAC"},
 				Networking: kubeadm.Networking{
 					ServiceSubnet: "10.96.0.1/12",
@@ -261,6 +342,10 @@ func TestValidateMasterConfiguration(t *testing.T) {
 			}, true},
 		{"valid master configuration using IPv6 service subnet",
 			&kubeadm.MasterConfiguration{
+				API: kubeadm.API{
+					AdvertiseAddress: "1:2:3::4",
+					BindPort:         3446,
+				},
 				AuthorizationModes: []string{"Node", "RBAC"},
 				Networking: kubeadm.Networking{
 					ServiceSubnet: "2001:db8::1/98",

cmd/kubeadm/app/cmd/init.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"strconv"
 	"text/template"
 	"time"
 
@@ -73,7 +72,7 @@ var (
 		You can now join any number of machines by running the following on each node
 		as root:
 
-		  kubeadm join --token {{.Token}} {{.MasterIP}}:{{.MasterPort}} --discovery-token-ca-cert-hash {{.CAPubKeyPin}}
+		  kubeadm join --token {{.Token}} {{.MasterHostPort}} --discovery-token-ca-cert-hash {{.CAPubKeyPin}}
 
 		`)))
 )
@@ -329,6 +328,9 @@ func (i *Init) Run(out io.Writer) error {
 
 	// Load the CA certificate from so we can pin its public key
 	caCert, err := pkiutil.TryLoadCertFromDisk(i.cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
+
+	// Generate the Master host/port pair used by initDoneTempl
+	masterHostPort, err := kubeadmutil.GetMasterHostPort(i.cfg)
 	if err != nil {
 		return err
 	}
@@ -338,8 +340,7 @@ func (i *Init) Run(out io.Writer) error {
 		"KubeConfigName": kubeadmconstants.AdminKubeConfigFileName,
 		"Token":          i.cfg.Token,
 		"CAPubKeyPin":    pubkeypin.Hash(caCert),
-		"MasterIP":       i.cfg.API.AdvertiseAddress,
-		"MasterPort":     strconv.Itoa(int(i.cfg.API.BindPort)),
+		"MasterHostPort": masterHostPort,
 	}
 	if i.skipTokenPrint {
 		ctx["Token"] = "<value withheld>"

cmd/kubeadm/app/phases/addons/proxy/proxy.go

@@ -49,10 +49,15 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte
 		return fmt.Errorf("error when creating kube-proxy service account: %v", err)
 	}
 
+	// Generate Master Enpoint kubeconfig file
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	if err != nil {
+		return err
+	}
+
 	proxyConfigMapBytes, err := kubeadmutil.ParseTemplate(KubeProxyConfigMap, struct{ MasterEndpoint string }{
 		// Fetch this value from the kubeconfig file
-		MasterEndpoint: fmt.Sprintf("https://%s:%d", cfg.API.AdvertiseAddress, cfg.API.BindPort),
-	})
+		MasterEndpoint: masterEndpoint})
 	if err != nil {
 		return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
 	}

cmd/kubeadm/app/phases/kubeconfig/BUILD

@@ -16,6 +16,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
+        "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
@@ -44,6 +45,7 @@ go_test(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/phases/certs/pkiutil:go_default_library",
+        "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/test:go_default_library",
         "//cmd/kubeadm/test/certs:go_default_library",
         "//cmd/kubeadm/test/kubeconfig:go_default_library",

cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go

@@ -32,6 +32,7 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 )
 
@@ -134,10 +135,15 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo
 		return nil, fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
 	}
 
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	if err != nil {
+		return nil, err
+	}
+
 	var kubeConfigSpec = map[string]*kubeConfigSpec{
 		kubeadmconstants.AdminKubeConfigFileName: {
 			CACert:     caCert,
-			APIServer:  cfg.GetMasterEndpoint(),
+			APIServer:  masterEndpoint,
 			ClientName: "kubernetes-admin",
 			ClientCertAuth: &clientCertAuth{
 				CAKey:         caKey,
@@ -146,7 +152,7 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo
 		},
 		kubeadmconstants.KubeletKubeConfigFileName: {
 			CACert:     caCert,
-			APIServer:  cfg.GetMasterEndpoint(),
+			APIServer:  masterEndpoint,
 			ClientName: fmt.Sprintf("system:node:%s", cfg.NodeName),
 			ClientCertAuth: &clientCertAuth{
 				CAKey:         caKey,
@@ -155,15 +161,15 @@ func getKubeConfigSpecs(cfg *kubeadmapi.MasterConfiguration) (map[string]*kubeCo
 		},
 		kubeadmconstants.ControllerManagerKubeConfigFileName: {
 			CACert:     caCert,
-			APIServer:  cfg.GetMasterEndpoint(),
+			APIServer:  masterEndpoint,
 			ClientName: kubeadmconstants.ControllerManagerUser,
 			ClientCertAuth: &clientCertAuth{
 				CAKey: caKey,
 			},
 		},
 		kubeadmconstants.SchedulerKubeConfigFileName: {
 			CACert:     caCert,
-			APIServer:  cfg.GetMasterEndpoint(),
+			APIServer:  masterEndpoint,
 			ClientName: kubeadmconstants.SchedulerUser,
 			ClientCertAuth: &clientCertAuth{
 				CAKey: caKey,
@@ -266,9 +272,14 @@ func WriteKubeConfigWithClientCert(out io.Writer, cfg *kubeadmapi.MasterConfigur
 		return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
 	}
 
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	if err != nil {
+		return err
+	}
+
 	spec := &kubeConfigSpec{
 		ClientName: clientName,
-		APIServer:  cfg.GetMasterEndpoint(),
+		APIServer:  masterEndpoint,
 		CACert:     caCert,
 		ClientCertAuth: &clientCertAuth{
 			CAKey: caKey,
@@ -287,9 +298,14 @@ func WriteKubeConfigWithToken(out io.Writer, cfg *kubeadmapi.MasterConfiguration
 		return fmt.Errorf("couldn't create a kubeconfig; the CA files couldn't be loaded: %v", err)
 	}
 
+	masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+	if err != nil {
+		return err
+	}
+
 	spec := &kubeConfigSpec{
 		ClientName: clientName,
-		APIServer:  cfg.GetMasterEndpoint(),
+		APIServer:  masterEndpoint,
 		CACert:     caCert,
 		TokenAuth: &tokenAuth{
 			Token: token,

cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go

@@ -34,6 +34,7 @@ import (
 
 	pkiutil "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs/pkiutil"
 
+	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	testutil "k8s.io/kubernetes/cmd/kubeadm/test"
 	certstestutil "k8s.io/kubernetes/cmd/kubeadm/test/certs"
 	kubeconfigtestutil "k8s.io/kubernetes/cmd/kubeadm/test/kubeconfig"
@@ -117,8 +118,12 @@ func TestGetKubeConfigSpecs(t *testing.T) {
 			}
 
 			// Asserts MasterConfiguration values injected into spec
-			if spec.APIServer != cfg.GetMasterEndpoint() {
-				t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer address into spec for %s", assertion.kubeConfigFile)
+			masterEndpoint, err := kubeadmutil.GetMasterEndpoint(cfg)
+			if err != nil {
+				t.Error(err)
+			}
+			if spec.APIServer != masterEndpoint {
+				t.Errorf("getKubeConfigSpecs didn't injected cfg.APIServer endpoint into spec for %s", assertion.kubeConfigFile)
 			}
 
 			// Asserts CA certs and CA keys loaded into specs

cmd/kubeadm/app/util/BUILD

@@ -9,11 +9,13 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "endpoint.go",
         "error.go",
         "template.go",
         "version.go",
     ],
     deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
     ],
@@ -22,12 +24,16 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "endpoint_test.go",
         "error_test.go",
         "template_test.go",
         "version_test.go",
     ],
     library = ":go_default_library",
-    deps = ["//cmd/kubeadm/app/preflight:go_default_library"],
+    deps = [
+        "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/preflight:go_default_library",
+    ],
 )
 
 filegroup(