feat: allow fqdn to be used when registering k8s node

This PR fixes a problem we had with AWS clusters. We now allow the kubelet to register using the full fqdn instead of just hostname. Signed-off-by: Spencer Smith <robertspencersmith@gmail.com>
siderolabs · Feb 1, 2021 · e4e6da3 · e4e6da3
1 parent 87ccf0e
commit e4e6da3
Show file tree

Hide file tree

Showing 9 changed files with 116 additions and 11 deletions.
diff --git a/internal/app/machined/pkg/runtime/runtime.go b/internal/app/machined/pkg/runtime/runtime.go
@@ -16,4 +16,5 @@ type Runtime interface {
 	State() State
 	Events() EventStream
 	Logging() LoggingManager
+	NodeName() (string, error)
 }
diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go
@@ -6,6 +6,9 @@ package v1alpha1
 
 import (
 	"fmt"
+	"log"
+	"os"
+	"syscall"
 
 	"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
 	"github.com/talos-systems/talos/pkg/machinery/config"
@@ -75,3 +78,45 @@ func (r *Runtime) Events() runtime.EventStream {
 func (r *Runtime) Logging() runtime.LoggingManager {
 	return r.l
 }
+
+// NodeName implements the Runtime interface.
+func (r *Runtime) NodeName() (string, error) {
+	// attempt to fetch hostname and domain name via syscalls and concat them if necessary
+	if r.Config().Machine().Kubelet().RegisterWithFQDN() {
+		var utsName syscall.Utsname
+		if err := syscall.Uname(&utsName); err != nil {
+			return "", err
+		}
+
+		nodeName := utsNameVarToString(utsName.Nodename)
+		log.Printf("Nodename is %s", nodeName)
+
+		domainName := utsNameVarToString(utsName.Domainname)
+		log.Printf("Domain name is %s", domainName)
+
+		// As odd as it looks, the Uname method sets domainName to this "(none)" string if not set.
+		if domainName != "(none)" {
+			return fmt.Sprintf("%s.%s", nodeName, domainName), nil
+		}
+
+		return nodeName, nil
+	}
+
+	// default to os.Hostname if we don't need to worry about fqdn.
+	return os.Hostname()
+}
+
+// converts int8 array to a string
+// borrowed from https://github.com/aisola/go-coreutils/blob/master/uname/uname.go#L98
+func utsNameVarToString(unameArray [65]int8) string {
+	var byteString [65]byte
+
+	var indexLength int
+
+	for unameArray[indexLength] != 0 {
+		byteString[indexLength] = uint8(unameArray[indexLength])
+		indexLength++
+	}
+
+	return string(byteString[:indexLength])
+}
diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go
@@ -1224,9 +1224,9 @@ func UnmountSystemDiskBindMounts(seq runtime.Sequence, data interface{}) (runtim
 //nolint: dupl
 func CordonAndDrainNode(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
 	return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
-		var hostname string
+		var nodename string
 
-		if hostname, err = os.Hostname(); err != nil {
+		if nodename, err = r.NodeName(); err != nil {
 			return err
 		}
 
@@ -1236,7 +1236,7 @@ func CordonAndDrainNode(seq runtime.Sequence, data interface{}) (runtime.TaskExe
 			return err
 		}
 
-		if err = kubeHelper.CordonAndDrain(hostname); err != nil {
+		if err = kubeHelper.CordonAndDrain(nodename); err != nil {
 			return err
 		}
 
@@ -1251,9 +1251,9 @@ func CordonAndDrainNode(seq runtime.Sequence, data interface{}) (runtime.TaskExe
 //nolint: dupl
 func UncordonNode(seq runtime.Sequence, data interface{}) (runtime.TaskExecutionFunc, string) {
 	return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) (err error) {
-		var hostname string
+		var nodename string
 
-		if hostname, err = os.Hostname(); err != nil {
+		if nodename, err = r.NodeName(); err != nil {
 			return err
 		}
 
@@ -1263,11 +1263,11 @@ func UncordonNode(seq runtime.Sequence, data interface{}) (runtime.TaskExecution
 			return err
 		}
 
-		if err = kubeHelper.WaitUntilReady(hostname); err != nil {
+		if err = kubeHelper.WaitUntilReady(nodename); err != nil {
 			return err
 		}
 
-		if err = kubeHelper.Uncordon(hostname, false); err != nil {
+		if err = kubeHelper.Uncordon(nodename, false); err != nil {
 			return err
 		}
 
@@ -1475,13 +1475,14 @@ func LabelNodeAsMaster(seq runtime.Sequence, data interface{}) (runtime.TaskExec
 			return err
 		}
 
-		hostname, err := os.Hostname()
-		if err != nil {
+		var nodename string
+
+		if nodename, err = r.NodeName(); err != nil {
 			return err
 		}
 
 		err = retry.Constant(constants.NodeReadyTimeout, retry.WithUnits(3*time.Second), retry.WithErrorLogging(true)).Retry(func() error {
-			if err = h.LabelNodeAsMaster(hostname, !r.Config().Cluster().ScheduleOnMasters()); err != nil {
+			if err = h.LabelNodeAsMaster(nodename, !r.Config().Cluster().ScheduleOnMasters()); err != nil {
 				return retry.ExpectedError(err)
 			}
 

diff --git a/internal/app/machined/pkg/system/services/kubelet.go b/internal/app/machined/pkg/system/services/kubelet.go
@@ -267,6 +267,11 @@ func newKubeletConfiguration(clusterDNS []string, dnsDomain string) *kubeletconf
 }
 
 func (k *Kubelet) args(r runtime.Runtime) ([]string, error) {
+	nodename, err := r.NodeName()
+	if err != nil {
+		return nil, err
+	}
+
 	denyListArgs := argsbuilder.Args{
 		"bootstrap-kubeconfig":       constants.KubeletBootstrapKubeconfig,
 		"kubeconfig":                 constants.KubeletKubeconfig,
@@ -277,6 +282,8 @@ func (k *Kubelet) args(r runtime.Runtime) ([]string, error) {
 
 		"cert-dir":     constants.KubeletPKIDir,
 		"cni-conf-dir": cni.DefaultNetDir,
+
+		"hostname-override": nodename,
 	}
 
 	extraArgs := argsbuilder.Args(r.Config().Machine().Kubelet().ExtraArgs())

diff --git a/pkg/machinery/config/provider.go b/pkg/machinery/config/provider.go
@@ -200,6 +200,7 @@ type Kubelet interface {
 	Image() string
 	ExtraArgs() map[string]string
 	ExtraMounts() []specs.Mount
+	RegisterWithFQDN() bool
 }
 
 // Registries defines the configuration for image fetching.

diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go
@@ -262,6 +262,11 @@ func (k *KubeletConfig) ExtraMounts() []specs.Mount {
 	return k.KubeletExtraMounts
 }
 
+// RegisterWithFQDN implements the config.Provider interface.
+func (k *KubeletConfig) RegisterWithFQDN() bool {
+	return k.KubeletRegisterWithFQDN
+}
+
 // Name implements the config.Provider interface.
 func (c *ClusterConfig) Name() string {
 	return c.ClusterName

diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go
@@ -677,6 +677,15 @@ type KubeletConfig struct {
 	//   examples:
 	//     - value: kubeletExtraMountsExample
 	KubeletExtraMounts []specs.Mount `yaml:"extraMounts,omitempty"`
+	//   description: |
+	//     The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.
+	//     This is required in clouds like AWS.
+	//   values:
+	//     - true
+	//     - yes
+	//     - false
+	//     - no
+	KubeletRegisterWithFQDN bool `yaml:"registerWithFQDN,omitempty"`
 }
 
 // NetworkConfig represents the machine's networking config values.

diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go
@@ -379,7 +379,7 @@ func init() {
 			FieldName: "kubelet",
 		},
 	}
-	KubeletConfigDoc.Fields = make([]encoder.Doc, 3)
+	KubeletConfigDoc.Fields = make([]encoder.Doc, 4)
 	KubeletConfigDoc.Fields[0].Name = "image"
 	KubeletConfigDoc.Fields[0].Type = "string"
 	KubeletConfigDoc.Fields[0].Note = ""
@@ -403,6 +403,17 @@ func init() {
 	KubeletConfigDoc.Fields[2].Comments[encoder.LineComment] = "The `extraMounts` field is used to add additional mounts to the kubelet container."
 
 	KubeletConfigDoc.Fields[2].AddExample("", kubeletExtraMountsExample)
+	KubeletConfigDoc.Fields[3].Name = "registerWithFQDN"
+	KubeletConfigDoc.Fields[3].Type = "bool"
+	KubeletConfigDoc.Fields[3].Note = ""
+	KubeletConfigDoc.Fields[3].Description = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.\nThis is required in clouds like AWS."
+	KubeletConfigDoc.Fields[3].Comments[encoder.LineComment] = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration."
+	KubeletConfigDoc.Fields[3].Values = []string{
+		"true",
+		"yes",
+		"false",
+		"no",
+	}
 
 	NetworkConfigDoc.Type = "NetworkConfig"
 	NetworkConfigDoc.Comments[encoder.LineComment] = "NetworkConfig represents the machine's networking config values."

diff --git a/website/content/docs/v0.9/Reference/configuration.md b/website/content/docs/v0.9/Reference/configuration.md
@@ -1286,6 +1286,31 @@ extraMounts:
 
 <hr />
 
+<div class="dd">
+
+<code>registerWithFQDN</code>  <i>bool</i>
+
+</div>
+<div class="dt">
+
+The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.
+This is required in clouds like AWS.
+
+
+Valid values:
+
+
+  - <code>true</code>
+
+  - <code>yes</code>
+
+  - <code>false</code>
+
+  - <code>no</code>
+</div>
+
+<hr />
+