feat: allow kubelet to be restarted and provide negative nodeIP subnets

Fixes #4407 fixes #4489 This PR started by enabling simple restart of the `kubelet` service via services API, but it turned out there's a problem: When kubelet restarts, CNI is already up, so there's an interface on the host with CNI node IP, the code which picks kubelet node IP finds it and tries to add it to the list of kubelet node IPs which completely breaks kubelet. Solution was easy: allow node IPs to be filtered out - e.g. we never want kubelet node IP to be from the pod CIDR. But this filtering feature is also useful in other cases, so I added that as well. Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
siderolabs · Nov 15, 2021 · a76f6d6 · a76f6d6
1 parent 189221d
commit a76f6d6
Show file tree

Hide file tree

Showing 13 changed files with 53 additions and 27 deletions.
diff --git a/go.mod b/go.mod
@@ -97,7 +97,7 @@ require (
 	github.com/talos-systems/go-retry v0.3.1
 	github.com/talos-systems/go-smbios v0.1.0
 	github.com/talos-systems/grpc-proxy v0.2.0
-	github.com/talos-systems/net v0.3.0
+	github.com/talos-systems/net v0.3.1-0.20211112122313-0abe5bdae8f8
 	github.com/talos-systems/talos/pkg/machinery v0.0.0-00010101000000-000000000000
 	github.com/u-root/u-root v7.0.0+incompatible
 	github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5

diff --git a/go.sum b/go.sum
@@ -1082,8 +1082,8 @@ github.com/talos-systems/go-smbios v0.1.0 h1:C6ooNSKyw2bzJxDTPeNbom5sri53h6bJgTW
 github.com/talos-systems/go-smbios v0.1.0/go.mod h1:HxhrzAoTZ7ed5Z5VvtCvnCIrOxyXDS7V2B5hCetAMW8=
 github.com/talos-systems/grpc-proxy v0.2.0 h1:DN75bLfaW4xfhq0r0mwFRnfGhSB+HPhK1LNzuMEs9Pw=
 github.com/talos-systems/grpc-proxy v0.2.0/go.mod h1:sm97Vc/z2cok3pu6ruNeszQej4KDxFrDgfWs4C1mtC4=
-github.com/talos-systems/net v0.3.0 h1:TG6PoiNdg9NmSeSjyecSgguUXzoJ8wp5a8RYlIdkq3Y=
-github.com/talos-systems/net v0.3.0/go.mod h1:VreSAyRmxMtqussAHSKMKkJQa1YwBTSVfkmE4Jydam4=
+github.com/talos-systems/net v0.3.1-0.20211112122313-0abe5bdae8f8 h1:oT2MASZ8V3DuZbhaJWJ8oZ373zfmgXpvw2xLHM5cOYk=
+github.com/talos-systems/net v0.3.1-0.20211112122313-0abe5bdae8f8/go.mod h1:zhcGixNJz9dgwFiUwc7gkkAqdVqXagU1SNNoIVXYKGo=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=

diff --git a/hack/release.toml b/hack/release.toml
@@ -79,6 +79,14 @@ cluster information that could help with future debugging in a single run.
 Output of the command is a `zip` archive with all talos service logs, kubernetes pod logs and manifests,
 talos resources manifests and so on.
 Generated archive does not contain any secret information so it is safe to send it for analysis to a third party.
+"""
+
+    [notes.kubelet]
+        title = "Kubelet"
+        description = """\
+Kubelet service can now be restarted with `talosctl service kubelet restart`.
+
+Kubelet node IP configuration (`.machine.kubelet.nodeIP.validSubnets`) can now include negative subnet matches (prefixed with `!`).
 """
 
 [make_deps]

diff --git a/internal/app/machined/pkg/system/services/kubelet.go b/internal/app/machined/pkg/system/services/kubelet.go
@@ -253,6 +253,11 @@ func (k *Kubelet) HealthSettings(runtime.Runtime) *health.Settings {
 	return &settings
 }
 
+// APIRestartAllowed implements APIRestartableService.
+func (k *Kubelet) APIRestartAllowed(runtime.Runtime) bool {
+	return true
+}
+
 func newKubeletConfiguration(clusterDNS []string, dnsDomain string) *kubeletconfig.KubeletConfiguration {
 	f := false
 	t := true
@@ -329,6 +334,11 @@ func (k *Kubelet) args(r runtime.Runtime) ([]string, error) {
 		}
 	}
 
+	// anyway filter out pod cidrs, they can't be node IPs
+	for _, cidr := range r.Config().Cluster().Network().PodCIDRs() {
+		validSubnets = append(validSubnets, "!"+cidr)
+	}
+
 	nodeIPs, err := pickNodeIPs(validSubnets)
 	if err != nil {
 		return nil, err
@@ -432,20 +442,5 @@ func pickNodeIPs(cidrs []string) ([]stdnet.IP, error) {
 		return nil, fmt.Errorf("failed to discover interface IP addresses: %w", err)
 	}
 
-	var result []stdnet.IP
-
-	for _, subnet := range cidrs {
-		network, err := net.ParseCIDR(subnet)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse subnet: %w", err)
-		}
-
-		for _, ip := range ips {
-			if network.Contains(ip) {
-				result = append(result, ip)
-			}
-		}
-	}
-
-	return result, nil
+	return net.FilterIPs(ips, cidrs)
 }
diff --git a/internal/integration/cli/config.go b/internal/integration/cli/config.go
@@ -69,7 +69,7 @@ func (suite *TalosconfigSuite) TestMerge() {
 
 // TestNew checks `talosctl config new`.
 func (suite *TalosconfigSuite) TestNew() {
-	stdout := suite.RunCLI([]string{"version", "--json"})
+	stdout := suite.RunCLI([]string{"version", "--json", "--nodes", suite.RandomDiscoveredNode()})
 
 	var v machineapi.Version
 	err := protojson.Unmarshal([]byte(stdout), &v)

diff --git a/internal/integration/cli/services.go b/internal/integration/cli/services.go
@@ -9,6 +9,7 @@ package cli
 
 import (
 	"regexp"
+	"time"
 
 	"github.com/talos-systems/talos/internal/integration/base"
 )
@@ -40,6 +41,17 @@ func (suite *ServicesSuite) TestStatus() {
 	)
 }
 
+// TestRestart verifies kubelet restart.
+func (suite *ServicesSuite) TestRestart() {
+	node := suite.RandomDiscoveredNode()
+
+	suite.RunCLI([]string{"service", "kubelet", "restart", "--nodes", node})
+
+	time.Sleep(200 * time.Millisecond)
+
+	suite.RunAndWaitForMatch([]string{"service", "-n", node, "kubelet"}, regexp.MustCompile(`EVENTS\s+\[Running\]: Health check successful`), 30*time.Second)
+}
+
 func init() {
 	allSuites = append(allSuites, new(ServicesSuite))
 }
diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go
@@ -470,6 +470,7 @@ metadata:
 	kubeletNodeIPExample = KubeletNodeIPConfig{
 		KubeletNodeIPValidSubnets: []string{
 			"10.0.0.0/8",
+			"!10.0.0.3/32",
 			"fdc7::/16",
 		},
 	}
@@ -894,7 +895,9 @@ type KubeletNodeIPConfig struct {
 	//  description: |
 	//    The `validSubnets` field configures the networks to pick kubelet node IP from.
 	//    For dual stack configuration, there should be two subnets: one for IPv4, another for IPv6.
-	//    If not specified, kubelet configures node IP automatically.
+	//    IPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.
+	//    Negative subnet matches should be specified last to filter out IPs picked by positive matches.
+	//    If not specified, node IP is picked based on cluster podCIDRs: IPv4/IPv6 address or both.
 	KubeletNodeIPValidSubnets []string `yaml:"validSubnets,omitempty"`
 }
 

diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go
diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go
@@ -690,6 +690,8 @@ func (k *KubeletConfig) Validate() ([]string, error) {
 	var result *multierror.Error
 
 	for _, cidr := range k.KubeletNodeIP.KubeletNodeIPValidSubnets {
+		cidr = strings.TrimPrefix(cidr, "!")
+
 		if _, _, err := net.ParseCIDR(cidr); err != nil {
 			result = multierror.Append(result, fmt.Errorf("kubelet nodeIP subnet is not valid: %q", cidr))
 		}

diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation_test.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation_test.go
@@ -845,6 +845,7 @@ func TestValidate(t *testing.T) {
 						KubeletNodeIP: v1alpha1.KubeletNodeIPConfig{
 							KubeletNodeIPValidSubnets: []string{
 								"10.0.0.0/8",
+								"!10.0.0.3/32",
 							},
 						},
 					},

diff --git a/pkg/machinery/go.mod b/pkg/machinery/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/talos-systems/crypto v0.3.4
 	github.com/talos-systems/go-blockdevice v0.2.4
 	github.com/talos-systems/go-debug v0.2.1
-	github.com/talos-systems/net v0.3.0
+	github.com/talos-systems/net v0.3.1-0.20211112122313-0abe5bdae8f8
 	google.golang.org/genproto v0.0.0-20211112145013-271947fe86fd
 	google.golang.org/grpc v1.42.0
 	google.golang.org/protobuf v1.27.1

diff --git a/pkg/machinery/go.sum b/pkg/machinery/go.sum
@@ -171,8 +171,8 @@ github.com/talos-systems/go-debug v0.2.1 h1:VSN8P1zXWeHWgUBZn4cVT3keBcecCAJBG9Up
 github.com/talos-systems/go-debug v0.2.1/go.mod h1:pR4NjsZQNFqGx3n4qkD4MIj1F2CxyIF8DCiO1+05JO0=
 github.com/talos-systems/go-retry v0.1.1-0.20201113203059-8c63d290a688/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM=
 github.com/talos-systems/go-retry v0.3.1/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM=
-github.com/talos-systems/net v0.3.0 h1:TG6PoiNdg9NmSeSjyecSgguUXzoJ8wp5a8RYlIdkq3Y=
-github.com/talos-systems/net v0.3.0/go.mod h1:VreSAyRmxMtqussAHSKMKkJQa1YwBTSVfkmE4Jydam4=
+github.com/talos-systems/net v0.3.1-0.20211112122313-0abe5bdae8f8 h1:oT2MASZ8V3DuZbhaJWJ8oZ373zfmgXpvw2xLHM5cOYk=
+github.com/talos-systems/net v0.3.1-0.20211112122313-0abe5bdae8f8/go.mod h1:zhcGixNJz9dgwFiUwc7gkkAqdVqXagU1SNNoIVXYKGo=
 github.com/unix4ever/yaml v0.0.0-20210315173758-8fb30b8e5a5b h1:8pnPjZJU0SYanlmHnhMTeR8OR148K9yStwBz1GsjBsQ=
 github.com/unix4ever/yaml v0.0.0-20210315173758-8fb30b8e5a5b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -324,7 +324,6 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=

diff --git a/website/content/docs/v0.14/Reference/configuration.md b/website/content/docs/v0.14/Reference/configuration.md
@@ -309,6 +309,7 @@ kubelet:
     #     # The `validSubnets` field configures the networks to pick kubelet node IP from.
     #     validSubnets:
     #         - 10.0.0.0/8
+    #         - '!10.0.0.3/32'
     #         - fdc7::/16
 ```
 
@@ -1438,6 +1439,7 @@ extraArgs:
 #     # The `validSubnets` field configures the networks to pick kubelet node IP from.
 #     validSubnets:
 #         - 10.0.0.0/8
+#         - '!10.0.0.3/32'
 #         - fdc7::/16
 ```
 
@@ -1586,6 +1588,7 @@ nodeIP:
     # The `validSubnets` field configures the networks to pick kubelet node IP from.
     validSubnets:
         - 10.0.0.0/8
+        - '!10.0.0.3/32'
         - fdc7::/16
 ```
 
@@ -1608,6 +1611,7 @@ Appears in:
 # The `validSubnets` field configures the networks to pick kubelet node IP from.
 validSubnets:
     - 10.0.0.0/8
+    - '!10.0.0.3/32'
     - fdc7::/16
 ```
 
@@ -1622,7 +1626,9 @@ validSubnets:
 
 The `validSubnets` field configures the networks to pick kubelet node IP from.
 For dual stack configuration, there should be two subnets: one for IPv4, another for IPv6.
-If not specified, kubelet configures node IP automatically.
+IPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.
+Negative subnet matches should be specified last to filter out IPs picked by positive matches.
+If not specified, node IP is picked based on cluster podCIDRs: IPv4/IPv6 address or both.
 
 </div>