feat: support rotating Kubernetes CA

Fixes #8440

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

api/resource/definitions/secrets/secrets.proto

@@ -36,9 +36,9 @@ message EtcdRootSpec {
 // KubeletSpec describes root Kubernetes secrets.
 message KubeletSpec {
   common.URL endpoint = 1;
-  common.PEMEncodedCertificateAndKey ca = 2;
   string bootstrap_token_id = 3;
   string bootstrap_token_secret = 4;
+  repeated common.PEMEncodedCertificate accepted_c_as = 5;
 }
 
 // KubernetesCertsSpec describes generated Kubernetes certificates.
@@ -63,14 +63,15 @@ message KubernetesRootSpec {
   common.URL local_endpoint = 3;
   repeated string cert_sa_ns = 4;
   string dns_domain = 6;
-  common.PEMEncodedCertificateAndKey ca = 7;
+  common.PEMEncodedCertificateAndKey issuing_ca = 7;
   common.PEMEncodedKey service_account = 8;
   common.PEMEncodedCertificateAndKey aggregator_ca = 9;
   string aescbc_encryption_secret = 10;
   string bootstrap_token_id = 11;
   string bootstrap_token_secret = 12;
   string secretbox_encryption_secret = 13;
   repeated common.NetIP api_server_ips = 14;
+  repeated common.PEMEncodedCertificate accepted_c_as = 15;
 }
 
 // MaintenanceRootSpec describes maintenance service CA.

cmd/talosctl/cmd/talos/containers.go

@@ -38,7 +38,7 @@ var containersCmd = &cobra.Command{
 				driver    common.ContainerDriver
 			)
 
-			if kubernetes {
+			if kubernetesFlag {
 				namespace = criconstants.K8sContainerdNamespace
 				driver = common.ContainerDriver_CRI
 			} else {
@@ -95,7 +95,7 @@ func containerRender(remotePeer *peer.Peer, resp *machineapi.ContainersResponse)
 }
 
 func init() {
-	containersCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace")
+	containersCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace")
 
 	containersCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver")
 	containersCmd.Flags().MarkHidden("use-cri") //nolint:errcheck

cmd/talosctl/cmd/talos/logs.go

@@ -41,11 +41,11 @@ var logsCmd = &cobra.Command{
 			return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp
 		}
 
-		if kubernetes {
-			return getContainersFromNode(kubernetes), cobra.ShellCompDirectiveNoFileComp
+		if kubernetesFlag {
+			return getContainersFromNode(kubernetesFlag), cobra.ShellCompDirectiveNoFileComp
 		}
 
-		return mergeSuggestions(getServiceFromNode(), getContainersFromNode(kubernetes), getLogsContainers()), cobra.ShellCompDirectiveNoFileComp
+		return mergeSuggestions(getServiceFromNode(), getContainersFromNode(kubernetesFlag), getLogsContainers()), cobra.ShellCompDirectiveNoFileComp
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return WithClient(func(ctx context.Context, c *client.Client) error {
@@ -54,7 +54,7 @@ var logsCmd = &cobra.Command{
 				driver    common.ContainerDriver
 			)
 
-			if kubernetes {
+			if kubernetesFlag {
 				namespace = criconstants.K8sContainerdNamespace
 				driver = common.ContainerDriver_CRI
 			} else {
@@ -230,7 +230,7 @@ func getLogsContainers() []string {
 }
 
 func init() {
-	logsCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace")
+	logsCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace")
 	logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "specify if the logs should be streamed")
 	logsCmd.Flags().Int32VarP(&tailLines, "tail", "", -1, "lines of log file to display (default is to show from the beginning)")
 

cmd/talosctl/cmd/talos/restart.go

@@ -27,7 +27,7 @@ var restartCmd = &cobra.Command{
 			return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp
 		}
 
-		return getContainersFromNode(kubernetes), cobra.ShellCompDirectiveNoFileComp
+		return getContainersFromNode(kubernetesFlag), cobra.ShellCompDirectiveNoFileComp
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return WithClient(func(ctx context.Context, c *client.Client) error {
@@ -36,7 +36,7 @@ var restartCmd = &cobra.Command{
 				driver    common.ContainerDriver
 			)
 
-			if kubernetes {
+			if kubernetesFlag {
 				namespace = criconstants.K8sContainerdNamespace
 				driver = common.ContainerDriver_CRI
 			} else {
@@ -54,7 +54,7 @@ var restartCmd = &cobra.Command{
 }
 
 func init() {
-	restartCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace")
+	restartCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace")
 
 	restartCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver")
 	restartCmd.Flags().MarkHidden("use-cri") //nolint:errcheck

cmd/talosctl/cmd/talos/root.go

@@ -27,7 +27,7 @@ import (
 	"github.com/siderolabs/talos/pkg/machinery/formatters"
 )
 
-var kubernetes bool
+var kubernetesFlag bool
 
 // GlobalArgs is the common arguments for the root command.
 var GlobalArgs global.Args

cmd/talosctl/cmd/talos/rotate-ca.go

@@ -11,29 +11,38 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/siderolabs/talos/pkg/cluster"
 	"github.com/siderolabs/talos/pkg/machinery/client"
 	clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config"
 	"github.com/siderolabs/talos/pkg/machinery/config"
 	"github.com/siderolabs/talos/pkg/machinery/config/encoder"
 	"github.com/siderolabs/talos/pkg/machinery/config/generate/secrets"
+	"github.com/siderolabs/talos/pkg/rotate/pki/kubernetes"
 	"github.com/siderolabs/talos/pkg/rotate/pki/talos"
 )
 
 var rotateCACmdFlags struct {
-	clusterState  clusterNodes
-	forceEndpoint string
-	output        string
-	withExamples  bool
-	withDocs      bool
-	dryRun        bool
+	clusterState     clusterNodes
+	forceEndpoint    string
+	output           string
+	withExamples     bool
+	withDocs         bool
+	dryRun           bool
+	rotateTalos      bool
+	rotateKubernetes bool
 }
 
 // rotateCACmd represents the rotate-ca command.
 var rotateCACmd = &cobra.Command{
 	Use:   "rotate-ca",
 	Short: "Rotate cluster CAs (Talos and Kubernetes APIs).",
-	Long:  `The command starts by generating new CAs, and gracefully applying it to the cluster.`,
-	Args:  cobra.NoArgs,
+	Long: `The command can rotate both Talos and Kubernetes root CAs (for the API).
+By default both CAs are rotated, but you can choose to rotate just one or another.
+The command starts by generating new CAs, and gracefully applying it to the cluster.
+
+For Kubernetes, the command only rotates the API server issuing CA, and other Kubernetes
+PKI can be rotated by applying machine config changes to the controlplane nodes.`,
+	Args: cobra.NoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		err := rotateCACmdFlags.clusterState.InitNodeInfos()
 		if err != nil {
@@ -44,13 +53,13 @@ var rotateCACmd = &cobra.Command{
 	},
 }
 
-func rotateCA(ctx context.Context, oldClient *client.Client) error {
+func rotateCA(ctx context.Context, c *client.Client) error {
 	commentsFlags := encoder.CommentsDisabled
-	if upgradeK8sCmdFlags.withDocs {
+	if rotateCACmdFlags.withDocs {
 		commentsFlags |= encoder.CommentsDocs
 	}
 
-	if upgradeK8sCmdFlags.withExamples {
+	if rotateCACmdFlags.withExamples {
 		commentsFlags |= encoder.CommentsExamples
 	}
 
@@ -61,9 +70,39 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error {
 		return err
 	}
 
+	newBundle, err := secrets.NewBundle(secrets.NewFixedClock(time.Now()), config.TalosVersionCurrent)
+	if err != nil {
+		return fmt.Errorf("error generating new Talos CA: %w", err)
+	}
+
+	if rotateCACmdFlags.rotateTalos {
+		var newTalosconfig *clientconfig.Config
+
+		newTalosconfig, err = rotateTalosCA(ctx, c, encoderOpt, clusterInfo, newBundle)
+		if err != nil {
+			return fmt.Errorf("error rotating Talos CA: %w", err)
+		}
+
+		// re-create client with new Talos PKI
+		c, err = client.New(ctx, client.WithConfig(newTalosconfig))
+		if err != nil {
+			return fmt.Errorf("failed to create new client with rotated Talos CA: %w", err)
+		}
+	}
+
+	if rotateCACmdFlags.rotateKubernetes {
+		if err = rotateKubernetesCA(ctx, c, encoderOpt, clusterInfo, newBundle); err != nil {
+			return fmt.Errorf("error rotating Kubernetes CA: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func rotateTalosCA(ctx context.Context, oldClient *client.Client, encoderOpt encoder.Option, clusterInfo cluster.Info, newBundle *secrets.Bundle) (*clientconfig.Config, error) {
 	oldTalosconfig, err := clientconfig.Open(GlobalArgs.Talosconfig)
 	if err != nil {
-		return fmt.Errorf("failed to open config file %q: %w", GlobalArgs.Talosconfig, err)
+		return nil, fmt.Errorf("failed to open config file %q: %w", GlobalArgs.Talosconfig, err)
 	}
 
 	configContext := oldTalosconfig.Context
@@ -72,11 +111,6 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error {
 		configContext = GlobalArgs.CmdContext
 	}
 
-	newBundle, err := secrets.NewBundle(secrets.NewFixedClock(time.Now()), config.TalosVersionCurrent)
-	if err != nil {
-		return fmt.Errorf("error generating new Talos CA: %w", err)
-	}
-
 	options := talos.Options{
 		DryRun: rotateCACmdFlags.dryRun,
 
@@ -95,6 +129,35 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error {
 
 	newTalosconfig, err := talos.Rotate(ctx, options)
 	if err != nil {
+		return nil, err
+	}
+
+	if rotateCACmdFlags.dryRun {
+		fmt.Println("> Dry-run mode enabled, no changes were made to the cluster, re-run with `--dry-run=false` to apply the changes.")
+
+		return nil, nil
+	}
+
+	fmt.Printf("> Writing new talosconfig to %q\n", rotateCACmdFlags.output)
+
+	return newTalosconfig, newTalosconfig.Save(rotateCACmdFlags.output)
+}
+
+func rotateKubernetesCA(ctx context.Context, c *client.Client, encoderOpt encoder.Option, clusterInfo cluster.Info, newBundle *secrets.Bundle) error {
+	options := kubernetes.Options{
+		DryRun: rotateCACmdFlags.dryRun,
+
+		TalosClient: c,
+		ClusterInfo: clusterInfo,
+
+		NewKubernetesCA: newBundle.Certs.K8s,
+
+		EncoderOption: encoderOpt,
+
+		Printf: func(format string, args ...any) { fmt.Printf(format, args...) },
+	}
+
+	if err := kubernetes.Rotate(ctx, options); err != nil {
 		return err
 	}
 
@@ -104,9 +167,9 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error {
 		return nil
 	}
 
-	fmt.Printf("> Writing new talosconfig to %q\n", rotateCACmdFlags.output)
+	fmt.Printf("> Kubernetes CA rotation done, new 'kubeconfig' can be fetched with `talosctl kubeconfig`.\n")
 
-	return newTalosconfig.Save(rotateCACmdFlags.output)
+	return nil
 }
 
 func init() {
@@ -119,4 +182,6 @@ func init() {
 	rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.withDocs, "with-docs", "", true, "patch all machine configs adding the documentation for each field")
 	rotateCACmd.Flags().StringVarP(&rotateCACmdFlags.output, "output", "o", "talosconfig", "path to the output new `talosconfig`")
 	rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.dryRun, "dry-run", "", true, "dry-run mode (no changes to the cluster)")
+	rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.rotateTalos, "talos", "", true, "rotate Talos API CA")
+	rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.rotateKubernetes, "kubernetes", "", true, "rotate Kubernetes API CA")
 }

cmd/talosctl/cmd/talos/stats.go

@@ -37,7 +37,7 @@ var statsCmd = &cobra.Command{
 				driver    common.ContainerDriver
 			)
 
-			if kubernetes {
+			if kubernetesFlag {
 				namespace = criconstants.K8sContainerdNamespace
 				driver = common.ContainerDriver_CRI
 			} else {
@@ -95,7 +95,7 @@ func statsRender(remotePeer *peer.Peer, resp *machineapi.StatsResponse) error {
 }
 
 func init() {
-	statsCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace")
+	statsCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace")
 
 	statsCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver")
 	statsCmd.Flags().MarkHidden("use-cri") //nolint:errcheck

internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"fmt"
 
-	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -47,7 +46,7 @@ func (r *NodeWatcher) Get() (*corev1.Node, error) {
 }
 
 // Watch starts watching Node state and notifies on updates via notify channel.
-func (r *NodeWatcher) Watch(ctx context.Context, logger *zap.Logger) (<-chan struct{}, func(), error) {
+func (r *NodeWatcher) Watch(ctx context.Context) (<-chan struct{}, <-chan error, func(), error) {
 	informerFactory := informers.NewSharedInformerFactoryWithOptions(
 		r.client.Clientset,
 		0,
@@ -59,6 +58,7 @@ func (r *NodeWatcher) Watch(ctx context.Context, logger *zap.Logger) (<-chan str
 	)
 
 	notifyCh := make(chan struct{}, 1)
+	watchErrCh := make(chan error, 1)
 
 	notify := func(_ any) {
 		select {
@@ -70,22 +70,25 @@ func (r *NodeWatcher) Watch(ctx context.Context, logger *zap.Logger) (<-chan str
 	r.nodes = informerFactory.Core().V1().Nodes()
 
 	if err := r.nodes.Informer().SetWatchErrorHandler(func(r *cache.Reflector, err error) {
-		logger.Error("node watch error", zap.Error(err))
+		select {
+		case watchErrCh <- err:
+		default:
+		}
 	}); err != nil {
-		return nil, nil, fmt.Errorf("failed to set watch error handler: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed to set watch error handler: %w", err)
 	}
 
 	if _, err := r.nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    notify,
 		DeleteFunc: notify,
 		UpdateFunc: func(_, _ any) { notify(nil) },
 	}); err != nil {
-		return nil, nil, fmt.Errorf("failed to add event handler: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed to add event handler: %w", err)
 	}
 
 	informerFactory.Start(ctx.Done())
 
 	informerFactory.WaitForCacheSync(ctx.Done())
 
-	return notifyCh, informerFactory.Shutdown, nil
+	return notifyCh, watchErrCh, informerFactory.Shutdown, nil
 }