From 7a68504b6b451018e3a5e2589d03aa029b4216f6 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 22 Mar 2024 18:29:34 +0400 Subject: [PATCH] feat: support rotating Kubernetes CA Fixes #8440 Signed-off-by: Andrey Smirnov --- api/api.descriptors | Bin 87469 -> 88509 bytes .../definitions/secrets/secrets.proto | 5 +- cmd/talosctl/cmd/talos/containers.go | 4 +- cmd/talosctl/cmd/talos/logs.go | 10 +- cmd/talosctl/cmd/talos/restart.go | 6 +- cmd/talosctl/cmd/talos/root.go | 2 +- cmd/talosctl/cmd/talos/rotate-ca.go | 103 ++++- cmd/talosctl/cmd/talos/stats.go | 4 +- .../k8s/internal/nodewatch/nodewatch.go | 15 +- .../pkg/controllers/k8s/kubelet_service.go | 74 ++-- .../pkg/controllers/k8s/kubelet_static_pod.go | 2 +- .../pkg/controllers/k8s/node_status.go | 23 +- .../k8s/render_secrets_static_pod.go | 4 +- .../pkg/controllers/secrets/kubelet.go | 13 +- .../pkg/controllers/secrets/kubelet_test.go | 2 +- .../pkg/controllers/secrets/kubernetes.go | 14 +- .../secrets/kubernetes_dynamic_certs.go | 2 +- .../secrets/kubernetes_dynamic_certs_test.go | 2 +- .../controllers/secrets/kubernetes_test.go | 2 +- .../machined/pkg/controllers/secrets/root.go | 16 +- .../pkg/controllers/secrets/root_test.go | 10 +- internal/integration/api/generate-config.go | 2 +- internal/integration/api/rotate.go | 55 +++ pkg/kubeconfig/generate.go | 23 +- pkg/kubeconfig/generate_test.go | 3 +- pkg/kubernetes/kubernetes.go | 2 +- .../definitions/secrets/secrets.pb.go | 235 ++++++------ .../definitions/secrets/secrets_vtproto.pb.go | 228 +++++++---- pkg/machinery/config/config/cluster.go | 3 +- .../config/generate/secrets/bundle.go | 2 +- .../config/schemas/config.schema.json | 13 + .../types/v1alpha1/v1alpha1_clusterconfig.go | 10 +- .../types/v1alpha1/v1alpha1_redact_test.go | 2 +- .../config/types/v1alpha1/v1alpha1_types.go | 9 + .../types/v1alpha1/v1alpha1_types_doc.go | 37 +- .../types/v1alpha1/v1alpha1_validation.go | 2 +- .../types/v1alpha1/zz_generated.deepcopy.go | 10 + .../resources/secrets/deep_copy.generated.go | 23 +- pkg/machinery/resources/secrets/kubelet.go | 2 +- .../resources/secrets/kubernetes_root.go | 3 +- pkg/rotate/pki/internal/helpers/helpers.go | 161 ++++++++ pkg/rotate/pki/kubernetes/kubernetes.go | 362 ++++++++++++++++++ pkg/rotate/pki/talos/helpers.go | 76 ---- pkg/rotate/pki/talos/talos.go | 9 +- website/content/v1.7/reference/api.md | 5 +- website/content/v1.7/reference/cli.md | 7 + .../configuration/v1alpha1/config.md | 1 + .../content/v1.7/schemas/config.schema.json | 13 + 48 files changed, 1229 insertions(+), 382 deletions(-) create mode 100644 pkg/rotate/pki/internal/helpers/helpers.go create mode 100644 pkg/rotate/pki/kubernetes/kubernetes.go delete mode 100644 pkg/rotate/pki/talos/helpers.go diff --git a/api/api.descriptors b/api/api.descriptors index 405289495ba3866b236636e757ee6ee4dccbe718..066442e1e09863fd7e3e18d51fca643aa8874d03 100644 GIT binary patch delta 992 zcmZuvO^ee|6y>GqM_#5M#L>n@wTuiif~XW1ZUi$^Z3|K)t*(TarqB8`O+u26j+-t- z{Q*UTAOpG*M0BW75Cj))-1!IG4dOp=C%$)6`Z2f(_nv!l-#h1>_wSNl#>tOQHS~M@ zOuLmrpGOC^Pr?&4dRc#vQ&w=M)z6Xswp5XAj(NB zcR_&Rnc-bE4d>w)y3P>g7*4UB#0k|5%;Ihsu00m+7dK8jrs zb&=Qnn1EQJ>l-lQUWhu5p%y0;ahkgEkXs_^DeA(2dAi^P)Xe9hI2h0_h-OCfnpNsk+)gfTo-yqe3)>JlKZy@HS4p*9J2gOK!%1tu4p)sM`FTRZ^p57XieeYZop z@lq)k;{gPWrFVpjV#bm5`8(d=H(sJaaO6X@HcOMTf#)K<2OHD9W#|_kJ$7A3r6wj z7kn7mwK4D@!Ki$oR(Sp^OizzvAx`8vJ4l{^R>%y3_UB#7=iHUK_bai*eBkW)W b6716(Js7o^n3A{adN9so+|K94xSk&XVKFGA diff --git a/api/resource/definitions/secrets/secrets.proto b/api/resource/definitions/secrets/secrets.proto index d9702c088b..d7d73c32ef 100755 --- a/api/resource/definitions/secrets/secrets.proto +++ b/api/resource/definitions/secrets/secrets.proto @@ -36,9 +36,9 @@ message EtcdRootSpec { // KubeletSpec describes root Kubernetes secrets. message KubeletSpec { common.URL endpoint = 1; - common.PEMEncodedCertificateAndKey ca = 2; string bootstrap_token_id = 3; string bootstrap_token_secret = 4; + repeated common.PEMEncodedCertificate accepted_c_as = 5; } // KubernetesCertsSpec describes generated Kubernetes certificates. @@ -63,7 +63,7 @@ message KubernetesRootSpec { common.URL local_endpoint = 3; repeated string cert_sa_ns = 4; string dns_domain = 6; - common.PEMEncodedCertificateAndKey ca = 7; + common.PEMEncodedCertificateAndKey issuing_ca = 7; common.PEMEncodedKey service_account = 8; common.PEMEncodedCertificateAndKey aggregator_ca = 9; string aescbc_encryption_secret = 10; @@ -71,6 +71,7 @@ message KubernetesRootSpec { string bootstrap_token_secret = 12; string secretbox_encryption_secret = 13; repeated common.NetIP api_server_ips = 14; + repeated common.PEMEncodedCertificate accepted_c_as = 15; } // MaintenanceRootSpec describes maintenance service CA. diff --git a/cmd/talosctl/cmd/talos/containers.go b/cmd/talosctl/cmd/talos/containers.go index cb41a1b56b..468a184b70 100644 --- a/cmd/talosctl/cmd/talos/containers.go +++ b/cmd/talosctl/cmd/talos/containers.go @@ -38,7 +38,7 @@ var containersCmd = &cobra.Command{ driver common.ContainerDriver ) - if kubernetes { + if kubernetesFlag { namespace = criconstants.K8sContainerdNamespace driver = common.ContainerDriver_CRI } else { @@ -95,7 +95,7 @@ func containerRender(remotePeer *peer.Peer, resp *machineapi.ContainersResponse) } func init() { - containersCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + containersCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") containersCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver") containersCmd.Flags().MarkHidden("use-cri") //nolint:errcheck diff --git a/cmd/talosctl/cmd/talos/logs.go b/cmd/talosctl/cmd/talos/logs.go index 79c426a389..ccea9de652 100644 --- a/cmd/talosctl/cmd/talos/logs.go +++ b/cmd/talosctl/cmd/talos/logs.go @@ -41,11 +41,11 @@ var logsCmd = &cobra.Command{ return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp } - if kubernetes { - return getContainersFromNode(kubernetes), cobra.ShellCompDirectiveNoFileComp + if kubernetesFlag { + return getContainersFromNode(kubernetesFlag), cobra.ShellCompDirectiveNoFileComp } - return mergeSuggestions(getServiceFromNode(), getContainersFromNode(kubernetes), getLogsContainers()), cobra.ShellCompDirectiveNoFileComp + return mergeSuggestions(getServiceFromNode(), getContainersFromNode(kubernetesFlag), getLogsContainers()), cobra.ShellCompDirectiveNoFileComp }, RunE: func(cmd *cobra.Command, args []string) error { return WithClient(func(ctx context.Context, c *client.Client) error { @@ -54,7 +54,7 @@ var logsCmd = &cobra.Command{ driver common.ContainerDriver ) - if kubernetes { + if kubernetesFlag { namespace = criconstants.K8sContainerdNamespace driver = common.ContainerDriver_CRI } else { @@ -230,7 +230,7 @@ func getLogsContainers() []string { } func init() { - logsCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + logsCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "specify if the logs should be streamed") logsCmd.Flags().Int32VarP(&tailLines, "tail", "", -1, "lines of log file to display (default is to show from the beginning)") diff --git a/cmd/talosctl/cmd/talos/restart.go b/cmd/talosctl/cmd/talos/restart.go index ebaefbcf7f..8d9f032c37 100644 --- a/cmd/talosctl/cmd/talos/restart.go +++ b/cmd/talosctl/cmd/talos/restart.go @@ -27,7 +27,7 @@ var restartCmd = &cobra.Command{ return nil, cobra.ShellCompDirectiveError | cobra.ShellCompDirectiveNoFileComp } - return getContainersFromNode(kubernetes), cobra.ShellCompDirectiveNoFileComp + return getContainersFromNode(kubernetesFlag), cobra.ShellCompDirectiveNoFileComp }, RunE: func(cmd *cobra.Command, args []string) error { return WithClient(func(ctx context.Context, c *client.Client) error { @@ -36,7 +36,7 @@ var restartCmd = &cobra.Command{ driver common.ContainerDriver ) - if kubernetes { + if kubernetesFlag { namespace = criconstants.K8sContainerdNamespace driver = common.ContainerDriver_CRI } else { @@ -54,7 +54,7 @@ var restartCmd = &cobra.Command{ } func init() { - restartCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + restartCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") restartCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver") restartCmd.Flags().MarkHidden("use-cri") //nolint:errcheck diff --git a/cmd/talosctl/cmd/talos/root.go b/cmd/talosctl/cmd/talos/root.go index 522345cd0f..84b53eec13 100644 --- a/cmd/talosctl/cmd/talos/root.go +++ b/cmd/talosctl/cmd/talos/root.go @@ -27,7 +27,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/formatters" ) -var kubernetes bool +var kubernetesFlag bool // GlobalArgs is the common arguments for the root command. var GlobalArgs global.Args diff --git a/cmd/talosctl/cmd/talos/rotate-ca.go b/cmd/talosctl/cmd/talos/rotate-ca.go index 6ff2960af6..9641baddd1 100644 --- a/cmd/talosctl/cmd/talos/rotate-ca.go +++ b/cmd/talosctl/cmd/talos/rotate-ca.go @@ -11,29 +11,38 @@ import ( "github.com/spf13/cobra" + "github.com/siderolabs/talos/pkg/cluster" "github.com/siderolabs/talos/pkg/machinery/client" clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" "github.com/siderolabs/talos/pkg/machinery/config" "github.com/siderolabs/talos/pkg/machinery/config/encoder" "github.com/siderolabs/talos/pkg/machinery/config/generate/secrets" + "github.com/siderolabs/talos/pkg/rotate/pki/kubernetes" "github.com/siderolabs/talos/pkg/rotate/pki/talos" ) var rotateCACmdFlags struct { - clusterState clusterNodes - forceEndpoint string - output string - withExamples bool - withDocs bool - dryRun bool + clusterState clusterNodes + forceEndpoint string + output string + withExamples bool + withDocs bool + dryRun bool + rotateTalos bool + rotateKubernetes bool } // rotateCACmd represents the rotate-ca command. var rotateCACmd = &cobra.Command{ Use: "rotate-ca", Short: "Rotate cluster CAs (Talos and Kubernetes APIs).", - Long: `The command starts by generating new CAs, and gracefully applying it to the cluster.`, - Args: cobra.NoArgs, + Long: `The command can rotate both Talos and Kubernetes root CAs (for the API). +By default both CAs are rotated, but you can choose to rotate just one or another. +The command starts by generating new CAs, and gracefully applying it to the cluster. + +For Kubernetes, the command only rotates the API server issuing CA, and other Kubernetes +PKI can be rotated by applying machine config changes to the controlplane nodes.`, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { err := rotateCACmdFlags.clusterState.InitNodeInfos() if err != nil { @@ -44,13 +53,13 @@ var rotateCACmd = &cobra.Command{ }, } -func rotateCA(ctx context.Context, oldClient *client.Client) error { +func rotateCA(ctx context.Context, c *client.Client) error { commentsFlags := encoder.CommentsDisabled - if upgradeK8sCmdFlags.withDocs { + if rotateCACmdFlags.withDocs { commentsFlags |= encoder.CommentsDocs } - if upgradeK8sCmdFlags.withExamples { + if rotateCACmdFlags.withExamples { commentsFlags |= encoder.CommentsExamples } @@ -61,9 +70,39 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error { return err } + newBundle, err := secrets.NewBundle(secrets.NewFixedClock(time.Now()), config.TalosVersionCurrent) + if err != nil { + return fmt.Errorf("error generating new Talos CA: %w", err) + } + + if rotateCACmdFlags.rotateTalos { + var newTalosconfig *clientconfig.Config + + newTalosconfig, err = rotateTalosCA(ctx, c, encoderOpt, clusterInfo, newBundle) + if err != nil { + return fmt.Errorf("error rotating Talos CA: %w", err) + } + + // re-create client with new Talos PKI + c, err = client.New(ctx, client.WithConfig(newTalosconfig)) + if err != nil { + return fmt.Errorf("failed to create new client with rotated Talos CA: %w", err) + } + } + + if rotateCACmdFlags.rotateKubernetes { + if err = rotateKubernetesCA(ctx, c, encoderOpt, clusterInfo, newBundle); err != nil { + return fmt.Errorf("error rotating Kubernetes CA: %w", err) + } + } + + return nil +} + +func rotateTalosCA(ctx context.Context, oldClient *client.Client, encoderOpt encoder.Option, clusterInfo cluster.Info, newBundle *secrets.Bundle) (*clientconfig.Config, error) { oldTalosconfig, err := clientconfig.Open(GlobalArgs.Talosconfig) if err != nil { - return fmt.Errorf("failed to open config file %q: %w", GlobalArgs.Talosconfig, err) + return nil, fmt.Errorf("failed to open config file %q: %w", GlobalArgs.Talosconfig, err) } configContext := oldTalosconfig.Context @@ -72,11 +111,6 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error { configContext = GlobalArgs.CmdContext } - newBundle, err := secrets.NewBundle(secrets.NewFixedClock(time.Now()), config.TalosVersionCurrent) - if err != nil { - return fmt.Errorf("error generating new Talos CA: %w", err) - } - options := talos.Options{ DryRun: rotateCACmdFlags.dryRun, @@ -95,6 +129,35 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error { newTalosconfig, err := talos.Rotate(ctx, options) if err != nil { + return nil, err + } + + if rotateCACmdFlags.dryRun { + fmt.Println("> Dry-run mode enabled, no changes were made to the cluster, re-run with `--dry-run=false` to apply the changes.") + + return nil, nil + } + + fmt.Printf("> Writing new talosconfig to %q\n", rotateCACmdFlags.output) + + return newTalosconfig, newTalosconfig.Save(rotateCACmdFlags.output) +} + +func rotateKubernetesCA(ctx context.Context, c *client.Client, encoderOpt encoder.Option, clusterInfo cluster.Info, newBundle *secrets.Bundle) error { + options := kubernetes.Options{ + DryRun: rotateCACmdFlags.dryRun, + + TalosClient: c, + ClusterInfo: clusterInfo, + + NewKubernetesCA: newBundle.Certs.K8s, + + EncoderOption: encoderOpt, + + Printf: func(format string, args ...any) { fmt.Printf(format, args...) }, + } + + if err := kubernetes.Rotate(ctx, options); err != nil { return err } @@ -104,9 +167,9 @@ func rotateCA(ctx context.Context, oldClient *client.Client) error { return nil } - fmt.Printf("> Writing new talosconfig to %q\n", rotateCACmdFlags.output) + fmt.Printf("> Kubernetes CA rotation done, new 'kubeconfig' can be fetched with `talosctl kubeconfig`.\n") - return newTalosconfig.Save(rotateCACmdFlags.output) + return nil } func init() { @@ -119,4 +182,6 @@ func init() { rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.withDocs, "with-docs", "", true, "patch all machine configs adding the documentation for each field") rotateCACmd.Flags().StringVarP(&rotateCACmdFlags.output, "output", "o", "talosconfig", "path to the output new `talosconfig`") rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.dryRun, "dry-run", "", true, "dry-run mode (no changes to the cluster)") + rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.rotateTalos, "talos", "", true, "rotate Talos API CA") + rotateCACmd.Flags().BoolVarP(&rotateCACmdFlags.rotateKubernetes, "kubernetes", "", true, "rotate Kubernetes API CA") } diff --git a/cmd/talosctl/cmd/talos/stats.go b/cmd/talosctl/cmd/talos/stats.go index 090cba9b46..4e864fcf95 100644 --- a/cmd/talosctl/cmd/talos/stats.go +++ b/cmd/talosctl/cmd/talos/stats.go @@ -37,7 +37,7 @@ var statsCmd = &cobra.Command{ driver common.ContainerDriver ) - if kubernetes { + if kubernetesFlag { namespace = criconstants.K8sContainerdNamespace driver = common.ContainerDriver_CRI } else { @@ -95,7 +95,7 @@ func statsRender(remotePeer *peer.Peer, resp *machineapi.StatsResponse) error { } func init() { - statsCmd.Flags().BoolVarP(&kubernetes, "kubernetes", "k", false, "use the k8s.io containerd namespace") + statsCmd.Flags().BoolVarP(&kubernetesFlag, "kubernetes", "k", false, "use the k8s.io containerd namespace") statsCmd.Flags().BoolP("use-cri", "c", false, "use the CRI driver") statsCmd.Flags().MarkHidden("use-cri") //nolint:errcheck diff --git a/internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go b/internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go index 60621a918a..71672749e2 100644 --- a/internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go +++ b/internal/app/machined/pkg/controllers/k8s/internal/nodewatch/nodewatch.go @@ -9,7 +9,6 @@ import ( "context" "fmt" - "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -47,7 +46,7 @@ func (r *NodeWatcher) Get() (*corev1.Node, error) { } // Watch starts watching Node state and notifies on updates via notify channel. -func (r *NodeWatcher) Watch(ctx context.Context, logger *zap.Logger) (<-chan struct{}, func(), error) { +func (r *NodeWatcher) Watch(ctx context.Context) (<-chan struct{}, <-chan error, func(), error) { informerFactory := informers.NewSharedInformerFactoryWithOptions( r.client.Clientset, 0, @@ -59,6 +58,7 @@ func (r *NodeWatcher) Watch(ctx context.Context, logger *zap.Logger) (<-chan str ) notifyCh := make(chan struct{}, 1) + watchErrCh := make(chan error, 1) notify := func(_ any) { select { @@ -70,9 +70,12 @@ func (r *NodeWatcher) Watch(ctx context.Context, logger *zap.Logger) (<-chan str r.nodes = informerFactory.Core().V1().Nodes() if err := r.nodes.Informer().SetWatchErrorHandler(func(r *cache.Reflector, err error) { - logger.Error("node watch error", zap.Error(err)) + select { + case watchErrCh <- err: + default: + } }); err != nil { - return nil, nil, fmt.Errorf("failed to set watch error handler: %w", err) + return nil, nil, nil, fmt.Errorf("failed to set watch error handler: %w", err) } if _, err := r.nodes.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -80,12 +83,12 @@ func (r *NodeWatcher) Watch(ctx context.Context, logger *zap.Logger) (<-chan str DeleteFunc: notify, UpdateFunc: func(_, _ any) { notify(nil) }, }); err != nil { - return nil, nil, fmt.Errorf("failed to add event handler: %w", err) + return nil, nil, nil, fmt.Errorf("failed to add event handler: %w", err) } informerFactory.Start(ctx.Done()) informerFactory.WaitForCacheSync(ctx.Done()) - return notifyCh, informerFactory.Shutdown, nil + return notifyCh, watchErrCh, informerFactory.Shutdown, nil } diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_service.go b/internal/app/machined/pkg/controllers/k8s/kubelet_service.go index d3de270431..9fa443024a 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_service.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_service.go @@ -22,7 +22,9 @@ import ( "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/safe" "github.com/cosi-project/runtime/pkg/state" + talosx509 "github.com/siderolabs/crypto/x509" "github.com/siderolabs/gen/optional" + "github.com/siderolabs/gen/xslices" "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -194,12 +196,8 @@ func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runt } } - // refresh certs only if we are managing the node name (not overridden by the user) - if cfgSpec.ExpectedNodename != "" { - err = ctrl.refreshKubeletCerts(cfgSpec.ExpectedNodename, logger) - if err != nil { - return err - } + if err = ctrl.refreshKubeletCerts(cfgSpec.ExpectedNodename, secretSpec.AcceptedCAs, logger); err != nil { + return err } if err = ctrl.handlePolicyChange(cfgSpec, logger); err != nil { @@ -210,7 +208,7 @@ func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runt return err } - if err = ctrl.updateKubeconfig(secretSpec.Endpoint, logger); err != nil { + if err = ctrl.updateKubeconfig(secretSpec.Endpoint, secretSpec.AcceptedCAs, logger); err != nil { return err } @@ -300,6 +298,8 @@ func getFromMap[T any](m map[string]any, key string) (optional.Optional[T], erro } func (ctrl *KubeletServiceController) writePKI(secretSpec *secrets.KubeletSpec) error { + acceptedCAs := bytes.Join(xslices.Map(secretSpec.AcceptedCAs, func(ca *talosx509.PEMEncodedCertificate) []byte { return ca.Crt }), nil) + cfg := struct { Server string CACert string @@ -307,7 +307,7 @@ func (ctrl *KubeletServiceController) writePKI(secretSpec *secrets.KubeletSpec) BootstrapTokenSecret string }{ Server: secretSpec.Endpoint.String(), - CACert: base64.StdEncoding.EncodeToString(secretSpec.CA.Crt), + CACert: base64.StdEncoding.EncodeToString(acceptedCAs), BootstrapTokenID: secretSpec.BootstrapTokenID, BootstrapTokenSecret: secretSpec.BootstrapTokenSecret, } @@ -328,7 +328,7 @@ func (ctrl *KubeletServiceController) writePKI(secretSpec *secrets.KubeletSpec) return err } - return os.WriteFile(constants.KubernetesCACert, secretSpec.CA.Crt, 0o400) + return os.WriteFile(constants.KubernetesCACert, acceptedCAs, 0o400) } var kubeletKubeConfigTemplate = []byte(`apiVersion: v1 @@ -403,7 +403,7 @@ func (ctrl *KubeletServiceController) writeKubeletCredentialProviderConfig(cfgSp } // updateKubeconfig updates the kubeconfig of kubelet with the given endpoint if it exists. -func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, logger *zap.Logger) error { +func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, acceptedCAs []*talosx509.PEMEncodedCertificate, logger *zap.Logger) error { config, err := clientcmd.LoadFromFile(constants.KubeletKubeconfig) if errors.Is(err, os.ErrNotExist) { return nil @@ -430,19 +430,18 @@ func (ctrl *KubeletServiceController) updateKubeconfig(newEndpoint *url.URL, log return nil } - if cluster.Server == newEndpoint.String() { - return nil - } - cluster.Server = newEndpoint.String() + cluster.CertificateAuthorityData = bytes.Join(xslices.Map(acceptedCAs, func(ca *talosx509.PEMEncodedCertificate) []byte { return ca.Crt }), nil) return clientcmd.WriteToFile(*config, constants.KubeletKubeconfig) } -// refreshKubeletCerts checks if the existing kubelet certificates match the node hostname. +// refreshKubeletCerts checks if the existing kubelet certificates match the node hostname and expected CA. // If they don't match, it clears the certificate directory and the removes kubelet's kubeconfig so that // they can be regenerated next time kubelet is started. -func (ctrl *KubeletServiceController) refreshKubeletCerts(nodename string, logger *zap.Logger) error { +// +//nolint:gocyclo +func (ctrl *KubeletServiceController) refreshKubeletCerts(expectedNodename string, acceptedCAs []*talosx509.PEMEncodedCertificate, logger *zap.Logger) error { cert, err := ctrl.readKubeletClientCertificate() if err != nil { return err @@ -452,20 +451,49 @@ func (ctrl *KubeletServiceController) refreshKubeletCerts(nodename string, logge return nil } - expectedCommonName := fmt.Sprintf("system:node:%s", nodename) + valid := true + + // refresh certs only if we are managing the node name (not overridden by the user) + if expectedNodename != "" { + expectedCommonName := fmt.Sprintf("system:node:%s", expectedNodename) + + valid = valid && expectedCommonName == cert.Subject.CommonName + + if !valid { + logger.Info("kubelet client certificate does not match expected nodename, removing", + zap.String("expected", expectedCommonName), + zap.String("actual", cert.Subject.CommonName), + ) + } + } + + // check against CAs + if valid { + rootCAs := x509.NewCertPool() + + for _, ca := range acceptedCAs { + if !rootCAs.AppendCertsFromPEM(ca.Crt) { + return fmt.Errorf("error adding CA to root pool: %w", err) + } + } + + _, verifyErr := cert.Verify(x509.VerifyOptions{ + Roots: rootCAs, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, + }) + + valid = valid && verifyErr == nil - valid := expectedCommonName == cert.Subject.CommonName + if !valid { + logger.Info("kubelet client certificate does not match any accepted CAs, removing", zap.NamedError("verify_error", verifyErr)) + } + } if valid { // certificate looks good, no need to refresh return nil } - logger.Info("kubelet client certificate does not match expected nodename, removing", - zap.String("expected", expectedCommonName), - zap.String("actual", cert.Subject.CommonName), - ) - // remove the pki directory err = os.RemoveAll(constants.KubeletPKIDir) if err != nil { diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go index e75b636fa3..0c79c9d260 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go @@ -159,7 +159,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru nodename.TypedSpec().Nodename, certs.APIServerKubeletClient.Crt, certs.APIServerKubeletClient.Key, - rootSecrets.TypedSpec().CA.Crt, + rootSecrets.TypedSpec().IssuingCA.Crt, ) if err != nil { return fmt.Errorf("error building kubelet client: %w", err) diff --git a/internal/app/machined/pkg/controllers/k8s/node_status.go b/internal/app/machined/pkg/controllers/k8s/node_status.go index 3e3b789738..799c34544f 100644 --- a/internal/app/machined/pkg/controllers/k8s/node_status.go +++ b/internal/app/machined/pkg/controllers/k8s/node_status.go @@ -24,6 +24,9 @@ import ( "github.com/siderolabs/talos/pkg/machinery/resources/k8s" ) +// watchErrorsThreshold is the number of consecutive watch errors before the controller stops watching. +const watchErrorsThreshold = 5 + // NodeStatusController pulls list of Affiliate resource from the Kubernetes registry. type NodeStatusController struct{} @@ -63,7 +66,9 @@ func (ctrl *NodeStatusController) Run(ctx context.Context, r controller.Runtime, nodewatcher *nodewatch.NodeWatcher watchCtxCancel context.CancelFunc notifyCh <-chan struct{} + watchErrCh <-chan error notifyCloser func() + watchErrors int ) closeWatcher := func() { @@ -76,6 +81,8 @@ func (ctrl *NodeStatusController) Run(ctx context.Context, r controller.Runtime, notifyCloser() notifyCloser = nil notifyCh = nil + watchErrCh = nil + watchErrors = 0 } if kubernetesClient != nil { @@ -95,6 +102,20 @@ func (ctrl *NodeStatusController) Run(ctx context.Context, r controller.Runtime, return nil case <-r.EventCh(): case <-notifyCh: + watchErrors = 0 + case watchErr := <-watchErrCh: + logger.Error("node watch error", zap.Error(watchErr), zap.Int("error_count", watchErrors)) + + watchErrors++ + + if watchErrors >= watchErrorsThreshold { + closeWatcher() + + watchErrors = 0 + } else { + // keep waiting + continue + } } nodename, err := safe.ReaderGetByID[*k8s.Nodename](ctx, r, k8s.NodenameID) @@ -137,7 +158,7 @@ func (ctrl *NodeStatusController) Run(ctx context.Context, r controller.Runtime, var watchCtx context.Context watchCtx, watchCtxCancel = context.WithCancel(ctx) //nolint:govet - notifyCh, notifyCloser, err = nodewatcher.Watch(watchCtx, logger) + notifyCh, watchErrCh, notifyCloser, err = nodewatcher.Watch(watchCtx) if err != nil { return fmt.Errorf("error setting up registry watcher: %w", err) //nolint:govet } diff --git a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go index 4f69b3df14..16f3249338 100644 --- a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go +++ b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go @@ -184,7 +184,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control keyFilename: "etcd-client.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.CA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.IssuingCA }, certFilename: "ca.crt", }, { @@ -231,7 +231,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control gid: constants.KubernetesControllerManagerRunGroup, secrets: []secret{ { - getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.CA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.IssuingCA }, certFilename: "ca.crt", keyFilename: "ca.key", }, diff --git a/internal/app/machined/pkg/controllers/secrets/kubelet.go b/internal/app/machined/pkg/controllers/secrets/kubelet.go index fb06e4c100..fbe352013f 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubelet.go +++ b/internal/app/machined/pkg/controllers/secrets/kubelet.go @@ -12,6 +12,7 @@ import ( "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/controller/generic/transform" + "github.com/siderolabs/crypto/x509" "github.com/siderolabs/gen/optional" "go.uber.org/zap" @@ -64,10 +65,16 @@ func NewKubeletController() *KubeletController { kubeletSecrets.Endpoint = cfgProvider.Cluster().Endpoint() } - kubeletSecrets.CA = cfgProvider.Cluster().CA() + kubeletSecrets.AcceptedCAs = nil - if kubeletSecrets.CA == nil { - return errors.New("missing cluster.CA secret") + if cfgProvider.Cluster().IssuingCA() != nil { + kubeletSecrets.AcceptedCAs = append(kubeletSecrets.AcceptedCAs, &x509.PEMEncodedCertificate{Crt: cfgProvider.Cluster().IssuingCA().Crt}) + } + + kubeletSecrets.AcceptedCAs = append(kubeletSecrets.AcceptedCAs, cfgProvider.Cluster().AcceptedCAs()...) + + if len(kubeletSecrets.AcceptedCAs) == 0 { + return errors.New("missing accepted Kubernetes CAs") } kubeletSecrets.BootstrapTokenID = cfgProvider.Cluster().Token().ID() diff --git a/internal/app/machined/pkg/controllers/secrets/kubelet_test.go b/internal/app/machined/pkg/controllers/secrets/kubelet_test.go index 83eb0675a4..939419bad8 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubelet_test.go +++ b/internal/app/machined/pkg/controllers/secrets/kubelet_test.go @@ -89,7 +89,7 @@ func (suite *KubeletSuite) TestReconcile() { spec := kubeletSecrets.TypedSpec() suite.Assert().Equal("https://foo:6443", spec.Endpoint.String()) - suite.Assert().Equal(k8sCA, spec.CA) + suite.Assert().Equal([]*x509.PEMEncodedCertificate{{Crt: k8sCA.Crt}}, spec.AcceptedCAs) suite.Assert().Equal("abc", spec.BootstrapTokenID) suite.Assert().Equal("def", spec.BootstrapTokenSecret) diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes.go b/internal/app/machined/pkg/controllers/secrets/kubernetes.go index 7565839501..1d0e5d4808 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes.go @@ -126,7 +126,8 @@ func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootS if err := kubeconfig.Generate(&kubeconfig.GenerateInput{ ClusterName: k8sRoot.Name, - CA: k8sRoot.CA, + IssuingCA: k8sRoot.IssuingCA, + AcceptedCAs: k8sRoot.AcceptedCAs, CertificateLifetime: KubernetesCertificateValidityDuration, CommonName: constants.KubernetesControllerManagerOrganization, @@ -146,7 +147,8 @@ func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.KubernetesRootS if err := kubeconfig.Generate(&kubeconfig.GenerateInput{ ClusterName: k8sRoot.Name, - CA: k8sRoot.CA, + IssuingCA: k8sRoot.IssuingCA, + AcceptedCAs: k8sRoot.AcceptedCAs, CertificateLifetime: KubernetesCertificateValidityDuration, CommonName: constants.KubernetesSchedulerOrganization, @@ -217,8 +219,12 @@ func (adapter *generateAdminAdapter) Endpoint() *url.URL { return adapter.endpoint } -func (adapter *generateAdminAdapter) CA() *x509.PEMEncodedCertificateAndKey { - return adapter.k8sRoot.CA +func (adapter *generateAdminAdapter) IssuingCA() *x509.PEMEncodedCertificateAndKey { + return adapter.k8sRoot.IssuingCA +} + +func (adapter *generateAdminAdapter) AcceptedCAs() []*x509.PEMEncodedCertificate { + return adapter.k8sRoot.AcceptedCAs } func (adapter *generateAdminAdapter) AdminKubeconfig() config.AdminKubeconfig { diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go index ffa950d6ee..21149bbdf2 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs.go @@ -171,7 +171,7 @@ func (ctrl *KubernetesDynamicCertsController) Run(ctx context.Context, r control func (ctrl *KubernetesDynamicCertsController) updateSecrets(k8sRoot *secrets.KubernetesRootSpec, k8sCerts *secrets.KubernetesDynamicCertsSpec, certSANs *secrets.CertSANSpec, ) error { - ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.CA) + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.IssuingCA) if err != nil { return fmt.Errorf("failed to parse CA certificate: %w", err) } diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go index d81f89804b..edbc0f5cb7 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_dynamic_certs_test.go @@ -67,7 +67,7 @@ func (suite *KubernetesDynamicCertsSuite) TestReconcile() { rootSecrets.TypedSpec().LocalEndpoint, err = url.Parse("https://localhost:6443/") suite.Require().NoError(err) - rootSecrets.TypedSpec().CA = &x509.PEMEncodedCertificateAndKey{ + rootSecrets.TypedSpec().IssuingCA = &x509.PEMEncodedCertificateAndKey{ Crt: k8sCA.CrtPEM, Key: k8sCA.KeyPEM, } diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go b/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go index e51f1b2118..f352351bc3 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes_test.go @@ -64,7 +64,7 @@ func (suite *KubernetesSuite) TestReconcile() { rootSecrets.TypedSpec().LocalEndpoint, err = url.Parse("https://localhost:6443/") suite.Require().NoError(err) - rootSecrets.TypedSpec().CA = &x509.PEMEncodedCertificateAndKey{ + rootSecrets.TypedSpec().IssuingCA = &x509.PEMEncodedCertificateAndKey{ Crt: k8sCA.CrtPEM, Key: k8sCA.KeyPEM, } diff --git a/internal/app/machined/pkg/controllers/secrets/root.go b/internal/app/machined/pkg/controllers/secrets/root.go index a213aaa752..ac7d4c5a11 100644 --- a/internal/app/machined/pkg/controllers/secrets/root.go +++ b/internal/app/machined/pkg/controllers/secrets/root.go @@ -113,9 +113,21 @@ func NewRootKubernetesController() *RootKubernetesController { return errors.New("missing cluster.aggregatorCA secret") } - k8sSecrets.CA = cfgProvider.Cluster().CA() + k8sSecrets.IssuingCA = cfgProvider.Cluster().IssuingCA() + k8sSecrets.AcceptedCAs = cfgProvider.Cluster().AcceptedCAs() - if k8sSecrets.CA == nil { + if k8sSecrets.IssuingCA != nil { + k8sSecrets.AcceptedCAs = append(k8sSecrets.AcceptedCAs, &x509.PEMEncodedCertificate{ + Crt: k8sSecrets.IssuingCA.Crt, + }) + } + + if len(k8sSecrets.IssuingCA.Key) == 0 { + // drop incomplete issuing CA, as the machine config for workers contains just the cert + k8sSecrets.IssuingCA = nil + } + + if len(k8sSecrets.AcceptedCAs) == 0 { return errors.New("missing cluster.CA secret") } diff --git a/internal/app/machined/pkg/controllers/secrets/root_test.go b/internal/app/machined/pkg/controllers/secrets/root_test.go index 7c1448f0fb..62fc0d8ad3 100644 --- a/internal/app/machined/pkg/controllers/secrets/root_test.go +++ b/internal/app/machined/pkg/controllers/secrets/root_test.go @@ -72,7 +72,15 @@ func (suite *RootSuite) TestReconcileControlPlane() { ) rtestutils.AssertResources(suite.Ctx(), suite.T(), suite.State(), []resource.ID{secrets.KubernetesRootID}, func(res *secrets.KubernetesRoot, asrt *assert.Assertions) { - asrt.Equal(res.TypedSpec().CA, cfg.Cluster().CA()) + asrt.Equal(res.TypedSpec().IssuingCA, cfg.Cluster().IssuingCA()) + asrt.Equal( + []*x509.PEMEncodedCertificate{ + { + Crt: cfg.Cluster().IssuingCA().Crt, + }, + }, + res.TypedSpec().AcceptedCAs, + ) }, ) diff --git a/internal/integration/api/generate-config.go b/internal/integration/api/generate-config.go index 6bd6950297..c26b2b0a31 100644 --- a/internal/integration/api/generate-config.go +++ b/internal/integration/api/generate-config.go @@ -179,7 +179,7 @@ func (suite *GenerateConfigSuite) TestGenerate() { config.Cluster().AESCBCEncryptionSecret(), joinedConfig.Cluster().AESCBCEncryptionSecret(), ) - suite.Require().EqualValues(config.Cluster().CA(), joinedConfig.Cluster().CA()) + suite.Require().EqualValues(config.Cluster().IssuingCA(), joinedConfig.Cluster().IssuingCA()) suite.Require().EqualValues(config.Cluster().Token(), joinedConfig.Cluster().Token()) suite.Require().EqualValues(config.Cluster().Etcd().CA(), config.Cluster().Etcd().CA()) } diff --git a/internal/integration/api/rotate.go b/internal/integration/api/rotate.go index e1149336de..d7ed914443 100644 --- a/internal/integration/api/rotate.go +++ b/internal/integration/api/rotate.go @@ -23,6 +23,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/constants" secretsres "github.com/siderolabs/talos/pkg/machinery/resources/secrets" "github.com/siderolabs/talos/pkg/provision/access" + "github.com/siderolabs/talos/pkg/rotate/pki/kubernetes" "github.com/siderolabs/talos/pkg/rotate/pki/talos" ) @@ -117,6 +118,60 @@ func (suite *RotateCASuite) TestTalos() { suite.AssertClusterHealthy(suite.ctx) } +// TestKubernetes updates Kubernetes CA in the cluster. +func (suite *RotateCASuite) TestKubernetes() { + if suite.Cluster == nil { + suite.T().Skip("cluster information is not available") + } + + if testing.Short() { + suite.T().Skip("skipping in short mode") + } + + suite.T().Logf("capturing current Kubernetes CA") + + nodeInternalIP := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane) + + // save k8sRoot + k8sRoot, err := safe.StateGetByID[*secretsres.KubernetesRoot](client.WithNode(suite.ctx, nodeInternalIP), suite.Client.COSI, secretsres.KubernetesRootID) + suite.Require().NoError(err) + + suite.T().Logf("rotating current CA -> new CA") + + newBundle, err := secrets.NewBundle(secrets.NewFixedClock(time.Now()), config.TalosVersionCurrent) + suite.Require().NoError(err) + + options := kubernetes.Options{ + TalosClient: suite.Client, + ClusterInfo: access.NewAdapter(suite.Cluster), + + NewKubernetesCA: newBundle.Certs.K8s, + + EncoderOption: encoder.WithComments(encoder.CommentsAll), + + Printf: suite.T().Logf, + } + + suite.Require().NoError(kubernetes.Rotate(suite.ctx, options)) + + suite.T().Logf("rotating back new CA -> old CA") + + options = kubernetes.Options{ + TalosClient: suite.Client, + ClusterInfo: access.NewAdapter(suite.Cluster), + + NewKubernetesCA: k8sRoot.TypedSpec().IssuingCA, + + EncoderOption: encoder.WithComments(encoder.CommentsAll), + + Printf: suite.T().Logf, + } + + suite.Require().NoError(kubernetes.Rotate(suite.ctx, options)) + + suite.AssertClusterHealthy(suite.ctx) +} + func (suite *RotateCASuite) restartAPIServices(c *client.Client) { suite.T().Logf("restarting API services") diff --git a/pkg/kubeconfig/generate.go b/pkg/kubeconfig/generate.go index fa40eab01f..f04553c5ba 100644 --- a/pkg/kubeconfig/generate.go +++ b/pkg/kubeconfig/generate.go @@ -5,6 +5,7 @@ package kubeconfig import ( + "bytes" stdlibx509 "crypto/x509" "encoding/base64" "fmt" @@ -14,6 +15,7 @@ import ( "time" "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/xslices" "github.com/siderolabs/talos/pkg/machinery/config/config" ) @@ -45,16 +47,24 @@ current-context: {{ .ContextName }}@{{ .ClusterName }} type GenerateAdminInput interface { Name() string Endpoint() *url.URL - CA() *x509.PEMEncodedCertificateAndKey + IssuingCA() *x509.PEMEncodedCertificateAndKey + AcceptedCAs() []*x509.PEMEncodedCertificate AdminKubeconfig() config.AdminKubeconfig } // GenerateAdmin generates admin kubeconfig for the cluster. func GenerateAdmin(config GenerateAdminInput, out io.Writer) error { + acceptedCAs := config.AcceptedCAs() + + if config.IssuingCA() != nil { + acceptedCAs = append(acceptedCAs, &x509.PEMEncodedCertificate{Crt: config.IssuingCA().Crt}) + } + return Generate( &GenerateInput{ ClusterName: config.Name(), - CA: config.CA(), + IssuingCA: config.IssuingCA(), + AcceptedCAs: acceptedCAs, CertificateLifetime: config.AdminKubeconfig().CertLifetime(), CommonName: config.AdminKubeconfig().CommonName(), @@ -72,7 +82,8 @@ func GenerateAdmin(config GenerateAdminInput, out io.Writer) error { type GenerateInput struct { ClusterName string - CA *x509.PEMEncodedCertificateAndKey + IssuingCA *x509.PEMEncodedCertificateAndKey + AcceptedCAs []*x509.PEMEncodedCertificate CertificateLifetime time.Duration CommonName string @@ -94,7 +105,7 @@ func Generate(in *GenerateInput, out io.Writer) error { return fmt.Errorf("error parsing kubeconfig template: %w", err) } - k8sCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(in.CA) + k8sCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(in.IssuingCA) if err != nil { return fmt.Errorf("error getting Kubernetes CA: %w", err) } @@ -115,6 +126,8 @@ func Generate(in *GenerateInput, out io.Writer) error { clientCertPEM := x509.NewCertificateAndKeyFromKeyPair(clientCert) + serverCAs := bytes.Join(xslices.Map(in.AcceptedCAs, func(ca *x509.PEMEncodedCertificate) []byte { return ca.Crt }), nil) + return tpl.Execute(out, struct { GenerateInput @@ -123,7 +136,7 @@ func Generate(in *GenerateInput, out io.Writer) error { ClientKey string }{ GenerateInput: *in, - CACert: string(in.CA.Crt), + CACert: string(serverCAs), ClientCert: string(clientCertPEM.Crt), ClientKey: string(clientCertPEM.Key), }) diff --git a/pkg/kubeconfig/generate_test.go b/pkg/kubeconfig/generate_test.go index bf35151606..d3783c13a5 100644 --- a/pkg/kubeconfig/generate_test.go +++ b/pkg/kubeconfig/generate_test.go @@ -70,7 +70,8 @@ func (suite *GenerateSuite) TestGenerate() { input := kubeconfig.GenerateInput{ ClusterName: "foo", - CA: k8sCA, + IssuingCA: k8sCA, + AcceptedCAs: []*x509.PEMEncodedCertificate{{Crt: k8sCA.Crt}}, CertificateLifetime: time.Hour, CommonName: "system:kube-controller-manager", diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go index acb411c135..0758009641 100644 --- a/pkg/kubernetes/kubernetes.go +++ b/pkg/kubernetes/kubernetes.go @@ -105,7 +105,7 @@ func NewTemporaryClientControlPlane(ctx context.Context, r controller.Reader) (c k8sRootSpec := k8sRoot.TypedSpec() - return NewTemporaryClientFromPKI(k8sRootSpec.CA, k8sRootSpec.LocalEndpoint) + return NewTemporaryClientFromPKI(k8sRootSpec.IssuingCA, k8sRootSpec.LocalEndpoint) } // NewTemporaryClientFromPKI initializes a Kubernetes client using a certificate diff --git a/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go b/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go index bd540b60a5..563234e1d0 100644 --- a/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go +++ b/pkg/machinery/api/resource/definitions/secrets/secrets.pb.go @@ -277,10 +277,10 @@ type KubeletSpec struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Endpoint *common.URL `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"` - Ca *common.PEMEncodedCertificateAndKey `protobuf:"bytes,2,opt,name=ca,proto3" json:"ca,omitempty"` - BootstrapTokenId string `protobuf:"bytes,3,opt,name=bootstrap_token_id,json=bootstrapTokenId,proto3" json:"bootstrap_token_id,omitempty"` - BootstrapTokenSecret string `protobuf:"bytes,4,opt,name=bootstrap_token_secret,json=bootstrapTokenSecret,proto3" json:"bootstrap_token_secret,omitempty"` + Endpoint *common.URL `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + BootstrapTokenId string `protobuf:"bytes,3,opt,name=bootstrap_token_id,json=bootstrapTokenId,proto3" json:"bootstrap_token_id,omitempty"` + BootstrapTokenSecret string `protobuf:"bytes,4,opt,name=bootstrap_token_secret,json=bootstrapTokenSecret,proto3" json:"bootstrap_token_secret,omitempty"` + AcceptedCAs []*common.PEMEncodedCertificate `protobuf:"bytes,5,rep,name=accepted_c_as,json=acceptedCAs,proto3" json:"accepted_c_as,omitempty"` } func (x *KubeletSpec) Reset() { @@ -322,13 +322,6 @@ func (x *KubeletSpec) GetEndpoint() *common.URL { return nil } -func (x *KubeletSpec) GetCa() *common.PEMEncodedCertificateAndKey { - if x != nil { - return x.Ca - } - return nil -} - func (x *KubeletSpec) GetBootstrapTokenId() string { if x != nil { return x.BootstrapTokenId @@ -343,6 +336,13 @@ func (x *KubeletSpec) GetBootstrapTokenSecret() string { return "" } +func (x *KubeletSpec) GetAcceptedCAs() []*common.PEMEncodedCertificate { + if x != nil { + return x.AcceptedCAs + } + return nil +} + // KubernetesCertsSpec describes generated Kubernetes certificates. type KubernetesCertsSpec struct { state protoimpl.MessageState @@ -490,7 +490,7 @@ type KubernetesRootSpec struct { LocalEndpoint *common.URL `protobuf:"bytes,3,opt,name=local_endpoint,json=localEndpoint,proto3" json:"local_endpoint,omitempty"` CertSaNs []string `protobuf:"bytes,4,rep,name=cert_sa_ns,json=certSaNs,proto3" json:"cert_sa_ns,omitempty"` DnsDomain string `protobuf:"bytes,6,opt,name=dns_domain,json=dnsDomain,proto3" json:"dns_domain,omitempty"` - Ca *common.PEMEncodedCertificateAndKey `protobuf:"bytes,7,opt,name=ca,proto3" json:"ca,omitempty"` + IssuingCa *common.PEMEncodedCertificateAndKey `protobuf:"bytes,7,opt,name=issuing_ca,json=issuingCa,proto3" json:"issuing_ca,omitempty"` ServiceAccount *common.PEMEncodedKey `protobuf:"bytes,8,opt,name=service_account,json=serviceAccount,proto3" json:"service_account,omitempty"` AggregatorCa *common.PEMEncodedCertificateAndKey `protobuf:"bytes,9,opt,name=aggregator_ca,json=aggregatorCa,proto3" json:"aggregator_ca,omitempty"` AescbcEncryptionSecret string `protobuf:"bytes,10,opt,name=aescbc_encryption_secret,json=aescbcEncryptionSecret,proto3" json:"aescbc_encryption_secret,omitempty"` @@ -498,6 +498,7 @@ type KubernetesRootSpec struct { BootstrapTokenSecret string `protobuf:"bytes,12,opt,name=bootstrap_token_secret,json=bootstrapTokenSecret,proto3" json:"bootstrap_token_secret,omitempty"` SecretboxEncryptionSecret string `protobuf:"bytes,13,opt,name=secretbox_encryption_secret,json=secretboxEncryptionSecret,proto3" json:"secretbox_encryption_secret,omitempty"` ApiServerIps []*common.NetIP `protobuf:"bytes,14,rep,name=api_server_ips,json=apiServerIps,proto3" json:"api_server_ips,omitempty"` + AcceptedCAs []*common.PEMEncodedCertificate `protobuf:"bytes,15,rep,name=accepted_c_as,json=acceptedCAs,proto3" json:"accepted_c_as,omitempty"` } func (x *KubernetesRootSpec) Reset() { @@ -567,9 +568,9 @@ func (x *KubernetesRootSpec) GetDnsDomain() string { return "" } -func (x *KubernetesRootSpec) GetCa() *common.PEMEncodedCertificateAndKey { +func (x *KubernetesRootSpec) GetIssuingCa() *common.PEMEncodedCertificateAndKey { if x != nil { - return x.Ca + return x.IssuingCa } return nil } @@ -623,6 +624,13 @@ func (x *KubernetesRootSpec) GetApiServerIps() []*common.NetIP { return nil } +func (x *KubernetesRootSpec) GetAcceptedCAs() []*common.PEMEncodedCertificate { + if x != nil { + return x.AcceptedCAs + } + return nil +} + // MaintenanceRootSpec describes maintenance service CA. type MaintenanceRootSpec struct { state protoimpl.MessageState @@ -914,93 +922,99 @@ var file_resource_definitions_secrets_secrets_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x06, 0x65, 0x74, 0x63, 0x64, 0x43, - 0x61, 0x22, 0xcf, 0x01, 0x0a, 0x0b, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x53, 0x70, 0x65, + 0x61, 0x22, 0xdd, 0x01, 0x0a, 0x0b, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x27, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, 0x4c, - 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x33, 0x0a, 0x02, 0x63, 0x61, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, - 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, - 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x02, 0x63, 0x61, 0x12, + 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x62, 0x6f, + 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, + 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x62, 0x6f, 0x6f, 0x74, + 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, + 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, + 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x41, + 0x0a, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x5f, 0x61, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, + 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x43, 0x41, + 0x73, 0x22, 0xf5, 0x01, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, + 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x31, 0x0a, 0x14, 0x73, 0x63, 0x68, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, + 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x1d, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x4d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x3c, 0x0a, 0x1a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x61, 0x64, + 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x41, + 0x64, 0x6d, 0x69, 0x6e, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x29, + 0x0a, 0x10, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x4b, + 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x86, 0x02, 0x0a, 0x1a, 0x4b, 0x75, + 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x43, + 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x42, 0x0a, 0x0a, 0x61, 0x70, 0x69, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, + 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, + 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, + 0x79, 0x52, 0x09, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x5e, 0x0a, 0x19, + 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x6c, + 0x65, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, + 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, + 0x64, 0x4b, 0x65, 0x79, 0x52, 0x16, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, + 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x44, 0x0a, 0x0b, + 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, + 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0a, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x22, 0xe6, 0x05, 0x0a, 0x12, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, + 0x73, 0x52, 0x6f, 0x6f, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x27, 0x0a, + 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x08, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, + 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x0d, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x0a, 0x63, 0x65, + 0x72, 0x74, 0x5f, 0x73, 0x61, 0x5f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, + 0x63, 0x65, 0x72, 0x74, 0x53, 0x61, 0x4e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x6e, 0x73, 0x5f, + 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x6e, + 0x73, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x42, 0x0a, 0x0a, 0x69, 0x73, 0x73, 0x75, 0x69, + 0x6e, 0x67, 0x5f, 0x63, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, + 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, + 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, + 0x52, 0x09, 0x69, 0x73, 0x73, 0x75, 0x69, 0x6e, 0x67, 0x43, 0x61, 0x12, 0x3e, 0x0a, 0x0f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, + 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x48, 0x0a, 0x0d, 0x61, + 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, + 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0c, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, + 0x74, 0x6f, 0x72, 0x43, 0x61, 0x12, 0x38, 0x0a, 0x18, 0x61, 0x65, 0x73, 0x63, 0x62, 0x63, 0x5f, + 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x61, 0x65, 0x73, 0x63, 0x62, 0x63, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, + 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, - 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x62, + 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x22, 0xf5, 0x01, 0x0a, 0x13, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, - 0x65, 0x73, 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x31, 0x0a, 0x14, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x73, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, - 0x0a, 0x1d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1b, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x72, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x3c, 0x0a, 0x1a, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x5f, - 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73, - 0x74, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x29, 0x0a, 0x10, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x61, 0x64, 0x6d, 0x69, - 0x6e, 0x4b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x86, 0x02, 0x0a, 0x1a, - 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, - 0x63, 0x43, 0x65, 0x72, 0x74, 0x73, 0x53, 0x70, 0x65, 0x63, 0x12, 0x42, 0x0a, 0x0a, 0x61, 0x70, - 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, - 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, - 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, - 0x4b, 0x65, 0x79, 0x52, 0x09, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x5e, - 0x0a, 0x19, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6b, 0x75, 0x62, - 0x65, 0x6c, 0x65, 0x74, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, - 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x16, 0x61, 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x4b, 0x75, 0x62, 0x65, 0x6c, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x44, - 0x0a, 0x0b, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, - 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, - 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0a, 0x66, 0x72, 0x6f, 0x6e, 0x74, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x22, 0x94, 0x05, 0x0a, 0x12, 0x4b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, - 0x74, 0x65, 0x73, 0x52, 0x6f, 0x6f, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x27, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x08, - 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x52, 0x4c, 0x52, 0x0d, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x0a, - 0x63, 0x65, 0x72, 0x74, 0x5f, 0x73, 0x61, 0x5f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x53, 0x61, 0x4e, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x6e, - 0x73, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x64, 0x6e, 0x73, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x33, 0x0a, 0x02, 0x63, 0x61, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, - 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x02, 0x63, 0x61, 0x12, 0x3e, - 0x0a, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x48, - 0x0a, 0x0d, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x61, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, - 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x41, 0x6e, 0x64, 0x4b, 0x65, 0x79, 0x52, 0x0c, 0x61, 0x67, 0x67, 0x72, - 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x43, 0x61, 0x12, 0x38, 0x0a, 0x18, 0x61, 0x65, 0x73, 0x63, - 0x62, 0x63, 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, - 0x63, 0x72, 0x65, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x61, 0x65, 0x73, 0x63, - 0x62, 0x63, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, - 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, - 0x12, 0x34, 0x0a, 0x16, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x14, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x3e, 0x0a, 0x1b, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, - 0x62, 0x6f, 0x78, 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x73, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x62, 0x6f, 0x78, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x33, 0x0a, 0x0e, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x70, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, - 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, 0x52, 0x0c, 0x61, - 0x70, 0x69, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x70, 0x73, 0x22, 0x4a, 0x0a, 0x13, 0x4d, + 0x72, 0x65, 0x74, 0x12, 0x3e, 0x0a, 0x1b, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x62, 0x6f, 0x78, + 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x72, + 0x65, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x62, 0x6f, 0x78, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x65, 0x63, + 0x72, 0x65, 0x74, 0x12, 0x33, 0x0a, 0x0e, 0x61, 0x70, 0x69, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x69, 0x70, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x63, 0x6f, + 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x65, 0x74, 0x49, 0x50, 0x52, 0x0c, 0x61, 0x70, 0x69, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x70, 0x73, 0x12, 0x41, 0x0a, 0x0d, 0x61, 0x63, 0x63, 0x65, + 0x70, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x5f, 0x61, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x1d, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, + 0x64, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x0b, + 0x61, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64, 0x43, 0x41, 0x73, 0x22, 0x4a, 0x0a, 0x13, 0x4d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x53, 0x70, 0x65, 0x63, 0x12, 0x33, 0x0a, 0x02, 0x63, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x45, 0x4d, 0x45, 0x6e, 0x63, 0x6f, 0x64, @@ -1091,29 +1105,30 @@ var file_resource_definitions_secrets_secrets_proto_depIdxs = []int32{ 12, // 7: talos.resource.definitions.secrets.EtcdCertsSpec.etcd_api_server:type_name -> common.PEMEncodedCertificateAndKey 12, // 8: talos.resource.definitions.secrets.EtcdRootSpec.etcd_ca:type_name -> common.PEMEncodedCertificateAndKey 15, // 9: talos.resource.definitions.secrets.KubeletSpec.endpoint:type_name -> common.URL - 12, // 10: talos.resource.definitions.secrets.KubeletSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 13, // 10: talos.resource.definitions.secrets.KubeletSpec.accepted_c_as:type_name -> common.PEMEncodedCertificate 12, // 11: talos.resource.definitions.secrets.KubernetesDynamicCertsSpec.api_server:type_name -> common.PEMEncodedCertificateAndKey 12, // 12: talos.resource.definitions.secrets.KubernetesDynamicCertsSpec.api_server_kubelet_client:type_name -> common.PEMEncodedCertificateAndKey 12, // 13: talos.resource.definitions.secrets.KubernetesDynamicCertsSpec.front_proxy:type_name -> common.PEMEncodedCertificateAndKey 15, // 14: talos.resource.definitions.secrets.KubernetesRootSpec.endpoint:type_name -> common.URL 15, // 15: talos.resource.definitions.secrets.KubernetesRootSpec.local_endpoint:type_name -> common.URL - 12, // 16: talos.resource.definitions.secrets.KubernetesRootSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 12, // 16: talos.resource.definitions.secrets.KubernetesRootSpec.issuing_ca:type_name -> common.PEMEncodedCertificateAndKey 16, // 17: talos.resource.definitions.secrets.KubernetesRootSpec.service_account:type_name -> common.PEMEncodedKey 12, // 18: talos.resource.definitions.secrets.KubernetesRootSpec.aggregator_ca:type_name -> common.PEMEncodedCertificateAndKey 14, // 19: talos.resource.definitions.secrets.KubernetesRootSpec.api_server_ips:type_name -> common.NetIP - 12, // 20: talos.resource.definitions.secrets.MaintenanceRootSpec.ca:type_name -> common.PEMEncodedCertificateAndKey - 12, // 21: talos.resource.definitions.secrets.MaintenanceServiceCertsSpec.ca:type_name -> common.PEMEncodedCertificateAndKey - 12, // 22: talos.resource.definitions.secrets.MaintenanceServiceCertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey - 12, // 23: talos.resource.definitions.secrets.OSRootSpec.issuing_ca:type_name -> common.PEMEncodedCertificateAndKey - 14, // 24: talos.resource.definitions.secrets.OSRootSpec.cert_sani_ps:type_name -> common.NetIP - 13, // 25: talos.resource.definitions.secrets.OSRootSpec.accepted_c_as:type_name -> common.PEMEncodedCertificate - 12, // 26: talos.resource.definitions.secrets.TrustdCertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey - 13, // 27: talos.resource.definitions.secrets.TrustdCertsSpec.accepted_c_as:type_name -> common.PEMEncodedCertificate - 28, // [28:28] is the sub-list for method output_type - 28, // [28:28] is the sub-list for method input_type - 28, // [28:28] is the sub-list for extension type_name - 28, // [28:28] is the sub-list for extension extendee - 0, // [0:28] is the sub-list for field type_name + 13, // 20: talos.resource.definitions.secrets.KubernetesRootSpec.accepted_c_as:type_name -> common.PEMEncodedCertificate + 12, // 21: talos.resource.definitions.secrets.MaintenanceRootSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 12, // 22: talos.resource.definitions.secrets.MaintenanceServiceCertsSpec.ca:type_name -> common.PEMEncodedCertificateAndKey + 12, // 23: talos.resource.definitions.secrets.MaintenanceServiceCertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey + 12, // 24: talos.resource.definitions.secrets.OSRootSpec.issuing_ca:type_name -> common.PEMEncodedCertificateAndKey + 14, // 25: talos.resource.definitions.secrets.OSRootSpec.cert_sani_ps:type_name -> common.NetIP + 13, // 26: talos.resource.definitions.secrets.OSRootSpec.accepted_c_as:type_name -> common.PEMEncodedCertificate + 12, // 27: talos.resource.definitions.secrets.TrustdCertsSpec.server:type_name -> common.PEMEncodedCertificateAndKey + 13, // 28: talos.resource.definitions.secrets.TrustdCertsSpec.accepted_c_as:type_name -> common.PEMEncodedCertificate + 29, // [29:29] is the sub-list for method output_type + 29, // [29:29] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name } func init() { file_resource_definitions_secrets_secrets_proto_init() } diff --git a/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go b/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go index 79f6cd20f3..849a7169d0 100644 --- a/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go +++ b/pkg/machinery/api/resource/definitions/secrets/secrets_vtproto.pb.go @@ -402,6 +402,30 @@ func (m *KubeletSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.AcceptedCAs) > 0 { + for iNdEx := len(m.AcceptedCAs) - 1; iNdEx >= 0; iNdEx-- { + if vtmsg, ok := interface{}(m.AcceptedCAs[iNdEx]).(interface { + MarshalToSizedBufferVT([]byte) (int, error) + }); ok { + size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + } else { + encoded, err := proto.Marshal(m.AcceptedCAs[iNdEx]) + if err != nil { + return 0, err + } + i -= len(encoded) + copy(dAtA[i:], encoded) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(encoded))) + } + i-- + dAtA[i] = 0x2a + } + } if len(m.BootstrapTokenSecret) > 0 { i -= len(m.BootstrapTokenSecret) copy(dAtA[i:], m.BootstrapTokenSecret) @@ -416,28 +440,6 @@ func (m *KubeletSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0x1a } - if m.Ca != nil { - if vtmsg, ok := interface{}(m.Ca).(interface { - MarshalToSizedBufferVT([]byte) (int, error) - }); ok { - size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) - } else { - encoded, err := proto.Marshal(m.Ca) - if err != nil { - return 0, err - } - i -= len(encoded) - copy(dAtA[i:], encoded) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(encoded))) - } - i-- - dAtA[i] = 0x12 - } if m.Endpoint != nil { if vtmsg, ok := interface{}(m.Endpoint).(interface { MarshalToSizedBufferVT([]byte) (int, error) @@ -653,6 +655,30 @@ func (m *KubernetesRootSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.AcceptedCAs) > 0 { + for iNdEx := len(m.AcceptedCAs) - 1; iNdEx >= 0; iNdEx-- { + if vtmsg, ok := interface{}(m.AcceptedCAs[iNdEx]).(interface { + MarshalToSizedBufferVT([]byte) (int, error) + }); ok { + size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + } else { + encoded, err := proto.Marshal(m.AcceptedCAs[iNdEx]) + if err != nil { + return 0, err + } + i -= len(encoded) + copy(dAtA[i:], encoded) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(encoded))) + } + i-- + dAtA[i] = 0x7a + } + } if len(m.ApiServerIps) > 0 { for iNdEx := len(m.ApiServerIps) - 1; iNdEx >= 0; iNdEx-- { if vtmsg, ok := interface{}(m.ApiServerIps[iNdEx]).(interface { @@ -749,8 +775,8 @@ func (m *KubernetesRootSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0x42 } - if m.Ca != nil { - if vtmsg, ok := interface{}(m.Ca).(interface { + if m.IssuingCa != nil { + if vtmsg, ok := interface{}(m.IssuingCa).(interface { MarshalToSizedBufferVT([]byte) (int, error) }); ok { size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) @@ -760,7 +786,7 @@ func (m *KubernetesRootSpec) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= size i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) } else { - encoded, err := proto.Marshal(m.Ca) + encoded, err := proto.Marshal(m.IssuingCa) if err != nil { return 0, err } @@ -1331,16 +1357,6 @@ func (m *KubeletSpec) SizeVT() (n int) { } n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.Ca != nil { - if size, ok := interface{}(m.Ca).(interface { - SizeVT() int - }); ok { - l = size.SizeVT() - } else { - l = proto.Size(m.Ca) - } - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } l = len(m.BootstrapTokenId) if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) @@ -1349,6 +1365,18 @@ func (m *KubeletSpec) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if len(m.AcceptedCAs) > 0 { + for _, e := range m.AcceptedCAs { + if size, ok := interface{}(e).(interface { + SizeVT() int + }); ok { + l = size.SizeVT() + } else { + l = proto.Size(e) + } + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } n += len(m.unknownFields) return n } @@ -1459,13 +1487,13 @@ func (m *KubernetesRootSpec) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.Ca != nil { - if size, ok := interface{}(m.Ca).(interface { + if m.IssuingCa != nil { + if size, ok := interface{}(m.IssuingCa).(interface { SizeVT() int }); ok { l = size.SizeVT() } else { - l = proto.Size(m.Ca) + l = proto.Size(m.IssuingCa) } n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } @@ -1517,6 +1545,18 @@ func (m *KubernetesRootSpec) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if len(m.AcceptedCAs) > 0 { + for _, e := range m.AcceptedCAs { + if size, ok := interface{}(e).(interface { + SizeVT() int + }); ok { + l = size.SizeVT() + } else { + l = proto.Size(e) + } + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } n += len(m.unknownFields) return n } @@ -2390,11 +2430,11 @@ func (m *KubeletSpec) UnmarshalVT(dAtA []byte) error { } } iNdEx = postIndex - case 2: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ca", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BootstrapTokenId", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -2404,39 +2444,27 @@ func (m *KubeletSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - if m.Ca == nil { - m.Ca = &common.PEMEncodedCertificateAndKey{} - } - if unmarshal, ok := interface{}(m.Ca).(interface { - UnmarshalVT([]byte) error - }); ok { - if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { - return err - } - } else { - if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.Ca); err != nil { - return err - } - } + m.BootstrapTokenId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 3: + case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field BootstrapTokenId", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BootstrapTokenSecret", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -2464,13 +2492,13 @@ func (m *KubeletSpec) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.BootstrapTokenId = string(dAtA[iNdEx:postIndex]) + m.BootstrapTokenSecret = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field BootstrapTokenSecret", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field AcceptedCAs", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return protohelpers.ErrIntOverflow @@ -2480,23 +2508,33 @@ func (m *KubeletSpec) UnmarshalVT(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return protohelpers.ErrInvalidLength } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { return io.ErrUnexpectedEOF } - m.BootstrapTokenSecret = string(dAtA[iNdEx:postIndex]) + m.AcceptedCAs = append(m.AcceptedCAs, &common.PEMEncodedCertificate{}) + if unmarshal, ok := interface{}(m.AcceptedCAs[len(m.AcceptedCAs)-1]).(interface { + UnmarshalVT([]byte) error + }); ok { + if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.AcceptedCAs[len(m.AcceptedCAs)-1]); err != nil { + return err + } + } iNdEx = postIndex default: iNdEx = preIndex @@ -3097,7 +3135,7 @@ func (m *KubernetesRootSpec) UnmarshalVT(dAtA []byte) error { iNdEx = postIndex case 7: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Ca", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field IssuingCa", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3124,17 +3162,17 @@ func (m *KubernetesRootSpec) UnmarshalVT(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Ca == nil { - m.Ca = &common.PEMEncodedCertificateAndKey{} + if m.IssuingCa == nil { + m.IssuingCa = &common.PEMEncodedCertificateAndKey{} } - if unmarshal, ok := interface{}(m.Ca).(interface { + if unmarshal, ok := interface{}(m.IssuingCa).(interface { UnmarshalVT([]byte) error }); ok { if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { return err } } else { - if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.Ca); err != nil { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.IssuingCa); err != nil { return err } } @@ -3397,6 +3435,48 @@ func (m *KubernetesRootSpec) UnmarshalVT(dAtA []byte) error { } } iNdEx = postIndex + case 15: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AcceptedCAs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AcceptedCAs = append(m.AcceptedCAs, &common.PEMEncodedCertificate{}) + if unmarshal, ok := interface{}(m.AcceptedCAs[len(m.AcceptedCAs)-1]).(interface { + UnmarshalVT([]byte) error + }); ok { + if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.AcceptedCAs[len(m.AcceptedCAs)-1]); err != nil { + return err + } + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/pkg/machinery/config/config/cluster.go b/pkg/machinery/config/config/cluster.go index c54b1d285a..b95ae691ca 100644 --- a/pkg/machinery/config/config/cluster.go +++ b/pkg/machinery/config/config/cluster.go @@ -27,7 +27,8 @@ type ClusterConfig interface { Endpoint() *url.URL Token() Token CertSANs() []string - CA() *x509.PEMEncodedCertificateAndKey + IssuingCA() *x509.PEMEncodedCertificateAndKey + AcceptedCAs() []*x509.PEMEncodedCertificate AggregatorCA() *x509.PEMEncodedCertificateAndKey ServiceAccount() *x509.PEMEncodedKey AESCBCEncryptionSecret() string diff --git a/pkg/machinery/config/generate/secrets/bundle.go b/pkg/machinery/config/generate/secrets/bundle.go index 4ddee426fe..586c4f9eb3 100644 --- a/pkg/machinery/config/generate/secrets/bundle.go +++ b/pkg/machinery/config/generate/secrets/bundle.go @@ -168,7 +168,7 @@ func NewBundleFromKubernetesPKI(pkiDir, bootstrapToken string, versionContract * // NewBundleFromConfig creates secrets bundle using existing config. func NewBundleFromConfig(clock Clock, c config.Config) *Bundle { certs := &Certs{ - K8s: c.Cluster().CA(), + K8s: c.Cluster().IssuingCA(), K8sAggregator: c.Cluster().AggregatorCA(), K8sServiceAccount: c.Cluster().ServiceAccount(), Etcd: c.Cluster().Etcd().CA(), diff --git a/pkg/machinery/config/schemas/config.schema.json b/pkg/machinery/config/schemas/config.schema.json index d9b4d4d8ed..1fe9d658ca 100644 --- a/pkg/machinery/config/schemas/config.schema.json +++ b/pkg/machinery/config/schemas/config.schema.json @@ -847,6 +847,19 @@ "markdownDescription": "The base64 encoded root certificate authority used by Kubernetes.", "x-intellij-html-description": "\u003cp\u003eThe base64 encoded root certificate authority used by Kubernetes.\u003c/p\u003e\n" }, + "acceptedCAs": { + "properties": { + "crt": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "title": "acceptedCAs", + "description": "The list of base64 encoded accepted certificate authorities used by Kubernetes.\n", + "markdownDescription": "The list of base64 encoded accepted certificate authorities used by Kubernetes.", + "x-intellij-html-description": "\u003cp\u003eThe list of base64 encoded accepted certificate authorities used by Kubernetes.\u003c/p\u003e\n" + }, "aggregatorCA": { "properties": { "crt": { diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_clusterconfig.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_clusterconfig.go index 078c2c2a68..9f1ac4c848 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_clusterconfig.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_clusterconfig.go @@ -8,6 +8,7 @@ import ( "fmt" "net/netip" "net/url" + "slices" "strings" "github.com/siderolabs/crypto/x509" @@ -81,11 +82,16 @@ func (c *ClusterConfig) CertSANs() []string { return c.APIServerConfig.CertSANs } -// CA implements the config.ClusterConfig interface. -func (c *ClusterConfig) CA() *x509.PEMEncodedCertificateAndKey { +// IssuingCA implements the config.ClusterConfig interface. +func (c *ClusterConfig) IssuingCA() *x509.PEMEncodedCertificateAndKey { return c.ClusterCA } +// AcceptedCAs implements the config.ClusterConfig interface. +func (c *ClusterConfig) AcceptedCAs() []*x509.PEMEncodedCertificate { + return slices.Clone(c.ClusterAcceptedCAs) +} + // AggregatorCA implements the config.ClusterConfig interface. func (c *ClusterConfig) AggregatorCA() *x509.PEMEncodedCertificateAndKey { return c.ClusterAggregatorCA diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_redact_test.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_redact_test.go index 2eaccf4964..d59b6ba0e2 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_redact_test.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_redact_test.go @@ -45,7 +45,7 @@ func TestRedactSecrets(t *testing.T) { require.Equal(t, "***", config.Cluster().Token().Secret()) require.Equal(t, "", config.Cluster().AESCBCEncryptionSecret()) require.Equal(t, replacement, config.Cluster().SecretboxEncryptionSecret()) - require.Equal(t, replacement, string(config.Cluster().CA().Key)) + require.Equal(t, replacement, string(config.Cluster().IssuingCA().Key)) require.Equal(t, replacement, string(config.Cluster().Etcd().CA().Key)) require.Equal(t, replacement, string(config.Cluster().ServiceAccount().Key)) } diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go index 43e44f37bd..d7a0a0ee91 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go @@ -391,6 +391,15 @@ type ClusterConfig struct { // type: string ClusterCA *x509.PEMEncodedCertificateAndKey `yaml:"ca,omitempty"` // description: | + // The list of base64 encoded accepted certificate authorities used by Kubernetes. + // schema: + // type: object + // additionalProperties: false + // properties: + // crt: + // type: string + ClusterAcceptedCAs []*x509.PEMEncodedCertificate `yaml:"acceptedCAs,omitempty"` + // description: | // The base64 encoded aggregator certificate authority used by Kubernetes for front-proxy certificate generation. // // This CA can be self-signed. diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go index b9cdfe44e3..07bc0e9ff8 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go @@ -406,6 +406,13 @@ func (ClusterConfig) Doc() *encoder.Doc { Description: "The base64 encoded root certificate authority used by Kubernetes.", Comments: [3]string{"" /* encoder.HeadComment */, "The base64 encoded root certificate authority used by Kubernetes." /* encoder.LineComment */, "" /* encoder.FootComment */}, }, + { + Name: "acceptedCAs", + Type: "[]PEMEncodedCertificate", + Note: "", + Description: "The list of base64 encoded accepted certificate authorities used by Kubernetes.", + Comments: [3]string{"" /* encoder.HeadComment */, "The list of base64 encoded accepted certificate authorities used by Kubernetes." /* encoder.LineComment */, "" /* encoder.FootComment */}, + }, { Name: "aggregatorCA", Type: "PEMEncodedCertificateAndKey", @@ -529,27 +536,27 @@ func (ClusterConfig) Doc() *encoder.Doc { doc.Fields[6].AddExample("Decryption secret example (do not use in production!).", "z01mye6j16bspJYtTB/5SFX8j7Ph4JXxM2Xuu4vsBPM=") doc.Fields[7].AddExample("Decryption secret example (do not use in production!).", "z01mye6j16bspJYtTB/5SFX8j7Ph4JXxM2Xuu4vsBPM=") doc.Fields[8].AddExample("ClusterCA example.", pemEncodedCertificateExample()) - doc.Fields[9].AddExample("AggregatorCA example.", pemEncodedCertificateExample()) - doc.Fields[10].AddExample("AggregatorCA example.", pemEncodedKeyExample()) - doc.Fields[11].AddExample("", clusterAPIServerExample()) - doc.Fields[12].AddExample("", clusterControllerManagerExample()) - doc.Fields[13].AddExample("", clusterProxyExample()) - doc.Fields[14].AddExample("", clusterSchedulerExample()) - doc.Fields[15].AddExample("", clusterDiscoveryExample()) - doc.Fields[16].AddExample("", clusterEtcdExample()) - doc.Fields[17].AddExample("", clusterCoreDNSExample()) - doc.Fields[18].AddExample("", clusterExternalCloudProviderConfigExample()) - doc.Fields[19].AddExample("", []string{ + doc.Fields[10].AddExample("AggregatorCA example.", pemEncodedCertificateExample()) + doc.Fields[11].AddExample("AggregatorCA example.", pemEncodedKeyExample()) + doc.Fields[12].AddExample("", clusterAPIServerExample()) + doc.Fields[13].AddExample("", clusterControllerManagerExample()) + doc.Fields[14].AddExample("", clusterProxyExample()) + doc.Fields[15].AddExample("", clusterSchedulerExample()) + doc.Fields[16].AddExample("", clusterDiscoveryExample()) + doc.Fields[17].AddExample("", clusterEtcdExample()) + doc.Fields[18].AddExample("", clusterCoreDNSExample()) + doc.Fields[19].AddExample("", clusterExternalCloudProviderConfigExample()) + doc.Fields[20].AddExample("", []string{ "https://www.example.com/manifest1.yaml", "https://www.example.com/manifest2.yaml", }) - doc.Fields[20].AddExample("", map[string]string{ + doc.Fields[21].AddExample("", map[string]string{ "Token": "1234567", "X-ExtraInfo": "info", }) - doc.Fields[21].AddExample("", clusterInlineManifestsExample()) - doc.Fields[22].AddExample("", clusterAdminKubeconfigExample()) - doc.Fields[24].AddExample("", true) + doc.Fields[22].AddExample("", clusterInlineManifestsExample()) + doc.Fields[23].AddExample("", clusterAdminKubeconfigExample()) + doc.Fields[25].AddExample("", true) return doc } diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go index a49c23ec7c..5b1a6ef0b7 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go @@ -143,7 +143,7 @@ func (c *Config) Validate(mode validation.RuntimeMode, options ...validation.Opt result = multierror.Append(result, errors.New("issuing Talos API CA key is not allowed on non-controlplane nodes (.machine.ca)")) } - if c.Cluster().CA() != nil && len(c.Cluster().CA().Key) > 0 { + if c.Cluster().IssuingCA() != nil && len(c.Cluster().IssuingCA().Key) > 0 { result = multierror.Append(result, errors.New("issuing Kubernetes API CA key is not allowed on non-controlplane nodes (.cluster.ca)")) } case machine.TypeUnknown: diff --git a/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go b/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go index 1c1b35620a..2bcd9c7893 100644 --- a/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go @@ -261,6 +261,16 @@ func (in *ClusterConfig) DeepCopyInto(out *ClusterConfig) { in, out := &in.ClusterCA, &out.ClusterCA *out = (*in).DeepCopy() } + if in.ClusterAcceptedCAs != nil { + in, out := &in.ClusterAcceptedCAs, &out.ClusterAcceptedCAs + *out = make([]*x509.PEMEncodedCertificate, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = (*in).DeepCopy() + } + } + } if in.ClusterAggregatorCA != nil { in, out := &in.ClusterAggregatorCA, &out.ClusterAggregatorCA *out = (*in).DeepCopy() diff --git a/pkg/machinery/resources/secrets/deep_copy.generated.go b/pkg/machinery/resources/secrets/deep_copy.generated.go index a3a1e8729b..1492231dfe 100644 --- a/pkg/machinery/resources/secrets/deep_copy.generated.go +++ b/pkg/machinery/resources/secrets/deep_copy.generated.go @@ -86,8 +86,14 @@ func (o KubeletSpec) DeepCopy() KubeletSpec { *cp.Endpoint.User = *o.Endpoint.User } } - if o.CA != nil { - cp.CA = o.CA.DeepCopy() + if o.AcceptedCAs != nil { + cp.AcceptedCAs = make([]*x509.PEMEncodedCertificate, len(o.AcceptedCAs)) + copy(cp.AcceptedCAs, o.AcceptedCAs) + for i2 := range o.AcceptedCAs { + if o.AcceptedCAs[i2] != nil { + cp.AcceptedCAs[i2] = o.AcceptedCAs[i2].DeepCopy() + } + } } return cp } @@ -140,8 +146,17 @@ func (o KubernetesRootSpec) DeepCopy() KubernetesRootSpec { cp.APIServerIPs = make([]netip.Addr, len(o.APIServerIPs)) copy(cp.APIServerIPs, o.APIServerIPs) } - if o.CA != nil { - cp.CA = o.CA.DeepCopy() + if o.IssuingCA != nil { + cp.IssuingCA = o.IssuingCA.DeepCopy() + } + if o.AcceptedCAs != nil { + cp.AcceptedCAs = make([]*x509.PEMEncodedCertificate, len(o.AcceptedCAs)) + copy(cp.AcceptedCAs, o.AcceptedCAs) + for i2 := range o.AcceptedCAs { + if o.AcceptedCAs[i2] != nil { + cp.AcceptedCAs[i2] = o.AcceptedCAs[i2].DeepCopy() + } + } } if o.ServiceAccount != nil { cp.ServiceAccount = o.ServiceAccount.DeepCopy() diff --git a/pkg/machinery/resources/secrets/kubelet.go b/pkg/machinery/resources/secrets/kubelet.go index c88c5d5387..9a24fd7b49 100644 --- a/pkg/machinery/resources/secrets/kubelet.go +++ b/pkg/machinery/resources/secrets/kubelet.go @@ -31,7 +31,7 @@ type Kubelet = typed.Resource[KubeletSpec, KubeletExtension] type KubeletSpec struct { Endpoint *url.URL `yaml:"endpoint" protobuf:"1"` - CA *x509.PEMEncodedCertificateAndKey `yaml:"ca" protobuf:"2"` + AcceptedCAs []*x509.PEMEncodedCertificate `yaml:"acceptedCAs" protobuf:"5"` BootstrapTokenID string `yaml:"bootstrapTokenID" protobuf:"3"` BootstrapTokenSecret string `yaml:"bootstrapTokenSecret" protobuf:"4"` diff --git a/pkg/machinery/resources/secrets/kubernetes_root.go b/pkg/machinery/resources/secrets/kubernetes_root.go index a50d8ec4b1..405251c828 100644 --- a/pkg/machinery/resources/secrets/kubernetes_root.go +++ b/pkg/machinery/resources/secrets/kubernetes_root.go @@ -37,7 +37,8 @@ type KubernetesRootSpec struct { APIServerIPs []netip.Addr `yaml:"apiServerIPs" protobuf:"14"` DNSDomain string `yaml:"dnsDomain" protobuf:"6"` - CA *x509.PEMEncodedCertificateAndKey `yaml:"ca" protobuf:"7"` + IssuingCA *x509.PEMEncodedCertificateAndKey `yaml:"issuingCA" protobuf:"7"` + AcceptedCAs []*x509.PEMEncodedCertificate `yaml:"acceptedCAs" protobuf:"15"` ServiceAccount *x509.PEMEncodedKey `yaml:"serviceAccount" protobuf:"8"` AggregatorCA *x509.PEMEncodedCertificateAndKey `yaml:"aggregatorCA" protobuf:"9"` diff --git a/pkg/rotate/pki/internal/helpers/helpers.go b/pkg/rotate/pki/internal/helpers/helpers.go new file mode 100644 index 0000000000..b052c3c1bd --- /dev/null +++ b/pkg/rotate/pki/internal/helpers/helpers.go @@ -0,0 +1,161 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package helpers provides helper functions for the rotate/pki package. +package helpers + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-retry/retry" + "google.golang.org/grpc/codes" + + "github.com/siderolabs/talos/pkg/cluster" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + configres "github.com/siderolabs/talos/pkg/machinery/resources/config" + v1alpha1res "github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1" +) + +// MapToInternalIP maps a slice of NodeInfo to a slice of internal IPs. +func MapToInternalIP(in []cluster.NodeInfo) []string { + return xslices.Map(in, func(i cluster.NodeInfo) string { + return i.InternalIP.String() + }) +} + +// PatchNodeConfig patches the node config for the given node. +func PatchNodeConfig(ctx context.Context, c *client.Client, node string, encoderOpt encoder.Option, patchFunc func(config *v1alpha1.Config) error) error { + return retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond), retry.WithErrorLogging(true)).RetryWithContext( + ctx, + func(ctx context.Context) error { + err := patchNodeConfigInternal(ctx, c, node, encoderOpt, patchFunc) + if err != nil { + if client.StatusCode(err) == codes.Unavailable || client.StatusCode(err) == codes.Canceled { + return retry.ExpectedError(err) + } + } + + return err + }, + ) +} + +// PatchNodeConfigWithKubeletRestart patches the node config for the given node waiting for the kubelet to be restarted. +// +//nolint:gocyclo,cyclop +func PatchNodeConfigWithKubeletRestart(ctx context.Context, c *client.Client, node string, encoderOpt encoder.Option, patchFunc func(config *v1alpha1.Config) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + ctx = client.WithNode(ctx, node) + + watchCh := make(chan safe.WrappedStateEvent[*v1alpha1res.Service]) + + if err := safe.StateWatch(ctx, c.COSI, resource.NewMetadata(v1alpha1res.NamespaceName, v1alpha1res.ServiceType, "kubelet", resource.VersionUndefined), watchCh); err != nil { + return fmt.Errorf("error watching service: %w", err) + } + + var ev safe.WrappedStateEvent[*v1alpha1res.Service] + + select { + case ev = <-watchCh: + case <-ctx.Done(): + return ctx.Err() + } + + if ev.Type() != state.Created { + return fmt.Errorf("unexpected event type: %s", ev.Type()) + } + + initialService, err := ev.Resource() + if err != nil { + return fmt.Errorf("error inspecting service: %w", err) + } + + if !initialService.TypedSpec().Running || !initialService.TypedSpec().Healthy { + return errors.New("kubelet is not healthy") + } + + if err = PatchNodeConfig(ctx, c, node, encoderOpt, patchFunc); err != nil { + return fmt.Errorf("error patching node config: %w", err) + } + + // first, wait for kubelet to go down + for { + select { + case ev = <-watchCh: + case <-ctx.Done(): + return ctx.Err() + } + + if ev.Type() == state.Destroyed { + break + } + } + + // now wait for kubelet to go up & healthy + for { + select { + case ev = <-watchCh: + case <-ctx.Done(): + return ctx.Err() + } + + if ev.Type() == state.Created || ev.Type() == state.Updated { + var service *v1alpha1res.Service + + service, err = ev.Resource() + if err != nil { + return fmt.Errorf("error inspecting service: %w", err) + } + + if service.TypedSpec().Running && service.TypedSpec().Healthy { + break + } + } + } + + return nil +} + +func patchNodeConfigInternal(ctx context.Context, c *client.Client, node string, encoderOpt encoder.Option, patchFunc func(config *v1alpha1.Config) error) error { + ctx = client.WithNode(ctx, node) + + mc, err := safe.StateGetByID[*configres.MachineConfig](ctx, c.COSI, configres.V1Alpha1ID) + if err != nil { + return fmt.Errorf("error fetching config resource: %w", err) + } + + provider := mc.Provider() + + newProvider, err := provider.PatchV1Alpha1(patchFunc) + if err != nil { + return fmt.Errorf("error patching config: %w", err) + } + + cfgBytes, err := newProvider.EncodeBytes(encoderOpt) + if err != nil { + return fmt.Errorf("error serializing config: %w", err) + } + + _, err = c.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{ + Data: cfgBytes, + Mode: machineapi.ApplyConfigurationRequest_NO_REBOOT, + }) + if err != nil { + return fmt.Errorf("error applying config: %w", err) + } + + return nil +} diff --git a/pkg/rotate/pki/kubernetes/kubernetes.go b/pkg/rotate/pki/kubernetes/kubernetes.go new file mode 100644 index 0000000000..23baf661ed --- /dev/null +++ b/pkg/rotate/pki/kubernetes/kubernetes.go @@ -0,0 +1,362 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package kubernetes implements safe Talos API PKI rotation for the cluster. +package kubernetes + +import ( + "bufio" + "bytes" + "context" + "fmt" + "slices" + "time" + + "github.com/cosi-project/runtime/pkg/safe" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/go-retry/retry" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/siderolabs/talos/pkg/cluster" + taloskubernetes "github.com/siderolabs/talos/pkg/kubernetes" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + secretsres "github.com/siderolabs/talos/pkg/machinery/resources/secrets" + "github.com/siderolabs/talos/pkg/rotate/pki/internal/helpers" +) + +// Options is the input to the Kubernetes API rotation process. +type Options struct { + // DryRun is the flag to enable dry-run mode. + // + // In dry-run mode, the rotation process will not make any changes to the cluster. + DryRun bool + + // TalosClient is a Talos API client + TalosClient *client.Client + // ClusterInfo provides information about cluster topology. + ClusterInfo cluster.Info + + // NewKubernetesCA is the new CA for Kubernetes API. + NewKubernetesCA *x509.PEMEncodedCertificateAndKey + + // EncoderOption is the option for encoding machine configuration (while patching). + EncoderOption encoder.Option + + // Printf is the function used to print messages. + Printf func(format string, args ...any) +} + +type rotator struct { + opts Options + + currentCA []byte + + talosClientProvider *cluster.ConfigClientProvider + currentKubernetes *cluster.KubernetesClient + newKubernetes *cluster.KubernetesClient +} + +// Rotate rotates the Kubernetes API PKI. +// +// The process overview: +// - fetch current information +// - verify connectivity with the existing PKI +// - add new Kubernetes CA as accepted +// - verify connectivity +// - make new CA issuing, old CA is still accepted +// - verify connectivity with the new PKI +// - remove old CA +// - verify connectivity with the new PKI. +func Rotate(ctx context.Context, opts Options) error { + r := rotator{ + opts: opts, + } + + defer func() { + if r.currentKubernetes != nil { + r.currentKubernetes.K8sClose() //nolint:errcheck + } + + if r.newKubernetes != nil { + r.newKubernetes.K8sClose() //nolint:errcheck + } + }() + + r.talosClientProvider = &cluster.ConfigClientProvider{ + DefaultClient: opts.TalosClient, + } + + return r.rotate(ctx) +} + +//nolint:gocyclo +func (r *rotator) rotate(ctx context.Context) error { + r.printIntro() + + if err := r.fetchClient(ctx, &r.currentKubernetes, "current"); err != nil { + return err + } + + if err := r.fetchCurrentCA(ctx); err != nil { + return err + } + + if err := r.printNewCA(); err != nil { + return err + } + + if err := r.verifyConnectivity(ctx, r.currentKubernetes, "existing PKI"); err != nil { + return err + } + + if err := r.addNewCAAccepted(ctx); err != nil { + return err + } + + if err := r.swapCAs(ctx); err != nil { + return err + } + + if err := r.fetchClient(ctx, &r.newKubernetes, "new"); err != nil { + return err + } + + if err := r.verifyConnectivity(ctx, r.newKubernetes, "new PKI"); err != nil { + return err + } + + if err := r.dropOldCA(ctx); err != nil { + return err + } + + if err := r.verifyConnectivity(ctx, r.newKubernetes, "new PKI"); err != nil { + return err + } + + return nil +} + +func (r *rotator) printIntro() { + r.opts.Printf("> Starting Kubernetes API PKI rotation, dry-run mode %v...\n", r.opts.DryRun) + + r.opts.Printf("> Cluster topology:\n") + + r.opts.Printf(" - control plane nodes: %q\n", + append( + helpers.MapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeInit)), + helpers.MapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeControlPlane))..., + ), + ) + r.opts.Printf(" - worker nodes: %q\n", + helpers.MapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeWorker)), + ) +} + +func (r *rotator) fetchClient(ctx context.Context, clientPtr **cluster.KubernetesClient, label string) error { + r.opts.Printf("> Building %s Kubernetes client...\n", label) + + firstNode := append( + r.opts.ClusterInfo.NodesByType(machine.TypeInit), + r.opts.ClusterInfo.NodesByType(machine.TypeControlPlane)..., + )[0] + + *clientPtr = &cluster.KubernetesClient{ + ClientProvider: r.talosClientProvider, + } + + _, err := (*clientPtr).K8sClient(client.WithNode(ctx, firstNode.InternalIP.String())) + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + return nil +} + +func (r *rotator) fetchCurrentCA(ctx context.Context) error { + r.opts.Printf("> Current Kubernetes CA:\n") + + firstNode := append( + r.opts.ClusterInfo.NodesByType(machine.TypeInit), + r.opts.ClusterInfo.NodesByType(machine.TypeControlPlane)..., + )[0] + + k8sRoot, err := safe.StateGetByID[*secretsres.KubernetesRoot](client.WithNode(ctx, firstNode.InternalIP.String()), r.opts.TalosClient.COSI, secretsres.KubernetesRootID) + if err != nil { + return fmt.Errorf("error fetching current Kubernetes CA: %w", err) + } + + r.currentCA = k8sRoot.TypedSpec().IssuingCA.Crt + + var b bytes.Buffer + + if err = yaml.NewEncoder(&b).Encode(k8sRoot.TypedSpec().IssuingCA); err != nil { + return fmt.Errorf("error encoding current Kubernetes CA: %w", err) + } + + for scanner := bufio.NewScanner(&b); scanner.Scan(); { + r.opts.Printf(" %s\n", scanner.Text()) + } + + return nil +} + +func (r *rotator) printNewCA() error { + r.opts.Printf("> New Kubernetes CA:\n") + + var b bytes.Buffer + + if err := yaml.NewEncoder(&b).Encode(r.opts.NewKubernetesCA); err != nil { + return fmt.Errorf("error encoding new Talos CA: %w", err) + } + + for scanner := bufio.NewScanner(&b); scanner.Scan(); { + r.opts.Printf(" %s\n", scanner.Text()) + } + + return nil +} + +func (r *rotator) verifyConnectivity(ctx context.Context, client *cluster.KubernetesClient, label string) error { + r.opts.Printf("> Verifying connectivity with %s...\n", label) + + if r.opts.DryRun { + r.opts.Printf(" - OK (dry-run mode)\n") + + return nil + } + + clientset, err := client.K8sClient(ctx) + if err != nil { + return fmt.Errorf("error building Kubernetes client: %w", err) + } + + return retry.Constant(3*time.Minute, retry.WithUnits(time.Second), retry.WithErrorLogging(true)).RetryWithContext(ctx, + func(ctx context.Context) error { + nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + if taloskubernetes.IsRetryableError(err) { + return retry.ExpectedError(err) + } + + return err + } + + var notReadyNodes []string + + for _, node := range nodes.Items { + for _, cond := range node.Status.Conditions { + if cond.Type == v1.NodeReady { + if cond.Status != v1.ConditionTrue { + notReadyNodes = append(notReadyNodes, node.Name) + + break + } + } + } + } + + if len(notReadyNodes) > 0 { + return retry.ExpectedErrorf("nodes not ready: %q", notReadyNodes) + } + + r.opts.Printf(" - OK (%d nodes ready)\n", len(nodes.Items)) + + return nil + }) +} + +func (r *rotator) addNewCAAccepted(ctx context.Context) error { + r.opts.Printf("> Adding new Kubernetes CA as accepted...\n") + + if err := r.patchAllNodes(ctx, + func(_ machine.Type, config *v1alpha1.Config) error { + config.ClusterConfig.ClusterAcceptedCAs = append( + config.ClusterConfig.ClusterAcceptedCAs, + &x509.PEMEncodedCertificate{ + Crt: r.opts.NewKubernetesCA.Crt, + }, + ) + + return nil + }); err != nil { + return fmt.Errorf("error patching all machine configs: %w", err) + } + + return nil +} + +func (r *rotator) swapCAs(ctx context.Context) error { + r.opts.Printf("> Making new Kubernetes CA the issuing CA, old Kubernetes CA the accepted CA...\n") + + if err := r.patchAllNodes(ctx, + func(machineType machine.Type, config *v1alpha1.Config) error { + config.ClusterConfig.ClusterAcceptedCAs = append( + config.ClusterConfig.ClusterAcceptedCAs, + &x509.PEMEncodedCertificate{ + Crt: r.currentCA, + }, + ) + config.ClusterConfig.ClusterAcceptedCAs = slices.DeleteFunc(config.Cluster().AcceptedCAs(), func(ca *x509.PEMEncodedCertificate) bool { + return bytes.Equal(ca.Crt, r.opts.NewKubernetesCA.Crt) + }) + + if machineType.IsControlPlane() { + config.ClusterConfig.ClusterCA = r.opts.NewKubernetesCA + } else { + config.ClusterConfig.ClusterCA = &x509.PEMEncodedCertificateAndKey{ + Crt: r.opts.NewKubernetesCA.Crt, + } + } + + return nil + }); err != nil { + return fmt.Errorf("error patching all machine configs: %w", err) + } + + return nil +} + +func (r *rotator) dropOldCA(ctx context.Context) error { + r.opts.Printf("> Removing old Kubernetes CA from the accepted CAs...\n") + + if err := r.patchAllNodes(ctx, + func(_ machine.Type, config *v1alpha1.Config) error { + config.ClusterConfig.ClusterAcceptedCAs = slices.DeleteFunc(config.Cluster().AcceptedCAs(), func(ca *x509.PEMEncodedCertificate) bool { + return bytes.Equal(ca.Crt, r.currentCA) + }) + + return nil + }); err != nil { + return fmt.Errorf("error patching all machine configs: %w", err) + } + + return nil +} + +func (r *rotator) patchAllNodes(ctx context.Context, patchFunc func(machineType machine.Type, config *v1alpha1.Config) error) error { + for _, machineType := range []machine.Type{machine.TypeInit, machine.TypeControlPlane, machine.TypeWorker} { + for _, node := range r.opts.ClusterInfo.NodesByType(machineType) { + if r.opts.DryRun { + r.opts.Printf(" - %s: skipped (dry-run)\n", node.InternalIP) + + continue + } + + if err := helpers.PatchNodeConfigWithKubeletRestart(ctx, r.opts.TalosClient, node.InternalIP.String(), r.opts.EncoderOption, func(config *v1alpha1.Config) error { + return patchFunc(machineType, config) + }); err != nil { + return fmt.Errorf("error patching node %s: %w", node.InternalIP, err) + } + + r.opts.Printf(" - %s: OK\n", node.InternalIP) + } + } + + return nil +} diff --git a/pkg/rotate/pki/talos/helpers.go b/pkg/rotate/pki/talos/helpers.go deleted file mode 100644 index 533ed166df..0000000000 --- a/pkg/rotate/pki/talos/helpers.go +++ /dev/null @@ -1,76 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package talos - -import ( - "context" - "fmt" - "time" - - "github.com/cosi-project/runtime/pkg/safe" - "github.com/siderolabs/gen/xslices" - "github.com/siderolabs/go-retry/retry" - "google.golang.org/grpc/codes" - - "github.com/siderolabs/talos/pkg/cluster" - machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" - "github.com/siderolabs/talos/pkg/machinery/client" - "github.com/siderolabs/talos/pkg/machinery/config/encoder" - "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" - configres "github.com/siderolabs/talos/pkg/machinery/resources/config" -) - -func mapToInternalIP(in []cluster.NodeInfo) []string { - return xslices.Map(in, func(i cluster.NodeInfo) string { - return i.InternalIP.String() - }) -} - -func patchNodeConfig(ctx context.Context, c *client.Client, node string, encoderOpt encoder.Option, patchFunc func(config *v1alpha1.Config) error) error { - return retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond), retry.WithErrorLogging(true)).RetryWithContext( - ctx, - func(ctx context.Context) error { - err := patchNodeConfigInternal(ctx, c, node, encoderOpt, patchFunc) - if err != nil { - if client.StatusCode(err) == codes.Unavailable || client.StatusCode(err) == codes.Canceled { - return retry.ExpectedError(err) - } - } - - return err - }, - ) -} - -func patchNodeConfigInternal(ctx context.Context, c *client.Client, node string, encoderOpt encoder.Option, patchFunc func(config *v1alpha1.Config) error) error { - ctx = client.WithNode(ctx, node) - - mc, err := safe.StateGetByID[*configres.MachineConfig](ctx, c.COSI, configres.V1Alpha1ID) - if err != nil { - return fmt.Errorf("error fetching config resource: %w", err) - } - - provider := mc.Provider() - - newProvider, err := provider.PatchV1Alpha1(patchFunc) - if err != nil { - return fmt.Errorf("error patching config: %w", err) - } - - cfgBytes, err := newProvider.EncodeBytes(encoderOpt) - if err != nil { - return fmt.Errorf("error serializing config: %w", err) - } - - _, err = c.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{ - Data: cfgBytes, - Mode: machineapi.ApplyConfigurationRequest_NO_REBOOT, - }) - if err != nil { - return fmt.Errorf("error applying config: %w", err) - } - - return nil -} diff --git a/pkg/rotate/pki/talos/talos.go b/pkg/rotate/pki/talos/talos.go index ede62477fc..bce9e82550 100644 --- a/pkg/rotate/pki/talos/talos.go +++ b/pkg/rotate/pki/talos/talos.go @@ -29,6 +29,7 @@ import ( "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" secretsres "github.com/siderolabs/talos/pkg/machinery/resources/secrets" "github.com/siderolabs/talos/pkg/machinery/role" + "github.com/siderolabs/talos/pkg/rotate/pki/internal/helpers" ) // Options is the input to the Talos API rotation process. @@ -157,12 +158,12 @@ func (r *rotator) printIntro() { r.opts.Printf(" - control plane nodes: %q\n", append( - mapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeInit)), - mapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeControlPlane))..., + helpers.MapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeInit)), + helpers.MapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeControlPlane))..., ), ) r.opts.Printf(" - worker nodes: %q\n", - mapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeWorker)), + helpers.MapToInternalIP(r.opts.ClusterInfo.NodesByType(machine.TypeWorker)), ) } @@ -369,7 +370,7 @@ func (r *rotator) patchAllNodes(ctx context.Context, c *client.Client, patchFunc continue } - if err := patchNodeConfig(ctx, c, node.InternalIP.String(), r.opts.EncoderOption, func(config *v1alpha1.Config) error { + if err := helpers.PatchNodeConfig(ctx, c, node.InternalIP.String(), r.opts.EncoderOption, func(config *v1alpha1.Config) error { return patchFunc(machineType, config) }); err != nil { return fmt.Errorf("error patching node %s: %w", node.InternalIP, err) diff --git a/website/content/v1.7/reference/api.md b/website/content/v1.7/reference/api.md index 9253aea59e..fd076315ba 100644 --- a/website/content/v1.7/reference/api.md +++ b/website/content/v1.7/reference/api.md @@ -4331,9 +4331,9 @@ KubeletSpec describes root Kubernetes secrets. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | | endpoint | [common.URL](#common.URL) | | | -| ca | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | | bootstrap_token_id | [string](#string) | | | | bootstrap_token_secret | [string](#string) | | | +| accepted_c_as | [common.PEMEncodedCertificate](#common.PEMEncodedCertificate) | repeated | | @@ -4388,7 +4388,7 @@ KubernetesRootSpec describes root Kubernetes secrets. | local_endpoint | [common.URL](#common.URL) | | | | cert_sa_ns | [string](#string) | repeated | | | dns_domain | [string](#string) | | | -| ca | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | +| issuing_ca | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | | service_account | [common.PEMEncodedKey](#common.PEMEncodedKey) | | | | aggregator_ca | [common.PEMEncodedCertificateAndKey](#common.PEMEncodedCertificateAndKey) | | | | aescbc_encryption_secret | [string](#string) | | | @@ -4396,6 +4396,7 @@ KubernetesRootSpec describes root Kubernetes secrets. | bootstrap_token_secret | [string](#string) | | | | secretbox_encryption_secret | [string](#string) | | | | api_server_ips | [common.NetIP](#common.NetIP) | repeated | | +| accepted_c_as | [common.PEMEncodedCertificate](#common.PEMEncodedCertificate) | repeated | | diff --git a/website/content/v1.7/reference/cli.md b/website/content/v1.7/reference/cli.md index 4a11865e21..d63cd8df38 100644 --- a/website/content/v1.7/reference/cli.md +++ b/website/content/v1.7/reference/cli.md @@ -2666,8 +2666,13 @@ Rotate cluster CAs (Talos and Kubernetes APIs). ### Synopsis +The command can rotate both Talos and Kubernetes root CAs (for the API). +By default both CAs are rotated, but you can choose to rotate just one or another. The command starts by generating new CAs, and gracefully applying it to the cluster. +For Kubernetes, the command only rotates the API server issuing CA, and other Kubernetes +PKI can be rotated by applying machine config changes to the controlplane nodes. + ``` talosctl rotate-ca [flags] ``` @@ -2680,7 +2685,9 @@ talosctl rotate-ca [flags] -h, --help help for rotate-ca --init-node string specify IPs of init node --k8s-endpoint string use endpoint instead of kubeconfig default + --kubernetes rotate Kubernetes API CA (default true) -o, --output talosconfig path to the output new talosconfig (default "talosconfig") + --talos rotate Talos API CA (default true) --with-docs patch all machine configs adding the documentation for each field (default true) --with-examples patch all machine configs with the commented examples (default true) --worker-nodes strings specify IPs of worker nodes diff --git a/website/content/v1.7/reference/configuration/v1alpha1/config.md b/website/content/v1.7/reference/configuration/v1alpha1/config.md index 52fc033c6c..fa295ff160 100644 --- a/website/content/v1.7/reference/configuration/v1alpha1/config.md +++ b/website/content/v1.7/reference/configuration/v1alpha1/config.md @@ -2852,6 +2852,7 @@ ca: crt: LS0tIEVYQU1QTEUgQ0VSVElGSUNBVEUgLS0t key: LS0tIEVYQU1QTEUgS0VZIC0tLQ== {{< /highlight >}} | | +|`acceptedCAs` |[]PEMEncodedCertificate |The list of base64 encoded accepted certificate authorities used by Kubernetes. | | |`aggregatorCA` |PEMEncodedCertificateAndKey |
The base64 encoded aggregator certificate authority used by Kubernetes for front-proxy certificate generation.
This CA can be self-signed.
Show example(s){{< highlight yaml >}} aggregatorCA: crt: LS0tIEVYQU1QTEUgQ0VSVElGSUNBVEUgLS0t diff --git a/website/content/v1.7/schemas/config.schema.json b/website/content/v1.7/schemas/config.schema.json index d9b4d4d8ed..1fe9d658ca 100644 --- a/website/content/v1.7/schemas/config.schema.json +++ b/website/content/v1.7/schemas/config.schema.json @@ -847,6 +847,19 @@ "markdownDescription": "The base64 encoded root certificate authority used by Kubernetes.", "x-intellij-html-description": "\u003cp\u003eThe base64 encoded root certificate authority used by Kubernetes.\u003c/p\u003e\n" }, + "acceptedCAs": { + "properties": { + "crt": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object", + "title": "acceptedCAs", + "description": "The list of base64 encoded accepted certificate authorities used by Kubernetes.\n", + "markdownDescription": "The list of base64 encoded accepted certificate authorities used by Kubernetes.", + "x-intellij-html-description": "\u003cp\u003eThe list of base64 encoded accepted certificate authorities used by Kubernetes.\u003c/p\u003e\n" + }, "aggregatorCA": { "properties": { "crt": {