feat: add a tool and package to convert self-hosted CP to static pods

This is required to upgrade from Talos 0.8.x to 0.9.x. After the cluster
is fully upgraded, control plane is still self-hosted (as it was
bootstrapped with bootkube).

Tool `talosctl convert-k8s` (and library behind it) performs the upgrade
to self-hosted version.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

.drone.jsonnet

@@ -354,9 +354,9 @@ local integration_provision_tests_prepare = Step("provision-tests-prepare", priv
 local integration_provision_tests_track_0 = Step("provision-tests-track-0", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"REGISTRY": local_registry});
 local integration_provision_tests_track_1 = Step("provision-tests-track-1", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"REGISTRY": local_registry});
 
-local integration_cilium = Step("e2e-cilium-1.8.5", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
+local integration_cilium = Step("e2e-cilium-1.9.4", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
         "SHORT_INTEGRATION_TEST": "yes",
-        "CUSTOM_CNI_URL": "https://raw.githubusercontent.com/cilium/cilium/v1.8.5/install/kubernetes/quick-install.yaml",
+        "CUSTOM_CNI_URL": "https://raw.githubusercontent.com/cilium/cilium/v1.9.4/install/kubernetes/quick-install.yaml",
         "REGISTRY": local_registry,
         "CLUSTER_CIDR": 2,
 });

Makefile

@@ -24,7 +24,7 @@ CLUSTERCTL_URL ?= https://github.com/kubernetes-sigs/cluster-api/releases/downlo
 SONOBUOY_VERSION ?= 0.19.0
 SONOBUOY_URL ?= https://github.com/heptio/sonobuoy/releases/download/v$(SONOBUOY_VERSION)/sonobuoy_$(SONOBUOY_VERSION)_$(OPERATING_SYSTEM)_amd64.tar.gz
 TESTPKGS ?= github.com/talos-systems/talos/...
-RELEASES ?= v0.7.1 v0.8.0
+RELEASES ?= v0.7.1 v0.8.3
 SHORT_INTEGRATION_TEST ?=
 CUSTOM_CNI_URL ?=
 

api/machine/machine.proto

@@ -47,6 +47,8 @@ service MachineService {
   rpc Rollback(RollbackRequest) returns (RollbackResponse);
   rpc Reset(ResetRequest) returns (ResetResponse);
   rpc Recover(RecoverRequest) returns (RecoverResponse);
+  rpc RemoveBootkubeInitializedKey(google.protobuf.Empty)
+      returns (RemoveBootkubeInitializedKeyResponse);
   rpc ServiceList(google.protobuf.Empty) returns (ServiceListResponse);
   rpc ServiceRestart(ServiceRestartRequest) returns (ServiceRestartResponse);
   rpc ServiceStart(ServiceStartRequest) returns (ServiceStartResponse);
@@ -64,6 +66,7 @@ service MachineService {
 message ApplyConfigurationRequest {
   bytes data = 1;
   bool on_reboot = 2;
+  bool immediate = 3;
 }
 
 // ApplyConfigurationResponse describes the response to a configuration request.
@@ -836,3 +839,7 @@ message GenerateConfiguration {
 message GenerateConfigurationResponse {
   repeated GenerateConfiguration messages = 1;
 }
+
+// RemoveBootkubeInitializedKeyResponse describes the response to a RemoveBootkubeInitializedKey request.
+message RemoveBootkubeInitializedKey { common.Metadata metadata = 1; }
+message RemoveBootkubeInitializedKeyResponse { repeated RemoveBootkubeInitializedKey messages = 1; }

cmd/talosctl/cmd/talos/apply-config.go

@@ -24,6 +24,7 @@ var applyConfigCmdFlags struct {
 	insecure         bool
 	interactive      bool
 	onReboot         bool
+	immediate        bool
 }
 
 // applyConfigCmd represents the applyConfiguration command.
@@ -134,8 +135,9 @@ var applyConfigCmd = &cobra.Command{
 			}
 
 			if _, err := c.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{
-				Data:     cfgBytes,
-				OnReboot: applyConfigCmdFlags.onReboot,
+				Data:      cfgBytes,
+				OnReboot:  applyConfigCmdFlags.onReboot,
+				Immediate: applyConfigCmdFlags.immediate,
 			}); err != nil {
 				return fmt.Errorf("error applying new configuration: %s", err)
 			}
@@ -151,6 +153,7 @@ func init() {
 	applyConfigCmd.Flags().StringSliceVar(&applyConfigCmdFlags.certFingerprints, "cert-fingerprint", nil, "list of server certificate fingeprints to accept (defaults to no check)")
 	applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.interactive, "interactive", false, "apply the config using text based interactive mode")
 	applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.onReboot, "on-reboot", false, "apply the config on reboot")
+	applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.immediate, "immediate", false, "apply the config immediately (without a reboot)")
 
 	// deprecated, to be removed in 0.10
 	applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.onReboot, "no-reboot", false, "apply the config only after the reboot")

cmd/talosctl/cmd/talos/convert-k8s.go

@@ -0,0 +1,57 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package talos
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"github.com/talos-systems/talos/pkg/cluster"
+	k8s "github.com/talos-systems/talos/pkg/cluster/kubernetes"
+	"github.com/talos-systems/talos/pkg/machinery/client"
+)
+
+// convertK8sCmd represents the convert-k8s command.
+var convertK8sCmd = &cobra.Command{
+	Use:   "convert-k8s",
+	Short: "Convert Kubernetes control plane from self-hosted (bootkube) to Talos-managed (static pods).",
+	Long: `Command converts control plane bootstrapped on Talos <= 0.8 to Talos-managed control plane (Talos >= 0.9).
+As part of the conversion process tool reads existing configuration of the control plane, updates
+Talos node configuration to reflect changes made since the boostrap time. Once config is updated,
+tool releases static pods and deletes self-hosted DaemonSets.`,
+	Args: cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return WithClient(convertKubernetes)
+	},
+}
+
+var convertOptions k8s.ConvertOptions
+
+func init() {
+	convertK8sCmd.Flags().StringVar(&convertOptions.ControlPlaneEndpoint, "endpoint", "", "the cluster control plane endpoint")
+	convertK8sCmd.Flags().BoolVar(&convertOptions.ForceYes, "force", false, "skip prompts, assume yes")
+	addCommand(convertK8sCmd)
+}
+
+func convertKubernetes(ctx context.Context, c *client.Client) error {
+	clientProvider := &cluster.ConfigClientProvider{
+		DefaultClient: c,
+	}
+	defer clientProvider.Close() //nolint: errcheck
+
+	state := struct {
+		cluster.ClientProvider
+		cluster.K8sProvider
+	}{
+		ClientProvider: clientProvider,
+		K8sProvider: &cluster.KubernetesClient{
+			ClientProvider: clientProvider,
+			ForceEndpoint:  convertOptions.ControlPlaneEndpoint,
+		},
+	}
+
+	return k8s.ConvertToStaticPods(ctx, &state, convertOptions)
+}

cmd/talosctl/cmd/talos/upgrade-k8s.go

@@ -6,6 +6,7 @@ package talos
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/spf13/cobra"
 
@@ -45,13 +46,24 @@ func upgradeKubernetes(ctx context.Context, c *client.Client) error {
 	defer clientProvider.Close() //nolint: errcheck
 
 	state := struct {
+		cluster.ClientProvider
 		cluster.K8sProvider
 	}{
+		ClientProvider: clientProvider,
 		K8sProvider: &cluster.KubernetesClient{
 			ClientProvider: clientProvider,
-			ForceEndpoint:  healthCmdFlags.forceEndpoint,
+			ForceEndpoint:  upgradeOptions.ControlPlaneEndpoint,
 		},
 	}
 
-	return k8s.Upgrade(ctx, &state, upgradeOptions)
+	selfHosted, err := k8s.IsSelfHostedControlPlane(ctx, &state, Nodes[0])
+	if err != nil {
+		return fmt.Errorf("error checking self-hosted status: %w", err)
+	}
+
+	if selfHosted {
+		return k8s.UpgradeSelfHosted(ctx, &state, upgradeOptions)
+	}
+
+	return k8s.UpgradeTalosManaged(ctx, &state, upgradeOptions)
 }

internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go

@@ -32,6 +32,7 @@ import (
 	"github.com/prometheus/procfs"
 	"github.com/rs/xid"
 	"github.com/talos-systems/go-blockdevice/blockdevice/partition/gpt"
+	"go.etcd.io/etcd/clientv3"
 	"go.etcd.io/etcd/clientv3/concurrency"
 	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
@@ -103,14 +104,33 @@ func (s *Server) Register(obj *grpc.Server) {
 }
 
 // ApplyConfiguration implements machine.MachineService.
-func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfigurationRequest) (reply *machine.ApplyConfigurationResponse, err error) {
-	if !in.OnReboot {
-		if err = s.Controller.Runtime().SetConfig(in.GetData()); err != nil {
+//
+//nolint: gocyclo
+func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfigurationRequest) (*machine.ApplyConfigurationResponse, error) {
+	log.Printf("apply config request: immediate %v, on reboot %v", in.Immediate, in.OnReboot)
+
+	switch {
+	// --immediate
+	case in.Immediate:
+		if err := s.Controller.Runtime().CanApplyImmediate(in.GetData()); err != nil {
+			return nil, err
+		}
+
+		if err := s.Controller.Runtime().SetConfig(in.GetData()); err != nil {
+			return nil, err
+		}
+
+		if err := ioutil.WriteFile(constants.ConfigPath, in.GetData(), 0o600); err != nil {
+			return nil, err
+		}
+	// default (no flags)
+	case !in.OnReboot:
+		if err := s.Controller.Runtime().SetConfig(in.GetData()); err != nil {
 			return nil, err
 		}
 
 		go func() {
-			if err = s.Controller.Run(runtime.SequenceApplyConfiguration, in); err != nil {
+			if err := s.Controller.Run(runtime.SequenceApplyConfiguration, in); err != nil {
 				if !runtime.IsRebootError(err) {
 					log.Println("apply configuration failed:", err)
 				}
@@ -120,7 +140,8 @@ func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfig
 				}
 			}
 		}()
-	} else {
+	// --no-reboot
+	case in.OnReboot:
 		cfg, err := s.Controller.Runtime().ValidateConfig(in.GetData())
 		if err != nil {
 			return nil, err
@@ -143,13 +164,11 @@ func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfig
 		}
 	}
 
-	reply = &machine.ApplyConfigurationResponse{
+	return &machine.ApplyConfigurationResponse{
 		Messages: []*machine.ApplyConfiguration{
 			{},
 		},
-	}
-
-	return reply, nil
+	}, nil
 }
 
 // GenerateConfiguration implements the machine.MachineServer interface.
@@ -1722,6 +1741,33 @@ func (s *Server) EtcdForfeitLeadership(ctx context.Context, in *machine.EtcdForf
 	return reply, nil
 }
 
+// RemoveBootkubeInitializedKey implements machine.MachineService.
+//
+// Temporary API only used when converting from self-hosted to Talos-managed control plane.
+// This API can be removed once the conversion process is no longer needed (Talos 0.11?).
+func (s *Server) RemoveBootkubeInitializedKey(ctx context.Context, in *empty.Empty) (*machine.RemoveBootkubeInitializedKeyResponse, error) {
+	client, err := etcd.NewLocalClient()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create etcd client: %w", err)
+	}
+
+	// nolint: errcheck
+	defer client.Close()
+
+	ctx = clientv3.WithRequireLeader(ctx)
+
+	_, err = client.Delete(ctx, constants.InitializedKey)
+	if err != nil {
+		return nil, fmt.Errorf("error deleting initialized key: %w", err)
+	}
+
+	return &machine.RemoveBootkubeInitializedKeyResponse{
+		Messages: []*machine.RemoveBootkubeInitializedKey{
+			{},
+		},
+	}, nil
+}
+
 func upgradeMutex(c *etcd.Client) (*concurrency.Mutex, error) {
 	sess, err := concurrency.NewSession(c.Client,
 		concurrency.WithTTL(MinimumEtcdUpgradeLeaseLockSeconds),

internal/app/machined/pkg/controllers/config/k8s_control_plane.go

@@ -16,11 +16,11 @@ import (
 	"github.com/talos-systems/os-runtime/pkg/resource"
 	"github.com/talos-systems/os-runtime/pkg/state"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
 	"github.com/talos-systems/talos/pkg/images"
 	talosconfig "github.com/talos-systems/talos/pkg/machinery/config"
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/resources/config"
 )
 
 // K8sControlPlaneController manages config.K8sControlPlane based on configuration.

internal/app/machined/pkg/controllers/config/machine_type.go

@@ -14,8 +14,8 @@ import (
 	"github.com/talos-systems/os-runtime/pkg/resource"
 	"github.com/talos-systems/os-runtime/pkg/state"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
+	"github.com/talos-systems/talos/pkg/resources/config"
 )
 
 // MachineTypeController manages config.MachineType based on configuration.

internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go

@@ -19,9 +19,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/resources/config"
+	"github.com/talos-systems/talos/pkg/resources/k8s"
 )
 
 // ControlPlaneStaticPodController manages k8s.StaticPod based on control plane configuration.

internal/app/machined/pkg/controllers/k8s/extra_manifest.go

@@ -20,9 +20,9 @@ import (
 	"github.com/talos-systems/os-runtime/pkg/resource"
 	"github.com/talos-systems/os-runtime/pkg/state"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
+	"github.com/talos-systems/talos/pkg/resources/config"
+	"github.com/talos-systems/talos/pkg/resources/k8s"
+	"github.com/talos-systems/talos/pkg/resources/v1alpha1"
 )
 
 // ExtraManifestController renders manifests based on templates and config/secrets.

internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go

@@ -22,11 +22,11 @@ import (
 	"github.com/talos-systems/os-runtime/pkg/state"
 	"gopkg.in/yaml.v3"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
 	"github.com/talos-systems/talos/pkg/kubernetes/kubelet"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/resources/k8s"
+	"github.com/talos-systems/talos/pkg/resources/secrets"
+	"github.com/talos-systems/talos/pkg/resources/v1alpha1"
 )
 
 // KubeletStaticPodController renders static pod definitions and manages k8s.StaticPodStatus.

internal/app/machined/pkg/controllers/k8s/manifest.go

@@ -16,9 +16,9 @@ import (
 	"github.com/talos-systems/os-runtime/pkg/resource"
 	"github.com/talos-systems/os-runtime/pkg/state"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
+	"github.com/talos-systems/talos/pkg/resources/config"
+	"github.com/talos-systems/talos/pkg/resources/k8s"
+	"github.com/talos-systems/talos/pkg/resources/secrets"
 )
 
 // ManifestController renders manifests based on templates and config/secrets.

internal/app/machined/pkg/controllers/k8s/manifest_apply.go

@@ -28,11 +28,11 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
 	"github.com/talos-systems/talos/internal/pkg/etcd"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/resources/k8s"
+	"github.com/talos-systems/talos/pkg/resources/secrets"
+	"github.com/talos-systems/talos/pkg/resources/v1alpha1"
 )
 
 // ManifestApplyController applies manifests via control plane endpoint.

internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go

@@ -19,9 +19,9 @@ import (
 	"github.com/talos-systems/os-runtime/pkg/resource"
 	"github.com/talos-systems/os-runtime/pkg/state"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/resources/k8s"
+	"github.com/talos-systems/talos/pkg/resources/secrets"
 )
 
 // RenderSecretsStaticPodController manages k8s.SecretsReady and renders secrets from secrets.Kubernetes.

internal/app/machined/pkg/controllers/secrets/kubernetes.go

@@ -18,14 +18,14 @@ import (
 	"github.com/talos-systems/os-runtime/pkg/resource"
 	"github.com/talos-systems/os-runtime/pkg/state"
 
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
-	"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
 	"github.com/talos-systems/talos/internal/pkg/etcd"
 	"github.com/talos-systems/talos/internal/pkg/kubeconfig"
 	talosconfig "github.com/talos-systems/talos/pkg/machinery/config"
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/resources/config"
+	"github.com/talos-systems/talos/pkg/resources/secrets"
+	"github.com/talos-systems/talos/pkg/resources/v1alpha1"
 )
 
 // KubernetesController manages secrets.Kubernetes based on configuration.