From 67f4b1a573feae49d3a4e4710f866cdce4844359 Mon Sep 17 00:00:00 2001 From: Kyle Robbertze Date: Wed, 12 Oct 2022 12:04:53 +0200 Subject: [PATCH] feat: drop etcd support --- README.md | 2 +- cmd/common_test.go | 1 - cmd/etcd.go | 59 ------------- main.go | 1 - pkg/api/postgres/types.go | 1 - pkg/provision/cluster.go | 8 +- pkg/provision/etcd.go | 177 ------------------------------------- pkg/provision/status.go | 1 - pkg/provision/terminate.go | 63 +------------ 9 files changed, 6 insertions(+), 307 deletions(-) delete mode 100644 cmd/etcd.go delete mode 100644 pkg/provision/etcd.go diff --git a/README.md b/README.md index 4a3b0e224..c10b4f1aa 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ * **Monitoring** ([Grafana](https://github.com/integr8ly/grafana-operator), [Prometheus](https://github.com/coreos/prometheus-operator), [Thanos](https://thanos.io/), [Karma](https://github.com/prymitive/karma), [Canary Checker](https://github.com/flanksource/canary-checker)) * **Multi-Tenancy** ([Namespace Configurator](https://github.com/redhat-cop/namespace-configuration-operator) Cluster Quotas, [Kiosk](https://github.com/kiosk-sh/kiosk)) * **Cluster Provisioning** framework for [Kind](https://karina.docs.flanksource.com/admin-guide/provisioning/kind/), [vSphere](https://karina.docs.flanksource.com/admin-guide/provisioning/vsphere/) and Cluster API (Coming Soon) -* **Operations focused CLI** for health checks, etcd maintenance, backup and restore, rolling updates, logging, etc.. +* **Operations focused CLI** for health checks, backup and restore, rolling updates, logging, etc.. karina leverages a number of other standalone operators built by flanksource: diff --git a/cmd/common_test.go b/cmd/common_test.go index 20f75384b..048e4dc12 100644 --- a/cmd/common_test.go +++ b/cmd/common_test.go @@ -30,7 +30,6 @@ func TestGetConfigSetDefaults(t *testing.T) { ControllerExtraArgs: map[string]string{}, SchedulerExtraArgs: map[string]string{}, KubeletExtraArgs: map[string]string{}, - EtcdExtraArgs: map[string]string{}, ContainerRuntime: "docker", })) } diff --git a/cmd/etcd.go b/cmd/etcd.go deleted file mode 100644 index 7536cb4a8..000000000 --- a/cmd/etcd.go +++ /dev/null @@ -1,59 +0,0 @@ -package cmd - -import ( - "github.com/flanksource/karina/pkg/provision" - "github.com/spf13/cobra" -) - -var Etcd = &cobra.Command{ - Use: "etcd", -} - -var etcdHost string - -func init() { - Etcd.PersistentFlags().StringVar(&etcdHost, "etcd-host", "", "Etcd hostname to connect through") - Etcd.AddCommand(&cobra.Command{ - Use: "status", - Run: func(cmd *cobra.Command, args []string) { - platform := getPlatform(cmd) - client, err := platform.GetClientset() - if err != nil { - platform.Fatalf("could not get client: %v", err) - } - if err := provision.GetEtcdClient(platform, client, etcdHost).PrintStatus(); err != nil { - platform.Fatalf("Failed to get etcd status: %v", err) - } - }, - }) - - Etcd.AddCommand(&cobra.Command{ - Use: "remove-member [name]", - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - platform := getPlatform(cmd) - client, err := platform.GetClientset() - if err != nil { - platform.Fatalf("could not get client: %v", err) - } - if err := provision.GetEtcdClient(platform, client, etcdHost).RemoveMember(args[0]); err != nil { - platform.Fatalf(err.Error()) - } - }, - }) - - Etcd.AddCommand(&cobra.Command{ - Use: "move-leader [name]", - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - platform := getPlatform(cmd) - client, err := platform.GetClientset() - if err != nil { - platform.Fatalf("could not get client: %v", err) - } - if err := provision.GetEtcdClient(platform, client, etcdHost).MoveLeader(args[0]); err != nil { - platform.Fatalf(err.Error()) - } - }, - }) -} diff --git a/main.go b/main.go index 55503f6c7..7950d5299 100644 --- a/main.go +++ b/main.go @@ -41,7 +41,6 @@ func main() { cmd.DNS, cmd.Exec, cmd.ExecNode, - cmd.Etcd, cmd.Harbor, cmd.Images, cmd.Logs, diff --git a/pkg/api/postgres/types.go b/pkg/api/postgres/types.go index 0b64850c2..689120a03 100644 --- a/pkg/api/postgres/types.go +++ b/pkg/api/postgres/types.go @@ -469,7 +469,6 @@ type OperatorLogicalBackupConfiguration struct { type OperatorConfigurationData struct { EnableCRDValidation *bool `json:"enable_crd_validation,omitempty"` EnableLazySpiloUpgrade bool `json:"enable_lazy_spilo_upgrade,omitempty"` - EtcdHost string `json:"etcd_host,omitempty"` KubernetesUseConfigMaps bool `json:"kubernetes_use_configmaps,omitempty"` DockerImage string `json:"docker_image,omitempty"` Workers uint32 `json:"workers,omitempty"` diff --git a/pkg/provision/cluster.go b/pkg/provision/cluster.go index 565943453..d1f50ce78 100644 --- a/pkg/provision/cluster.go +++ b/pkg/provision/cluster.go @@ -69,7 +69,6 @@ type Cluster struct { Nodes NodeMachines Orphans []types.Machine Kubernetes kubernetes.Interface - Etcd *EtcdClient } func GetCluster(platform *platform.Platform) (*Cluster, error) { @@ -117,21 +116,16 @@ func GetCluster(platform *platform.Platform) (*Cluster, error) { Nodes: nodes, Orphans: orphans, Kubernetes: client, - Etcd: GetEtcdClient(platform, client, ""), } cluster.Platform = platform return cluster, nil } func (cluster *Cluster) Terminate(node types.Machine) error { - terminate(cluster.Platform, cluster.Etcd, node) + terminate(cluster.Platform, node) return nil } -func (cluster *Cluster) GetHealth(node v1.Node) string { - return cluster.Etcd.GetHealth(node) -} - func (cluster *Cluster) Cordon(node v1.Node) error { return cluster.Platform.Cordon(node.Name) } diff --git a/pkg/provision/etcd.go b/pkg/provision/etcd.go deleted file mode 100644 index 719fb5559..000000000 --- a/pkg/provision/etcd.go +++ /dev/null @@ -1,177 +0,0 @@ -package provision - -import ( - "context" - "fmt" - "os" - "text/tabwriter" - - "github.com/flanksource/commons/collections" - "github.com/flanksource/karina/pkg/phases/kubeadm" - "github.com/flanksource/karina/pkg/platform" - "github.com/flanksource/kommons" - "github.com/flanksource/kommons/etcd" - "go.etcd.io/etcd/clientv3" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -type EtcdClient struct { - *platform.Platform - Kubernetes kubernetes.Interface - PreferredHostname string - Etcd *etcd.EtcdClientGenerator -} - -func GetEtcdClient(platform *platform.Platform, client kubernetes.Interface, PreferredHostname string) *EtcdClient { - return &EtcdClient{Platform: platform, Kubernetes: client, PreferredHostname: PreferredHostname} -} - -func (cluster *EtcdClient) connectToEtcd() error { - if cluster.Etcd != nil { - return nil - } - - // upload etcd certs so that we can connect to etcd - cert, err := kubeadm.UploadEtcdCerts(cluster.Platform) - if err != nil { - return err - } - - etcdClientGenerator, err := cluster.Platform.GetEtcdClientGenerator(cert) - if err != nil { - return err - } - cluster.Etcd = etcdClientGenerator - return nil -} - -func (cluster *EtcdClient) getMemberStatus(name string) (*clientv3.StatusResponse, error) { - if err := cluster.connectToEtcd(); err != nil { - return nil, err - } - etcdClient, err := cluster.Etcd.ForNode(context.Background(), name) - if err != nil { - return nil, err - } - return etcdClient.EtcdClient.Status(context.Background(), etcdClient.EtcdClient.Endpoints()[0]) -} - -func (cluster *EtcdClient) GetHealth(node v1.Node) string { - status, err := cluster.getMemberStatus(node.Name) - if err != nil { - return err.Error() - } - s := fmt.Sprintf("v%s (%s) ", status.Version, size(status.DbSize)) - return s -} - -func (cluster *EtcdClient) RemoveMember(name string) error { - client, err := cluster.GetEtcdLeader() - if err != nil { - return err - } - members, err := client.Members(context.TODO()) - if err != nil { - return err - } - for _, member := range members { - if member.Name == name { - return client.RemoveMember(context.TODO(), member.ID) - } - } - return fmt.Errorf("member not found: %s", name) -} - -func (cluster *EtcdClient) MoveLeader(name string) error { - client, err := cluster.GetEtcdLeader() - if err != nil { - return err - } - members, err := client.Members(context.TODO()) - if err != nil { - return err - } - - for _, member := range members { - if member.Name == name { - return client.MoveLeader(context.TODO(), member.ID) - } - } - - return fmt.Errorf("member not found: %s", name) -} - -func (cluster *EtcdClient) GetEtcdClient(node v1.Node) (*etcd.Client, error) { - if err := cluster.connectToEtcd(); err != nil { - return nil, err - } - return cluster.Etcd.ForNode(context.TODO(), node.Name) -} - -func (cluster *EtcdClient) GetOrphans() ([]string, error) { - var orphans []string - return orphans, nil -} - -func (cluster *EtcdClient) GetEtcdLeader() (*etcd.Client, error) { - if err := cluster.connectToEtcd(); err != nil { - return nil, err - } - - if cluster.PreferredHostname != "" { - return cluster.Etcd.ForNode(context.TODO(), cluster.PreferredHostname) - } - list, err := cluster.Kubernetes.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return nil, err - } - - return cluster.Etcd.ForLeader(context.TODO(), list) -} - -func (cluster *EtcdClient) PrintStatus() error { - if err := cluster.connectToEtcd(); err != nil { - return fmt.Errorf("failed to connect to etcd: %v", err) - } - client, err := cluster.GetEtcdLeader() - if err != nil { - return fmt.Errorf("failed to connect to etcd: %v", err) - } - - w := tabwriter.NewWriter(os.Stdout, 3, 2, 3, ' ', tabwriter.DiscardEmptyColumns) - fmt.Fprintf(w, "ID\tNAME\tK8S\tSTATUS\tVERSION\tSIZE\tALARMS \n") - members, err := client.Members(context.TODO()) - if err != nil { - return fmt.Errorf("failed to get members: %v", err) - } - for _, member := range members { - fmt.Fprintf(w, "%d\t", member.ID) - fmt.Fprintf(w, "%s\t", member.Name) - node, err := cluster.Kubernetes.CoreV1().Nodes().Get(context.TODO(), member.Name, metav1.GetOptions{}) - if err == nil { - fmt.Fprintf(w, "%s\t", kommons.GetNodeStatus(*node)) - } else { - fmt.Fprintf(w, "MISSING\t") - } - s := "" - if member.ID == client.LeaderID { - s = "LEADER" - } else if member.IsLearner { - s = "LEARNER" - } - status, err := cluster.getMemberStatus(member.Name) - if err != nil { - s += " " + err.Error() - fmt.Fprintf(w, "%s\t\t\t", s) - } else if status != nil { - fmt.Fprintf(w, "%s\t%s\t%s\t", s, status.Version, size(status.DbSizeInUse)) - } - - fmt.Fprintf(w, "%s\t", collections.ToString(member.Alarms)) - fmt.Fprintf(w, "\n") - } - _ = w.Flush() - return nil -} diff --git a/pkg/provision/status.go b/pkg/provision/status.go index 87f5a8578..425d1abf8 100644 --- a/pkg/provision/status.go +++ b/pkg/provision/status.go @@ -94,7 +94,6 @@ func Status(p *platform.Platform) error { fmt.Fprintf(w, "%s\t", kommons.GetNodeStatus(node)) if kommons.IsMasterNode(node) { fmt.Fprintf(w, "%s\t", kubeadm.GetNodeVersion(p, nodeMachine.Node)) - fmt.Fprintf(w, "%s\t", cluster.GetHealth(node)) } else { fmt.Fprintf(w, "\t\t") } diff --git a/pkg/provision/terminate.go b/pkg/provision/terminate.go index d3251a752..4b92fb050 100644 --- a/pkg/provision/terminate.go +++ b/pkg/provision/terminate.go @@ -9,7 +9,6 @@ import ( "github.com/flanksource/karina/pkg/platform" "github.com/flanksource/karina/pkg/types" "github.com/flanksource/kommons" - "github.com/flanksource/kommons/etcd" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -59,7 +58,7 @@ func TerminateNodes(platform *platform.Platform, nodes []string) error { return nil } -func terminate(platform *platform.Platform, etcd *EtcdClient, vm types.Machine) { +func terminate(platform *platform.Platform, vm types.Machine) { if err := platform.ProvisionHook.BeforeTerminate(platform, vm); err != nil { platform.Warnf("[%s] failed to call before terminate: %v", vm, err) return @@ -79,11 +78,11 @@ func terminate(platform *platform.Platform, etcd *EtcdClient, vm types.Machine) if err != nil { // we always attempt to terminate a node as a master if we don't know // to ensure it is always removed from etcd - if err := terminateMaster(platform, etcd, vm.Name()); err != nil { + if err := terminateMaster(platform, vm.Name()); err != nil { platform.Warnf("Failed to terminate master %v", err) } } else if kommons.IsMasterNode(*node) { - if err := terminateMaster(platform, etcd, node.Name); err != nil { + if err := terminateMaster(platform, node.Name); err != nil { platform.Warnf("Failed to terminate master %v", err) } } @@ -103,52 +102,6 @@ func terminate(platform *platform.Platform, etcd *EtcdClient, vm types.Machine) } } -func terminateEtcd(platform *platform.Platform, etcdClient *EtcdClient, name string) error { - ctx := context.TODO() - // we always interact via the etcd leader, as a previous node may have become unavailable - leaderClient, err := etcdClient.GetEtcdLeader() - if err != nil { - return err - } - platform.Infof("etcd leader is: %s", leaderClient.Name) - - members, err := leaderClient.Members(ctx) - if err != nil { - return err - } - var etcdMember *etcd.Member - var candidateLeader *etcd.Member - for _, member := range members { - if member.Name == name { - // find the etcd member for the node - etcdMember = member - } - if member.Name != leaderClient.Name { - // choose a potential candidate to move the etcd leader - candidateLeader = member - } - } - if etcdMember == nil { - platform.Warnf("%s has already been removed from etcd cluster", name) - } else if candidateLeader == nil { - platform.Warnf("%s is the only member left of the etcd cluster", name) - } else { - if etcdMember.ID == leaderClient.MemberID { - platform.Infof("Moving etcd leader from %s to %s", name, candidateLeader.Name) - if err := leaderClient.MoveLeader(ctx, candidateLeader.ID); err != nil { - return fmt.Errorf("failed to move leader: %v", err) - } - } - - platform.Infof("Removing etcd member %s", name) - if err := leaderClient.RemoveMember(ctx, etcdMember.ID); err != nil { - return fmt.Errorf("failed to remove member: %v", err) - } - } - - return nil -} - func terminateConsul(platform *platform.Platform, name string) error { if platform.Consul != "" { // proactively remove server from consul so that we can get a new connection to k8s @@ -159,15 +112,7 @@ func terminateConsul(platform *platform.Platform, name string) error { return nil } -func terminateMaster(platform *platform.Platform, etcdClient *EtcdClient, name string) error { - // if we are terminating the cluster then we don't need to worry about etcd - if !platform.Terminating { - if err := backoff(func() error { - return terminateEtcd(platform, etcdClient, name) - }, platform.Logger, nil); err != nil { - return err - } - } +func terminateMaster(platform *platform.Platform, name string) error { if err := backoff(func() error { return terminateConsul(platform, name) }, platform.Logger, nil); err != nil {