Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

The roleGroups field supports to be customized #1018

Merged
merged 1 commit into from
Jan 20, 2022
Merged

The roleGroups field supports to be customized #1018

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
216 changes: 64 additions & 152 deletions apis/kubekey/v1alpha2/cluster_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package v1alpha2

import (
"fmt"
"github.com/kubesphere/kubekey/pkg/core/connector"
"regexp"
"strconv"
"strings"
Expand All @@ -39,7 +40,7 @@ type ClusterSpec struct {

// Foo is an example field of Cluster. Edit Cluster_types.go to remove/update
Hosts []HostCfg `yaml:"hosts" json:"hosts,omitempty"`
RoleGroups RoleGroups `yaml:"roleGroups" json:"roleGroups,omitempty"`
RoleGroups map[string][]string `yaml:"roleGroups" json:"roleGroups,omitempty"`
ControlPlaneEndpoint ControlPlaneEndpoint `yaml:"controlPlaneEndpoint" json:"controlPlaneEndpoint,omitempty"`
System System `yaml:"system" json:"system,omitempty"`
Kubernetes Kubernetes `yaml:"kubernetes" json:"kubernetes,omitempty"`
Expand Down Expand Up @@ -148,31 +149,8 @@ type HostCfg struct {
PrivateKeyPath string `yaml:"privateKeyPath,omitempty" json:"privateKeyPath,omitempty"`
Arch string `yaml:"arch,omitempty" json:"arch,omitempty"`

Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"`
ID string `yaml:"id,omitempty" json:"id,omitempty"`
Index int `json:"-"`
IsEtcd bool `json:"-"`
IsMaster bool `json:"-"`
IsWorker bool `json:"-"`
IsRegistry bool `json:"-"`
}

// RoleGroups defines the grouping of role for hosts (etcd / master / worker / registry).
type RoleGroups struct {
Etcd []string `yaml:"etcd" json:"etcd,omitempty"`
Master []string `yaml:"master" json:"master,omitempty"`
Worker []string `yaml:"worker" json:"worker,omitempty"`
Registry []string `yaml:"registry" json:"registry,omitempty"`
}

// HostGroups defines the grouping of hosts for cluster (all / etcd / master / worker / k8s).
type HostGroups struct {
All []HostCfg
Etcd []HostCfg
Master []HostCfg
Worker []HostCfg
K8s []HostCfg
Registry []HostCfg
Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"`
ID string `yaml:"id,omitempty" json:"id,omitempty"`
}

// ControlPlaneEndpoint defines the control plane endpoint information for cluster.
Expand Down Expand Up @@ -246,99 +224,49 @@ func (cfg *ClusterSpec) GenerateCertSANs() []string {
}

// GroupHosts is used to group hosts according to the configuration file.s
func (cfg *ClusterSpec) GroupHosts() (*HostGroups, error) {
clusterHostsGroups := HostGroups{}

hostList := map[string]string{}
for _, host := range cfg.Hosts {
hostList[host.Name] = host.Name
func (cfg *ClusterSpec) GroupHosts() (map[string][]*connector.BaseHost, error) {
hostMap := make(map[string]*connector.BaseHost)
for _, hostCfg := range cfg.Hosts {
host := toHosts(hostCfg)
hostMap[host.Name] = host
}

etcdGroup, masterGroup, workerGroup, registryGroup, err := cfg.ParseRolesList(hostList)
roleGroups, err := cfg.ParseRolesList(hostMap)
if err != nil {
return nil, err
}
for index, host := range cfg.Hosts {
host.Index = index
if len(etcdGroup) > 0 {
for _, hostName := range etcdGroup {
if host.Name == hostName {
host.IsEtcd = true
break
}
}
}

if len(masterGroup) > 0 {
for _, hostName := range masterGroup {
if host.Name == hostName {
host.IsMaster = true
break
}
}
}

if len(workerGroup) > 0 {
for _, hostName := range workerGroup {
if hostName != "" && host.Name == hostName {
host.IsWorker = true
break
}
}
}

if len(registryGroup) > 0 {
for _, hostName := range registryGroup {
if hostName != "" && host.Name == hostName {
host.IsRegistry = true
}
}
}

if host.IsEtcd {
clusterHostsGroups.Etcd = append(clusterHostsGroups.Etcd, host)
}
if host.IsMaster {
clusterHostsGroups.Master = append(clusterHostsGroups.Master, host)
}
if host.IsWorker {
clusterHostsGroups.Worker = append(clusterHostsGroups.Worker, host)
}
if host.IsMaster || host.IsWorker {
clusterHostsGroups.K8s = append(clusterHostsGroups.K8s, host)
}
if host.IsRegistry {
clusterHostsGroups.Registry = append(clusterHostsGroups.Registry, host)
}

clusterHostsGroups.All = append(clusterHostsGroups.All, host)
}

//Check that the parameters under roleGroups are incorrect
if len(masterGroup) == 0 {
logger.Log.Fatal(errors.New("The number of master cannot be 0"))
if len(roleGroups[Master]) == 0 && len(roleGroups[ControlPlane]) == 0 {
logger.Log.Fatal(errors.New("The number of master/control-plane cannot be 0"))
}
if len(etcdGroup) == 0 {
if len(roleGroups[Etcd]) == 0 {
logger.Log.Fatal(errors.New("The number of etcd cannot be 0"))
}
if len(registryGroup) > 1 {
if len(roleGroups[Registry]) > 1 {
logger.Log.Fatal(errors.New("The number of registry node cannot be greater than 1."))
}

if len(masterGroup) != len(clusterHostsGroups.Master) {
return nil, errors.New("Incorrect nodeName under roleGroups/master in the configuration file")
}
if len(etcdGroup) != len(clusterHostsGroups.Etcd) {
return nil, errors.New("Incorrect nodeName under roleGroups/etcd in the configuration file")
}
if len(workerGroup) != len(clusterHostsGroups.Worker) {
return nil, errors.New("Incorrect nodeName under roleGroups/work in the configuration file")
}
if len(registryGroup) != len(clusterHostsGroups.Registry) {
return nil, errors.New("Incorrect nodeName under roleGroups/registry in the configuration file")
for _, host := range roleGroups[ControlPlane] {
host.SetRole(Master)
roleGroups[Master] = append(roleGroups[Master], host)
}

return &clusterHostsGroups, nil
return roleGroups, nil
}

func toHosts(cfg HostCfg) *connector.BaseHost {
host := connector.NewHost()
host.Name = cfg.Name
host.Address = cfg.Address
host.InternalAddress = cfg.InternalAddress
host.Port = cfg.Port
host.User = cfg.User
host.Password = cfg.Password
host.PrivateKey = cfg.PrivateKey
host.PrivateKeyPath = cfg.PrivateKeyPath
host.Arch = cfg.Arch
return host
}

// ClusterIP is used to get the kube-apiserver service address inside the cluster.
Expand All @@ -361,77 +289,61 @@ func (cfg *ClusterSpec) ClusterDNS() string {
}

// ParseRolesList is used to parse the host grouping list.
func (cfg *ClusterSpec) ParseRolesList(hostList map[string]string) ([]string, []string, []string, []string, error) {
etcdGroupList := make([]string, 0)
masterGroupList := make([]string, 0)
workerGroupList := make([]string, 0)
registryGroupList := make([]string, 0)

for _, host := range cfg.RoleGroups.Etcd {
if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") {
etcdGroupList = append(etcdGroupList, getHostsRange(host, hostList, "etcd")...)
} else {
if err := hostVerify(hostList, host, "etcd"); err != nil {
logger.Log.Fatal(err)
}
etcdGroupList = append(etcdGroupList, host)
}
}

for _, host := range cfg.RoleGroups.Master {
if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") {
masterGroupList = append(masterGroupList, getHostsRange(host, hostList, "master")...)
} else {
if err := hostVerify(hostList, host, "master"); err != nil {
logger.Log.Fatal(err)
func (cfg *ClusterSpec) ParseRolesList(hostMap map[string]*connector.BaseHost) (map[string][]*connector.BaseHost, error) {
roleGroupLists := make(map[string][]*connector.BaseHost)
for role, hosts := range cfg.RoleGroups {
roleGroup := make([]string, 0)
for _, host := range hosts {
h := make([]string, 0)
if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") {
rangeHosts := getHostsRange(host, hostMap, role)
h = append(h, rangeHosts...)
} else {
if err := hostVerify(hostMap, host, role); err != nil {
logger.Log.Fatal(err)
}
h = append(h, host)
}
masterGroupList = append(masterGroupList, host)
}
}

for _, host := range cfg.RoleGroups.Worker {
if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") {
workerGroupList = append(workerGroupList, getHostsRange(host, hostList, "worker")...)
} else {
if err := hostVerify(hostList, host, "worker"); err != nil {
logger.Log.Fatal(err)
roleGroup = append(roleGroup, h...)
for _, hostName := range h {
if h, ok := hostMap[hostName]; ok {
roleGroupAppend(roleGroupLists, role, h)
} else {
return roleGroupLists, fmt.Errorf("incorrect nodeName under roleGroups/%s in the configuration file", role)
}
}
workerGroupList = append(workerGroupList, host)
}
}

for _, host := range cfg.RoleGroups.Registry {
if strings.Contains(host, "[") && strings.Contains(host, "]") && strings.Contains(host, ":") {
registryGroupList = append(registryGroupList, getHostsRange(host, hostList, "registry")...)
} else {
if err := hostVerify(hostList, host, "registry"); err != nil {
logger.Log.Fatal(err)
}
registryGroupList = append(registryGroupList, host)
}
}
return roleGroupLists, nil
}

return etcdGroupList, masterGroupList, workerGroupList, registryGroupList, nil
func roleGroupAppend(roleGroupLists map[string][]*connector.BaseHost, role string, host *connector.BaseHost) {
host.SetRole(role)
r := roleGroupLists[role]
r = append(r, host)
roleGroupLists[role] = r
}

func getHostsRange(rangeStr string, hostList map[string]string, group string) []string {
func getHostsRange(rangeStr string, hostMap map[string]*connector.BaseHost, group string) []string {
hostRangeList := make([]string, 0)
r := regexp.MustCompile(`\[(\d+)\:(\d+)\]`)
nameSuffix := r.FindStringSubmatch(rangeStr)
namePrefix := strings.Split(rangeStr, nameSuffix[0])[0]
nameSuffixStart, _ := strconv.Atoi(nameSuffix[1])
nameSuffixEnd, _ := strconv.Atoi(nameSuffix[2])
for i := nameSuffixStart; i <= nameSuffixEnd; i++ {
if err := hostVerify(hostList, fmt.Sprintf("%s%d", namePrefix, i), group); err != nil {
if err := hostVerify(hostMap, fmt.Sprintf("%s%d", namePrefix, i), group); err != nil {
logger.Log.Fatal(err)
}
hostRangeList = append(hostRangeList, fmt.Sprintf("%s%d", namePrefix, i))
}
return hostRangeList
}

func hostVerify(hostList map[string]string, hostName string, group string) error {
if _, ok := hostList[hostName]; !ok {
func hostVerify(hostMap map[string]*connector.BaseHost, hostName string, group string) error {
if _, ok := hostMap[hostName]; !ok {
return fmt.Errorf("[%s] is in [%s] group, but not in hosts list", hostName, group)
}
return nil
Expand Down
13 changes: 8 additions & 5 deletions apis/kubekey/v1alpha2/default.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package v1alpha2

import (
"fmt"
"github.com/kubesphere/kubekey/pkg/core/connector"
"github.com/kubesphere/kubekey/pkg/core/util"
"os"
"strings"
Expand Down Expand Up @@ -63,8 +64,10 @@ const (
DefaultIsulaEndpoint = "unix:///var/run/isulad.sock"
Etcd = "etcd"
Master = "master"
ControlPlane = "control-plane"
Worker = "worker"
K8s = "k8s"
Registry = "registry"
DefaultEtcdBackupDir = "/var/backups/kube_etcd"
DefaultEtcdBackupPeriod = 30
DefaultKeepBackNumber = 5
Expand All @@ -84,16 +87,16 @@ const (
Haproxy = "haproxy"
)

func (cfg *ClusterSpec) SetDefaultClusterSpec(incluster bool) (*ClusterSpec, *HostGroups, error) {
func (cfg *ClusterSpec) SetDefaultClusterSpec(incluster bool) (*ClusterSpec, map[string][]*connector.BaseHost, error) {
clusterCfg := ClusterSpec{}

clusterCfg.Hosts = SetDefaultHostsCfg(cfg)
clusterCfg.RoleGroups = cfg.RoleGroups
hostGroups, err := clusterCfg.GroupHosts()
roleGroups, err := clusterCfg.GroupHosts()
if err != nil {
return nil, nil, err
}
clusterCfg.ControlPlaneEndpoint = SetDefaultLBCfg(cfg, hostGroups.Master, incluster)
clusterCfg.ControlPlaneEndpoint = SetDefaultLBCfg(cfg, roleGroups[Master], incluster)
clusterCfg.Network = SetDefaultNetworkCfg(cfg)
clusterCfg.System = cfg.System
clusterCfg.Kubernetes = SetDefaultClusterCfg(cfg)
Expand All @@ -116,7 +119,7 @@ func (cfg *ClusterSpec) SetDefaultClusterSpec(incluster bool) (*ClusterSpec, *Ho
if cfg.Kubernetes.ProxyMode == "" {
clusterCfg.Kubernetes.ProxyMode = DefaultProxyMode
}
return &clusterCfg, hostGroups, nil
return &clusterCfg, roleGroups, nil
}

func SetDefaultHostsCfg(cfg *ClusterSpec) []HostCfg {
Expand Down Expand Up @@ -155,7 +158,7 @@ func SetDefaultHostsCfg(cfg *ClusterSpec) []HostCfg {
return hostCfg
}

func SetDefaultLBCfg(cfg *ClusterSpec, masterGroup []HostCfg, incluster bool) ControlPlaneEndpoint {
func SetDefaultLBCfg(cfg *ClusterSpec, masterGroup []*connector.BaseHost, incluster bool) ControlPlaneEndpoint {
if !incluster {
//The detection is not an HA environment, and the address at LB does not need input
if len(masterGroup) == 1 && cfg.ControlPlaneEndpoint.Address != "" {
Expand Down