fix etcd member count

yunionio · Jul 21, 2020 · 8f5c7b5 · 8f5c7b5
1 parent 802d630
commit 8f5c7b5
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 18 deletions.
diff --git a/pkg/apis/constants/constants.go b/pkg/apis/constants/constants.go
@@ -35,6 +35,10 @@ const (
 	InstanceLabelKey string = "app.kubernetes.io/instance"
 	AppLabelKey      string = "app"
 
+	// LabelNodeRoleMaster specifies that a node is a control-plane
+	// This is a duplicate definition of the constant in pkg/controller/service/service_controller.go
+	LabelNodeRoleMaster string = "node-role.kubernetes.io/master"
+
 	ServiceAccountOnecloudOperator string = "onecloud-operator"
 )
 

diff --git a/pkg/controller/onecloud_control.go b/pkg/controller/onecloud_control.go
@@ -28,6 +28,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -494,14 +495,11 @@ func (c keystoneComponent) getWebAccessUrl() (string, error) {
 	}
 	var masterAddress string
 	for _, node := range nodes.Items {
-		if length := len(node.Status.Conditions); length > 0 {
-			if node.Status.Conditions[length-1].Type == v1.NodeReady &&
-				node.Status.Conditions[length-1].Status == v1.ConditionTrue {
-				for _, addr := range node.Status.Addresses {
-					if addr.Type == v1.NodeInternalIP {
-						masterAddress = addr.Address
-						break
-					}
+		if k8sutil.IsNodeReady(node) {
+			for _, addr := range node.Status.Addresses {
+				if addr.Type == v1.NodeInternalIP {
+					masterAddress = addr.Address
+					break
 				}
 			}
 		}

diff --git a/pkg/manager/component/etcd.go b/pkg/manager/component/etcd.go
@@ -26,6 +26,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
@@ -107,13 +108,29 @@ func (m *etcdManager) setUnsync() {
 }
 
 func (m *etcdManager) fixEtcdSize(oc *v1alpha1.OnecloudCluster) (bool, error) {
-	nodes, err := m.nodeLister.List(labels.NewSelector())
+	// list nodes by master node selector
+	masterNodeSelector := labels.NewSelector()
+	r, err := labels.NewRequirement(
+		constants.LabelNodeRoleMaster, selection.Exists, nil)
 	if err != nil {
 		return false, err
 	}
+	masterNodeSelector = masterNodeSelector.Add(*r)
+
+	nodes, err := m.nodeLister.List(masterNodeSelector)
+	if err != nil {
+		return false, err
+	}
+	readyMasterCount := 0
+	for _, node := range nodes {
+		if k8sutil.IsNodeReady(*node) {
+			readyMasterCount += 1
+		}
+	}
+	log.Infof("Ready master node count %d", readyMasterCount)
 
 	oldSize := oc.Spec.Etcd.Size
-	if len(nodes) < 3 {
+	if readyMasterCount < 3 {
 		oc.Spec.Etcd.Size = 1
 	} else {
 		if oc.Spec.Etcd.Size < constants.EtcdDefaultClusterSize {

diff --git a/pkg/manager/component/region.go b/pkg/manager/component/region.go
@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
 
 	"yunion.io/x/onecloud/pkg/compute/options"
 
@@ -103,14 +104,11 @@ func (m *regionManager) setBaremetalPrepareConfigure(oc *v1alpha1.OnecloudCluste
 	}
 	var masterAddress string
 	for _, node := range nodes {
-		if length := len(node.Status.Conditions); length > 0 {
-			if node.Status.Conditions[length-1].Type == v1.NodeReady &&
-				node.Status.Conditions[length-1].Status == v1.ConditionTrue {
-				for _, addr := range node.Status.Addresses {
-					if addr.Type == v1.NodeInternalIP {
-						masterAddress = addr.Address
-						break
-					}
+		if k8sutil.IsNodeReady(*node) {
+			for _, addr := range node.Status.Addresses {
+				if addr.Type == v1.NodeInternalIP {
+					masterAddress = addr.Address
+					break
 				}
 			}
 		}