Fix node filter to check etcd in-sync status properly

Signed-off-by: morimoto-cybozu <kenji_morimoto@cybozu.co.jp>
cybozu-go · Dec 28, 2022 · cfd2165 · cfd2165
1 parent 86050e3
commit cfd2165
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/server/node_filter.go b/server/node_filter.go
@@ -142,7 +142,12 @@ func (nf *NodeFilter) EtcdIsGood() bool {
 	if !st.IsHealthy {
 		return false
 	}
-	return len(st.Members) == len(st.InSyncMembers)
+	for k := range st.Members {
+		if !st.InSyncMembers[k] {
+			return false
+		}
+	}
+	return true
 }
 
 // EtcdStoppedMembers returns control plane nodes that are not running etcd.

diff --git a/server/strategy_test.go b/server/strategy_test.go
@@ -1893,20 +1893,23 @@ func TestDecideOps(t *testing.T) {
 			Name: "EtcdRemoveNonClusterMember",
 			Input: newData().withAllServices().with(func(d testData) {
 				d.Status.Etcd.Members["10.0.0.100"] = &etcdserverpb.Member{Name: "10.0.0.100", ID: 3}
+				d.Status.Etcd.InSyncMembers["10.0.0.100"] = false
 			}),
 			ExpectedOps: []string{"etcd-remove-member"},
 		},
 		{
 			Name: "SkipEtcdRemoveNonClusterMember",
 			Input: newData().withAllServices().with(func(d testData) {
 				d.Status.Etcd.Members["10.0.0.100"] = &etcdserverpb.Member{Name: "10.0.0.100", ID: 3}
+				d.Status.Etcd.InSyncMembers["10.0.0.100"] = false
 			}).withSSHNotConnectedNodes(),
 			ExpectedOps: []string{"wait-kubernetes"},
 		},
 		{
 			Name: "EtcdDestroyNonCPMember",
 			Input: newData().withAllServices().with(func(d testData) {
 				d.Status.Etcd.Members["10.0.0.14"] = &etcdserverpb.Member{Name: "10.0.0.14", ID: 3}
+				d.Status.Etcd.InSyncMembers["10.0.0.14"] = false
 			}),
 			ExpectedOps:        []string{"etcd-destroy-member"},
 			ExpectedTargetNums: map[string]int{"etcd-destroy-member": 1},
@@ -1915,6 +1918,7 @@ func TestDecideOps(t *testing.T) {
 			Name: "EtcdDestroyNonCPMemberSSHNotConnected",
 			Input: newData().withAllServices().with(func(d testData) {
 				d.Status.Etcd.Members["10.0.0.14"] = &etcdserverpb.Member{Name: "10.0.0.14", ID: 3}
+				d.Status.Etcd.InSyncMembers["10.0.0.14"] = false
 			}).withSSHNotConnectedNonCPWorker(3),
 			ExpectedOps:        []string{"etcd-destroy-member"},
 			ExpectedTargetNums: map[string]int{"etcd-destroy-member": 0},
@@ -1924,6 +1928,7 @@ func TestDecideOps(t *testing.T) {
 			Input: newData().withAllServices().with(func(d testData) {
 				d.Status.Etcd.Members["10.0.0.13"].Name = ""
 				d.Status.Etcd.Members["10.0.0.13"].ID = 0
+				d.Status.Etcd.InSyncMembers["10.0.0.13"] = false
 			}),
 			ExpectedOps: []string{"etcd-add-member"},
 		},
@@ -1934,7 +1939,7 @@ func TestDecideOps(t *testing.T) {
 				delete(d.Status.Etcd.Members, "10.0.0.13")
 				delete(d.Status.Etcd.InSyncMembers, "10.0.0.13")
 				// but the cluster is not good enough
-				delete(d.Status.Etcd.InSyncMembers, "10.0.0.12")
+				d.Status.Etcd.InSyncMembers["10.0.0.12"] = false
 			}),
 			ExpectedOps: nil,
 		},
@@ -2214,7 +2219,7 @@ func TestDecideOps(t *testing.T) {
 					Status: cke.RebootStatusQueued,
 				},
 			}).with(func(d testData) {
-				delete(d.Status.Etcd.InSyncMembers, "10.0.0.11")
+				d.Status.Etcd.InSyncMembers["10.0.0.11"] = false
 			}),
 			ExpectedOps:        nil,
 			ExpectedTargetNums: nil,