This repository has been archived by the owner on Nov 10, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 31
/
component_condition.go
177 lines (160 loc) · 5.15 KB
/
component_condition.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
package controllers
import (
"context"
"fmt"
"github.com/milvus-io/milvus-operator/apis/milvus.io/v1beta1"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
//go:generate mockgen -package=controllers -source=component_condition.go -destination=component_condition_mock.go ComponentConditionGetter
type ComponentConditionGetter interface {
GetMilvusInstanceCondition(ctx context.Context, cli client.Client, mc v1beta1.Milvus) (v1beta1.MilvusCondition, error)
}
type ComponentConditionGetterImpl struct{}
func (c ComponentConditionGetterImpl) GetMilvusInstanceCondition(ctx context.Context, cli client.Client, mc v1beta1.Milvus) (v1beta1.MilvusCondition, error) {
if mc.Spec.IsStopping() {
reason := v1beta1.ReasonMilvusStopping
msg := MessageMilvusStopped
stopped, err := CheckMilvusStopped(ctx, cli, mc)
if err != nil {
return v1beta1.MilvusCondition{}, err
}
if stopped {
reason = v1beta1.ReasonMilvusStopped
msg = MessageMilvusStopping
}
return v1beta1.MilvusCondition{
Type: v1beta1.MilvusReady,
Status: corev1.ConditionFalse,
Reason: reason,
Message: msg,
}, nil
}
if !IsDependencyReady(mc.Status.Conditions) {
notReadyConditions := GetNotReadyDependencyConditions(mc.Status.Conditions)
reason := v1beta1.ReasonDependencyNotReady
var msg string
for depType, notReadyCondition := range notReadyConditions {
if notReadyCondition != nil {
msg += fmt.Sprintf("dep[%s]: %s;", depType, notReadyCondition.Message)
} else {
msg = "condition not probed yet"
}
}
ctrl.LoggerFrom(ctx).Info("milvus dependency unhealty", "reason", reason, "msg", msg)
}
deployList := &appsv1.DeploymentList{}
opts := &client.ListOptions{
Namespace: mc.Namespace,
}
opts.LabelSelector = labels.SelectorFromSet(map[string]string{
AppLabelInstance: mc.GetName(),
AppLabelName: "milvus",
})
if err := cli.List(ctx, deployList, opts); err != nil {
return v1beta1.MilvusCondition{}, err
}
allComponents := GetComponentsBySpec(mc.Spec)
var notReadyComponents []string
var errDetail *ComponentErrorDetail
var err error
componentDeploy := makeComponentDeploymentMap(mc, deployList.Items)
hasReadyReplica := false
for _, component := range allComponents {
deployment := componentDeploy[component.Name]
if deployment != nil && DeploymentReady(deployment.Status) {
if deployment.Status.ReadyReplicas > 0 {
hasReadyReplica = true
}
continue
}
notReadyComponents = append(notReadyComponents, component.Name)
if errDetail == nil {
errDetail, err = getComponentErrorDetail(ctx, cli, component.Name, deployment)
if err != nil {
return v1beta1.MilvusCondition{}, errors.Wrap(err, "failed to get component err detail")
}
}
}
cond := v1beta1.MilvusCondition{
Type: v1beta1.MilvusReady,
}
if len(notReadyComponents) == 0 {
if !hasReadyReplica {
return v1beta1.MilvusCondition{}, nil
}
cond.Status = corev1.ConditionTrue
cond.Reason = v1beta1.ReasonMilvusHealthy
cond.Message = MessageMilvusHealthy
} else {
cond.Status = corev1.ConditionFalse
cond.Reason = v1beta1.ReasonMilvusComponentNotHealthy
cond.Message = fmt.Sprintf("%s not ready, detail: %s", notReadyComponents, errDetail)
ctrl.LoggerFrom(ctx).Info("milvus unhealty", "reason", cond.Reason, "msg", cond.Message)
}
return cond, nil
}
var getComponentErrorDetail = func(ctx context.Context, cli client.Client, component string, deploy *appsv1.Deployment) (*ComponentErrorDetail, error) {
ret := &ComponentErrorDetail{ComponentName: component}
if deploy == nil {
return ret, nil
}
if deploy.Status.ObservedGeneration < deploy.Generation {
ret.NotObserved = true
return ret, nil
}
var err error
ret.Deployment, err = GetDeploymentFalseCondition(*deploy)
if err != nil {
return ret, err
}
pods := &corev1.PodList{}
opts := &client.ListOptions{
Namespace: deploy.Namespace,
}
opts.LabelSelector = labels.SelectorFromSet(deploy.Spec.Selector.MatchLabels)
if err := cli.List(ctx, pods, opts); err != nil {
return nil, errors.Wrap(err, "list pods")
}
if len(pods.Items) == 0 {
return ret, nil
}
for _, pod := range pods.Items {
if !PodReady(pod) {
podCondition, err := GetPodFalseCondition(pod)
if err != nil {
return nil, err
}
ret.PodName = pod.Name
ret.Pod = podCondition
ret.Container = getFirstNotReadyContainerStatus(pod.Status.ContainerStatuses)
return ret, nil
}
}
return ret, nil
}
func GetComponentConditionGetter() ComponentConditionGetter {
return singletonComponentConditionGetter
}
var singletonComponentConditionGetter ComponentConditionGetter = ComponentConditionGetterImpl{}
func CheckMilvusStopped(ctx context.Context, cli client.Client, mc v1beta1.Milvus) (bool, error) {
podList := &corev1.PodList{}
opts := &client.ListOptions{
Namespace: mc.Namespace,
}
opts.LabelSelector = labels.SelectorFromSet(map[string]string{
AppLabelInstance: mc.GetName(),
AppLabelName: "milvus",
})
if err := cli.List(ctx, podList, opts); err != nil {
return false, err
}
if len(podList.Items) > 0 {
return false, nil
}
return true, nil
}