/
statuses.go
160 lines (136 loc) · 5.47 KB
/
statuses.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
package controllers
import (
"context"
trustyaiopendatahubiov1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
// IsAllReady checks if all the necessary readiness fields are true.
func (rs *AvailabilityStatus) IsAllReady() bool {
return rs.PVCReady && rs.DeploymentReady && rs.RouteReady
}
// AvailabilityStatus holds the readiness status of various resources.
type AvailabilityStatus struct {
PVCReady bool
DeploymentReady bool
RouteReady bool
InferenceServiceReady bool
}
func (r *TrustyAIServiceReconciler) updateStatus(ctx context.Context, original *trustyaiopendatahubiov1alpha1.TrustyAIService, update func(saved *trustyaiopendatahubiov1alpha1.TrustyAIService),
) (*trustyaiopendatahubiov1alpha1.TrustyAIService, error) {
saved := &trustyaiopendatahubiov1alpha1.TrustyAIService{}
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
err := r.Client.Get(ctx, client.ObjectKeyFromObject(original), saved)
if err != nil {
return err
}
// Update status here
update(saved)
// Try to update
err = r.Client.Status().Update(ctx, saved)
return err
})
if err != nil {
log.FromContext(ctx).Error(err, "Failed to update TrustyAIService status")
}
return saved, err
}
// reconcileStatuses checks the readiness status of PVC, Deployment, Route and Inference Services
func (r *TrustyAIServiceReconciler) reconcileStatuses(ctx context.Context, instance *trustyaiopendatahubiov1alpha1.TrustyAIService) (ctrl.Result, error) {
var err error
status := AvailabilityStatus{}
// Check for PVC readiness
status.PVCReady, err = r.checkPVCReady(ctx, instance)
if err != nil || !status.PVCReady {
// PVC not ready, requeue
return Requeue()
}
// Check for deployment readiness
status.DeploymentReady, err = r.checkDeploymentReady(ctx, instance)
if err != nil || !status.DeploymentReady {
// Deployment not ready, requeue
return Requeue()
}
// Check for route readiness
status.RouteReady, err = r.checkRouteReady(ctx, instance)
if err != nil || !status.RouteReady {
// Route not ready, requeue
return Requeue()
}
// Check if InferenceServices present
status.InferenceServiceReady, err = r.checkInferenceServicesPresent(ctx, instance.Namespace)
// All checks passed, resources are ready
if status.IsAllReady() {
_, updateErr := r.updateStatus(ctx, instance, func(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
if status.InferenceServiceReady {
UpdateInferenceServicePresent(saved)
} else {
UpdateInferenceServiceNotPresent(saved)
}
UpdatePVCAvailable(saved)
UpdateRouteAvailable(saved)
UpdateTrustyAIServiceAvailable(saved)
saved.Status.Phase = "Ready"
saved.Status.Ready = v1.ConditionTrue
})
if updateErr != nil {
return RequeueWithErrorMessage(ctx, err, "Failed to update status")
}
} else {
_, updateErr := r.updateStatus(ctx, instance, func(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
if status.InferenceServiceReady {
UpdateInferenceServicePresent(saved)
} else {
UpdateInferenceServiceNotPresent(saved)
}
if status.PVCReady {
UpdatePVCAvailable(saved)
} else {
UpdatePVCNotAvailable(saved)
}
if status.RouteReady {
UpdateRouteAvailable(saved)
} else {
UpdateRouteNotAvailable(saved)
}
UpdateTrustyAIServiceNotAvailable(saved)
saved.Status.Phase = "Ready"
saved.Status.Ready = v1.ConditionFalse
})
if updateErr != nil {
return RequeueWithErrorMessage(ctx, err, "Failed to update status")
}
}
// All resources are reconciled, return no error and do not requeue
return ctrl.Result{}, nil
}
func UpdateInferenceServiceNotPresent(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypeInferenceServicesPresent, StatusReasonInferenceServicesNotFound, "InferenceServices not found", v1.ConditionFalse)
saved.Status.Ready = v1.ConditionFalse
}
func UpdateInferenceServicePresent(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypeInferenceServicesPresent, StatusReasonInferenceServicesFound, "InferenceServices found", v1.ConditionTrue)
}
func UpdatePVCNotAvailable(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypePVCAvailable, StatusReasonPVCNotFound, "PersistentVolumeClaim not found", v1.ConditionFalse)
saved.Status.Phase = "Not Ready"
saved.Status.Ready = v1.ConditionFalse
}
func UpdatePVCAvailable(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypePVCAvailable, StatusReasonPVCFound, "PersistentVolumeClaim found", v1.ConditionTrue)
}
func UpdateRouteAvailable(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypeRouteAvailable, StatusReasonRouteFound, "Route found", v1.ConditionTrue)
}
func UpdateRouteNotAvailable(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypeRouteAvailable, StatusReasonRouteNotFound, "Route not found", v1.ConditionFalse)
}
func UpdateTrustyAIServiceAvailable(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypeAvailable, StatusAvailable, StatusAvailable, v1.ConditionTrue)
}
func UpdateTrustyAIServiceNotAvailable(saved *trustyaiopendatahubiov1alpha1.TrustyAIService) {
saved.SetStatus(StatusTypeAvailable, StatusNotAvailable, "Not all components available", v1.ConditionFalse)
}