-
Notifications
You must be signed in to change notification settings - Fork 327
/
chart_health.go
109 lines (97 loc) · 3.1 KB
/
chart_health.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
/*
* Tencent is pleased to support the open source community by making TKEStack
* available.
*
* Copyright (C) 2012-2019 Tencent. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use
* this file except in compliance with the License. You may obtain a copy of the
* License at
*
* https://opensource.org/licenses/Apache-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package chart
import (
"context"
"sync"
"time"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
registryv1 "tkestack.io/tke/api/registry/v1"
"tkestack.io/tke/pkg/util/log"
)
type chartHealth struct {
mu sync.Mutex
charts sets.String
}
func (s *chartHealth) Exist(key string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.charts.Has(key)
}
func (s *chartHealth) Del(key string) {
s.mu.Lock()
defer s.mu.Unlock()
s.charts.Delete(key)
}
func (s *chartHealth) Set(key string) {
s.mu.Lock()
defer s.mu.Unlock()
s.charts.Insert(key)
}
func (c *Controller) startChartHealthCheck(ctx context.Context, key string) {
if !c.health.Exist(key) {
c.health.Set(key)
go func() {
if err := wait.PollImmediateUntil(1*time.Minute, c.watchChartHealth(ctx, key), c.stopCh); err != nil {
log.Error("Failed to wait poll immediate until", log.Err(err))
}
}()
log.Info("Chart phase start new health check", log.String("chart", key))
} else {
log.Info("Chart phase health check exit", log.String("chart", key))
}
}
// for PollImmediateUntil, when return true ,an err while exit
func (c *Controller) watchChartHealth(ctx context.Context, key string) func() (bool, error) {
return func() (bool, error) {
log.Debug("Check chart health", log.String("key", key))
if !c.health.Exist(key) {
return true, nil
}
chartGroupName, chartName, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
log.Error("Failed to split meta chart key", log.String("key", key))
c.health.Del(key)
return true, nil
}
chart, err := c.client.RegistryV1().Charts(chartGroupName).Get(ctx, chartName, metav1.GetOptions{})
if err != nil && errors.IsNotFound(err) {
log.Error("Chart not found, to exit the health check loop",
log.String("chartName", chartName))
c.health.Del(key)
return true, nil
}
if err != nil {
log.Error("Check chart health, chart get failed",
log.String("chartName", chartName), log.Err(err))
return false, nil
}
// if status is terminated,to exit the health check loop
if chart.Status.Phase == registryv1.ChartTerminating || chart.Status.Phase == registryv1.ChartPending {
log.Warn("Chart status is terminated, to exit the health check loop",
log.String("chartName", chartName))
c.health.Del(key)
return true, nil
}
return false, nil
}
}