-
Notifications
You must be signed in to change notification settings - Fork 0
/
kubernetes.go
252 lines (220 loc) · 7.89 KB
/
kubernetes.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
package kubernetes
import (
"context"
"errors"
"fmt"
"log"
"strings"
"time"
v1 "k8s.io/api/core/v1"
e "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
type K8sClient interface {
DeletePod(ctx context.Context, pod, namespace string) error
GenerateToBeDeletedPodList(ctx context.Context, namespace, eventReason, errorMessage string, counter, pollingInterval int) (map[string]string, error)
PodChecks(ctx context.Context, podName, podNamespace string) error
}
// NewK8sClient discover if kubeconfig creds are inside a Pod or outside the cluster and return a clientSet
func NewK8sClient(kubeconfig string) (*kubeClient, error) {
// read and parse kubeconfig
config, err := rest.InClusterConfig() // creates the in-cluster config
if err != nil {
config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) // creates the out-cluster config
if err != nil {
msg := fmt.Sprintf("The kubeconfig cannot be loaded: %v\n", err)
return nil, errors.New(msg)
}
log.Println("Running from OUTSIDE the cluster")
} else {
log.Println("Running from INSIDE the cluster")
}
// create the clientset for in-cluster/out-cluster config
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
msg := fmt.Sprintf("The clientset cannot be created: %v\n", err)
return nil, errors.New(msg)
}
return &kubeClient{
clientSet: clientset,
}, nil
}
// listPods returns a list with all the Pods in the Cluster
func (c *kubeClient) listPods(ctx context.Context, namespace string) (*[]PodDetails, error) {
api := c.clientSet.CoreV1()
var podData PodDetails
var podsData []PodDetails
// list all Pods in Pending state
pods, err := api.Pods(namespace).List(
ctx,
metav1.ListOptions{
TypeMeta: metav1.TypeMeta{Kind: "Pod"},
// FieldSelector: "status.phase=Pending",
},
)
if err != nil {
msg := fmt.Sprintf("Could not get a list of Pods: \n%v", err)
return &podsData, errors.New(msg)
}
for _, pod := range pods.Items {
podData = PodDetails{
UID: pod.ObjectMeta.UID,
PodName: pod.ObjectMeta.Name,
PodNamespace: pod.ObjectMeta.Namespace,
ResourceVersion: pod.ObjectMeta.ResourceVersion,
Phase: pod.Status.Phase,
ContainerStatuses: pod.Status.ContainerStatuses,
OwnerReferences: pod.ObjectMeta.OwnerReferences,
CreationTimestamp: pod.ObjectMeta.CreationTimestamp.Time,
DeletionTimestamp: pod.ObjectMeta.DeletionTimestamp,
}
podsData = append(podsData, podData)
}
log.Printf("There is a TOTAL of %d Pods in the cluster\n", len(podsData))
return &podsData, nil
}
// GetEvents returns a list of namespaced Events that match Reason
func (c *kubeClient) GetEvents(ctx context.Context, namespace, eventReason, errorMessage string) ([]PodEvent, error) {
api := c.clientSet.CoreV1()
var podEvents []PodEvent
eventList, err := api.Events(namespace).List(
ctx,
metav1.ListOptions{
TypeMeta: metav1.TypeMeta{Kind: "Pod"},
// ResourceVersion: "46641835",
})
if err != nil {
msg := fmt.Sprintf("Could not get Events in namespace: %s\n%s", namespace, err)
return podEvents, errors.New(msg)
}
// keep only Events that match event Reason (eg: FailedCreatePodSandBox)
// keep only Events that have errorMessage
// TO ADD filter out Events older than polling interval
for _, item := range eventList.Items {
if item.Reason == eventReason && strings.Contains(item.Message, errorMessage) {
podEventData := PodEvent{
UID: item.InvolvedObject.UID,
PodName: item.InvolvedObject.Name,
PodNamespace: item.InvolvedObject.Namespace,
ResourceVersion: item.InvolvedObject.ResourceVersion,
Reason: item.Reason,
EventType: item.Type,
Message: item.Message,
FirstTimestamp: item.FirstTimestamp.Time,
LastTimestamp: item.LastTimestamp.Time,
}
podEvents = append(podEvents, podEventData)
}
}
return podEvents, nil
}
// getPodEvents returns Pod Events
func (c *kubeClient) getPodEvents(ctx context.Context, pod, namespace string) ([]PodEvent, error) {
api := c.clientSet.CoreV1()
var podEvents []PodEvent
// get Pod events
eventsStruct, err := api.Events(namespace).List(
ctx,
metav1.ListOptions{
FieldSelector: fmt.Sprintf("involvedObject.name=%s", pod),
TypeMeta: metav1.TypeMeta{Kind: "Pod"},
})
if err != nil {
msg := fmt.Sprintf("Could not go through Pod's Events: %s/%s\n%s", namespace, pod, err)
return podEvents, errors.New(msg)
}
for _, item := range eventsStruct.Items {
podEventData := PodEvent{
UID: item.InvolvedObject.UID,
PodName: item.InvolvedObject.Name,
PodNamespace: item.InvolvedObject.Namespace,
ResourceVersion: item.InvolvedObject.ResourceVersion,
Reason: item.Reason,
EventType: item.Type,
Message: item.Message,
FirstTimestamp: item.FirstTimestamp.Time,
LastTimestamp: item.LastTimestamp.Time,
}
podEvents = append(podEvents, podEventData)
}
if len(podEvents) == 0 {
msg := fmt.Sprintf(
"Pod has 0 Events. Probably it does not exist or it does not have any events in the last hour: %s/%s",
namespace, pod,
)
return podEvents, errors.New(msg)
}
return podEvents, nil
}
// GetPodDetails returns Pod details
func (c *kubeClient) GetPodDetails(ctx context.Context, pod, namespace string) (*PodDetails, error) {
api := c.clientSet.CoreV1()
var item *v1.Pod
var podData PodDetails
var err error
item, err = api.Pods(namespace).Get(
ctx,
pod,
metav1.GetOptions{},
)
if e.IsNotFound(err) {
msg := fmt.Sprintf("Pod %s/%s does not exist anymore", namespace, pod)
return &podData, errors.New(msg)
} else if statusError, isStatus := err.(*e.StatusError); isStatus {
msg := fmt.Sprintf("Error getting pod %s/%s: %v",
namespace, pod, statusError.ErrStatus.Message)
return &podData, errors.New(msg)
} else if err != nil {
msg := fmt.Sprintf("Pod %s/%s has a problem: %v", namespace, pod, err)
return &podData, errors.New(msg)
}
podData = PodDetails{
UID: item.ObjectMeta.UID,
PodName: item.ObjectMeta.Name,
PodNamespace: item.ObjectMeta.Namespace,
ResourceVersion: item.ObjectMeta.ResourceVersion,
Phase: item.Status.Phase,
ContainerStatuses: item.Status.ContainerStatuses,
OwnerReferences: item.ObjectMeta.OwnerReferences,
CreationTimestamp: item.ObjectMeta.CreationTimestamp.Time,
DeletionTimestamp: item.ObjectMeta.DeletionTimestamp,
}
return &podData, nil
}
// DeletePod deletes a Pod
func (c *kubeClient) DeletePod(ctx context.Context, pod, namespace string) error {
api := c.clientSet.CoreV1()
err := api.Pods(namespace).Delete(
ctx,
pod,
metav1.DeleteOptions{},
)
if err != nil {
return err
}
log.Printf("DELETED Pod %s/%s", namespace, pod)
return nil
}
// GenerateToBeDeletedPodList generates a map of Pods that match Event Reason and Error Message
func (c *kubeClient) GenerateToBeDeletedPodList(ctx context.Context, namespace, eventReason, errorMessage string, counter, pollingInterval int) (map[string]string, error) {
var uniquePodList = make(map[string]string)
// get a list of Events that match Reason
eventList, err := c.GetEvents(ctx, namespace, eventReason, errorMessage)
if err != nil {
return uniquePodList, err
}
// Filter out Events that are older than polling interval
eventMaxAge := time.Now().Add(-time.Duration(pollingInterval) * time.Second)
if counter > 0 {
eventList = removeOlderEvents(eventList, eventMaxAge)
}
log.Printf("There is a total of %d Events with Reason: %s", len(eventList), eventReason) // DEBUG
// generate a unique list of Pods that match Event Reason
// we do this because a Pod might have multiple Events with the same Reason
uniquePodList = getUniqueListOfPods(eventList)
log.Printf("There is a total of %d Pods with Reason: %s", len(uniquePodList), eventReason) // DEBUG
return uniquePodList, nil
}