-
Notifications
You must be signed in to change notification settings - Fork 2
/
jobs_cleanup_controller.go
85 lines (75 loc) · 2.25 KB
/
jobs_cleanup_controller.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
package state
import (
"context"
"errors"
"fmt"
"time"
"github.com/castai/kvisor/pkg/logging"
"github.com/samber/lo"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
)
type JobsCleanupConfig struct {
CleanupInterval time.Duration `validate:"required"`
CleanupJobAge time.Duration `validate:"required"`
Namespace string `validate:"required"`
}
func NewJobsCleanupController(log *logging.Logger, clientset kubernetes.Interface, cfg JobsCleanupConfig) *JobsCleanupController {
if cfg.CleanupInterval == 0 {
cfg.CleanupInterval = 10 * time.Minute
}
if cfg.CleanupJobAge == 0 {
cfg.CleanupJobAge = 10 * time.Minute
}
return &JobsCleanupController{
log: log.WithField("component", "jobs_cleanup"),
clientset: clientset,
cfg: cfg,
}
}
type JobsCleanupController struct {
log *logging.Logger
clientset kubernetes.Interface
cfg JobsCleanupConfig
}
func (c *JobsCleanupController) Run(ctx context.Context) error {
c.log.Info("running")
defer c.log.Infof("stopping")
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(c.cfg.CleanupInterval):
func() {
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
defer cancel()
if err := c.cleanupJobs(ctx); err != nil && !errors.Is(err, context.Canceled) {
c.log.Errorf("jobs cleanup: %v", err)
}
}()
}
}
}
func (c *JobsCleanupController) cleanupJobs(ctx context.Context) error {
selector := labels.Set{"app.kubernetes.io/managed-by": "castai"}.String()
jobs, err := c.clientset.BatchV1().Jobs(c.cfg.Namespace).List(ctx, metav1.ListOptions{
LabelSelector: selector,
})
if err != nil {
return fmt.Errorf("list jobs for cleanup: %w", err)
}
cleanupOlderThan := time.Now().UTC().Add(-c.cfg.CleanupJobAge)
for _, job := range jobs.Items {
if job.CreationTimestamp.Time.UTC().Before(cleanupOlderThan) {
if err := c.clientset.BatchV1().Jobs(c.cfg.Namespace).Delete(ctx, job.Name, metav1.DeleteOptions{
GracePeriodSeconds: lo.ToPtr(int64(0)),
PropagationPolicy: lo.ToPtr(metav1.DeletePropagationBackground),
}); err != nil {
return fmt.Errorf("deleting old job: %w", err)
}
c.log.Infof("deleted old job %q", job.Name)
}
}
return nil
}