/
set.go
119 lines (106 loc) · 3.42 KB
/
set.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package resources
import (
"bytes"
"encoding/json"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/golang/glog"
s3path "k8s.io/test-infra/maintenance/aws-janitor/s3"
)
// Set keeps track of the first time we saw a particular
// ARN, and the global TTL. See Mark() for more details.
type Set struct {
firstSeen map[string]time.Time // ARN -> first time we saw
marked map[string]bool // ARN -> seen this run
swept []string // List of resources we attempted to sweep (to summarize)
ttl time.Duration
}
func LoadSet(sess *session.Session, p *s3path.Path, ttl time.Duration) (*Set, error) {
s := &Set{firstSeen: make(map[string]time.Time), marked: make(map[string]bool), ttl: ttl}
svc := s3.New(sess, &aws.Config{Region: aws.String(p.Region)})
resp, err := svc.GetObject(&s3.GetObjectInput{Bucket: aws.String(p.Bucket), Key: aws.String(p.Key)})
if err != nil {
if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchKey" {
return s, nil
}
return nil, err
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&s.firstSeen); err != nil {
return nil, err
}
return s, nil
}
func (s *Set) Save(sess *session.Session, p *s3path.Path) error {
b, err := json.MarshalIndent(s.firstSeen, "", " ")
if err != nil {
return err
}
svc := s3.New(sess, &aws.Config{Region: aws.String(p.Region)})
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(p.Bucket),
Key: aws.String(p.Key),
Body: bytes.NewReader(b),
CacheControl: aws.String("max-age=0"),
})
return err
}
// Mark marks a particular resource as currently present, and advises
// on whether it should be deleted. If Mark(r) returns true, the TTL
// has expired for r and it should be deleted.
func (s *Set) Mark(r Interface) bool {
key := r.ResourceKey()
now := time.Now()
s.marked[key] = true
if t, ok := s.firstSeen[key]; ok {
since := now.Sub(t)
if since > s.ttl {
s.swept = append(s.swept, key)
return true
}
glog.V(1).Infof("%s: seen for %v", key, since)
return false
}
s.firstSeen[key] = now
glog.V(1).Infof("%s: first seen", key)
if s.ttl == 0 {
// If the TTL is 0, it should be deleted now.
s.swept = append(s.swept, key)
return true
}
return false
}
// MarkComplete figures out which ARNs were in previous passes but not
// this one, and eliminates them. It should only be run after all
// resources have been marked.
func (s *Set) MarkComplete() int {
var gone []string
for key := range s.firstSeen {
if !s.marked[key] {
gone = append(gone, key)
}
}
for _, key := range gone {
glog.V(1).Infof("%s: deleted since last run", key)
delete(s.firstSeen, key)
}
if len(s.swept) > 0 {
glog.Errorf("%d resources swept: %v", len(s.swept), s.swept)
}
return len(s.swept)
}