-
Notifications
You must be signed in to change notification settings - Fork 56
/
retrying_k8s_client.go
108 lines (92 loc) · 3.2 KB
/
retrying_k8s_client.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
package authorization
import (
"context"
"time"
"code.cloudfoundry.org/korifi/api/correlation"
"code.cloudfoundry.org/korifi/controllers/webhooks"
"github.com/go-logr/logr"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func NewDefaultBackoff() wait.Backoff {
return wait.Backoff{
Duration: 5 * time.Millisecond,
Factor: 2,
Steps: 10,
}
}
type AuthRetryingClient struct {
client.WithWatch
backoff wait.Backoff
logger logr.Logger
}
func NewAuthRetryingClient(c client.WithWatch, backoff wait.Backoff) client.WithWatch {
return AuthRetryingClient{
WithWatch: c,
backoff: backoff,
logger: ctrl.Log.WithName("retrying-client"),
}
}
func (a AuthRetryingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error {
return a.retryOnForbidden(ctx, func() error {
return a.WithWatch.Get(ctx, key, obj)
}, "get")
}
func (a AuthRetryingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
return a.retryOnForbidden(ctx, func() error {
return a.WithWatch.List(ctx, list, opts...)
}, "list")
}
func (a AuthRetryingClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
return a.retryOnForbidden(ctx, func() error {
return a.WithWatch.Create(ctx, obj, opts...)
}, "create")
}
func (a AuthRetryingClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
return a.retryOnForbidden(ctx, func() error {
return a.WithWatch.Delete(ctx, obj, opts...)
}, "delete")
}
func (a AuthRetryingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
return a.retryOnForbidden(ctx, func() error {
return a.WithWatch.Update(ctx, obj, opts...)
}, "update")
}
func (a AuthRetryingClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
return a.retryOnForbidden(ctx, func() error {
return a.WithWatch.Patch(ctx, obj, patch, opts...)
}, "patch")
}
func (a AuthRetryingClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
return a.retryOnForbidden(ctx, func() error {
return a.WithWatch.DeleteAllOf(ctx, obj, opts...)
}, "deleteAllOf")
}
func (a AuthRetryingClient) retryOnForbidden(ctx context.Context, fn func() error, op string) error {
count := 0
return retry.OnError(a.backoff, isForbidden, func() error {
logger := correlation.AddCorrelationIDToLogger(ctx, a.logger)
err := fn()
if err != nil {
count++
logger.Info("k8s client returned forbidden", "op", op, "count", count, "error", err)
}
return err
})
}
// isForbidden returns true for forbidden errors that are NOT korifi webhook
// validation errors, false otherwise upon webhook validation errors it makes
// no sense to retry the operation as the webhook is expected to consistently
// return the same validation error
func isForbidden(err error) bool {
if !k8serrors.IsForbidden(err) {
return false
}
if _, isValidationErr := webhooks.WebhookErrorToValidationError(err); isValidationErr {
return false
}
return true
}