-
Notifications
You must be signed in to change notification settings - Fork 5.3k
/
utils.go
470 lines (409 loc) · 15.8 KB
/
utils.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
package utils
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"io"
"os"
"reflect"
"regexp"
"sort"
"strings"
"text/template"
"unsafe"
"github.com/Masterminds/sprig/v3"
"github.com/valyala/fasttemplate"
"sigs.k8s.io/yaml"
log "github.com/sirupsen/logrus"
argoappsv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
)
var sprigFuncMap = sprig.GenericFuncMap() // a singleton for better performance
func init() {
// Avoid allowing the user to learn things about the environment.
delete(sprigFuncMap, "env")
delete(sprigFuncMap, "expandenv")
delete(sprigFuncMap, "getHostByName")
sprigFuncMap["normalize"] = SanitizeName
sprigFuncMap["toYaml"] = toYAML
sprigFuncMap["fromYaml"] = fromYAML
sprigFuncMap["fromYamlArray"] = fromYAMLArray
}
type Renderer interface {
RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy *argoappsv1.ApplicationSetSyncPolicy, params map[string]interface{}, useGoTemplate bool, goTemplateOptions []string) (*argoappsv1.Application, error)
}
type Render struct {
}
func copyValueIntoUnexported(destination, value reflect.Value) {
reflect.NewAt(destination.Type(), unsafe.Pointer(destination.UnsafeAddr())).
Elem().
Set(value)
}
func copyUnexported(copy, original reflect.Value) {
var unexported = reflect.NewAt(original.Type(), unsafe.Pointer(original.UnsafeAddr())).Elem()
copyValueIntoUnexported(copy, unexported)
}
func IsJSONStr(str string) bool {
str = strings.TrimSpace(str)
return len(str) > 0 && str[0] == '{'
}
func ConvertYAMLToJSON(str string) (string, error) {
if !IsJSONStr(str) {
jsonStr, err := yaml.YAMLToJSON([]byte(str))
if err != nil {
return str, err
}
return string(jsonStr), nil
}
return str, nil
}
// This function is in charge of searching all String fields of the object recursively and apply templating
// thanks to https://gist.github.com/randallmlough/1fd78ec8a1034916ca52281e3b886dc7
func (r *Render) deeplyReplace(copy, original reflect.Value, replaceMap map[string]interface{}, useGoTemplate bool, goTemplateOptions []string) error {
switch original.Kind() {
// The first cases handle nested structures and translate them recursively
// If it is a pointer we need to unwrap and call once again
case reflect.Ptr:
// To get the actual value of the original we have to call Elem()
// At the same time this unwraps the pointer so we don't end up in
// an infinite recursion
originalValue := original.Elem()
// Check if the pointer is nil
if !originalValue.IsValid() {
return nil
}
// Allocate a new object and set the pointer to it
if originalValue.CanSet() {
copy.Set(reflect.New(originalValue.Type()))
} else {
copyUnexported(copy, original)
}
// Unwrap the newly created pointer
if err := r.deeplyReplace(copy.Elem(), originalValue, replaceMap, useGoTemplate, goTemplateOptions); err != nil {
// Not wrapping the error, since this is a recursive function. Avoids excessively long error messages.
return err
}
// If it is an interface (which is very similar to a pointer), do basically the
// same as for the pointer. Though a pointer is not the same as an interface so
// note that we have to call Elem() after creating a new object because otherwise
// we would end up with an actual pointer
case reflect.Interface:
// Get rid of the wrapping interface
originalValue := original.Elem()
// Create a new object. Now new gives us a pointer, but we want the value it
// points to, so we have to call Elem() to unwrap it
if originalValue.IsValid() {
reflectType := originalValue.Type()
reflectValue := reflect.New(reflectType)
copyValue := reflectValue.Elem()
if err := r.deeplyReplace(copyValue, originalValue, replaceMap, useGoTemplate, goTemplateOptions); err != nil {
// Not wrapping the error, since this is a recursive function. Avoids excessively long error messages.
return err
}
copy.Set(copyValue)
}
// If it is a struct we translate each field
case reflect.Struct:
for i := 0; i < original.NumField(); i += 1 {
var currentType = fmt.Sprintf("%s.%s", original.Type().Field(i).Name, original.Type().PkgPath())
// specific case time
if currentType == "time.Time" {
copy.Field(i).Set(original.Field(i))
} else if currentType == "Raw.k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" || currentType == "Raw.k8s.io/apimachinery/pkg/runtime" {
var unmarshaled interface{}
originalBytes := original.Field(i).Bytes()
convertedToJson, err := ConvertYAMLToJSON(string(originalBytes))
if err != nil {
return fmt.Errorf("error while converting template to json %q: %w", convertedToJson, err)
}
err = json.Unmarshal([]byte(convertedToJson), &unmarshaled)
if err != nil {
return fmt.Errorf("failed to unmarshal JSON field: %w", err)
}
jsonOriginal := reflect.ValueOf(&unmarshaled)
jsonCopy := reflect.New(jsonOriginal.Type()).Elem()
err = r.deeplyReplace(jsonCopy, jsonOriginal, replaceMap, useGoTemplate, goTemplateOptions)
if err != nil {
return fmt.Errorf("failed to deeply replace JSON field contents: %w", err)
}
jsonCopyInterface := jsonCopy.Interface().(*interface{})
data, err := json.Marshal(jsonCopyInterface)
if err != nil {
return fmt.Errorf("failed to marshal templated JSON field: %w", err)
}
copy.Field(i).Set(reflect.ValueOf(data))
} else if err := r.deeplyReplace(copy.Field(i), original.Field(i), replaceMap, useGoTemplate, goTemplateOptions); err != nil {
// Not wrapping the error, since this is a recursive function. Avoids excessively long error messages.
return err
}
}
// If it is a slice we create a new slice and translate each element
case reflect.Slice:
if copy.CanSet() {
copy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
} else {
copyValueIntoUnexported(copy, reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))
}
for i := 0; i < original.Len(); i += 1 {
if err := r.deeplyReplace(copy.Index(i), original.Index(i), replaceMap, useGoTemplate, goTemplateOptions); err != nil {
// Not wrapping the error, since this is a recursive function. Avoids excessively long error messages.
return err
}
}
// If it is a map we create a new map and translate each value
case reflect.Map:
if copy.CanSet() {
copy.Set(reflect.MakeMap(original.Type()))
} else {
copyValueIntoUnexported(copy, reflect.MakeMap(original.Type()))
}
for _, key := range original.MapKeys() {
originalValue := original.MapIndex(key)
if originalValue.Kind() != reflect.String && isNillable(originalValue) && originalValue.IsNil() {
continue
}
// New gives us a pointer, but again we want the value
copyValue := reflect.New(originalValue.Type()).Elem()
if err := r.deeplyReplace(copyValue, originalValue, replaceMap, useGoTemplate, goTemplateOptions); err != nil {
// Not wrapping the error, since this is a recursive function. Avoids excessively long error messages.
return err
}
// Keys can be templated as well as values (e.g. to template something into an annotation).
if key.Kind() == reflect.String {
templatedKey, err := r.Replace(key.String(), replaceMap, useGoTemplate, goTemplateOptions)
if err != nil {
// Not wrapping the error, since this is a recursive function. Avoids excessively long error messages.
return err
}
key = reflect.ValueOf(templatedKey)
}
copy.SetMapIndex(key, copyValue)
}
// Otherwise we cannot traverse anywhere so this finishes the recursion
// If it is a string translate it (yay finally we're doing what we came for)
case reflect.String:
strToTemplate := original.String()
templated, err := r.Replace(strToTemplate, replaceMap, useGoTemplate, goTemplateOptions)
if err != nil {
// Not wrapping the error, since this is a recursive function. Avoids excessively long error messages.
return err
}
if copy.CanSet() {
copy.SetString(templated)
} else {
copyValueIntoUnexported(copy, reflect.ValueOf(templated))
}
return nil
// And everything else will simply be taken from the original
default:
if copy.CanSet() {
copy.Set(original)
} else {
copyUnexported(copy, original)
}
}
return nil
}
// isNillable returns true if the value is something which may be set to nil. This function is meant to guard against a
// panic from calling IsNil on a non-pointer type.
func isNillable(v reflect.Value) bool {
switch v.Kind() {
case reflect.Map, reflect.Pointer, reflect.UnsafePointer, reflect.Interface, reflect.Slice:
return true
}
return false
}
func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy *argoappsv1.ApplicationSetSyncPolicy, params map[string]interface{}, useGoTemplate bool, goTemplateOptions []string) (*argoappsv1.Application, error) {
if tmpl == nil {
return nil, fmt.Errorf("application template is empty")
}
if len(params) == 0 {
return tmpl, nil
}
original := reflect.ValueOf(tmpl)
copy := reflect.New(original.Type()).Elem()
if err := r.deeplyReplace(copy, original, params, useGoTemplate, goTemplateOptions); err != nil {
return nil, err
}
replacedTmpl := copy.Interface().(*argoappsv1.Application)
// Add the 'resources-finalizer' finalizer if:
// The template application doesn't have any finalizers, and:
// a) there is no syncPolicy, or
// b) there IS a syncPolicy, but preserveResourcesOnDeletion is set to false
// See TestRenderTemplateParamsFinalizers in util_test.go for test-based definition of behaviour
if (syncPolicy == nil || !syncPolicy.PreserveResourcesOnDeletion) &&
((*replacedTmpl).ObjectMeta.Finalizers == nil || len((*replacedTmpl).ObjectMeta.Finalizers) == 0) {
(*replacedTmpl).ObjectMeta.Finalizers = []string{"resources-finalizer.argocd.argoproj.io"}
}
return replacedTmpl, nil
}
func (r *Render) RenderGeneratorParams(gen *argoappsv1.ApplicationSetGenerator, params map[string]interface{}, useGoTemplate bool, goTemplateOptions []string) (*argoappsv1.ApplicationSetGenerator, error) {
if gen == nil {
return nil, fmt.Errorf("generator is empty")
}
if len(params) == 0 {
return gen, nil
}
original := reflect.ValueOf(gen)
copy := reflect.New(original.Type()).Elem()
if err := r.deeplyReplace(copy, original, params, useGoTemplate, goTemplateOptions); err != nil {
return nil, fmt.Errorf("failed to replace parameters in generator: %w", err)
}
replacedGen := copy.Interface().(*argoappsv1.ApplicationSetGenerator)
return replacedGen, nil
}
var isTemplatedRegex = regexp.MustCompile(".*{{.*}}.*")
// Replace executes basic string substitution of a template with replacement values.
// remaining in the substituted template.
func (r *Render) Replace(tmpl string, replaceMap map[string]interface{}, useGoTemplate bool, goTemplateOptions []string) (string, error) {
if useGoTemplate {
template, err := template.New("").Funcs(sprigFuncMap).Parse(tmpl)
if err != nil {
return "", fmt.Errorf("failed to parse template %s: %w", tmpl, err)
}
for _, option := range goTemplateOptions {
template = template.Option(option)
}
var replacedTmplBuffer bytes.Buffer
if err = template.Execute(&replacedTmplBuffer, replaceMap); err != nil {
return "", fmt.Errorf("failed to execute go template %s: %w", tmpl, err)
}
return replacedTmplBuffer.String(), nil
}
if !isTemplatedRegex.MatchString(tmpl) {
return tmpl, nil
}
fstTmpl, err := fasttemplate.NewTemplate(tmpl, "{{", "}}")
if err != nil {
return "", fmt.Errorf("invalid template: %w", err)
}
replacedTmpl := fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) {
trimmedTag := strings.TrimSpace(tag)
replacement, ok := replaceMap[trimmedTag].(string)
if len(trimmedTag) == 0 || !ok {
return w.Write([]byte(fmt.Sprintf("{{%s}}", tag)))
}
return w.Write([]byte(replacement))
})
return replacedTmpl, nil
}
// Log a warning if there are unrecognized generators
func CheckInvalidGenerators(applicationSetInfo *argoappsv1.ApplicationSet) error {
hasInvalidGenerators, invalidGenerators := invalidGenerators(applicationSetInfo)
var errorMessage error
if len(invalidGenerators) > 0 {
gnames := []string{}
for n := range invalidGenerators {
gnames = append(gnames, n)
}
sort.Strings(gnames)
aname := applicationSetInfo.ObjectMeta.Name
msg := "ApplicationSet %s contains unrecognized generators: %s"
errorMessage = fmt.Errorf(msg, aname, strings.Join(gnames, ", "))
log.Warnf(msg, aname, strings.Join(gnames, ", "))
} else if hasInvalidGenerators {
name := applicationSetInfo.ObjectMeta.Name
msg := "ApplicationSet %s contains unrecognized generators"
errorMessage = fmt.Errorf(msg, name)
log.Warnf(msg, name)
}
return errorMessage
}
// Return true if there are unknown generators specified in the application set. If we can discover the names
// of these generators, return the names as the keys in a map
func invalidGenerators(applicationSetInfo *argoappsv1.ApplicationSet) (bool, map[string]bool) {
names := make(map[string]bool)
hasInvalidGenerators := false
for index, generator := range applicationSetInfo.Spec.Generators {
v := reflect.Indirect(reflect.ValueOf(generator))
found := false
for i := 0; i < v.NumField(); i++ {
field := v.Field(i)
if !field.CanInterface() {
continue
}
if !reflect.ValueOf(field.Interface()).IsNil() {
found = true
break
}
}
if !found {
hasInvalidGenerators = true
addInvalidGeneratorNames(names, applicationSetInfo, index)
}
}
return hasInvalidGenerators, names
}
func addInvalidGeneratorNames(names map[string]bool, applicationSetInfo *argoappsv1.ApplicationSet, index int) {
// The generator names are stored in the "kubectl.kubernetes.io/last-applied-configuration" annotation
config := applicationSetInfo.ObjectMeta.Annotations["kubectl.kubernetes.io/last-applied-configuration"]
var values map[string]interface{}
err := json.Unmarshal([]byte(config), &values)
if err != nil {
log.Warnf("couldn't unmarshal kubectl.kubernetes.io/last-applied-configuration: %+v", config)
return
}
spec, ok := values["spec"].(map[string]interface{})
if !ok {
log.Warn("coundn't get spec from kubectl.kubernetes.io/last-applied-configuration annotation")
return
}
generators, ok := spec["generators"].([]interface{})
if !ok {
log.Warn("coundn't get generators from kubectl.kubernetes.io/last-applied-configuration annotation")
return
}
if index >= len(generators) {
log.Warnf("index %d out of range %d for generator in kubectl.kubernetes.io/last-applied-configuration", index, len(generators))
return
}
generator, ok := generators[index].(map[string]interface{})
if !ok {
log.Warn("coundn't get generator from kubectl.kubernetes.io/last-applied-configuration annotation")
return
}
for key := range generator {
names[key] = true
break
}
}
func NormalizeBitbucketBasePath(basePath string) string {
if strings.HasSuffix(basePath, "/rest/") {
return strings.TrimSuffix(basePath, "/")
}
if !strings.HasSuffix(basePath, "/rest") {
return basePath + "/rest"
}
return basePath
}
func getTlsConfigWithCACert(scmRootCAPath string) *tls.Config {
tlsConfig := &tls.Config{}
if scmRootCAPath != "" {
_, err := os.Stat(scmRootCAPath)
if os.IsNotExist(err) {
log.Errorf("scmRootCAPath '%s' specified does not exist: %s", scmRootCAPath, err)
return tlsConfig
}
rootCA, err := os.ReadFile(scmRootCAPath)
if err != nil {
log.Errorf("error reading certificate from file '%s', proceeding without custom rootCA : %s", scmRootCAPath, err)
return tlsConfig
}
certPool := x509.NewCertPool()
ok := certPool.AppendCertsFromPEM([]byte(rootCA))
if !ok {
log.Errorf("failed to append certificates from PEM: proceeding without custom rootCA")
} else {
tlsConfig.RootCAs = certPool
}
}
return tlsConfig
}
func GetTlsConfig(scmRootCAPath string, insecure bool) *tls.Config {
tlsConfig := getTlsConfigWithCACert(scmRootCAPath)
if insecure {
tlsConfig.InsecureSkipVerify = true
}
return tlsConfig
}