forked from fluxcd/flux
/
load.go
201 lines (179 loc) · 5.43 KB
/
load.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
package resource
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/pkg/errors"
"github.com/weaveworks/flux/resource"
)
// Load takes paths to directories or files, and creates an object set
// based on the file(s) therein. Resources are named according to the
// file content, rather than the file name of directory structure.
func Load(base string, paths []string) (map[string]resource.Resource, error) {
objs := map[string]resource.Resource{}
charts, err := newChartTracker(base)
if err != nil {
return nil, errors.Wrapf(err, "walking %q for chartdirs", base)
}
for _, root := range paths {
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return errors.Wrapf(err, "walking %q for yamels", path)
}
if charts.isDirChart(path) {
return filepath.SkipDir
}
if charts.isPathInChart(path) {
return nil
}
if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" {
bytes, err := ioutil.ReadFile(path)
if err != nil {
return errors.Wrapf(err, "unable to read file at %q", path)
}
source, err := filepath.Rel(base, path)
if err != nil {
return errors.Wrapf(err, "path to scan %q is not under base %q", path, base)
}
docsInFile, err := ParseMultidoc(bytes, source)
if err != nil {
return err
}
for id, obj := range docsInFile {
if alreadyDefined, ok := objs[id]; ok {
return fmt.Errorf(`duplicate definition of '%s' (in %s and %s)`, id, alreadyDefined.Source(), source)
}
objs[id] = obj
}
}
return nil
})
if err != nil {
return objs, err
}
}
return objs, nil
}
type chartTracker map[string]bool
func newChartTracker(root string) (chartTracker, error) {
var chartdirs = make(map[string]bool)
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
return errors.Wrapf(err, "walking %q for charts", path)
}
if info.IsDir() && looksLikeChart(path) {
chartdirs[path] = true
return filepath.SkipDir
}
return nil
})
if err != nil {
return nil, err
}
return chartTracker(chartdirs), nil
}
func (c chartTracker) isDirChart(path string) bool {
return c[path]
}
func (c chartTracker) isPathInChart(path string) bool {
p := path
root := fmt.Sprintf("%c", filepath.Separator)
for p != root {
if c[p] {
return true
}
p = filepath.Dir(p)
}
return false
}
// looksLikeChart returns `true` if the path `dir` (assumed to be a
// directory) looks like it contains a Helm chart, rather than
// manifest files.
func looksLikeChart(dir string) bool {
// These are the two mandatory parts of a chart. If they both
// exist, chances are it's a chart. See
// https://github.com/kubernetes/helm/blob/master/docs/charts.md#the-chart-file-structure
chartpath := filepath.Join(dir, "Chart.yaml")
valuespath := filepath.Join(dir, "values.yaml")
if _, err := os.Stat(chartpath); err != nil && os.IsNotExist(err) {
return false
}
if _, err := os.Stat(valuespath); err != nil && os.IsNotExist(err) {
return false
}
return true
}
// ParseMultidoc takes a dump of config (a multidoc YAML) and
// constructs an object set from the resources represented therein.
func ParseMultidoc(multidoc []byte, source string) (map[string]resource.Resource, error) {
objs := map[string]resource.Resource{}
chunks := bufio.NewScanner(bytes.NewReader(multidoc))
initialBuffer := make([]byte, 4096) // Matches startBufSize in bufio/scan.go
chunks.Buffer(initialBuffer, 1024*1024) // Allow growth to 1MB
chunks.Split(splitYAMLDocument)
var obj resource.Resource
var err error
for chunks.Scan() {
// It's not guaranteed that the return value of Bytes() will not be mutated later:
// https://golang.org/pkg/bufio/#Scanner.Bytes
// But we will be snaffling it away, so make a copy.
bytes := chunks.Bytes()
bytes2 := make([]byte, len(bytes), cap(bytes))
copy(bytes2, bytes)
if obj, err = unmarshalObject(source, bytes2); err != nil {
return nil, errors.Wrapf(err, "parsing YAML doc from %q", source)
}
if obj == nil {
continue
}
// Lists must be treated specially, since it's the
// contained resources we are after.
if list, ok := obj.(*List); ok {
for _, item := range list.Items {
objs[item.ResourceID().String()] = item
}
} else {
objs[obj.ResourceID().String()] = obj
}
}
if err := chunks.Err(); err != nil {
return objs, errors.Wrapf(err, "scanning multidoc from %q", source)
}
return objs, nil
}
// ---
// Taken directly from https://github.com/kubernetes/apimachinery/blob/master/pkg/util/yaml/decoder.go.
const yamlSeparator = "\n---"
// splitYAMLDocument is a bufio.SplitFunc for splitting YAML streams into individual documents.
func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
sep := len([]byte(yamlSeparator))
if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
// We have a potential document terminator
i += sep
after := data[i:]
if len(after) == 0 {
// we can't read any more characters
if atEOF {
return len(data), data[:len(data)-sep], nil
}
return 0, nil, nil
}
if j := bytes.IndexByte(after, '\n'); j >= 0 {
return i + j + 1, data[0 : i-sep], nil
}
return 0, nil, nil
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
}
// ---