-
-
Notifications
You must be signed in to change notification settings - Fork 307
/
metadata.go
325 lines (282 loc) · 12.8 KB
/
metadata.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
// Mgmt
// Copyright (C) 2013-2021+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package interfaces
import (
"fmt"
"io"
"io/ioutil"
"strings"
"github.com/purpleidea/mgmt/util/errwrap"
"gopkg.in/yaml.v2"
)
const (
// MetadataFilename is the filename for the metadata storage. This is
// the ideal entry point for any running code.
MetadataFilename = "metadata.yaml"
// FileNameExtension is the filename extension used for languages files.
FileNameExtension = "mcl" // alternate suggestions welcome!
// DotFileNameExtension is the filename extension with a dot prefix.
DotFileNameExtension = "." + FileNameExtension
// MainFilename is the default filename for code to start running from.
MainFilename = "main" + DotFileNameExtension
// PathDirectory is the path directory name we search for modules in.
PathDirectory = "path/"
// FilesDirectory is the files directory name we include alongside
// modules. It can store any useful files that we'd like.
FilesDirectory = "files/"
// ModuleDirectory is the default module directory name. It gets
// appended to whatever the running prefix is or relative to the base
// dir being used for deploys.
ModuleDirectory = "modules/"
)
// Metadata is a data structure representing the module metadata. Since it can
// get moved around to different filesystems, it should only contain relative
// paths.
type Metadata struct {
// Main is the path to the entry file where we start reading code.
// Normally this is main.mcl or the value of the MainFilename constant.
Main string `yaml:"main"`
// Path is the relative path to the local module search path directory
// that we should look in. This is similar to golang's vendor directory.
// If a module wishes to include this directory, it's recommended that
// it have the contained directory be a `git submodule` if possible.
Path string `yaml:"path"`
// Files is the location of the files/ directory which can contain some
// useful additions that might get used in the modules. You can store
// templates, or any other data that you'd like.
// TODO: also allow storing files alongside the .mcl files in their dir!
Files string `yaml:"files"`
// License is the listed license of the module. Use the short names, eg:
// LGPLv3+, or MIT.
License string `yaml:"license"`
// ParentPathBlock specifies whether we're allowed to search in parent
// metadata file Path settings for modules. We always search in the
// global path if we don't find others first. This setting defaults to
// false, which is important because the downloader uses it to decide
// where to put downloaded modules. It is similar to the equivalent of
// a `require vendoring` flag in golang if such a thing existed. If a
// module sets this to true, and specifies a Path value, then only that
// path will be used as long as imports are present there. Otherwise it
// will fall-back on the global modules directory. If a module sets this
// to true, and does not specify a Path value, then the global modules
// directory is automatically chosen for the import location for this
// module. When this is set to true, in no scenario will an import come
// from a directory other than the one specified here, or the global
// modules directory. Module authors should use this sparingly when they
// absolutely need a specific import vendored, otherwise they might
// rouse the ire of module consumers. Keep in mind that you can specify
// a Path directory, and include a git submodule in it, which will be
// used by default, without specifying this option. In that scenario,
// the consumer can decide to not recursively clone your submodule if
// they wish to override it higher up in the module search locations.
ParentPathBlock bool `yaml:"parentpathblock"`
// Metadata stores a link to the parent metadata structure if it exists.
Metadata *Metadata // this does *NOT* get a yaml struct tag
// metadataPath stores the absolute path to this metadata file as it is
// parsed. This is useful when we search upwards for parent Path values.
metadataPath string // absolute path that this file was found in
// TODO: is this needed anymore?
defaultMain *string // set this to pick a default Main when decoding
// bug395 is a flag to workaround the terrible yaml parser resetting all
// the default struct field values when it finds an empty yaml document.
// We set this value to have a default of true, which enables us to know
// if the document was empty or not, and if so, then we know this struct
// was emptied, so we should then return a new struct with all defaults.
// See: https://github.com/go-yaml/yaml/issues/395 for more information.
bug395 bool
}
// DefaultMetadata returns the default metadata that is used for absent values.
func DefaultMetadata() *Metadata {
return &Metadata{ // the defaults
Main: MainFilename, // main.mcl
// This MUST be empty for a top-level default, because if it's
// not, then an undefined Path dir at a lower level won't search
// upwards to find a suitable path, and we'll nest forever...
//Path: PathDirectory, // do NOT set this!
Files: FilesDirectory, // files/
//License: "", // TODO: ???
bug395: true, // workaround, lol
}
}
// SetAbsSelfPath sets the absolute directory path to this metadata file. This
// method is used on a built metadata file so that it can internally know where
// it is located.
func (obj *Metadata) SetAbsSelfPath(p string) error {
obj.metadataPath = p
return nil
}
// ToBytes marshals the struct into a byte array and returns it.
func (obj *Metadata) ToBytes() ([]byte, error) {
return yaml.Marshal(obj) // TODO: obj or *obj ?
}
// NOTE: this is not currently needed, but here for reference.
//// MarshalYAML modifies the struct before it is used to build the raw output.
//func (obj *Metadata) MarshalYAML() (interface{}, error) {
// // The Marshaler interface may be implemented by types to customize
// // their behavior when being marshaled into a YAML document. The
// // returned value is marshaled in place of the original value
// // implementing Marshaler.
//
// if obj.metadataPath == "" { // make sure metadataPath isn't saved!
// return obj, nil
// }
// md := obj.Copy() // TODO: implement me
// md.metadataPath = "" // if set, blank it out before save
// return md, nil
//}
// UnmarshalYAML is the standard unmarshal method for this struct.
func (obj *Metadata) UnmarshalYAML(unmarshal func(interface{}) error) error {
type indirect Metadata // indirection to avoid infinite recursion
def := DefaultMetadata()
// support overriding
if x := obj.defaultMain; x != nil {
def.Main = *x
}
raw := indirect(*def) // convert; the defaults go here
if err := unmarshal(&raw); err != nil {
return err
}
*obj = Metadata(raw) // restore from indirection with type conversion!
return nil
}
// ParseMetadata reads from some input and returns a *Metadata struct that
// contains plausible values to be used.
func ParseMetadata(reader io.Reader) (*Metadata, error) {
metadata := DefaultMetadata() // populate this
//main := MainFilename // set a custom default here if you want
//metadata.defaultMain = &main
// does not work in all cases :/ (fails with EOF files, ioutil does not)
//decoder := yaml.NewDecoder(reader)
////decoder.SetStrict(true) // TODO: consider being strict?
//if err := decoder.Decode(metadata); err != nil {
// return nil, errwrap.Wrapf(err, "can't parse metadata")
//}
b, err := ioutil.ReadAll(reader)
if err != nil {
return nil, errwrap.Wrapf(err, "can't read metadata")
}
if err := yaml.Unmarshal(b, metadata); err != nil {
return nil, errwrap.Wrapf(err, "can't parse metadata")
}
if !metadata.bug395 { // workaround, lol
// we must have gotten an empty document, so use a new default!
metadata = DefaultMetadata()
}
// FIXME: search for unclean paths containing ../ or similar and error!
if strings.HasPrefix(metadata.Main, "/") || strings.HasSuffix(metadata.Main, "/") {
return nil, fmt.Errorf("the Main field must be a relative file path")
}
if metadata.Path != "" && (strings.HasPrefix(metadata.Path, "/") || !strings.HasSuffix(metadata.Path, "/")) {
return nil, fmt.Errorf("the Path field must be undefined or be a relative dir path")
}
if metadata.Files != "" && (strings.HasPrefix(metadata.Files, "/") || !strings.HasSuffix(metadata.Files, "/")) {
return nil, fmt.Errorf("the Files field must be undefined or be a relative dir path")
}
// TODO: add more validation
return metadata, nil
}
// FindModulesPath returns an absolute path to the Path dir where modules can be
// found. This can vary, because the current metadata file might not specify a
// Path value, meaning we'd have to return the global modules path.
// Additionally, we can search upwards for a path if our metadata file allows
// this. It searches with respect to the calling base directory, and uses the
// ParentPathBlock field to determine if we're allowed to search upwards. It
// does logically without doing any filesystem operations.
func FindModulesPath(metadata *Metadata, base, modules string) (string, error) {
ret := func(s string) (string, error) { // return helper function
// don't return an empty string without an error!!!
if s == "" {
return "", fmt.Errorf("can't find a module path")
}
return s, nil
}
m := metadata // start
b := base // absolute base path current metadata file is in
for m != nil {
if m.metadataPath == "" { // a top-level module might be empty!
return ret(modules) // so return this, there's nothing else!
}
if m.metadataPath != b { // these should be the same if no bugs!
return "", fmt.Errorf("metadata inconsistency: `%s` != `%s`", m.metadataPath, b)
}
// does metadata specify where to look ?
// search in the module specific space
if m.Path != "" { // use this path, since it was specified!
if !strings.HasSuffix(m.Path, "/") {
return "", fmt.Errorf("metadata inconsistency: path `%s` has no trailing slash", m.Path)
}
return ret(b + m.Path) // join w/o cleaning trailing slash
}
// are we allowed to search incrementally upwards?
if m.ParentPathBlock {
break
}
// search upwards (search in parent dirs upwards recursively...)
m = m.Metadata // might be nil
if m != nil {
b = m.metadataPath // get new parent path
}
}
// by now we haven't found a metadata path, so we use the global path...
return ret(modules) // often comes from an ENV or a default
}
// FindModulesPathList does what FindModulesPath does, except this function
// returns the entirely linear string of possible module locations until it gets
// to the root. This can be useful if you'd like to know which possible
// locations are valid, so that you can search through them to see if there is
// downloaded code available.
func FindModulesPathList(metadata *Metadata, base, modules string) ([]string, error) {
found := []string{}
ret := func(s []string) ([]string, error) { // return helper function
// don't return an empty list without an error!!!
if s == nil || len(s) == 0 {
return nil, fmt.Errorf("can't find any module paths")
}
return s, nil
}
m := metadata // start
b := base // absolute base path current metadata file is in
for m != nil {
if m.metadataPath == "" { // a top-level module might be empty!
return ret([]string{modules}) // so return this, there's nothing else!
}
if m.metadataPath != b { // these should be the same if no bugs!
return nil, fmt.Errorf("metadata inconsistency: `%s` != `%s`", m.metadataPath, b)
}
// does metadata specify where to look ?
// search in the module specific space
if m.Path != "" { // use this path, since it was specified!
if !strings.HasSuffix(m.Path, "/") {
return nil, fmt.Errorf("metadata inconsistency: path `%s` has no trailing slash", m.Path)
}
p := b + m.Path // join w/o cleaning trailing slash
found = append(found, p) // add to list
}
// are we allowed to search incrementally upwards?
if m.ParentPathBlock {
break
}
// search upwards (search in parent dirs upwards recursively...)
m = m.Metadata // might be nil
if m != nil {
b = m.metadataPath // get new parent path
}
}
// add the global path to everything we've found...
found = append(found, modules) // often comes from an ENV or a default
return ret(found)
}