-
Notifications
You must be signed in to change notification settings - Fork 3
/
config.go
295 lines (267 loc) · 7.73 KB
/
config.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
// Copyright 2023 Block, Inc.
package config
import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
human "github.com/dustin/go-humanize"
"gopkg.in/yaml.v2"
"github.com/square/finch"
)
// stageFile represents the stage file layout:
//
// stage:
// name: read-only
// runtime: 60s
// workload:
// - clients: 1
//
// This is purely "decorative" because I like the explicit "stage" at top
// because it makes it clear that what follows is a stage config.
type stageFile struct {
Stage Stage `yaml:"stage"`
}
func Load(stageFiles []string, kvparams []string, dsn, db string) ([]Stage, error) {
var err error
base := map[string]Base{}
stages := []Stage{}
// Get and restore current working dir because we chdir to validate stage stageFiles
cwd, err := os.Getwd()
if err != nil {
return nil, err
}
finch.Debug("cwd %s", cwd)
defer func() {
os.Chdir(cwd)
}()
params := map[string]string{}
for _, kv := range kvparams {
f := strings.SplitN(kv, "=", 2)
if len(f) != 2 {
log.Printf("Ignoring invalid --param %s: split into %d fields, expected 2\n", kv, len(f))
continue
}
params[f[0]] = f[1]
}
for n, fileName := range stageFiles {
// Load base file (_all.yaml) once for the dir, if it exists
dir := filepath.Dir(fileName)
b, ok := base[dir]
if !ok {
baseFile := filepath.Join(dir, "_all.yaml")
if FileExists(baseFile) {
finch.Debug("base: %s", baseFile)
bytes, err := read(baseFile)
if err != nil {
return nil, err
} else {
var newb Base
if err := yaml.UnmarshalStrict(bytes, &newb); err != nil {
return nil, fmt.Errorf("cannot decode YAML in %s: %s", fileName, err)
}
base[dir] = newb
b = newb
finch.Debug("base: %+v", b)
}
} else {
finch.Debug("base: none in %s", dir)
}
// --param foo=bar on command line overrides .params in stage files
if len(b.Params) > 0 {
if b.Params == nil {
b.Params = map[string]string{}
}
for k, v := range params {
b.Params[k] = v
}
}
base[dir] = b
}
// Load stage file, which includes and overwrite the optional base config (b)
bytes, err := read(fileName)
if err != nil {
return nil, err
}
absFile, err := filepath.Abs(fileName)
if err != nil {
return nil, err
}
f := &stageFile{Stage: Stage{
File: absFile,
N: uint(n + 1),
}}
if err := yaml.UnmarshalStrict(bytes, f); err != nil {
return nil, fmt.Errorf("cannot decode YAML in %s: %s", fileName, err)
}
// Set stage with defaults (base)
f.Stage.With(b)
// --dsn and --database on command line override config files
f.Stage.CommandLine(dsn, db)
// interpolate $vars -> values (see Vars func below)
if err := f.Stage.Vars(); err != nil {
return nil, fmt.Errorf("in %s: %s", fileName, err)
}
// Chdir to confit file so relative trx file paths in the config work,
// e.g. "trx.file: trx/foo.sql" where trx/ is relative to the dir where
// the config file is located.
os.Chdir(filepath.Dir(fileName))
// Validate config now that it's final (interpolated vars and chdir)
if err := f.Stage.Validate(); err != nil {
return nil, fmt.Errorf("%s invalid: %s", fileName, err)
}
stages = append(stages, f.Stage)
finch.Debug("%+v", f.Stage)
os.Chdir(cwd)
}
return stages, nil
}
func read(filePath string) ([]byte, error) {
finch.Debug("read %s", filePath)
file, err := filepath.Abs(filePath)
if err != nil {
return nil, err
}
if _, err := os.Stat(file); err != nil {
return nil, err
}
return ioutil.ReadFile(file)
}
// ValidFreq validates the freq value for the given config section and returns
// nil if valid, else returns an error.
func ValidFreq(freq, config string) error {
if freq == "" {
return nil
}
if freq == "0" {
return fmt.Errorf("invalid config.%s: 0: must be greater than zero", config)
}
d, err := time.ParseDuration(freq)
if err != nil {
return fmt.Errorf("invalid config.%s: %s: %s", config, freq, err)
}
if d <= 0 {
return fmt.Errorf("invalid config.%s: %s (%d): value <= 0; must be greater than zero", config, freq, d)
}
return nil
}
// True returns true if b is non-nil and true.
// This is convenience function related to *bool files in config structs,
// which is required for knowing when a bool config is explicitily set
// or not. If set, it's not changed; if not, it's set to the default value.
// That makes a good config experience but a less than ideal code experience
// because !*b will panic if b is nil, hence the need for this func.
func True(b *bool) bool {
if b == nil {
return false
}
return *b
}
func FileExists(filePath string) bool {
file, err := filepath.Abs(filePath)
if err != nil {
return false
}
_, err = os.Stat(file)
return err == nil
}
var varRE = []*regexp.Regexp{
regexp.MustCompile(`\${([^}]+)}`), // ${param.foo} for "hello${param.foo}bar"
regexp.MustCompile(`\$([^\s"']+)`), // $param.foo for standalone value
}
var reHumanNumber = regexp.MustCompile(`([\d,]*\d+(?i:[MKGBI]*))`) // 1M or 1,000,000 -> 1000000
var reAllDigits = regexp.MustCompile(`^\d+$`)
// Vars changes $params.foo and $FOO to param values and environment variable
// values, respectively, and human numbers to integers (1k -> 1000).
// "${var}" is also valid but YAML requires string quotes around {}.
func Vars(s string, params map[string]string, numbers bool) (string, error) {
for _, r := range varRE {
m := r.FindAllStringSubmatch(s, -1)
if len(m) == 0 {
continue
}
rep := make([]string, 0, len(m)*2)
for i := range m {
v := m[i]
//p := strings.Trim(v[1], "{}")
p := v[1]
switch {
case strings.HasPrefix(p, "params."):
k := strings.TrimPrefix(p, "params.")
val, ok := params[k]
if !ok {
return "", fmt.Errorf("%s not defined (is it spelled correctly?)", p)
}
rep = append(rep, v[0], val)
finch.Debug("param: %s -> %v (user-defined)", s, rep)
case strings.HasPrefix(p, "sys."):
k := strings.TrimPrefix(p, "sys.")
val, ok := finch.SystemParams[k]
if !ok {
return "", fmt.Errorf("%s not defined (is it spelled correctly?)", p)
}
rep = append(rep, v[0], val)
finch.Debug("param: %s -> %v (built-in)", s, rep)
default:
val, ok := os.LookupEnv(p)
if !ok {
return "", fmt.Errorf("environment variable %s not set (is it spelled correctly?)", p)
}
rep = append(rep, v[0], val)
finch.Debug("param: %s -> %v (env var)", s, rep)
}
}
r := strings.NewReplacer(rep...)
s = r.Replace(s)
}
if !numbers {
return s, nil
}
// Look for human numbers like 1k and 1,000
m := reHumanNumber.FindAllStringSubmatch(s, -1)
if len(m) == 0 {
return s, nil // no human numbers
}
rep := []string{}
for i := range m {
// To keep the regex simple, reHumanNumber also matches ints like 1000,
// but don't waste time parsing something that's already machine number.
if reAllDigits.MatchString(m[i][0]) {
continue
}
d, err := human.ParseBytes(m[i][1])
if err != nil {
return "", fmt.Errorf("invalid human-readable number: %s: %s", m[i][1], err)
}
rep = append(rep, m[i][0], strconv.FormatUint(d, 10))
}
if len(rep) == 0 { // all machine numbers (see comment above)
return s, nil
}
finch.Debug("var: %s -> %v", s, rep)
r := strings.NewReplacer(rep...)
return r.Replace(s), nil
}
// setBool sets c to the value of b if c is nil (not set). Pointers are required
// because var b bool is false by default whether set or not, so we can't tell if
// it's explicitly set in config file. But var b *bool is only false if explicitly
// set b=false, else it's nil, so no we can tell if the var is set or not.
func setBool(c *bool, b *bool) *bool {
if c == nil && b != nil {
c = new(bool)
*c = *b
}
return c
}
func parseInt(s string) error {
if s == "" {
return nil
}
_, err := strconv.ParseUint(s, 10, 32)
return err
}