/
collect_variables.go
343 lines (297 loc) · 11.1 KB
/
collect_variables.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
package inputvars
import (
"fmt"
"github.com/turbot/steampipe/pkg/error_helpers"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
"log"
"os"
"regexp"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/spf13/viper"
"github.com/turbot/steampipe/pkg/constants"
"github.com/turbot/steampipe/pkg/filepaths"
"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig/var_config"
"github.com/turbot/terraform-components/tfdiags"
)
// CollectVariableValues inspects the various places that configuration input variable
// values can come from and constructs a map ready to be passed to the
// backend as part of a Operation.
//
// This method returns diagnostics relating to the collection of the values,
// but the values themselves may produce additional diagnostics when finally
// parsed.
func CollectVariableValues(workspacePath string, variableFileArgs []string, variablesArgs []string, workspaceMod *modconfig.Mod) (map[string]UnparsedVariableValue, error) {
workspaceModName := workspaceMod.ShortName
var modNames = make(map[string]struct{})
for _, m := range workspaceMod.ResourceMaps.Mods {
modNames[m.ShortName] = struct{}{}
}
ret := map[string]UnparsedVariableValue{}
// First we'll deal with environment variables
// since they have the lowest precedence.
// (apart from values in the mod Require proeprty, which are handled separately later)
{
env := os.Environ()
for _, raw := range env {
if !strings.HasPrefix(raw, constants.EnvInputVarPrefix) {
continue
}
raw = raw[len(constants.EnvInputVarPrefix):] // trim the prefix
eq := strings.Index(raw, "=")
if eq == -1 {
// Seems invalid, so we'll ignore it.
continue
}
name := raw[:eq]
rawVal := raw[eq+1:]
ret[name] = unparsedVariableValueString{
str: rawVal,
name: name,
sourceType: ValueFromEnvVar,
}
log.Printf("[INFO] adding value for variable '%s' from environment", name)
}
}
// Next up we have some implicit files that are loaded automatically
// if they are present. There's the original terraform.tfvars
// (constants.DefaultVarsFilename) along with the later-added search for all files
// ending in .auto.spvars.
defaultVarsPath := filepaths.DefaultVarsFilePath(workspacePath)
if _, err := os.Stat(defaultVarsPath); err == nil {
log.Printf("[INFO] adding values from %s", defaultVarsPath)
diags := addVarsFromFile(defaultVarsPath, ValueFromAutoFile, ret)
if diags.HasErrors() {
return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", defaultVarsPath), diags)
}
}
if infos, err := os.ReadDir("."); err == nil {
// "infos" is already sorted by name, so we just need to filter it here.
for _, info := range infos {
name := info.Name()
if !isAutoVarFile(name) {
continue
}
log.Printf("[INFO] adding values from %s", name)
diags := addVarsFromFile(name, ValueFromAutoFile, ret)
if diags.HasErrors() {
return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", name), diags)
}
}
}
// Finally we process values given explicitly on the command line, either
// as individual literal settings or as additional files to read.
for _, fileArg := range variableFileArgs {
log.Printf("[INFO] adding values from %s", fileArg)
diags := addVarsFromFile(fileArg, ValueFromNamedFile, ret)
if diags.HasErrors() {
return nil, error_helpers.DiagsToError(fmt.Sprintf("failed to load variables from '%s'", fileArg), diags)
}
}
var diags tfdiags.Diagnostics
for _, variableArg := range variablesArgs {
// Value should be in the form "name=value", where value is a
// raw string whose interpretation will depend on the variable's
// parsing mode.
raw := variableArg
eq := strings.Index(raw, "=")
if eq == -1 {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
fmt.Sprintf("The given --var option %q is not correctly specified. It must be a variable name and value separated an equals sign: --var key=value", raw),
"",
))
continue
}
name := raw[:eq]
rawVal := raw[eq+1:]
ret[name] = unparsedVariableValueString{
str: rawVal,
name: name,
sourceType: ValueFromCLIArg,
}
log.Printf("[INFO] adding value for variable '%s' from command line arg", name)
}
if diags.HasErrors() {
return nil, error_helpers.DiagsToError("failed to evaluate var args:", diags)
}
// check viper for any interactively added variables
if varMap := viper.GetStringMap(constants.ConfigInteractiveVariables); varMap != nil {
for name, rawVal := range varMap {
// Value should be in the form "name=value", where value is a
// raw string whose interpretation will depend on the variable's
// parsing mode.
ret[name] = UnparsedInteractiveVariableValue{
Name: name,
RawValue: rawVal.(string),
}
log.Printf("[INFO] adding value for variable '%s' specified on interactive prompt", name)
}
}
// now map any variable names of form <modname>.<variablename> to <modname>.var.<varname>
// - if any var value is qualified with the workspace mod, remove the qualification
// - remove any variables which are not in the root mod or first level dependencies
ret = transformVarNames(ret, workspaceModName, modNames)
return ret, nil
}
// map any variable names of form <modname>.<variablename> to <modname>.var.<varname>
func transformVarNames(rawValues map[string]UnparsedVariableValue, workspaceModName string, modNames map[string]struct{}) map[string]UnparsedVariableValue {
ret := make(map[string]UnparsedVariableValue, len(rawValues))
for k, v := range rawValues {
parts := strings.Split(k, ".")
if len(parts) > 1 {
if _, ok := modNames[parts[0]]; !ok {
// NOTE: skip any variables which are not in the root mod or first level dependencies
continue
}
if parts[0] == workspaceModName {
k = parts[1]
} else {
k = fmt.Sprintf("%s.var.%s", parts[0], parts[1])
}
}
ret[k] = v
}
return ret
}
func addVarsFromFile(filename string, sourceType ValueSourceType, to map[string]UnparsedVariableValue) tfdiags.Diagnostics {
var diags tfdiags.Diagnostics
src, err := os.ReadFile(filename)
if err != nil {
if os.IsNotExist(err) {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to read variables file",
fmt.Sprintf("Given variables file %s does not exist.", filename),
))
} else {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to read variables file",
fmt.Sprintf("Error while reading %s: %s.", filename, err),
))
}
return diags
}
// replace syntax `<modname>.<varname>=<var_value>` with `___steampipe_<modname>_<varname>=<var_value>
sanitisedSrc, depVarAliases := sanitiseVariableNames(src)
var f *hcl.File
var hclDiags hcl.Diagnostics
// attempt to parse the config
f, hclDiags = hclsyntax.ParseConfig(sanitisedSrc, filename, hcl.Pos{Line: 1, Column: 1})
diags = diags.Append(hclDiags)
if f == nil || f.Body == nil {
return diags
}
// Before we do our real decode, we'll probe to see if there are any blocks
// of type "variable" in this body, since it's a common mistake for new
// users to put variable declarations in tfvars rather than variable value
// definitions, and otherwise our error message for that case is not so
// helpful.
{
content, _, _ := f.Body.PartialContent(&hcl.BodySchema{
Blocks: []hcl.BlockHeaderSchema{
{
Type: "variable",
LabelNames: []string{"name"},
},
},
})
for _, block := range content.Blocks {
name := block.Labels[0]
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Variable declaration in .tfvars file",
Detail: fmt.Sprintf("A .tfvars file is used to assign values to variables that have already been declared in .tf files, not to declare new variables. To declare variable %q, place this block in one of your .tf files, such as variables.tf.\n\nTo set a value for this variable in %s, use the definition syntax instead:\n %s = <value>", name, block.TypeRange.Filename, name),
Subject: &block.TypeRange,
})
}
if diags.HasErrors() {
// If we already found problems then JustAttributes below will find
// the same problems with less-helpful messages, so we'll bail for
// now to let the user focus on the immediate problem.
return diags
}
}
attrs, hclDiags := f.Body.JustAttributes()
diags = diags.Append(hclDiags)
for name, attr := range attrs {
// check for aliases
if alias, ok := depVarAliases[name]; ok {
name = alias
}
to[name] = unparsedVariableValueExpression{
expr: attr.Expr,
sourceType: sourceType,
}
log.Printf("[INFO] adding value for variable '%s' from var file", name)
}
return diags
}
func sanitiseVariableNames(src []byte) ([]byte, map[string]string) {
// replace syntax `<modname>.<varname>=<var_value>` with `____steampipe_mod_<modname>_<varname>____=<var_value>
lines := strings.Split(string(src), "\n")
// make map of varname aliases
var depVarAliases = make(map[string]string)
for i, line := range lines {
r := regexp.MustCompile(`^ ?(([a-z0-9\-_]+)\.([a-z0-9\-_]+)) ?=`)
captureGroups := r.FindStringSubmatch(line)
if captureGroups != nil && len(captureGroups) == 4 {
fullVarName := captureGroups[1]
mod := captureGroups[2]
varName := captureGroups[3]
aliasedName := fmt.Sprintf("____steampipe_mod_%s_variable_%s____", mod, varName)
depVarAliases[aliasedName] = fullVarName
lines[i] = strings.Replace(line, fullVarName, aliasedName, 1)
}
}
// now try again
src = []byte(strings.Join(lines, "\n"))
return src, depVarAliases
}
// unparsedVariableValueLiteral is a UnparsedVariableValue
// implementation that was actually already parsed (!). This is
// intended to deal with expressions inside "tfvars" files.
type unparsedVariableValueExpression struct {
expr hcl.Expression
sourceType ValueSourceType
}
func (v unparsedVariableValueExpression) ParseVariableValue(mode var_config.VariableParsingMode) (*InputValue, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
val, hclDiags := v.expr.Value(nil) // nil because no function calls or variable references are allowed here
diags = diags.Append(hclDiags)
rng := tfdiags.SourceRangeFromHCL(v.expr.Range())
return &InputValue{
Value: val,
SourceType: v.sourceType,
SourceRange: rng,
}, diags
}
// unparsedVariableValueString is a UnparsedVariableValue
// implementation that parses its value from a string. This can be used
// to deal with values given directly on the command line and via environment
// variables.
type unparsedVariableValueString struct {
str string
name string
sourceType ValueSourceType
}
func (v unparsedVariableValueString) ParseVariableValue(mode var_config.VariableParsingMode) (*InputValue, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
val, hclDiags := mode.Parse(v.name, v.str)
diags = diags.Append(hclDiags)
return &InputValue{
Value: val,
SourceType: v.sourceType,
}, diags
}
// isAutoVarFile determines if the file ends with .auto.spvars or .auto.spvars.json
func isAutoVarFile(path string) bool {
for _, ext := range constants.AutoVariablesExtensions {
if strings.HasSuffix(path, ext) {
return true
}
}
return false
}