-
Notifications
You must be signed in to change notification settings - Fork 263
/
run_context.go
444 lines (382 loc) · 13.8 KB
/
run_context.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
package parse
import (
"fmt"
"strings"
"github.com/hashicorp/hcl/v2"
"github.com/stevenle/topsort"
filehelpers "github.com/turbot/go-kit/files"
"github.com/turbot/go-kit/helpers"
"github.com/turbot/steampipe/steampipeconfig/hclhelpers"
"github.com/turbot/steampipe/steampipeconfig/modconfig"
"github.com/turbot/steampipe/steampipeconfig/versionmap"
"github.com/zclconf/go-cty/cty"
)
const rootDependencyNode = "rootDependencyNode"
type ParseModFlag uint32
const (
CreateDefaultMod ParseModFlag = 1 << iota
CreatePseudoResources
)
/* ReferenceTypeValueMap is the raw data used to build the evaluation context
When resolving hcl references like :
- query.q1
- var.v1
- mod1.query.my_query.sql
ReferenceTypeValueMap is keyed by resource type, then by resource name
*/
type ReferenceTypeValueMap map[string]map[string]cty.Value
type RunContext struct {
// the mod which is currently being parsed
CurrentMod *modconfig.Mod
// the workspace lock data
WorkspaceLock *versionmap.WorkspaceLock
UnresolvedBlocks map[string]*unresolvedBlock
FileData map[string][]byte
// the eval context used to decode references in HCL
EvalCtx *hcl.EvalContext
Flags ParseModFlag
ListOptions *filehelpers.ListOptions
LoadedDependencyMods modconfig.ModMap
RootEvalPath string
// if set, only decode these blocks
BlockTypes []string
// if set, exclude these block types
BlockTypeExclusions []string
Variables map[string]*modconfig.Variable
dependencyGraph *topsort.Graph
// map of ReferenceTypeValueMaps keyed by mod
// NOTE: all values from root mod are keyed with "local"
referenceValues map[string]ReferenceTypeValueMap
blocks hcl.Blocks
}
func NewRunContext(workspaceLock *versionmap.WorkspaceLock, rootEvalPath string, flags ParseModFlag, listOptions *filehelpers.ListOptions) *RunContext {
c := &RunContext{
Flags: flags,
RootEvalPath: rootEvalPath,
WorkspaceLock: workspaceLock,
ListOptions: listOptions,
LoadedDependencyMods: make(modconfig.ModMap),
UnresolvedBlocks: make(map[string]*unresolvedBlock),
referenceValues: map[string]ReferenceTypeValueMap{
"local": make(ReferenceTypeValueMap),
},
}
// add root node - this will depend on all other nodes
c.dependencyGraph = c.newDependencyGraph()
c.buildEvalContext()
return c
}
func (r *RunContext) EnsureWorkspaceLock(mod *modconfig.Mod) error {
// if the mod has dependencies, there must a workspace lock object in the run context
// (mod MUST be the workspace mod, not a dependency, as we would hit this error as soon as we parse it)
if mod.HasDependentMods() && (r.WorkspaceLock.Empty() || r.WorkspaceLock.Incomplete()) {
return fmt.Errorf("not all dependencies are installed - run 'steampipe mod install'")
}
return nil
}
// VariableValueMap converts a map of variables to a map of the underlying cty value
func VariableValueMap(variables map[string]*modconfig.Variable) map[string]cty.Value {
ret := make(map[string]cty.Value, len(variables))
for k, v := range variables {
ret[k] = v.Value
}
return ret
}
// AddVariables adds variables to the run context.
// We load and evaluate variables before loading the workspace
func (r *RunContext) AddVariables(inputVariables map[string]*modconfig.Variable) {
r.Variables = inputVariables
// NOTE: we add with the name "var" not "variable" as that is how variables are referenced
r.referenceValues["local"]["var"] = VariableValueMap(inputVariables)
// do not reload variables as we already have them
r.BlockTypeExclusions = []string{modconfig.BlockTypeVariable}
}
// AddMod is used to add a mod with any pseudo resources to the eval context
// - in practice this will be a shell mod with just pseudo resources - other resources will be added as they are parsed
func (r *RunContext) AddMod(mod *modconfig.Mod) hcl.Diagnostics {
if len(r.UnresolvedBlocks) > 0 {
// should never happen
panic("calling SetContent on runContext but there are unresolved blocks from a previous parse")
}
var diags hcl.Diagnostics
moreDiags := r.storeResourceInCtyMap(mod)
diags = append(diags, moreDiags...)
resourceFunc := func(item modconfig.HclResource) bool {
// add all mod resources except variables into cty map
if _, ok := item.(*modconfig.Variable); !ok {
moreDiags := r.storeResourceInCtyMap(item)
diags = append(diags, moreDiags...)
}
// continue walking
return true
}
mod.WalkResources(resourceFunc)
// rebuild the eval context
r.buildEvalContext()
return diags
}
func (r *RunContext) SetDecodeContent(content *hcl.BodyContent, fileData map[string][]byte) {
r.blocks = content.Blocks
r.FileData = fileData
}
func (r *RunContext) ShouldIncludeBlock(block *hcl.Block) bool {
if len(r.BlockTypes) > 0 && !helpers.StringSliceContains(r.BlockTypes, block.Type) {
return false
}
if len(r.BlockTypeExclusions) > 0 && helpers.StringSliceContains(r.BlockTypeExclusions, block.Type) {
return false
}
return true
}
func (r *RunContext) ClearDependencies() {
r.UnresolvedBlocks = make(map[string]*unresolvedBlock)
r.dependencyGraph = r.newDependencyGraph()
}
// AddDependencies :: the block could not be resolved as it has dependencies
// 1) store block as unresolved
// 2) add dependencies to our tree of dependencies
func (r *RunContext) AddDependencies(block *hcl.Block, name string, dependencies []*modconfig.ResourceDependency) hcl.Diagnostics {
var diags hcl.Diagnostics
// store unresolved block
r.UnresolvedBlocks[name] = &unresolvedBlock{Name: name, Block: block, Dependencies: dependencies}
// store dependency in tree - d
if !r.dependencyGraph.ContainsNode(name) {
r.dependencyGraph.AddNode(name)
}
// add root dependency
if err := r.dependencyGraph.AddEdge(rootDependencyNode, name); err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "failed to add root dependency to graph",
Detail: err.Error()})
}
for _, dep := range dependencies {
// each dependency object may have multiple traversals
for _, t := range dep.Traversals {
parsedPropertyPath, err := modconfig.ParseResourcePropertyPath(hclhelpers.TraversalAsString(t))
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "failed to parse dependency",
Detail: err.Error()})
continue
}
// 'd' may be a property path - when storing dependencies we only care about the resource names
dependencyResourceName := parsedPropertyPath.ToResourceName()
if !r.dependencyGraph.ContainsNode(dependencyResourceName) {
r.dependencyGraph.AddNode(dependencyResourceName)
}
if err := r.dependencyGraph.AddEdge(name, dependencyResourceName); err != nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "failed to add dependency to graph",
Detail: err.Error()})
}
}
}
return diags
}
// BlocksToDecode builds a list of blocks to decode, the order of which is determined by the depdnency order
func (r *RunContext) BlocksToDecode() (hcl.Blocks, error) {
depOrder, err := r.getDependencyOrder()
if err != nil {
return nil, err
}
if len(depOrder) == 0 {
return r.blocks, nil
}
// NOTE: a block may appear more than once in unresolved blocks
// if it defines multiple unresolved resources, e.g a locals block
// make a map of blocks we have already included, keyed by the block def range
blocksMap := make(map[string]bool)
var blocksToDecode hcl.Blocks
for _, name := range depOrder {
// depOrder is all the blocks required to resolve dependencies.
// if this one is unparsed, added to list
block, ok := r.UnresolvedBlocks[name]
if ok && !blocksMap[block.Block.DefRange.String()] {
blocksToDecode = append(blocksToDecode, block.Block)
// add to map
blocksMap[block.Block.DefRange.String()] = true
}
}
return blocksToDecode, nil
}
// EvalComplete returns whether all elements in the dependency tree fully evaluated
func (r *RunContext) EvalComplete() bool {
return len(r.UnresolvedBlocks) == 0
}
// ShouldCreateDefaultMod returns whether the flag is set to create a default mod if no mod definition exists
func (r *RunContext) ShouldCreateDefaultMod() bool {
return r.Flags&CreateDefaultMod == CreateDefaultMod
}
// CreatePseudoResources returns whether the flag is set to create pseudo resources
func (r *RunContext) CreatePseudoResources() bool {
return r.Flags&CreatePseudoResources == CreatePseudoResources
}
// AddResource stores this resource as a variable to be added to the eval context. It alse
func (r *RunContext) AddResource(resource modconfig.HclResource) hcl.Diagnostics {
diagnostics := r.storeResourceInCtyMap(resource)
if diagnostics.HasErrors() {
return diagnostics
}
// rebuild the eval context
r.buildEvalContext()
return nil
}
func (r *RunContext) FormatDependencies() string {
// first get the dependency order
dependencyOrder, err := r.getDependencyOrder()
if err != nil {
return err.Error()
}
// build array of dependency strings - processes dependencies in reverse order for presentation reasons
numDeps := len(dependencyOrder)
depStrings := make([]string, numDeps)
for i := 0; i < len(dependencyOrder); i++ {
srcIdx := len(dependencyOrder) - i - 1
resourceName := dependencyOrder[srcIdx]
// find dependency
dep, ok := r.UnresolvedBlocks[resourceName]
if ok {
depStrings[i] = dep.String()
} else {
// this could happen if there is a dependency on a missing item
depStrings[i] = fmt.Sprintf(" MISSING: %s", resourceName)
}
}
return helpers.Tabify(strings.Join(depStrings, "\n"), " ")
}
func (r *RunContext) newDependencyGraph() *topsort.Graph {
dependencyGraph := topsort.NewGraph()
// add root node - this will depend on all other nodes
dependencyGraph.AddNode(rootDependencyNode)
return dependencyGraph
}
// return the optimal run order required to resolve dependencies
func (r *RunContext) getDependencyOrder() ([]string, error) {
rawDeps, err := r.dependencyGraph.TopSort(rootDependencyNode)
if err != nil {
return nil, err
}
// now remove the variable names and dedupe
var deps []string
for _, d := range rawDeps {
if d == rootDependencyNode {
continue
}
propertyPath, err := modconfig.ParseResourcePropertyPath(d)
if err != nil {
return nil, err
}
dep := modconfig.BuildModResourceName(propertyPath.ItemType, propertyPath.Name)
if !helpers.StringSliceContains(deps, dep) {
deps = append(deps, dep)
}
}
return deps, nil
}
// eval functions
func (r *RunContext) buildEvalContext() {
// convert variables to cty values
variables := make(map[string]cty.Value)
// now for each mod add all the values
for mod, modMap := range r.referenceValues {
if mod == "local" {
for k, v := range modMap {
variables[k] = cty.ObjectVal(v)
}
continue
}
// mod map is map[string]map[string]cty.Value
// for each element (i.e. map[string]cty.Value) convert to cty object
refTypeMap := make(map[string]cty.Value)
for refType, typeValueMap := range modMap {
refTypeMap[refType] = cty.ObjectVal(typeValueMap)
}
// now convert the cty map to a cty object
variables[mod] = cty.ObjectVal(refTypeMap)
}
// create evaluation context
r.EvalCtx = &hcl.EvalContext{
Variables: variables,
// use the mod path as the file root for functions
Functions: ContextFunctions(r.RootEvalPath),
}
}
// update the cached cty value for the given resource, as long as itr does not already exist
func (r *RunContext) storeResourceInCtyMap(resource modconfig.HclResource) hcl.Diagnostics {
// add resource to variable map
ctyValue, err := resource.CtyValue()
if err != nil {
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("failed to convert resource '%s'to its cty value", resource.Name()),
Detail: err.Error(),
Subject: resource.GetDeclRange(),
}}
}
// add into the reference value map
if diags := r.addReferenceValue(resource, ctyValue); diags.HasErrors() {
return diags
}
// remove this resource from unparsed blocks
delete(r.UnresolvedBlocks, resource.Name())
return nil
}
func (r *RunContext) addReferenceValue(resource modconfig.HclResource, value cty.Value) hcl.Diagnostics {
parsedName, err := modconfig.ParseResourceName(resource.Name())
if err != nil {
return hcl.Diagnostics{&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: fmt.Sprintf("failed to parse resource name %s", resource.Name()),
Detail: err.Error(),
Subject: resource.GetDeclRange(),
}}
}
// TODO validate mod name clashes
// TODO mod reserved names
// TODO handle aliases
key := parsedName.Name
typeString := parsedName.ItemType
// the resource name will not have a mod - but the run context knows which mod we are parsing
mod := r.CurrentMod
modName := mod.ShortName
if mod.ModPath == r.RootEvalPath {
modName = "local"
}
variablesForMod, ok := r.referenceValues[modName]
// do we have a map of reference values for this dep mod?
if !ok {
// no - create one
variablesForMod = make(ReferenceTypeValueMap)
r.referenceValues[modName] = variablesForMod
}
// do we have a map of reference values for this type
variablesForType, ok := variablesForMod[typeString]
if !ok {
// no - create one
variablesForType = make(map[string]cty.Value)
}
// DO NOT update the cached cty values if the value already exists
// this can happen in the case of variables where we initialise the context with values read from file
// or passed on the command line, // does this item exist in the map
if _, ok := variablesForType[key]; !ok {
variablesForType[key] = value
variablesForMod[typeString] = variablesForType
r.referenceValues[modName] = variablesForMod
}
return nil
}
func (r *RunContext) GetMod(modShortName string) *modconfig.Mod {
if modShortName == r.CurrentMod.ShortName {
return r.CurrentMod
}
// we need to iterate through dependency mods - we cannot use modShortNameas key as it is short name
for _, dep := range r.LoadedDependencyMods {
if dep.ShortName == modShortName {
return dep
}
}
return nil
}