-
Notifications
You must be signed in to change notification settings - Fork 59
/
execute_context.go
580 lines (501 loc) · 17.3 KB
/
execute_context.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
// Copyright 2023 The CUE Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime/debug"
"sort"
"strings"
"sync"
"testing"
"cuelang.org/go/cue"
"cuelang.org/go/cue/load"
"cuelang.org/go/cue/parser"
preprocessembed "github.com/cue-lang/cuelang.org"
)
type executeContext struct {
// executor is the underlying executor behind this context
executor *executor
// filter is a set of absolute path directories that should be considered as
// part of an execute step. If filter is nil, then all directories are
// considered.
filter map[string]bool
// order captures the order in which we discovered index pages. That is,
// order contains the directories in the order in which we walked the
// directory to discover the page roots. It provides a consistent order for
// the subsequent processing of pages.
order []string
// pages is a set of all the pages discovered by the executor. The string
// key is the full directory path of the page source.
pages map[string]*page
errorContext
*executionContext
}
func (ec *executeContext) Format(state fmt.State, verb rune) {
fmt.Fprintf(state, "%v", ec.executor)
}
func (e *executor) newExecuteContext(filter map[string]bool) *executeContext {
return &executeContext{
executor: e,
pages: make(map[string]*page),
filter: filter,
executionContext: e.executionContext,
errorContext: e.errorContext,
}
}
func (ec *executeContext) execute() error {
// Determine the buildID or version information of self, which will act as
// input into the caching calculation.
if err := ec.deriveHashOfSelf(); err != nil {
return err
}
ec.debugf(ec.debugCache, "%v: selfHash: %s", ec, ec.selfHash)
// Recursively walk wd to find $lang.md and _$lang.md files for all
// supported languages.
if err := ec.findPages(); err != nil {
return err
}
inputs := ec.findSiteCUE()
// If we are in error or have the --check or --ls flag we don't want to do anything else
if ec.isInError() || flagCheck.Bool(ec.executor.cmd) || flagList.Bool(ec.executor.cmd) {
return errorIfInError(ec)
}
// At this point we know we are not in --check mode, check we are not in
// serve mode and delete hugo/content as a temporary(ish) measure to ensure
// that we don't leave any stale files around.
if !flagServe.Bool(ec.executor.cmd) {
hugoContent := filepath.Join(ec.executor.root, "hugo", "content")
if err := os.RemoveAll(hugoContent); err != nil {
return ec.errorf("%v: failed to remove %s: %v", ec, hugoContent, err)
}
}
// Load all the CUE in one go and validate against the preprocessor schema.
cfg := &load.Config{
Dir: ec.executor.root,
}
bps := load.Instances(inputs, cfg)
if l := len(bps); l != 1 {
return ec.errorf("%v: expected 1 build instance; saw %d", ec, l)
}
v := ec.executor.ctx.BuildInstance(bps[0])
if err := v.Err(); err != nil {
return ec.errorf("%v: error in site configuration: %v", ec, err)
}
v = v.Unify(ec.executor.siteSchema)
if err := v.Err(); err != nil {
return ec.errorf("%v: site failed to validate against schema: %v", ec, err)
}
ec.config = v
// Load the CUE versions configured as part of the site
ec.cueVersions = make(map[string]cueVersion)
ec.cueEnvVersions = make(map[string]string)
cueVersionsPath := cue.ParsePath("versions.cue")
cueVersions := ec.config.LookupPath(cueVersionsPath)
if cueVersions.Exists() {
if err := cueVersions.Decode(&ec.cueVersions); err != nil {
return ec.errorf("%v: failed to decode %v: %v", cueVersionsPath, err)
}
for _, v := range ec.cueVersions {
ec.cueEnvVersions[v.Var] = v.V
}
}
// Build up a list of the test users we need for all the pages
testUsers := make(map[string]bool)
// Now load config per page
for _, d := range ec.order {
p := ec.pages[d]
p.loadConfig()
if p.config.TestUserAuthn != nil {
for _, u := range p.config.TestUserAuthn {
testUsers[u] = true
}
}
// This doesn't feel very elegant in the non-concurrent mode
ec.updateInError(p.isInError())
ec.logf("%s", p.bytes())
}
// If the user (of the preprocessor) has not provided config via --testuserauthn
// or the PREPROCESSOR_TEST_USER_AUTHN env var, we "fall back" to asking for these
// credentials via a special Central Registry endpoint
if ec.testUserAuthn == nil {
fetcher := centralRegistryTestUserFetcher{
requiredUsers: testUsers,
}
ec.testUserAuthn = sync.OnceValues(fetcher.fetch)
}
// At this point we have config for each page, which means we know
// which pages needs which test user.
if ec.isInError() {
return errorIfInError(ec)
}
var pageWaits []<-chan struct{}
// Process the pages we found.
for _, d := range ec.order {
p := ec.pages[d]
// v := vs[i]
// if err := v.Validate(); err != nil {
// return fmt.Errorf("failed to validate CUE package %s: %w", p.relPath, err)
// }
done := make(chan struct{})
go func() {
p.process()
close(done)
}()
pageWaits = append(pageWaits, done)
}
for i, d := range ec.order {
p := ec.pages[d]
w := pageWaits[i]
<-w
ec.updateInError(p.isInError())
ec.logf("%s", p.bytes())
}
return errorIfInError(ec)
}
// findSiteCUE finds all the CUE inputs that contribute to the site
// configuration. Because we have a non-standard load mode, this is a useful
// step to standardise. If `execute --check` is set, findSiteCUE also ensures
// that page CUE files contain configuration that only define values in the
// "namespace" defined by the page root directory. For example, for
// content/docs/howto/find-a-guide/page.cue it will ensure that fields are
// defined only within the content.docs.howto."find-a-guide" struct.
//
// If execute --list is set, then the CUE files that contribute to the site
// configurate are simply printed to stdout.
//
// Note that we know as a precondition of execute that the process working
// directory is contained by the project root.
func (ec *executeContext) findSiteCUE() (inputs []string) {
// Loop through the root and content CUE files that belong to the site
// package. Ensure that the content/**/*.cue files that are in page roots
// are well structured.
order := append([]string{ec.executor.root}, ec.order...)
dirs:
for _, absDir := range order {
// Load the files in the directory (assuming they all belong
// to the same package)
var filenames []string
filenames, err := filepath.Glob(filepath.Join(absDir, "*.cue"))
if err != nil {
ec.errorf("%s: failed to read: %v", absDir, err)
continue
}
doList := flagList.Bool(ec.executor.cmd)
var siteFilenames []string
// We only want the files that are part of the "site" package
for _, fn := range filenames {
f, err := parser.ParseFile(fn, nil)
if err != nil {
ec.errorf("%v: failed to parse %s: %v", ec, fn, err)
continue
}
if f.PackageName() != sitePackage {
continue
}
if doList {
fmt.Printf("%s\n", fn)
} else {
siteFilenames = append(siteFilenames, fn)
}
}
if doList {
continue
}
// No CUE files is also fine. We don't require CUE anywhere
if len(siteFilenames) == 0 {
continue
}
inputs = append(inputs, siteFilenames...)
bps := load.Instances(siteFilenames, &load.Config{
Dir: absDir,
})
if l := len(bps); l != 1 {
ec.errorf("%s: expected 1 build package; saw %d", absDir, l)
continue
}
v := ec.ctx.BuildInstance(bps[0])
// If we have an error at this stage we can't be
// sure things are fine. Bail early
if err := v.Err(); err != nil {
ec.errorf("%s: error loading .cue files: %v", absDir, err)
continue
}
// Now only do a structure check if we are not in the root of the site
if absDir == ec.executor.root {
continue
}
// derive the relative dirPath of d to the root, in canonical dirPath format
// (i.e. not OS-specific)
relDir := strings.TrimPrefix(absDir, ec.executor.root+string(os.PathSeparator))
dirPath := filepath.ToSlash(filepath.Clean(relDir))
parts := strings.Split(dirPath, "/")
// We now want to walk down into v to ensure that the only fields that
// exist at each "level" are consistent with the elements of path
var selectors []cue.Selector
for _, elem := range parts {
path := cue.MakePath(selectors...)
toCheck := v.LookupPath(path)
fieldIter, err := toCheck.Fields(cue.Definitions(true), cue.Hidden(true))
if err != nil {
ec.errorf("%v: %s: failed to create iterator over CUE value at path %v: %v", ec, absDir, path, err)
continue dirs
}
// Could be multiple bad fields at this level, report them all
var inError bool
for fieldIter.Next() {
sel := fieldIter.Selector()
if sel.LabelType() != cue.StringLabel || sel.Unquoted() != elem {
inError = true
val := fieldIter.Value()
badPath := cue.MakePath(append(selectors, sel)...)
// val.Pos() is the position of the _value_ (the RHS), not the
// field name. Hence we need to construct the format string by
// hand.
//
// TODO: work out whether we can get the location(s) of the
// label for this value in a more principled way.
pos := val.Pos()
ec.errorf("%v:%d: %v: field not allowed; expected %q", pos.Filename(), pos.Line(), badPath, elem)
}
}
if inError {
// No point descending further at this point
continue dirs
}
selectors = append(selectors, cue.Str(elem))
}
}
return inputs
}
func (ec *executeContext) findPages() error {
dirsToWalk := []string{ec.executor.wd}
var dir string
for len(dirsToWalk) > 0 {
dir, dirsToWalk = dirsToWalk[0], dirsToWalk[1:]
entries, err := os.ReadDir(dir)
if err != nil {
return ec.errorf("%v: failed to read dir %s: %v", ec, dir, err)
}
searchForRootFiles := ec.filter == nil || ec.filter[dir]
var p *page
for _, entry := range entries {
name := entry.Name()
// Add child directories to our list of dirs to walk if they don't
// begin with '.' or '_'.
if entry.IsDir() {
if name[0] != '_' && name[0] != '.' {
dirsToWalk = append(dirsToWalk, filepath.Join(dir, name))
}
continue
}
// name is now know to be a file. Only proceed if the filter
// was nil or dir matched the filter
if !searchForRootFiles {
continue
}
// Is the file path a page root?
match := pageRootFileRegexp.FindStringSubmatch(name)
if match == nil {
continue
}
// We found a page root! Extract key parts of the match
prefix := match[1]
lang := lang(match[2])
ext := match[3]
if p == nil {
// Keep the order so we process the resulting pages in the same order.
// For now this doesn't really serve any specific purpose, other than to
// make the result of execute deterministic order-wise.
ec.order = append(ec.order, dir)
relDir, err := filepath.Rel(ec.executor.wd, dir)
if err != nil {
return ec.errorf("%v: failed to determine %s relative to %s: %v", ec, dir, ec.executor.wd, err)
}
p, err = ec.newPage(dir, relDir)
if err != nil {
return ec.errorf("%v: failed to create page in dir %s: %v", ec, dir, err)
}
ec.pages[dir] = p
}
// Register the root file with the page
rf := p.newRootFile(name, lang, prefix, ext)
p.debugf(ec.debugGeneral, "%v: added root file named %s", p, name)
p.rootFiles[name] = rf
rootPages := p.langTargets[lang]
rootPages = append(rootPages, rf)
p.langTargets[lang] = rootPages
}
}
return nil
}
// deriveHashOfSelf is responsible for setting ec.selfHash to a value that
// represents a hash of the running preprocessor binary.
func (ec *executeContext) deriveHashOfSelf() (err error) {
// Post condition: ec.selfHash must be non-empty
defer func() {
if err == nil && ec.selfHash == "" {
err = ec.errorf("%v: failed to compute non-empty hash of self", ec)
}
}()
// In the special case of the preprocessor being tested, we want a stable
// value to avoid a change in the preprocessor itself affecting the contents
// of any gen_cache.cue testscript golden files.
if testing.Testing() {
ec.selfHash = "testing self"
return nil
}
// If we have buildinfo, with a main package module which has version and sum
// information we use that
bi, ok := debug.ReadBuildInfo()
if !ok {
return ec.errorf("%v: failed to read buildinfo from self", ec)
}
// If the main module has been replaced, read the replacement
if bi.Main.Replace != nil {
bi.Main = *bi.Main.Replace
}
// Iff the resulting main package module has sum (and therefore version)
// information we can use that.
if bi.Main.Sum != "" {
ec.selfHash = bi.Main.Version + " " + bi.Main.Sum
return nil
}
// A simple sanity check to ensure we actually do some work hashing self.
// Otherwise it's an indicator that we have no embedded files.
didWork := false
files := preprocessembed.Files
// This fallback only works if the main module is is
// github.com/cue-lang/cuelang.org. (It might be possible to relax this
// constraint, but a tight constraint works for now).
hash, _ := ec.executionContext.createHash()
err = fs.WalkDir(files, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
// Skip certain files we cannot exclude via the embed globs
if strings.HasSuffix(path, "_test.go") {
return nil
}
f, err := files.ReadFile(path)
if err != nil {
return err
}
fmt.Fprintf(hash, "hashing %s:\n%s", path, f)
didWork = true
return nil
})
if err != nil {
return ec.errorf("%v: failed to hash the files of self: %v", ec, err)
}
if !didWork {
ec.fatalf("%v: did no work computing hash of self", ec)
}
ec.selfHash = base64.StdEncoding.EncodeToString(hash.Sum(nil))
return nil
}
// centralRegistryTestUserFetcher exists as a separate type to provide the
// context to its single method, fetch. This makes the process of "wrapping" a
// call to centralRegistryTestUserFetcher.fetch with a sync.Values call simpler
// than closing over a variable for the args.
//
// TODO: have the fetch method fully support OAuth2, with refresh tokens etc.
type centralRegistryTestUserFetcher struct {
requiredUsers map[string]bool
}
func (c centralRegistryTestUserFetcher) fetch() (res map[string]wireToken, err error) {
const centralRegistryHost = "registry.cue.works"
// Fallback to calling the central registry
userAuthnSrc := fmt.Sprintf("https://%s/_api/test_user_tokens", centralRegistryHost)
configDir, err := os.UserConfigDir()
if err != nil {
return nil, fmt.Errorf("failed to locate config directory: %v", err)
}
loginsJsonPath := filepath.Join(configDir, "cue", "logins.json")
loginsJson, err := os.ReadFile(loginsJsonPath)
if err != nil {
return nil, fmt.Errorf("failed to read config file %s: %v", loginsJsonPath, err)
}
var logins struct {
Registries map[string]wireToken
}
if err := json.Unmarshal(loginsJson, &logins); err != nil {
return nil, fmt.Errorf("failed to JSON unmarshal %s: %v", loginsJsonPath, err)
}
var centralRegistryWireToken *wireToken
if logins.Registries != nil {
v, ok := logins.Registries[centralRegistryHost]
if ok {
centralRegistryWireToken = &v
}
}
if centralRegistryWireToken == nil {
return nil, fmt.Errorf("failed to find auth credentials for %s in %s; run `cue login`", centralRegistryHost, loginsJsonPath)
}
type testuserTokensArgs struct {
Usernames []string `json:"usernames"`
}
var args testuserTokensArgs
for u := range c.requiredUsers {
args.Usernames = append(args.Usernames, u)
}
sort.Strings(args.Usernames)
argsByts, err := json.Marshal(args)
if err != nil {
return nil, fmt.Errorf("failed to marshal args for call: %v", err)
}
client := new(http.Client)
req, err := http.NewRequest("POST", userAuthnSrc, nil)
if err != nil {
return nil, fmt.Errorf("failed to create reqest to %s: %v", userAuthnSrc, err)
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", centralRegistryWireToken.AccessToken))
req.Body = io.NopCloser(bytes.NewReader(argsByts))
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to make call for test user tokens: %v", err)
}
defer resp.Body.Close()
userAuthnByts, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response for test user tokens: %v", err)
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return nil, fmt.Errorf("failed to get test user tokens: %d: %s", resp.StatusCode, userAuthnByts)
}
if err := json.Unmarshal(userAuthnByts, &res); err != nil {
return nil, fmt.Errorf("failed to decode user authn map from %s: %v", userAuthnSrc, err)
}
return
}
// dockerImageChecker is a sync.Once checker for ensuring that the image
// dockerImageTag exists before a start/run command.
var dockerImageChecker = sync.OnceValue(func() error {
cmd := exec.Command("docker", "image", "inspect", dockerImageTag)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to find docker image %s: %s", dockerImageTag, out)
}
return nil
})