forked from vanadium-archive/go.devtools
-
Notifications
You must be signed in to change notification settings - Fork 0
/
go.go
435 lines (414 loc) · 14.8 KB
/
go.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package golib defines utilities for using the Go toolchain to build
// Vanadium binaries.
package golib
import (
"bufio"
"bytes"
"fmt"
"os"
"os/user"
"regexp"
"sort"
"strings"
"time"
"v.io/jiri"
"v.io/jiri/collect"
"v.io/jiri/gitutil"
"v.io/jiri/project"
"v.io/jiri/runutil"
"v.io/x/devtools/internal/buildinfo"
"v.io/x/lib/lookpath"
"v.io/x/lib/metadata"
"v.io/x/lib/set"
)
// ExtraLDFlagsFlagDescription describes the --extra-ldflags flag, to be added
// to any tool that allows adding extra ldflags to those automatically
// generated.
const ExtraLDFlagsFlagDescription = `This tool sets some ldflags automatically, e.g. to set binary metadata. The extra-ldflags are appended to the end of those automatically generated ldflags. Note that if your go command line specifies -ldflags explicitly, it will override both the automatically generated ldflags as well as the extra-ldflags.`
var goEnvVars = map[string]bool{
"CC": true,
"CGO_ENABLED": true,
"CXX": true,
"GOARCH": true,
"GOBIN": true,
"GOEXE": true,
"GOGCCFLAGS": true,
"GOHOSTARCH": true,
"GOHOSTOS": true,
"GOOS": true,
"GOPATH": true,
"GORACE": true,
"GOROOT": true,
"GOTOOLDIR": true,
"GO15VENDOREXPERIMENT": true,
}
// PrepareGo runs recommended checks on the environment and related commands
// before execution of the Go toolchain. The Go toolchain should use the
// returned args. PrepareGo for the 'env' strips any enviornment variables
// that the go command doesn't understand.
//
// For example, it ensures that all Go files generated by the VDL compiler are
// up-to-date. It also generates flags so that build information can be embedded
// in resulting binaries.
func PrepareGo(jirix *jiri.X, env map[string]string, args []string, extraLDFlags, installSuffix string) ([]string, error) {
switch args[0] {
case "env":
rargs := []string{"env"}
for _, v := range args[1:] {
if !goEnvVars[v] {
fmt.Fprintf(jirix.Stdout(), "%s\n", env[v])
} else {
rargs = append(rargs, v)
}
}
return rargs, nil
case "build", "install":
// Provide default ldflags to populate build info metadata in the
// binary. Any manual specification of ldflags already in the args
// will override this.
var err error
if args, err = setBuildInfoFlags(jirix, args, env, extraLDFlags, installSuffix); err != nil {
return nil, err
}
fallthrough
case "generate", "run", "test":
// Check that all non-master branches have been merged with the
// master branch to make sure the vdl tool is not run against
// out-of-date code base.
if err := reportOutdatedBranches(jirix); err != nil {
return nil, err
}
// Generate vdl files, if necessary.
if err := generateVDL(jirix, env, args[0], args[1:]); err != nil {
return nil, err
}
}
return args, nil
}
// getPlatform identifies the target platform by querying the go tool
// for the values of the GOARCH and GOOS environment variables.
func getPlatform(jirix *jiri.X, env map[string]string) (string, error) {
goBin, err := lookpath.Look(env, "go")
if err != nil {
return "", err
}
s := jirix.NewSeq()
var out bytes.Buffer
if err = s.Env(env).Capture(&out, nil).Last(goBin, "env", "GOARCH"); err != nil {
return "", err
}
arch := strings.TrimSpace(out.String())
out.Reset()
if err = s.Env(env).Capture(&out, nil).Last(goBin, "env", "GOOS"); err != nil {
return "", err
}
os := strings.TrimSpace(out.String())
return fmt.Sprintf("%s-%s", arch, os), nil
}
// setBuildInfoFlags augments the list of arguments with flags for the
// go compiler that encoded the build information expected by the
// v.io/x/lib/metadata package.
func setBuildInfoFlags(jirix *jiri.X, args []string, env map[string]string, extraLDFlags, installSuffix string) ([]string, error) {
info := buildinfo.T{Time: time.Now()}
// Compute the "platform" value.
platform, err := getPlatform(jirix, env)
if err != nil {
return nil, err
}
info.Platform = platform
// Compute the "manifest" value.
latestManifest := jirix.UpdateHistoryLatestLink()
manifest, err := project.ManifestFromFile(jirix, latestManifest)
if err != nil {
if !runutil.IsNotExist(err) {
return nil, err
}
fmt.Fprintf(jirix.Stderr(), `WARNING: Could not find %s.
The contents of this file are stored as metadata in binaries the jiri
tool builds. To fix this problem, please run "jiri update".
`, latestManifest)
manifest = &project.Manifest{}
}
info.Manifest = *manifest
// Compute the "pristine" value.
states, err := project.GetProjectStates(jirix, true)
if err != nil {
return nil, err
}
info.Pristine = true
for _, state := range states {
if state.CurrentBranch != "master" || state.HasUncommitted || state.HasUntracked {
info.Pristine = false
break
}
}
// Compute the "user" value.
if currUser, err := user.Current(); err == nil {
info.User = currUser.Name
}
// Encode buildinfo as metadata and extract the appropriate ldflags.
md, err := info.ToMetaData()
if err != nil {
return nil, err
}
ldflags := "-ldflags=" + metadata.LDFlag(md)
if extraLDFlags != "" {
ldflags += " " + extraLDFlags
}
args = append([]string{args[0], ldflags}, args[1:]...)
if installSuffix != "" {
args = append([]string{args[0], "-installsuffix=" + installSuffix}, args[1:]...)
}
return args, nil
}
// generateVDL generates VDL for the transitive Go package dependencies.
//
// Note that the vdl tool takes VDL packages as input, but we're supplying Go
// packages. We're assuming the package paths for the VDL packages we want to
// generate have the same path names as the Go package paths. Some of the Go
// package paths may not correspond to a valid VDL package, so we silently
// ignore these paths.
//
// It's fine if the VDL packages have dependencies not reflected in the Go
// packages; the vdl tool will compute the transitive closure of VDL package
// dependencies, as usual.
//
// TODO(toddw): Change the vdl tool to return vdl packages given the full Go
// dependencies, after vdl config files are implemented.
func generateVDL(jirix *jiri.X, env map[string]string, cmd string, args []string) error {
// Compute which VDL-based Go packages might need to be regenerated.
goPkgs, goFiles, goTags := processGoCmdAndArgs(cmd, args)
goDeps, err := computeGoDeps(jirix, env, append(goPkgs, goFiles...), goTags, cmd == "test")
if err != nil {
return err
}
// Regenerate the VDL-based Go packages.
// -ignore_unknown: Silently ignore unknown package paths.
vdlArgs := []string{"-ignore_unknown", "generate", "-lang=go"}
vdlArgs = append(vdlArgs, goDeps...)
vdlBin, err := lookpath.Look(env, "vdl")
if err != nil {
return err
}
var out bytes.Buffer
if err := jirix.NewSeq().Env(env).Capture(&out, &out).Last(vdlBin, vdlArgs...); err != nil {
return fmt.Errorf("failed to generate vdl: %v\n%s", err, out.String())
}
return nil
}
// reportOutdatedProjects checks if the currently checked out branches
// are up-to-date with respect to the local master branch. For each
// branch that is not, a notification is printed.
func reportOutdatedBranches(jirix *jiri.X) (e error) {
cwd, err := os.Getwd()
if err != nil {
return err
}
defer collect.Error(func() error { return jirix.NewSeq().Chdir(cwd).Done() }, &e)
projects, err := project.LocalProjects(jirix, false)
if err != nil {
return err
}
s := jirix.NewSeq()
for _, project := range projects {
if err := s.Chdir(project.Path).Done(); err != nil {
return err
}
switch project.Protocol {
case "git":
branches, _, err := gitutil.New(jirix.NewSeq()).GetBranches("--merged")
if err != nil {
return err
}
found := false
for _, branch := range branches {
if branch == "master" {
found = true
break
}
}
merging, err := gitutil.New(jirix.NewSeq()).MergeInProgress()
if err != nil {
return err
}
if !found && !merging {
fmt.Fprintf(jirix.Stderr(), "NOTE: project=%q path=%q\n", project.Name, project.Path)
fmt.Fprintf(jirix.Stderr(), "This project is on a non-master branch that is out of date.\n")
fmt.Fprintf(jirix.Stderr(), "Please update this branch using %q.\n", "git merge master")
fmt.Fprintf(jirix.Stderr(), "Until then the %q tool might not function properly.\n", "jiri")
}
}
}
return nil
}
// processGoCmdAndArgs is given the cmd and args for the go tool, filters out
// flags, and returns the PACKAGES or GOFILES that were specified in args, as
// well as "foo" if -tags=foo was specified in the args. Note that all commands
// that accept PACKAGES also accept GOFILES.
//
// go build [build flags] [-o out] [PACKAGES]
// go generate [-run regexp] [PACKAGES]
// go install [build flags] [PACKAGES]
// go run [build flags] [-exec prog] [GOFILES] [run args]
// go test [build flags] [test flags] [-exec prog] [PACKAGES] [testbin flags]
//
// Sadly there's no way to do this syntactically. It's easy for single token
// -flag and -flag=x, but non-boolean flags may be two tokens "-flag x".
//
// We keep track of all non-boolean flags F, and skip every token that starts
// with - or --, and also skip the next token if the flag is in F and isn't of
// the form -flag=x. If we forget to update F, we'll still handle the -flag and
// -flag=x cases correctly, but we'll get "-flag x" wrong.
func processGoCmdAndArgs(cmd string, args []string) ([]string, []string, string) {
var goTags string
var nonBool map[string]bool
switch cmd {
case "build":
nonBool = nonBoolGoBuild
case "generate":
nonBool = nonBoolGoGenerate
case "install":
nonBool = nonBoolGoInstall
case "run":
nonBool = nonBoolGoRun
case "test":
nonBool = nonBoolGoTest
}
// Move start to the start of PACKAGES or GOFILES, by skipping flags.
start := 0
for start < len(args) {
// Handle special-case terminator --
if args[start] == "--" {
start++
break
}
match := goFlagRE.FindStringSubmatch(args[start])
if match == nil {
break
}
// Skip this flag, and maybe skip the next token for the "-flag x" case.
// match[1] is the flag name
// match[2] is the optional "=" for the -flag=x case
start++
if nonBool[match[1]] && match[2] == "" {
start++
}
// Grab the value of -tags, if it is specified.
if match[1] == "tags" {
if match[2] == "=" {
goTags = match[3]
} else {
goTags = args[start-1]
}
}
}
// Move end to the end of PACKAGES or GOFILES.
var end int
switch cmd {
case "test":
// Any arg starting with - is a testbin flag.
// https://golang.org/cmd/go/#hdr-Test_packages
for end = start; end < len(args); end++ {
if strings.HasPrefix(args[end], "-") {
break
}
}
case "run":
// Go run takes gofiles, which are defined as a file ending in ".go".
// https://golang.org/cmd/go/#hdr-Compile_and_run_Go_program
for end = start; end < len(args); end++ {
if !strings.HasSuffix(args[end], ".go") {
break
}
}
default:
end = len(args)
}
// Decide whether these are packages or files.
switch {
case start == end:
return nil, nil, goTags
case (start < len(args) && strings.HasSuffix(args[start], ".go")):
return nil, args[start:end], goTags
default:
return args[start:end], nil, goTags
}
}
var (
goFlagRE = regexp.MustCompile(`^--?([^=]+)(=?)(.*)`)
nonBoolBuild = []string{
"p", "asmflags", "buildmode", "ccflags", "compiler", "gccgoflags", "gcflags", "installsuffix", "ldflags", "pkgdir", "tags", "toolexec",
}
nonBoolTest = []string{
"bench", "benchtime", "blockprofile", "blockprofilerate", "count", "covermode", "coverpkg", "coverprofile", "cpu", "cpuprofile", "memprofile", "memprofilerate", "outputdir", "parallel", "run", "timeout", "trace",
}
nonBoolGoBuild = set.StringBool.FromSlice(append(nonBoolBuild, "o"))
nonBoolGoGenerate = set.StringBool.FromSlice([]string{"run"})
nonBoolGoInstall = set.StringBool.FromSlice(nonBoolBuild)
nonBoolGoRun = set.StringBool.FromSlice(append(nonBoolBuild, "exec"))
nonBoolGoTest = set.StringBool.FromSlice(append(append(nonBoolBuild, nonBoolTest...), "exec", "o"))
)
// computeGoDeps computes the transitive Go package dependencies for the given
// set of pkgs. The strategy is to run "go list <pkgs>" with a special format
// string that dumps the specified pkgs and all deps as space / newline
// separated tokens. The pkgs may be in any format recognized by "go list"; dir
// paths, import paths, or go files.
func computeGoDeps(jirix *jiri.X, env map[string]string, pkgs []string, tags string, test bool) ([]string, error) {
if len(pkgs) == 0 {
pkgs = []string{"."}
}
goBin, err := lookpath.Look(env, "go")
if err != nil {
return nil, err
}
if test {
// In order to compute the test dependencies, we need to first grab the
// direct test imports, and use the resulting set of packages to capture the
// transitive dependencies. We can't do this with a single run of "go
// list", since unlike Dep, TestImports and XTestImports don't include
// transitive dependencies.
testDeps, err := runGoList(jirix, goBin, env, pkgs, tags, `{{join .TestImports " "}} {{join .XTestImports " "}}`)
if err != nil {
return nil, err
}
pkgs = append(pkgs, testDeps...)
}
return runGoList(jirix, goBin, env, pkgs, tags, `{{.ImportPath}} {{join .Deps " "}}`)
}
func runGoList(jirix *jiri.X, goBin string, env map[string]string, pkgs []string, tags, format string) ([]string, error) {
goListArgs := []string{`list`, `-f`, format}
if tags != "" {
goListArgs = append(goListArgs, "-tags="+tags)
}
goListArgs = append(goListArgs, pkgs...)
var stdout, stderr bytes.Buffer
// TODO(jsimsa): Avoid buffering all of the output in memory
// either by extending the runutil API to support piping of
// output, or by writing the output to a temporary file
// instead of an in-memory buffer.
// TODO(cnicolaou): the sequence code in runutil streams using a pipe
// internally so that could probably be taken advantage of here by having
// stdout be a pipe that the scanner reads below.
if err := jirix.NewSeq().Env(env).Capture(&stdout, &stderr).Last(goBin, goListArgs...); err != nil {
return nil, fmt.Errorf("failed to compute go deps: %v\n%s\n%v", err, stderr.String(), pkgs)
}
scanner := bufio.NewScanner(&stdout)
scanner.Split(bufio.ScanWords)
depsMap := make(map[string]bool)
for scanner.Scan() {
// Ignore bad packages:
// command-line-arguments is the dummy import path for "go run".
if dep := scanner.Text(); dep != "command-line-arguments" {
depsMap[dep] = true
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("Scan() failed: %v", err)
}
deps := set.StringBool.ToSlice(depsMap)
sort.Strings(deps)
return deps, nil
}