forked from golang/build
/
env.go
306 lines (274 loc) · 8.88 KB
/
env.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"golang.org/x/tools/go/vcs"
)
// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
// setup sets up the builder environment and returns the directory to run the buildCmd in.
setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}
// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
goos, goarch string
}
func (b *Builder) crossCompile() bool {
switch b.goos {
case "android", "nacl":
return true
case "darwin":
return b.goarch == "arm" || b.goarch == "arm64" // iOS
default:
return false
}
}
func (b *Builder) envv() []string {
if runtime.GOOS == "windows" {
return b.envvWindows()
}
var e []string
if *buildTool == "go" {
e = []string{
"GOOS=" + b.goos,
"GOARCH=" + b.goarch,
}
if !b.crossCompile() {
// If we are building, for example, linux/386 on a linux/amd64 machine we want to
// make sure that the whole build is done as a if this were compiled on a real
// linux/386 machine. In other words, we want to not do a cross compilation build.
// To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
//
// The exception to this rule is when we are doing nacl/android builds. These are by
// definition always cross compilation, and we have support built into cmd/go to be
// able to handle this case.
e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
}
}
for _, k := range extraEnv() {
if s, ok := getenvOk(k); ok {
e = append(e, k+"="+s)
}
}
return e
}
func (b *Builder) envvWindows() []string {
var start map[string]string
if *buildTool == "go" {
start = map[string]string{
"GOOS": b.goos,
"GOHOSTOS": b.goos,
"GOARCH": b.goarch,
"GOHOSTARCH": b.goarch,
"GOBUILDEXIT": "1", // exit all.bat with completion status.
}
}
for _, name := range extraEnv() {
if s, ok := getenvOk(name); ok {
start[name] = s
}
}
if b.goos == "windows" {
switch b.goarch {
case "amd64":
start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
case "386":
start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
}
}
skip := map[string]bool{
"GOBIN": true,
"GOPATH": true,
"GOROOT": true,
"INCLUDE": true,
"LIB": true,
}
var e []string
for name, v := range start {
e = append(e, name+"="+v)
skip[name] = true
}
for _, kv := range os.Environ() {
s := strings.SplitN(kv, "=", 2)
name := strings.ToUpper(s[0])
switch {
case name == "":
// variables, like "=C:=C:\", just copy them
e = append(e, kv)
case !skip[name]:
e = append(e, kv)
skip[name] = true
}
}
return e
}
// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
goworkpath := filepath.Join(workpath, "go")
if err := repo.Export(goworkpath, hash); err != nil {
return "", fmt.Errorf("error exporting repository: %s", err)
}
return filepath.Join(goworkpath, "src"), nil
}
// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}
// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
gccpath := filepath.Join(repo.Path, "gcc")
// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
git := vcs.ByCmd("git")
// only pull down gcc if we don't have a local copy.
if _, err := os.Stat(gccpath); err != nil {
if err := timeout(*cmdTimeout, func() error {
// pull down a working copy of GCC.
cloneCmd := []string{
"clone",
// This is just a guess since there are ~6000 commits to
// GCC per year. It's likely there will be enough history
// to cross-reference the Gofrontend commit against GCC.
// The disadvantage would be if the commit being built is more than
// a year old; in this case, the user should make a clone that has
// the full history.
"--depth", "6000",
// We only care about the master branch.
"--branch", "master", "--single-branch",
*gccPath,
}
// Clone Kind Clone Time(Dry run) Clone Size
// ---------------------------------------------------------------
// Full Clone 10 - 15 min 2.2 GiB
// Master Branch 2 - 3 min 1.5 GiB
// Full Clone(shallow) 1 min 900 MiB
// Master Branch(shallow) 40 sec 900 MiB
//
// The shallow clones have the same size, which is expected,
// but the full shallow clone will only have 6000 commits
// spread across all branches. There are ~50 branches.
return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
}); err != nil {
return "", err
}
}
if err := git.Download(gccpath); err != nil {
return "", err
}
// get the modified files for this commit.
var buf bytes.Buffer
if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
}
modifiedFiles := strings.Split(buf.String(), "\n")
var isMirrored bool
for _, f := range modifiedFiles {
if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
isMirrored = true
break
}
}
// use git log to find the corresponding commit to sync to in the gcc mirror.
// If the files modified in the gofrontend are mirrored to gcc, we expect a
// commit with a similar description in the gcc mirror. If the files modified are
// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
// occurred before those files were modified to verify gccgo's status at that point.
logCmd := []string{
"log",
"-1",
"--format=%H",
}
var errMsg string
if isMirrored {
commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
if err != nil {
return "", err
}
quotedDesc := regexp.QuoteMeta(string(commitDesc))
logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
} else {
commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
if err != nil {
return "", err
}
logCmd = append(logCmd, "--before", string(commitDate))
errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
}
buf.Reset()
if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
return "", fmt.Errorf("%s: %s", errMsg, err)
}
gccRev := buf.String()
if gccRev == "" {
return "", fmt.Errorf(errMsg)
}
// checkout gccRev
// TODO(cmang): Fix this to work in parallel mode.
if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
}
// make objdir to work in
gccobjdir := filepath.Join(workpath, "gcc-objdir")
if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
return "", err
}
// configure GCC with substituted gofrontend and libgo
if err := run(exec.Command(filepath.Join(gccpath, "configure"),
"--enable-languages=c,c++,go",
"--disable-bootstrap",
"--disable-multilib",
), runEnv(envv), runDir(gccobjdir)); err != nil {
return "", fmt.Errorf("Failed to configure GCC: %v", err)
}
// build gcc
if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
return "", fmt.Errorf("Failed to build GCC: %s", err)
}
return gccobjdir, nil
}
func getenvOk(k string) (v string, ok bool) {
v = os.Getenv(k)
if v != "" {
return v, true
}
keq := k + "="
for _, kv := range os.Environ() {
if kv == keq {
return "", true
}
}
return "", false
}
// extraEnv returns environment variables that need to be copied from
// the gobuilder's environment to the envv of its subprocesses.
func extraEnv() []string {
extra := []string{
"GOARM",
"GO386",
"GOROOT_BOOTSTRAP", // See https://golang.org/s/go15bootstrap
"CGO_ENABLED",
"CC",
"CC_FOR_TARGET",
"PATH",
"TMPDIR",
"USER",
"GO_TEST_TIMEOUT_SCALE", // increase test timeout for slow builders
}
if runtime.GOOS == "plan9" {
extra = append(extra, "objtype", "cputype", "path")
}
return extra
}