/
builders.go
2879 lines (2763 loc) · 98.4 KB
/
builders.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package dashboard contains shared configuration and logic used by various
// pieces of the Go continuous build system.
package dashboard
import (
"fmt"
"os"
"sort"
"strconv"
"strings"
"time"
"golang.org/x/build/buildenv"
"golang.org/x/build/maintner/maintnerd/maintapi/version"
"golang.org/x/build/types"
)
// slowBotAliases maps short names from TRY= comments to which builder to run.
//
// TODO: we'll likely expand this, or move it, or change the matching
// syntax entirely. This is a first draft.
var slowBotAliases = map[string]string{
// Known missing builders:
"ios-amd64": "", // There is no builder for the iOS Simulator. See issues 42100 and 42177.
"386": "linux-386",
"aix": "aix-ppc64",
"amd64": "linux-amd64",
"android": "android-amd64-emu",
"android-386": "android-386-emu",
"android-amd64": "android-amd64-emu",
"android-arm": "android-arm-corellium",
"android-arm64": "android-arm64-corellium",
"arm": "linux-arm-aws",
"arm64": "linux-arm64-aws",
"darwin": "darwin-amd64-10_14",
"darwin-amd64": "darwin-amd64-10_14",
"darwin-arm64": "darwin-arm64-11_0-toothrot",
"ios-arm64": "ios-arm64-corellium",
"dragonfly": "dragonfly-amd64",
"freebsd": "freebsd-amd64-12_2",
"freebsd-386": "freebsd-386-12_2",
"freebsd-amd64": "freebsd-amd64-12_2",
"freebsd-arm": "freebsd-arm-paulzhol",
"freebsd-arm64": "freebsd-arm64-dmgk",
"illumos": "illumos-amd64",
"ios": "ios-arm64-corellium",
"js": "js-wasm",
"linux": "linux-amd64",
"linux-arm": "linux-arm-aws",
"linux-arm64": "linux-arm64-aws",
"linux-mips": "linux-mips-rtrk",
"linux-mips64": "linux-mips64-rtrk",
"linux-mips64le": "linux-mips64le-mengzhuo",
"linux-mipsle": "linux-mipsle-rtrk",
"linux-ppc64": "linux-ppc64-buildlet",
"linux-ppc64le": "linux-ppc64le-buildlet",
"linux-ppc64le-power9": "linux-ppc64le-power9osu",
"linux-riscv64": "linux-riscv64-jsing",
"linux-s390x": "linux-s390x-ibm",
"longtest": "linux-amd64-longtest",
"mac": "darwin-amd64-10_14",
"macos": "darwin-amd64-10_14",
"mips": "linux-mips-rtrk",
"mips64": "linux-mips64-rtrk",
"mips64le": "linux-mips64le-mengzhuo",
"mipsle": "linux-mipsle-rtrk",
"netbsd": "netbsd-amd64-9_0",
"netbsd-386": "netbsd-386-9_0",
"netbsd-amd64": "netbsd-amd64-9_0",
"netbsd-arm": "netbsd-arm-bsiegert",
"netbsd-arm64": "netbsd-arm64-bsiegert",
"nocgo": "linux-amd64-nocgo",
"openbsd": "openbsd-amd64-68",
"openbsd-386": "openbsd-386-68",
"openbsd-amd64": "openbsd-amd64-68",
"openbsd-arm": "openbsd-arm-jsing",
"openbsd-arm64": "openbsd-arm64-jsing",
"openbsd-mips64": "openbsd-mips64-jsing",
"plan9": "plan9-arm",
"plan9-386": "plan9-386-0intro",
"plan9-amd64": "plan9-amd64-0intro",
"ppc64": "linux-ppc64-buildlet",
"ppc64le": "linux-ppc64le-buildlet",
"ppc64lep9": "linux-ppc64le-power9osu",
"riscv64": "linux-riscv64-jsing",
"s390x": "linux-s390x-ibm",
"solaris": "solaris-amd64-oraclerel",
"solaris-amd64": "solaris-amd64-oraclerel",
"wasm": "js-wasm",
"windows": "windows-amd64-2016",
"windows-386": "windows-386-2008",
"windows-amd64": "windows-amd64-2016",
"windows-arm": "windows-arm-zx2c4",
"windows-arm64": "windows-arm64-10",
}
// Builders are the different build configurations.
// The keys are like "darwin-amd64" or "linux-386-387".
// This map should not be modified by other packages.
// Initialization happens below, via calls to addBuilder.
var Builders = map[string]*BuildConfig{}
// Hosts contains the names and configs of all the types of
// buildlets. They can be VMs, containers, or dedicated machines.
var Hosts = map[string]*HostConfig{
"host-linux-bullseye": &HostConfig{
Notes: "Debian Bullseye",
ContainerImage: "linux-x86-bullseye:latest",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-buster": &HostConfig{
Notes: "Debian Buster",
ContainerImage: "linux-x86-buster:latest",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-jessie": &HostConfig{
Notes: "Debian Jessie, our standard Linux container image.",
ContainerImage: "linux-x86-jessie:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-stretch": &HostConfig{
Notes: "Debian Stretch",
ContainerImage: "linux-x86-stretch:latest",
machineType: "n1-standard-4", // 4 vCPUs, 15 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-stretch-morecpu": &HostConfig{
Notes: "Debian Stretch, but on n1-highcpu-16",
ContainerImage: "linux-x86-stretch:latest",
machineType: "n1-highcpu-16", // 16 vCPUs, 14.4 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-stretch-vmx": &HostConfig{
Notes: "Debian Stretch w/ Nested Virtualization (VMX CPU bit) enabled, for testing",
ContainerImage: "linux-x86-stretch:latest",
NestedVirt: true,
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-armhf-cross": &HostConfig{
Notes: "Debian with armhf cross-compiler, built from env/crosscompile/linux-armhf",
ContainerImage: "linux-armhf-cross:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-armel-cross": &HostConfig{
Notes: "Debian with armel cross-compiler, from env/crosscompile/linux-armel",
ContainerImage: "linux-armel-cross:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-amd64-localdev": &HostConfig{
IsReverse: true,
ExpectNum: 0,
Notes: "for localhost development of buildlets/gomote/coordinator only",
SSHUsername: os.Getenv("USER"),
},
"host-js-wasm": &HostConfig{
Notes: "Container with node.js for testing js/wasm.",
ContainerImage: "js-wasm:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-s390x-cross": &HostConfig{
Notes: "Container with s390x cross-compiler.",
ContainerImage: "linux-s390x-cross:latest",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
},
"host-linux-x86-alpine": &HostConfig{
Notes: "Alpine container",
ContainerImage: "linux-x86-alpine:latest",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.linux-amd64-static",
env: []string{"GOROOT_BOOTSTRAP=/usr/lib/go"},
SSHUsername: "root",
},
"host-linux-clang": &HostConfig{
Notes: "Container with clang.",
ContainerImage: "linux-x86-clang:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-sid": &HostConfig{
Notes: "Debian sid, updated occasionally.",
ContainerImage: "linux-x86-sid:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-fedora": &HostConfig{
Notes: "Fedora 30",
ContainerImage: "linux-x86-fedora:latest",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/goboot"},
SSHUsername: "root",
},
"host-linux-arm5spacemonkey": &HostConfig{
IsReverse: true,
ExpectNum: 3,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
OwnerGithub: "esnolte", // https://github.com/golang/go/issues/34973#issuecomment-543836871
},
"host-linux-riscv64-joelsing": &HostConfig{
Notes: "SiFive HiFive Unleashed RISC-V board. 8 GB RAM, 4 cores.",
IsReverse: true,
ExpectNum: 1,
OwnerGithub: "4a6f656c",
env: []string{"GOROOT_BOOTSTRAP=/usr/local/goboot"},
},
"host-linux-riscv64-unleashed": &HostConfig{
Notes: "SiFive HiFive Unleashed RISC-V board. 8 GB RAM, 4 cores.",
IsReverse: true,
ExpectNum: 1, // for now. Joel's board might join the party later.
OwnerGithub: "bradfitz", // at home
env: []string{"GOROOT_BOOTSTRAP=/usr/local/goboot"},
},
"host-openbsd-amd64-64": &HostConfig{
VMImage: "openbsd-amd64-64-190129a",
MinCPUPlatform: "Intel Skylake", // for better TSC? Maybe? see Issue 29223. builds faster at least.
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-amd64-go1_12.tar.gz",
Notes: "OpenBSD 6.4 with hw.smt=1; GCE VM is built from script in build/env/openbsd-amd64",
SSHUsername: "gopher",
},
"host-openbsd-386-64": &HostConfig{
VMImage: "openbsd-386-64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-386-go1_12.tar.gz",
Notes: "OpenBSD 6.4; GCE VM is built from script in build/env/openbsd-386",
SSHUsername: "gopher",
},
"host-openbsd-amd64-68": &HostConfig{
VMImage: "openbsd-amd64-68-v3", // v3 adds 009_exit syspatch; see golang.org/cl/278732.
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-amd64-go1_12.tar.gz",
Notes: "OpenBSD 6.8 (with 009_exit syspatch); GCE VM is built from script in build/env/openbsd-amd64",
SSHUsername: "gopher",
},
"host-openbsd-386-68": &HostConfig{
VMImage: "openbsd-386-68-v3", // v3 adds 009_exit syspatch; see golang.org/cl/278732.
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.openbsd-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-openbsd-386-go1_12.tar.gz",
Notes: "OpenBSD 6.8 (with 009_exit syspatch); GCE VM is built from script in build/env/openbsd-386",
SSHUsername: "gopher",
},
"host-openbsd-arm-joelsing": &HostConfig{
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
OwnerGithub: "4a6f656c",
},
"host-openbsd-arm64-joelsing": &HostConfig{
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
OwnerGithub: "4a6f656c",
},
"host-openbsd-mips64-joelsing": &HostConfig{
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
OwnerGithub: "4a6f656c",
},
"host-freebsd-11_2": &HostConfig{
VMImage: "freebsd-amd64-112",
Notes: "FreeBSD 11.2; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-freebsd-11_4": &HostConfig{
VMImage: "freebsd-amd64-114",
Notes: "FreeBSD 11.4; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-freebsd-12_2": &HostConfig{
VMImage: "freebsd-amd64-122",
Notes: "FreeBSD 12.2; GCE VM is built from script in build/env/freebsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-freebsd-12_2-big": &HostConfig{
VMImage: "freebsd-amd64-122",
Notes: "Same as host-freebsd-12_2, but on n1-highcpu-16",
machineType: "n1-highcpu-16", // 16 vCPUs, 14.4 GB mem
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.freebsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-freebsd-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-netbsd-amd64-9_0": &HostConfig{
VMImage: "netbsd-amd64-9-0-2019q4",
Notes: "NetBSD 9.0; GCE VM is built from script in build/env/netbsd-amd64",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.netbsd-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-netbsd-amd64-2da6b33.tar.gz",
SSHUsername: "root",
},
"host-netbsd-386-9_0": &HostConfig{
VMImage: "netbsd-i386-9-0-2019q4",
Notes: "NetBSD 9.0; GCE VM is built from script in build/env/netbsd-386",
machineType: "n1-highcpu-4",
buildletURLTmpl: "https://storage.googleapis.com/$BUCKET/buildlet.netbsd-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-netbsd-386-0b3b511.tar.gz",
SSHUsername: "root",
},
"host-netbsd-arm-bsiegert": &HostConfig{
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/usr/pkg/go112"},
OwnerGithub: "bsiegert",
},
"host-netbsd-arm64-bsiegert": &HostConfig{
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/usr/pkg/go114"},
OwnerGithub: "bsiegert",
},
"host-dragonfly-amd64-master": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "DragonFly BSD master, run by DragonFly team",
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go"},
SSHUsername: "root",
OwnerGithub: "tuxillo",
},
"host-freebsd-arm-paulzhol": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "Cubiboard2 1Gb RAM dual-core Cortex-A7 (Allwinner A20), FreeBSD 11.1-RELEASE",
env: []string{"GOROOT_BOOTSTRAP=/usr/home/paulzhol/go1.4"},
OwnerGithub: "paulzhol",
},
"host-freebsd-arm64-dmgk": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "AWS EC2 a1.large 2 vCPU 4GiB RAM, FreeBSD 12.1-STABLE",
env: []string{"GOROOT_BOOTSTRAP=/usr/home/builder/gobootstrap"},
OwnerGithub: "dmgk",
},
"host-plan9-arm-0intro": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "Raspberry Pi 3 Model B, Plan 9 from Bell Labs",
OwnerGithub: "0intro",
},
"host-plan9-amd64-0intro": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "QEMU VM, Plan 9 from Bell Labs, 9k kernel",
OwnerGithub: "0intro",
},
"host-plan9-386-0intro": &HostConfig{
IsReverse: true,
ExpectNum: 1,
Notes: "QEMU VM, Plan 9 from Bell Labs",
OwnerGithub: "0intro",
},
"host-plan9-386-gce": &HostConfig{
VMImage: "plan9-386-v7",
Notes: "Plan 9 from 0intro; GCE VM is built from script in build/env/plan9-386",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.plan9-386",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-plan9-386.tar.gz",
// We *were* using n1-standard-1 because Plan 9 can only
// reliably use a single CPU. Using 2 or 4 and we see
// test failures. See:
// https://golang.org/issue/8393
// https://golang.org/issue/9491
// n1-standard-1 has 3.6 GB of memory which WAS (see below)
// overkill (userspace probably only sees 2GB anyway),
// but it's the cheapest option. And plenty to keep
// our ~250 MB of inputs+outputs in its ramfs.
//
// But the docs says "For the n1 series of machine
// types, a virtual CPU is implemented as a single
// hyperthread on a 2.6GHz Intel Sandy Bridge Xeon or
// Intel Ivy Bridge Xeon (or newer) processor. This
// means that the n1-standard-2 machine type will see
// a whole physical core."
//
// ... so we used n1-highcpu-2 (1.80 RAM, still
// plenty), just so we can get 1 whole core for the
// single-core Plan 9. It will see 2 virtual cores and
// only use 1, but we hope that 1 will be more powerful
// and we'll stop timing out on tests.
machineType: "n1-highcpu-4",
env: []string{"GO_TEST_TIMEOUT_SCALE=3"},
},
"host-windows-amd64-2008": &HostConfig{
VMImage: "windows-amd64-server-2008r2-v7",
machineType: "n1-highcpu-4", // 4 vCPUs, 3.6 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-windows-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-windows-amd64-2012": &HostConfig{
VMImage: "windows-amd64-server-2012r2-v7",
machineType: "n1-highcpu-4", // 4 vCPUs, 3.6 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-windows-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-windows-amd64-2016": &HostConfig{
VMImage: "windows-amd64-server-2016-v7",
machineType: "n1-highcpu-4", // 4 vCPUs, 3.6 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-windows-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-windows-amd64-2016-big": &HostConfig{
Notes: "Same as host-windows-amd64-2016, but on n1-highcpu-16",
VMImage: "windows-amd64-server-2016-v7",
machineType: "n1-highcpu-16", // 16 vCPUs, 14.4 GB mem
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-amd64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/go1.4-windows-amd64.tar.gz",
SSHUsername: "gopher",
},
"host-windows-arm64-zx2c4": &HostConfig{
IsReverse: true,
ExpectNum: 1,
OwnerGithub: "zx2c4",
env: []string{"GOROOT_BOOTSTRAP=C:\\Program Files (Arm)\\Go"},
},
"host-windows-arm64-mini": &HostConfig{
Notes: "macOS hosting Windows 10 in qemu with HVM acceleration.",
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.windows-arm64",
goBootstrapURLTmpl: "https://storage.googleapis.com/$BUCKET/gobootstrap-windows-arm64-f22ec5.tar.gz",
IsReverse: true,
ExpectNum: 7,
},
"host-darwin-10_12": &HostConfig{
IsReverse: true,
ExpectNum: 3,
Notes: "MacStadium OS X 10.12 VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/go1.4",
},
SSHUsername: "gopher",
HermeticReverse: true, // we destroy the VM when done & let cmd/makemac recreate
},
"host-darwin-10_14": &HostConfig{
IsReverse: true,
ExpectNum: 3,
Notes: "MacStadium macOS Mojave (10.14) VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/goboot", // Go 1.12.1
},
SSHUsername: "gopher",
HermeticReverse: true, // we destroy the VM when done & let cmd/makemac recreate
},
"host-darwin-10_15": &HostConfig{
IsReverse: true,
ExpectNum: 5,
Notes: "MacStadium macOS Catalina (10.15) VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/goboot", // Go 1.12.1
},
SSHUsername: "gopher",
HermeticReverse: true, // we destroy the VM when done & let cmd/makemac recreate
},
"host-darwin-amd64-11_0": &HostConfig{
IsReverse: true,
ExpectNum: 5,
Notes: "MacStadium macOS Big Sur (11.0) VM under VMWare ESXi",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/goboot", // Go 1.13.4
},
SSHUsername: "gopher",
HermeticReverse: true, // we destroy the VM when done & let cmd/makemac recreate
},
"host-darwin-arm64-11_0-toothrot": &HostConfig{
IsReverse: true,
ExpectNum: 2,
Notes: "macOS Big Sur (11.0) ARM64 (M1). Mac mini",
env: []string{
"GOROOT_BOOTSTRAP=/Users/gopher/goboot",
},
SSHUsername: "gopher",
},
"host-linux-s390x": &HostConfig{
Notes: "run by IBM",
OwnerGithub: "ruixin-bao",
IsReverse: true,
env: []string{"GOROOT_BOOTSTRAP=/var/buildlet/go-linux-s390x-bootstrap"},
},
"host-linux-ppc64-osu": &HostConfig{
Notes: "Debian jessie; run by Go team on osuosl.org",
IsReverse: true,
ExpectNum: 5,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
SSHUsername: "root",
HermeticReverse: false, // TODO: run in chroots with overlayfs? https://github.com/golang/go/issues/34830#issuecomment-543386764
},
"host-linux-ppc64le-osu": &HostConfig{
Notes: "Debian Buster; run by Go team on osuosl.org; see x/build/env/linux-ppc64le/osuosl",
IsReverse: true,
ExpectNum: 5,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
SSHUsername: "root",
HermeticReverse: true,
},
"host-linux-ppc64le-power9-osu": &HostConfig{
Notes: "Debian Buster; run by Go team on osuosl.org; see x/build/env/linux-ppc64le/osuosl",
IsReverse: true,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap", "GOPPC64=power9"},
SSHUsername: "root",
HermeticReverse: true,
},
"host-linux-arm64-packet": &HostConfig{
Notes: "On 96 core packet.net host (Xenial) in Docker containers (Debian Buster); run by Go team. See x/build/env/linux-arm64/packet",
IsReverse: true,
HermeticReverse: true,
ExpectNum: 8,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
SSHUsername: "root",
},
"host-linux-arm64-aws": &HostConfig{
Notes: "Debian Buster, EC2 arm64 instance. See x/build/env/linux-arm64/aws",
VMImage: "ami-03089323a1d38e652",
ContainerImage: "gobuilder-arm64-aws:latest",
machineType: "m6g.xlarge",
isEC2: true,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-arm64",
SSHUsername: "root",
},
"host-linux-arm-aws": &HostConfig{
Notes: "Debian Buster, EC2 arm instance. See x/build/env/linux-arm/aws",
VMImage: "ami-07409163bccd5ac4d",
ContainerImage: "gobuilder-arm-aws:latest",
machineType: "m6g.xlarge",
isEC2: true,
env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-arm",
SSHUsername: "root",
},
"host-illumos-amd64-jclulow": &HostConfig{
Notes: "SmartOS base64@19.1.0 zone",
Owner: "josh@sysmgr.org",
OwnerGithub: "jclulow",
IsReverse: true,
ExpectNum: 1,
SSHUsername: "gobuild",
},
"host-solaris-oracle-amd64-oraclerel": &HostConfig{
Notes: "Oracle Solaris amd64 Release System",
Owner: "",
OwnerGithub: "rorth", // https://github.com/golang/go/issues/15581#issuecomment-550368581
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/opt/golang/go-solaris-amd64-bootstrap"},
},
"host-linux-mipsle-mengzhuo": &HostConfig{
Notes: "Loongson 3A Box hosted by Meng Zhuo; actually MIPS64 despite the name",
OwnerGithub: "mengzhuo",
IsReverse: true,
ExpectNum: 1,
env: []string{
"GOROOT_BOOTSTRAP=/usr/lib/golang",
"GOMIPS64=hardfloat",
},
},
"host-linux-mips64le-rtrk": &HostConfig{
Notes: "cavium,rhino_utm8 board hosted at RT-RK.com; quad-core cpu, 8GB of ram and 240GB ssd disks.",
OwnerGithub: "bogojevic", // and @milanknezevic. https://github.com/golang/go/issues/31217#issuecomment-547004892
IsReverse: true,
ExpectNum: 1,
env: []string{
"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap",
},
},
"host-linux-mips64-rtrk": &HostConfig{
Notes: "cavium,rhino_utm8 board hosted at RT-RK.com; quad-core cpu, 8GB of ram and 240GB ssd disks.",
OwnerGithub: "bogojevic", // and @milanknezevic. https://github.com/golang/go/issues/31217#issuecomment-547004892
IsReverse: true,
ExpectNum: 1,
env: []string{
"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap",
},
},
"host-ios-arm64-corellium-ios": &HostConfig{
Notes: "Virtual iOS devices hosted by Zenly on Corellium; see issues 31722 and 40523",
OwnerGithub: "znly",
IsReverse: true,
ExpectNum: 3,
env: []string{
"GOROOT_BOOTSTRAP=/var/mobile/go-ios-arm64-bootstrap",
},
},
"host-android-arm64-corellium-android": &HostConfig{
Notes: "Virtual Android devices hosted by Zenly on Corellium; see issues 31722 and 40523",
OwnerGithub: "znly",
IsReverse: true,
ExpectNum: 3,
env: []string{
"GOROOT_BOOTSTRAP=/data/data/com.termux/files/home/go-android-arm64-bootstrap",
},
},
"host-aix-ppc64-osuosl": &HostConfig{
Notes: "AIX 7.2 VM on OSU; run by Tony Reix",
OwnerGithub: "trex58",
IsReverse: true,
ExpectNum: 1,
env: []string{"GOROOT_BOOTSTRAP=/opt/freeware/lib/golang"},
},
"host-android-amd64-emu": &HostConfig{
Notes: "Debian Buster w/ Android SDK + emulator (use nested virt)",
ContainerImage: "android-amd64-emu:bff27c0c9263",
KonletVMImage: "android-amd64-emu",
NestedVirt: true,
buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64",
env: []string{"GOROOT_BOOTSTRAP=/go1.4"},
SSHUsername: "root",
},
"host-linux-amd64-wsl": &HostConfig{
Notes: "Windows 10 WSL2 Ubuntu",
OwnerGithub: "mengzhuo",
IsReverse: true,
ExpectNum: 2,
env: []string{"GOROOT_BOOTSTRAP=/usr/lib/go"},
},
}
// CrossCompileConfig describes how to cross-compile a build on a
// faster host.
type CrossCompileConfig struct {
// CompileHostType is the host type to use for compilation
CompileHostType string
// CCForTarget is the CC_FOR_TARGET environment variable.
CCForTarget string
// GOARM is any GOARM= environment variable.
GOARM string
// AlwaysCrossCompile controls whether this builder always
// cross compiles. Otherwise it's only done for trybot runs.
AlwaysCrossCompile bool
}
func init() {
for key, c := range Hosts {
if key == "" {
panic("empty string key in Hosts")
}
if c.HostType == "" {
c.HostType = key
}
if c.HostType != key {
panic(fmt.Sprintf("HostType %q != key %q", c.HostType, key))
}
nSet := 0
if c.VMImage != "" {
nSet++
}
if c.ContainerImage != "" && !c.isEC2 {
nSet++
}
if c.IsReverse {
nSet++
}
if nSet != 1 {
panic(fmt.Sprintf("exactly one of VMImage, ContainerImage, IsReverse must be set for host %q; got %v", key, nSet))
}
if c.buildletURLTmpl == "" && (c.VMImage != "" || c.ContainerImage != "") {
panic(fmt.Sprintf("missing buildletURLTmpl for host type %q", key))
}
}
}
// A HostConfig describes the available ways to obtain buildlets of
// different types. Some host configs can serve multiple
// builders. For example, a host config of "host-linux-jessie" can
// serve linux-amd64, linux-amd64-race, linux-386, linux-386-387, etc.
type HostConfig struct {
// HostType is the unique name of this host config. It is also
// the key in the Hosts map.
HostType string
// buildletURLTmpl is the URL "template" ($BUCKET is auto-expanded)
// for the URL to the buildlet binary.
// This field is required for VM and Container builders. It's not
// needed for reverse buildlets because in that case, the buildlets
// are already running and their stage0 should know how to update it
// it automatically.
buildletURLTmpl string
// Exactly 1 of these must be set (with the exception of EC2 instances).
// An EC2 instance may run a container inside a VM. In that case, a VMImage
// and ContainerImage will both be set.
VMImage string // e.g. "openbsd-amd64-60"
ContainerImage string // e.g. "linux-buildlet-std:latest" (suffix after "gcr.io/<PROJ>/")
IsReverse bool // if true, only use the reverse buildlet pool
// GCE options, if VMImage != ""
machineType string // optional GCE instance type
RegularDisk bool // if true, use spinning disk instead of SSD
MinCPUPlatform string // optional; https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform
// EC2 options
isEC2 bool // if true, the instance is configured to run on EC2
// ReverseOptions:
ExpectNum int // expected number of reverse buildlets of this type
HermeticReverse bool // whether reverse buildlet has fresh env per conn
// Container image options, if ContainerImage != "":
NestedVirt bool // container requires VMX nested virtualization
KonletVMImage string // optional VM image (containing konlet) to use instead of default
// Optional base env. GOROOT_BOOTSTRAP should go here if the buildlet
// has Go 1.4+ baked in somewhere.
env []string
// These template URLs may contain $BUCKET which is expanded to the
// relevant Cloud Storage bucket as specified by the build environment.
goBootstrapURLTmpl string // optional URL to a built Go 1.4+ tar.gz
Owner string // optional email of owner; "bradfitz@golang.org", empty means golang-dev
OwnerGithub string // optional GitHub username of owner
Notes string // notes for humans
SSHUsername string // username to ssh as, empty means not supported
}
// A BuildConfig describes how to run a builder.
type BuildConfig struct {
// Name is the unique name of the builder, in the form of
// "GOOS-GOARCH" or "GOOS-GOARCH-suffix". For example,
// "darwin-386", "linux-386-387", "linux-amd64-race". Some
// suffixes are well-known and carry special meaning, such as
// "-race".
Name string
// HostType is the required key into the Hosts map, describing
// the type of host this build will run on.
// For example, "host-linux-jessie".
HostType string
// KnownIssue is a non-zero golang.org/issue/nnn number for a builder
// that may fail due to a known issue, such as because it is a new
// builder still in development/testing, or because the feature
// or port that it's meant to test hasn't been added yet, etc.
//
// A non-zero value here means that failures on this builder should not
// be considered a serious regression and don't need investigation beyond
// what is already in scope of the listed issue.
KnownIssue int
Notes string // notes for humans
// tryBot optionally specifies a policy func for whether trybots are enabled.
// nil means off. Even if tryBot returns true, BuildConfig.BuildsRepo must also
// return true. See the implementation of BuildConfig.BuildsRepoTryBot.
// The proj is "go", "net", etc. The branch is proj's branch.
// The goBranch is the same as branch for proj "go", else it's the go branch
// ("master, "release-branch.go1.12", etc).
tryBot func(proj, branch, goBranch string) bool
tryOnly bool // only used for trybots, and not regular builds
CompileOnly bool // if true, compile tests, but don't run them
FlakyNet bool // network tests are flaky (try anyway, but ignore some failures)
// buildsRepo optionally specifies whether this builder does
// builds (of any type) for the given repo ("go", "net", etc.)
// and its branch ("master", "release-branch.go1.12", "dev.link", etc.).
// goBranch is the branch of "go" to build against. If repo == "go",
// goBranch == branch.
//
// If nil, a default set of repos as reported by buildRepoByDefault
// is built. See buildsRepoAtAll method for details.
//
// To implement a minor change to the default policy, create a
// function that uses buildRepoByDefault. For example:
//
// buildsRepo: func(repo, branch, goBranch string) bool {
// b := buildRepoByDefault(repo)
// // ... modify b from the default value as needed ...
// return b
// }
//
buildsRepo func(repo, branch, goBranch string) bool
// MinimumGoVersion optionally specifies the minimum Go version
// this builder is allowed to use. It can be useful for skipping
// builders that are too new and no longer support some supported
// Go versions. It doesn't need to be set for builders that support
// all supported Go versions.
//
// Note: This field currently has effect on trybot runs only.
//
// TODO: unexport this and make buildsRepoAtAll return false on too-old
// of repos. The callers in coordinator will need updating.
MinimumGoVersion types.MajorMinor
// SkipSnapshot, if true, means to not fetch a tarball
// snapshot of the world post-make.bash from the buildlet (and
// thus to not write it to Google Cloud Storage). This is
// incompatible with sharded tests, and should only be used
// for very slow builders or networks, unable to transfer
// the tarball in under ~5 minutes.
SkipSnapshot bool
// RunBench causes the coordinator to run benchmarks on this buildlet type.
RunBench bool
// StopAfterMake causes the build to stop after the make
// script completes, returning its result as the result of the
// whole build. It does not run or compile any of the tests,
// nor does it write a snapshot of the world to cloud
// storage. This option is only supported for builders whose
// BuildConfig.SplitMakeRun returns true.
StopAfterMake bool
// needsGoProxy is whether this builder should have GOPROXY set.
// Currently this is only for the longtest builder, which needs
// to run cmd/go tests fetching from the network.
needsGoProxy bool
// InstallRacePackages controls which packages to "go install
// -race <pkgs>" after running make.bash (or equivalent). If
// the builder ends in "-race", the default if non-nil is just
// "std".
InstallRacePackages []string
// GoDeps is a list of of git sha1 commits that must be in the
// commit to be tested's history. If absent, this builder is
// not run for that commit.
GoDeps []string
// CrossCompileConfig optionally specifies whether and how
// this build is cross compiled.
CrossCompileConfig *CrossCompileConfig
// distTestAdjust optionally specifies a function that can
// adjust the cmd/dist test policy for this builder.
//
// The BuildConfig.ShouldRunDistTest method implements the
// default cmd/dist test policy, and then calls distTestAdjust
// to adjust that decision further, if distTestAdjust is not nil.
//
// The initial value of the run parameter is what the default
// policy said. The returned value from distTestAdjust is what
// BuildConfig.ShouldRunDistTest reports to the caller.
//
// For example:
//
// distTestAdjust: func(run bool, distTest string, isNormalTry bool) bool {
// // ... modify run from the initial value as needed ...
// return run
// }
//
distTestAdjust func(run bool, distTest string, isNormalTry bool) bool
// numTestHelpers is the number of _additional_ buildlets
// past the first one to help out with sharded tests.
// For TryBots and SlowBots, the numTryHelpers value is used,
// unless it's zero, in which case numTestHelpers is used.
numTestHelpers int
numTryTestHelpers int // For TryBots/SlowBots. If 0, numTestHelpers is used.
env []string // extra environment ("key=value") pairs
allScriptArgs []string
testHostConf *HostConfig // override HostConfig for testing, at least for now
}
// Env returns the environment variables this builder should run with.
func (c *BuildConfig) Env() []string {
env := []string{"GO_BUILDER_NAME=" + c.Name}
if c.FlakyNet {
env = append(env, "GO_BUILDER_FLAKY_NET=1")
}
if c.IsLongTest() {
// Set a private hook in cmd/dist to run main Go repository tests
// without the default -short flag. See golang.org/issue/12508.
env = append(env, "GO_TEST_SHORT=0")
}
env = append(env, c.HostConfig().env...)
return append(env, c.env...)
}
// ModulesEnv returns the extra module-specific environment variables
// to append to this builder as a function of the repo being built
// ("go", "oauth2", "net", etc).
func (c *BuildConfig) ModulesEnv(repo string) (env []string) {
// EC2 and reverse builders should set the public module proxy
// address instead of the internal proxy.
if (c.HostConfig().isEC2 || c.IsReverse()) && repo != "go" {
env = append(env, "GOPROXY=https://proxy.golang.org")
}
switch repo {
case "go":
if !c.OutboundNetworkAllowed() {
env = append(env, "GOPROXY=off")
}
case "oauth2", "build", "perf", "website":
env = append(env, "GO111MODULE=on")
}
return env
}
// ShouldTestPackageInGOPATHMode is used to control whether the package
// with the specified import path should be tested in GOPATH mode.
//
// When running tests for all golang.org/* repositories in GOPATH mode,
// this method is called repeatedly with the full import path of each
// package that is found and is being considered for testing in GOPATH
// mode. It's not used and has no effect on import paths in the main
// "go" repository. It has no effect on tests done in module mode.
//
// When considering making changes here, keep the release policy in mind:
//
// https://golang.org/doc/devel/release.html#policy
//
func (*BuildConfig) ShouldTestPackageInGOPATHMode(importPath string) bool {
if importPath == "golang.org/x/tools/gopls" ||
strings.HasPrefix(importPath, "golang.org/x/tools/gopls/") {
// Don't test golang.org/x/tools/gopls/... in GOPATH mode.
return false
}
if importPath == "golang.org/x/net/http2/h2demo" {
// Don't test golang.org/x/net/http2/h2demo in GOPATH mode.
//
// It was never tested before golang.org/issue/34361 because it
// had a +build h2demo constraint. But that build constraint is
// being removed, so explicitly skip testing it in GOPATH mode.
//
// The package is supported only in module mode now, since
// it requires third-party dependencies.
return false
}
// Test everything else in GOPATH mode as usual.
return true
}
func (c *BuildConfig) IsReverse() bool { return c.HostConfig().IsReverse }
func (c *BuildConfig) IsContainer() bool { return c.HostConfig().IsContainer() }
func (c *HostConfig) IsContainer() bool { return c.ContainerImage != "" }
func (c *BuildConfig) IsVM() bool { return c.HostConfig().IsVM() }
// IsVM reports whether the instance running the job is ultimately a VM. Hosts where
// a VM is used only to initiate a container are considered a container, not a VM.
// EC2 instances may be configured to run in containers that are running
// on custom AMIs.
func (c *HostConfig) IsVM() bool {
if c.isEC2 {
return c.VMImage != "" && c.ContainerImage == ""
}
return c.VMImage != ""
}
func (c *BuildConfig) GOOS() string { return c.Name[:strings.Index(c.Name, "-")] }
func (c *BuildConfig) GOARCH() string {
arch := c.Name[strings.Index(c.Name, "-")+1:]
i := strings.Index(arch, "-")
if i == -1 {
return arch
}
return arch[:i]
}
// MatchesSlowBotTerm reports whether some provided term from a
// TRY=... comment on a Run-TryBot+1 vote on Gerrit should match this
// build config.
func (c *BuildConfig) MatchesSlowBotTerm(term string) bool {
return term != "" && (term == c.Name || slowBotAliases[term] == c.Name)
}