forked from mutagen-io/mutagen
/
build.go
468 lines (410 loc) · 14.2 KB
/
build.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
package main
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"time"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"github.com/havoc-io/mutagen/cmd"
"github.com/havoc-io/mutagen/pkg/mutagen"
)
const (
agentPackage = "github.com/havoc-io/mutagen/cmd/mutagen-agent"
cliPackage = "github.com/havoc-io/mutagen/cmd/mutagen"
agentBaseName = "mutagen-agent"
cliBaseName = "mutagen"
bundleBaseName = "mutagen-agents.tar.gz"
// If we're compiling for arm, then specify support for ARMv5. This will
// enable software-based floating point. For our use case, this is totally
// fine, because we don't have any numeric code, and the resulting binary
// bloat is very minimal. This won't apply for arm64, which always has
// hardware-based floating point support. For more information, see:
// https://github.com/golang/go/wiki/GoArm.
minimumARMSupport = "5"
)
var GOPATH, GOBIN string
func init() {
// Compute the GOPATH.
if gopath, ok := os.LookupEnv("GOPATH"); !ok {
panic("unable to determine GOPATH")
} else {
GOPATH = gopath
}
// Compute the GOPATH bin directory and ensure it exists.
GOBIN = filepath.Join(GOPATH, "bin")
if err := os.MkdirAll(GOBIN, 0700); err != nil {
panic(errors.Wrap(err, "unable to ensure GOPATH bin directory exists"))
}
}
type Target struct {
GOOS string
GOARCH string
}
func (t Target) String() string {
return fmt.Sprintf("%s/%s", t.GOOS, t.GOARCH)
}
func (t Target) Name() string {
return fmt.Sprintf("%s_%s", t.GOOS, t.GOARCH)
}
func (t Target) ExecutableName(base string) string {
if t.GOOS == "windows" {
return fmt.Sprintf("%s.exe", base)
}
return base
}
func (t Target) goEnv() []string {
// Duplicate the existing environment.
result := os.Environ()
// Override GOOS/GOARCH.
result = append(result, fmt.Sprintf("GOOS=%s", t.GOOS))
result = append(result, fmt.Sprintf("GOARCH=%s", t.GOARCH))
// Set up ARM target support. See notes for definition of minimumARMSupport.
if t.GOOS == "arm" {
result = append(result, fmt.Sprintf("GOARM=%s", minimumARMSupport))
}
// Done.
return result
}
func (t Target) Get(url string) error {
// Execute the build. We use the "-s -w" linker flags to omit the symbol
// table and debugging information. This shaves off about 25% of the binary
// size and only disables debugging (stack traces are still intact). For
// more information, see:
// https://blog.filippo.io/shrink-your-go-binaries-with-this-one-weird-trick
getter := exec.Command("go", "get", "-v", "-ldflags=-s -w", url)
getter.Env = t.goEnv()
getter.Stdin = os.Stdin
getter.Stdout = os.Stdout
getter.Stderr = os.Stderr
if err := getter.Run(); err != nil {
return errors.Wrap(err, "compilation failed")
}
// Success.
return nil
}
func (t Target) Cross() bool {
return t.GOOS != runtime.GOOS || t.GOARCH != runtime.GOARCH
}
func (t Target) ExecutableBuildPath() string {
// Compute the path to the Go bin directory.
result := filepath.Join(GOPATH, "bin")
// If we're cross-compiling, then add the target subdirectory.
if t.Cross() {
result = filepath.Join(result, t.Name())
}
// Done.
return result
}
// targets encodes which combinations of GOOS and GOARCH we want to use for
// building agent and CLI binaries. We don't build every target at the moment,
// but we do list all potential targets here and comment out those we don't
// support. This list is created from https://golang.org/doc/install/source.
// Unfortunately there's no automated way to construct this list, but that's
// fine since we have to manually groom it anyway.
var targets = []Target{
// We completely disable Android because it doesn't provide a useful shell
// or SSH server.
// {"android", "arm"},
// We completely disable darwin/386 because Go only supports macOS 10.7+,
// which is always able to run amd64 binaries.
// {"darwin", "386"},
{"darwin", "amd64"},
// We completely disble darwin/arm and darwin/arm64 because no ARM-based
// Darwin platforms (iOS, watchOS, tvOS) provide a useful shell or SSH
// server.
// {"darwin", "arm"},
// TODO: Figure out why darwin/arm64 doesn't compile in any case anyway.
// We're seeing this issue: https://github.com/golang/go/issues/16445. The
// "resolution" there makes sense, except that darwin/arm compiles fine.
// According to https://golang.org/cmd/cgo/, CGO is automatically disabled
// when cross-compiling. It's not clear if CGO is being disabled for
// darwin/arm (and not for darwin/arm64) or if the environment is just
// broken for darwin/arm64. In either case, there's some bug that needs to
// be fixed. Either CGO needs to be disabled automatically in both cases for
// consistency, or the cross-compilation environment needs to be fixed.
// {"darwin", "arm64"},
{"dragonfly", "amd64"},
{"freebsd", "386"},
{"freebsd", "amd64"},
{"freebsd", "arm"},
{"linux", "386"},
{"linux", "amd64"},
{"linux", "arm"},
{"linux", "arm64"},
{"linux", "ppc64"},
{"linux", "ppc64le"},
{"linux", "mips"},
{"linux", "mipsle"},
{"linux", "mips64"},
{"linux", "mips64le"},
// TODO: This combination is valid but not listed on the "Installing Go from
// source" page. Perhaps we should open a pull request to change that?
{"linux", "s390x"},
{"netbsd", "386"},
{"netbsd", "amd64"},
{"netbsd", "arm"},
{"openbsd", "386"},
{"openbsd", "amd64"},
{"openbsd", "arm"},
// We completely disable Plan 9 because it is just missing too many
// facilities for Mutagen to build easily, even just the agent component.
// TODO: We might be able to get Plan 9 functioning as an agent, but it's
// going to take some serious playing around with source file layouts and
// build tags. To get started looking into this, look for the !plan9 build
// tag and see where the gaps are. Most of the problems revolve around the
// syscall package, but none of that is necessary for the agent, so it can
// probably be built.
// {"plan9", "386"},
// {"plan9", "amd64"},
{"solaris", "amd64"},
{"windows", "386"},
{"windows", "amd64"},
}
// TODO: Figure out if we should set this on a per-machine basis. This value is
// taken from Go's io.Copy method, which defaults to allocating a 32k buffer if
// none is provided.
const archiveBuilderCopyBufferSize = 32 * 1024
type ArchiveBuilder struct {
file *os.File
compressor *gzip.Writer
archiver *tar.Writer
copyBuffer []byte
}
func NewArchiveBuilder(bundlePath string) (*ArchiveBuilder, error) {
// Open the underlying file.
file, err := os.Create(bundlePath)
if err != nil {
return nil, errors.Wrap(err, "unable to create target file")
}
// Create the compressor.
compressor, err := gzip.NewWriterLevel(file, gzip.BestCompression)
if err != nil {
file.Close()
return nil, errors.Wrap(err, "unable to create compressor")
}
// Success.
return &ArchiveBuilder{
file: file,
compressor: compressor,
archiver: tar.NewWriter(compressor),
copyBuffer: make([]byte, archiveBuilderCopyBufferSize),
}, nil
}
func (b *ArchiveBuilder) Close() error {
// Close in the necessary order to trigger flushes.
if err := b.archiver.Close(); err != nil {
b.compressor.Close()
b.file.Close()
return errors.Wrap(err, "unable to close archiver")
} else if err := b.compressor.Close(); err != nil {
b.file.Close()
return errors.Wrap(err, "unable to close compressor")
} else if err := b.file.Close(); err != nil {
return errors.Wrap(err, "unable to close file")
}
// Success.
return nil
}
func (b *ArchiveBuilder) Add(name, path string, mode int64) error {
// If the name is empty, use the base name.
if name == "" {
name = filepath.Base(path)
}
// Open the file and ensure its cleanup.
file, err := os.Open(path)
if err != nil {
return errors.Wrap(err, "unable to open file")
}
defer file.Close()
// Compute its size.
stat, err := file.Stat()
if err != nil {
return errors.Wrap(err, "unable to determine file size")
}
size := stat.Size()
// Write the header for the entry.
header := &tar.Header{
Name: name,
Mode: mode,
Size: size,
ModTime: time.Now(),
}
if err := b.archiver.WriteHeader(header); err != nil {
return errors.Wrap(err, "unable to write archive header")
}
// Copy the file contents.
if _, err := io.CopyBuffer(b.archiver, file, b.copyBuffer); err != nil {
return errors.Wrap(err, "unable to write archive entry")
}
// Success.
return nil
}
func buildAgentForTargetInTesting(target Target) bool {
return !target.Cross() ||
target.GOOS == "darwin" ||
target.GOOS == "windows" ||
(target.GOOS == "linux" && (target.GOARCH == "amd64" || target.GOARCH == "arm")) ||
(target.GOOS == "freebsd" && target.GOARCH == "amd64")
}
var usage = `usage: build [-h|--help] [-m|--mode=<mode>] [-s|--skip-bundles]
The mode flag takes three values: 'slim', 'testing', and 'release'. 'slim' will
build binaries only for the current platform. 'testing' will build the CLI
binary for only the current platform and agents for a small subset of platforms.
Both 'slim' and 'testing' will place their build output (CLI binary and agent
bundle) in the '$GOPATH/bin' directory. 'release' will build CLI and agent
binaries for all platforms and package them in the 'build' subdirectory of the
Mutagen source tree. The default mode is 'slim'. If 'release' mode is specified
with the -s/--skip-bundles flag, then all release artifacts will be built but
release bundles won't be produced.
`
func main() {
// Parse command line arguments.
flagSet := pflag.NewFlagSet("build", pflag.ContinueOnError)
flagSet.SetOutput(ioutil.Discard)
var mode string
var skipBundles bool
flagSet.StringVarP(&mode, "mode", "m", "slim", "specify the build mode")
flagSet.BoolVarP(&skipBundles, "skip-bundles", "s", false, "skip release bundle building")
if err := flagSet.Parse(os.Args[1:]); err != nil {
if err == pflag.ErrHelp {
fmt.Fprint(os.Stdout, usage)
return
} else {
cmd.Fatal(errors.Wrap(err, "unable to parse command line"))
}
}
if mode != "slim" && mode != "testing" && mode != "release" {
cmd.Fatal(errors.New("invalid build mode"))
}
// The only platform really suited to cross-compiling for every other
// platform at the moment is macOS. This is because its DNS resolution
// really has to be done through the system's DNS resolver in order to
// function properly and because FSEvents is used for file monitoring and
// that is a C-based API, not accessible purely via system calls. All of the
// other platforms can survive with pure Go compilation.
if runtime.GOOS != "darwin" {
if mode == "release" {
cmd.Fatal(errors.New("macOS required for release builds"))
} else if mode == "testing" {
cmd.Warning("macOS agents will be built without cgo support")
}
}
// Verify that this script is being run from the Mutagen source directory
// located inside the user's GOPATH. This is just a sanity check to ensure
// that people know what they're building.
_, scriptPath, _, ok := runtime.Caller(0)
if !ok {
cmd.Fatal(errors.New("unable to compute script path"))
}
sourcePath, err := filepath.Abs(filepath.Dir(filepath.Dir(scriptPath)))
if err != nil {
cmd.Fatal(errors.Wrap(err, "unable to determine source path"))
}
expectedSourcePath, err := filepath.Abs(filepath.Join(
GOPATH,
"src",
"github.com",
"havoc-io",
"mutagen",
))
if err != nil {
cmd.Fatal(errors.Wrap(err, "unable to determine expected source path"))
}
if sourcePath != expectedSourcePath {
cmd.Fatal(errors.New("script not invoked from target source path"))
}
// Compute the release build path and ensure it exists if we're in release
// mode.
var releasePath string
if mode == "release" {
releasePath = filepath.Join(sourcePath, "build")
if err := os.MkdirAll(releasePath, 0700); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to create release directory"))
}
}
// Create the agent bundle builder
agentBundlePath := filepath.Join(GOBIN, bundleBaseName)
agentBundle, err := NewArchiveBuilder(agentBundlePath)
if err != nil {
cmd.Fatal(errors.Wrap(err, "unable to create agent bundle builder"))
}
// Build and add agent binaries.
for _, target := range targets {
// Skip agent targets that aren't appropriate for this build mode.
if mode == "slim" && target.Cross() {
continue
} else if mode == "testing" && !buildAgentForTargetInTesting(target) {
continue
}
// Print information.
fmt.Println("Building agent for", target)
// Build.
if err := target.Get(agentPackage); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to build agent"))
}
// Add to bundle.
agentBuildPath := filepath.Join(
target.ExecutableBuildPath(),
target.ExecutableName(agentBaseName),
)
if err := agentBundle.Add(target.Name(), agentBuildPath, 0700); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to add agent to bundle"))
}
}
// Close the agent bundle.
if err := agentBundle.Close(); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to finalize agent bundle"))
}
// Build CLI binaries.
for _, target := range targets {
// If we're not in release mode, we don't do any cross-compilation.
if mode != "release" && target.Cross() {
continue
}
// Print information.
fmt.Println("Building CLI components for", target)
// Build.
if err := target.Get(cliPackage); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to build CLI"))
}
// If we're in release mode, create the release bundle, unless we're
// explicitly requested not to.
if mode == "release" && !skipBundles {
// Print information.
fmt.Println("Building release bundle for", target)
// Compute CLI component paths.
cliBuildPath := filepath.Join(
target.ExecutableBuildPath(),
target.ExecutableName(cliBaseName),
)
// Compute the bundle path.
bundlePath := filepath.Join(
releasePath,
fmt.Sprintf("mutagen_%s_v%s.tar.gz", target.Name(), mutagen.Version),
)
// Create the bundle.
bundle, err := NewArchiveBuilder(bundlePath)
if err != nil {
cmd.Fatal(errors.Wrap(err, "unable to create release bundle"))
}
// Add contents.
if err := bundle.Add("", cliBuildPath, 0700); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to bundle CLI"))
}
if err := bundle.Add("", agentBundlePath, 0600); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to bundle agent bundle"))
}
// Close the bundle.
if err := bundle.Close(); err != nil {
cmd.Fatal(errors.Wrap(err, "unable to finalize release bundle"))
}
}
}
}