diff --git a/mage/main.go b/mage/main.go index c036fa95..71c90d4c 100644 --- a/mage/main.go +++ b/mage/main.go @@ -87,6 +87,8 @@ type Invocation struct { Keep bool // tells mage to keep the generated main file after compiling Timeout time.Duration // tells mage to set a timeout to running the targets CompileOut string // tells mage to compile a static binary to this path, but not execute + GOOS string // sets the GOOS when producing a binary with -compileout + GOARCH string // sets the GOARCH when producing a binary with -compileout Stdout io.Writer // writer to write stdout messages to Stderr io.Writer // writer to write stderr messages to Stdin io.Reader // reader to read stdin from @@ -148,24 +150,31 @@ func Parse(stderr, stdout io.Writer, args []string) (inv Invocation, cmd Command inv.Stdout = stdout fs := flag.FlagSet{} fs.SetOutput(stdout) + + // options flags + fs.BoolVar(&inv.Force, "f", false, "force recreation of compiled magefile") fs.BoolVar(&inv.Debug, "debug", mg.Debug(), "turn on debug messages") fs.BoolVar(&inv.Verbose, "v", mg.Verbose(), "show verbose output when running mage targets") - fs.BoolVar(&inv.List, "l", false, "list mage targets in this directory") fs.BoolVar(&inv.Help, "h", false, "show this help") fs.DurationVar(&inv.Timeout, "t", 0, "timeout in duration parsable format (e.g. 5m30s)") fs.BoolVar(&inv.Keep, "keep", false, "keep intermediate mage files around after running") fs.StringVar(&inv.Dir, "d", ".", "run magefiles in the given directory") + fs.StringVar(&inv.GoCmd, "gocmd", mg.GoCmd(), "use the given go binary to compile the output") + fs.StringVar(&inv.GOOS, "goos", "", "set GOOS for binary produced with -compile") + fs.StringVar(&inv.GOARCH, "goarch", "", "set GOARCH for binary produced with -compile") + + // commands below + + fs.BoolVar(&inv.List, "l", false, "list mage targets in this directory") var showVersion bool fs.BoolVar(&showVersion, "version", false, "show version info for the mage binary") - var mageInit bool fs.BoolVar(&mageInit, "init", false, "create a starting template if no mage files exist") var clean bool fs.BoolVar(&clean, "clean", false, "clean out old generated binaries from CACHE_DIR") var compileOutPath string fs.StringVar(&compileOutPath, "compile", "", "output a static binary to the given path") - fs.StringVar(&inv.GoCmd, "gocmd", mg.GoCmd(), "use the given go binary to compile the output") fs.Usage = func() { fmt.Fprint(stdout, ` @@ -184,16 +193,18 @@ Commands: Options: -d - run magefiles in the given directory (default ".") - -debug turn on debug messages - -h show description of a target - -f force recreation of compiled magefile - -keep keep intermediate mage files around after running + run magefiles in the given directory (default ".") + -debug turn on debug messages + -h show description of a target + -f force recreation of compiled magefile + -keep keep intermediate mage files around after running -gocmd - use the given go binary to compile the output (default: "go") + use the given go binary to compile the output (default: "go") + -goos sets the GOOS for the binary created by -compile (default: current OS) + -goarch sets the GOARCH for the binary created by -compile (default: current arch) -t - timeout in duration parsable format (e.g. 5m30s) - -v show verbose output when running mage targets + timeout in duration parsable format (e.g. 5m30s) + -v show verbose output when running mage targets `[1:]) } err = fs.Parse(args) @@ -207,21 +218,21 @@ Options: return inv, cmd, flag.ErrHelp } - numFlags := 0 + numCommands := 0 switch { case mageInit: - numFlags++ + numCommands++ cmd = Init case compileOutPath != "": - numFlags++ + numCommands++ cmd = CompileStatic inv.CompileOut = compileOutPath inv.Force = true case showVersion: - numFlags++ + numCommands++ cmd = Version case clean: - numFlags++ + numCommands++ cmd = Clean if fs.NArg() > 0 { // Temporary dupe of below check until we refactor the other commands to use this check @@ -230,7 +241,7 @@ Options: } } if inv.Help { - numFlags++ + numCommands++ } if inv.Debug { @@ -239,16 +250,24 @@ Options: inv.CacheDir = mg.CacheDir() - if numFlags > 1 { - debug.Printf("%d commands defined", numFlags) + if numCommands > 1 { + debug.Printf("%d commands defined", numCommands) return inv, cmd, errors.New("-h, -init, -clean, -compile and -version cannot be used simultaneously") } + if cmd != CompileStatic && (inv.GOARCH != "" || inv.GOOS != "") { + return inv, cmd, errors.New("-goos and -goarch only apply when running with -compile") + } + inv.Args = fs.Args() if inv.Help && len(inv.Args) > 1 { return inv, cmd, errors.New("-h can only show help for a single target") } + if len(inv.Args) > 0 && cmd != None { + return inv, cmd, fmt.Errorf("unexpected arguments to command: %q", inv.Args) + } + return inv, cmd, err } @@ -265,7 +284,7 @@ func Invoke(inv Invocation) int { inv.CacheDir = mg.CacheDir() } - files, err := Magefiles(inv.Dir, inv.GoCmd, inv.Stderr, inv.Debug) + files, err := Magefiles(inv.Dir, inv.GOOS, inv.GOARCH, inv.GoCmd, inv.Stderr, inv.Debug) if err != nil { errlog.Println("Error determining list of magefiles:", err) return 1 @@ -276,13 +295,13 @@ func Invoke(inv Invocation) int { return 1 } debug.Printf("found magefiles: %s", strings.Join(files, ", ")) - exePath, err := ExeName(inv.GoCmd, inv.CacheDir, files) - if err != nil { - errlog.Println("Error getting exe name:", err) - return 1 - } - if inv.CompileOut != "" { - exePath = inv.CompileOut + exePath := inv.CompileOut + if inv.CompileOut == "" { + exePath, err = ExeName(inv.GoCmd, inv.CacheDir, files) + if err != nil { + errlog.Println("Error getting exe name:", err) + return 1 + } } debug.Println("output exe is ", exePath) @@ -378,7 +397,7 @@ func Invoke(inv Invocation) int { defer os.RemoveAll(main) } files = append(files, main) - if err := Compile(inv.Dir, inv.GoCmd, exePath, files, inv.Debug, inv.Stderr, inv.Stdout); err != nil { + if err := Compile(inv.GOOS, inv.GOARCH, inv.Dir, inv.GoCmd, exePath, files, inv.Debug, inv.Stderr, inv.Stdout); err != nil { errlog.Println("Error:", err) return 1 } @@ -491,22 +510,8 @@ type mainfileTemplateData struct { BinaryName string } -func goVersion(gocmd string) (string, error) { - cmd := exec.Command(gocmd, "version") - out, stderr := &bytes.Buffer{}, &bytes.Buffer{} - cmd.Stdout = out - cmd.Stderr = stderr - if err := cmd.Run(); err != nil { - if s := stderr.String(); s != "" { - return "", fmt.Errorf("failed to run `go version`: %s", s) - } - return "", fmt.Errorf("failed to run `go version`: %v", err) - } - return out.String(), nil -} - // Magefiles returns the list of magefiles in dir. -func Magefiles(magePath, goCmd string, stderr io.Writer, isDebug bool) ([]string, error) { +func Magefiles(magePath, goos, goarch, goCmd string, stderr io.Writer, isDebug bool) ([]string, error) { start := time.Now() defer func() { debug.Println("time to scan for Magefiles:", time.Since(start)) @@ -515,11 +520,16 @@ func Magefiles(magePath, goCmd string, stderr io.Writer, isDebug bool) ([]string return nil, err } + env, err := envWithGOOS(goos, goarch) + if err != nil { + return nil, err + } + debug.Println("getting all non-mage files in", magePath) // // first, grab all the files with no build tags specified.. this is actually // // our exclude list of things without the mage build tag. cmd := exec.Command(goCmd, "list", "-e", "-f", `{{join .GoFiles "||"}}`) - + cmd.Env = env if isDebug { cmd.Stderr = stderr } @@ -539,6 +549,8 @@ func Magefiles(magePath, goCmd string, stderr io.Writer, isDebug bool) ([]string } debug.Println("getting all files plus mage files") cmd = exec.Command(goCmd, "list", "-tags=mage", "-e", "-f", `{{join .GoFiles "||"}}`) + cmd.Env = env + if isDebug { cmd.Stderr = stderr } @@ -562,26 +574,29 @@ func Magefiles(magePath, goCmd string, stderr io.Writer, isDebug bool) ([]string } // Compile uses the go tool to compile the files into an executable at path. -func Compile(magePath, goCmd, compileTo string, gofiles []string, isDebug bool, stderr, stdout io.Writer) error { +func Compile(goos, goarch, magePath, goCmd, compileTo string, gofiles []string, isDebug bool, stderr, stdout io.Writer) error { debug.Println("compiling to", compileTo) debug.Println("compiling using gocmd:", goCmd) if isDebug { runDebug(goCmd, "version") runDebug(goCmd, "env") } - + environ, err := envWithGOOS(goos, goarch) + if err != nil { + return err + } // strip off the path since we're setting the path in the build command for i := range gofiles { gofiles[i] = filepath.Base(gofiles[i]) } debug.Printf("running %s build -o %s %s", goCmd, compileTo, strings.Join(gofiles, " ")) c := exec.Command(goCmd, append([]string{"build", "-o", compileTo}, gofiles...)...) - c.Env = os.Environ() + c.Env = environ c.Stderr = stderr c.Stdout = stdout c.Dir = magePath start := time.Now() - err := c.Run() + err = c.Run() debug.Println("time to compile Magefile:", time.Since(start)) if err != nil { return errors.New("error compiling magefiles") @@ -590,11 +605,15 @@ func Compile(magePath, goCmd, compileTo string, gofiles []string, isDebug bool, } func runDebug(cmd string, args ...string) error { + env, err := envWithCurrentGOOS() + if err != nil { + return err + } buf := &bytes.Buffer{} errbuf := &bytes.Buffer{} debug.Println("running", cmd, strings.Join(args, " ")) c := exec.Command(cmd, args...) - c.Env = os.Environ() + c.Env = env c.Stderr = errbuf c.Stdout = buf if err := c.Run(); err != nil { @@ -606,11 +625,15 @@ func runDebug(cmd string, args ...string) error { } func outputDebug(cmd string, args ...string) (string, error) { + env, err := envWithCurrentGOOS() + if err != nil { + return "", err + } buf := &bytes.Buffer{} errbuf := &bytes.Buffer{} debug.Println("running", cmd, strings.Join(args, " ")) c := exec.Command(cmd, args...) - c.Env = os.Environ() + c.Env = env c.Stderr = errbuf c.Stdout = buf if err := c.Run(); err != nil { @@ -672,7 +695,7 @@ func ExeName(goCmd, cacheDir string, files []string) (string, error) { // binary. hashes = append(hashes, fmt.Sprintf("%x", sha1.Sum([]byte(mageMainfileTplString)))) sort.Strings(hashes) - ver, err := goVersion(goCmd) + ver, err := outputDebug(goCmd, "version") if err != nil { return "", err } @@ -723,6 +746,8 @@ func RunCompiled(inv Invocation, exePath string, errlog *log.Logger) int { c.Stdout = inv.Stdout c.Stdin = inv.Stdin c.Dir = inv.Dir + // intentionally pass through unaltered os.Environ here.. your magefile has + // to deal with it. c.Env = os.Environ() if inv.Verbose { c.Env = append(c.Env, "MAGEFILE_VERBOSE=1") @@ -757,6 +782,8 @@ func filter(list []string, prefix string) []string { return out } +// removeContents removes all files but not any subdirectories in the given +// directory. func removeContents(dir string) error { debug.Println("removing all files in", dir) files, err := ioutil.ReadDir(dir) @@ -776,4 +803,62 @@ func removeContents(dir string) error { } } return nil + +} + +// splitEnv takes the results from os.Environ() (a []string of foo=bar values) +// and makes a map[string]string out of it. +func splitEnv(env []string) (map[string]string, error) { + out := map[string]string{} + + for _, s := range env { + parts := strings.SplitN(s, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("badly formatted environment variable: %v", s) + } + out[parts[0]] = parts[1] + } + return out, nil +} + +// joinEnv converts the given map into a list of foo=bar environment variables, +// such as that outputted by os.Environ(). +func joinEnv(env map[string]string) []string { + vals := make([]string, 0, len(env)) + for k, v := range env { + vals = append(vals, k+"="+v) + } + return vals +} + +// envWithCurrentGOOS returns a copy of os.Environ with the GOOS and GOARCH set +// to runtime.GOOS and runtime.GOARCH. +func envWithCurrentGOOS() ([]string, error) { + vals, err := splitEnv(os.Environ()) + if err != nil { + return nil, err + } + vals["GOOS"] = runtime.GOOS + vals["GOARCH"] = runtime.GOARCH + return joinEnv(vals), nil +} + +// envWithGOOS retuns the os.Environ() values with GOOS and/or GOARCH either set +// to their runtime value, or the given value if non-empty. +func envWithGOOS(goos, goarch string) ([]string, error) { + env, err := splitEnv(os.Environ()) + if err != nil { + return nil, err + } + if goos == "" { + env["GOOS"] = runtime.GOOS + } else { + env["GOOS"] = goos + } + if goarch == "" { + env["GOARCH"] = runtime.GOARCH + } else { + env["GOARCH"] = goarch + } + return joinEnv(env), nil } diff --git a/mage/main_test.go b/mage/main_test.go index 2f87113c..c992b977 100644 --- a/mage/main_test.go +++ b/mage/main_test.go @@ -2,11 +2,14 @@ package mage import ( "bytes" + "debug/macho" + "debug/pe" "flag" "fmt" "go/build" "go/parser" "go/token" + "io" "io/ioutil" "log" "os" @@ -106,7 +109,7 @@ func TestTransitiveDepCache(t *testing.T) { func TestListMagefilesMain(t *testing.T) { buf := &bytes.Buffer{} - files, err := Magefiles("testdata/mixed_main_files", "go", buf, false) + files, err := Magefiles("testdata/mixed_main_files", "", "", "go", buf, false) if err != nil { t.Errorf("error from magefile list: %v: %s", err, buf) } @@ -116,9 +119,112 @@ func TestListMagefilesMain(t *testing.T) { } } +func TestListMagefilesIgnoresGOOS(t *testing.T) { + buf := &bytes.Buffer{} + if runtime.GOOS == "windows" { + os.Setenv("GOOS", "linux") + } else { + os.Setenv("GOOS", "windows") + } + defer os.Setenv("GOOS", runtime.GOOS) + files, err := Magefiles("testdata/goos_magefiles", "", "", "go", buf, false) + if err != nil { + t.Errorf("error from magefile list: %v: %s", err, buf) + } + var expected []string + if runtime.GOOS == "windows" { + expected = []string{"testdata/goos_magefiles/magefile_windows.go"} + } else { + expected = []string{"testdata/goos_magefiles/magefile_nonwindows.go"} + } + if !reflect.DeepEqual(files, expected) { + t.Fatalf("expected %q but got %q", expected, files) + } +} + +func TestListMagefilesIgnoresRespectsGOOSArg(t *testing.T) { + buf := &bytes.Buffer{} + var goos string + if runtime.GOOS == "windows" { + goos = "linux" + } else { + goos = "windows" + } + files, err := Magefiles("testdata/goos_magefiles", goos, "", "go", buf, false) + if err != nil { + t.Errorf("error from magefile list: %v: %s", err, buf) + } + var expected []string + if goos == "windows" { + expected = []string{"testdata/goos_magefiles/magefile_windows.go"} + } else { + expected = []string{"testdata/goos_magefiles/magefile_nonwindows.go"} + } + if !reflect.DeepEqual(files, expected) { + t.Fatalf("expected %q but got %q", expected, files) + } +} + +func TestCompileDiffGoosGoarch(t *testing.T) { + target, err := ioutil.TempDir("./testdata", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(target) + + // intentionally choose an arch and os to build that are not our current one. + + goos := "windows" + if runtime.GOOS == "windows" { + goos = "darwin" + } + goarch := "amd64" + if runtime.GOARCH == "amd64" { + goarch = "386" + } + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + inv := Invocation{ + Stderr: stderr, + Stdout: stdout, + Debug: true, + Dir: "testdata", + // this is relative to the Dir above + CompileOut: filepath.Join(".", filepath.Base(target), "output"), + GOOS: goos, + GOARCH: goarch, + } + code := Invoke(inv) + if code != 0 { + t.Fatalf("got code %v, err: %s", code, stderr) + } + os, arch, err := fileData(filepath.Join(target, "output")) + if err != nil { + t.Fatal(err) + } + if goos == "windows" { + if os != winExe { + t.Error("ran with GOOS=windows but did not produce a windows exe") + } + } else { + if os != macExe { + t.Error("ran with GOOS=darwin but did not a mac exe") + } + } + if goarch == "amd64" { + if arch != arch64 { + t.Error("ran with GOARCH=amd64 but did not produce a 64 bit exe") + } + } else { + if arch != arch32 { + t.Error("rand with GOARCH=386 but did not produce a 32 bit exe") + } + } +} + func TestListMagefilesLib(t *testing.T) { buf := &bytes.Buffer{} - files, err := Magefiles("testdata/mixed_lib_files", "go", buf, false) + files, err := Magefiles("testdata/mixed_lib_files", "", "", "go", buf, false) if err != nil { t.Errorf("error from magefile list: %v: %s", err, buf) } @@ -1024,7 +1130,7 @@ func TestGoCmd(t *testing.T) { buf := &bytes.Buffer{} stderr := &bytes.Buffer{} - if err := Compile(dir, os.Args[0], name, []string{}, false, stderr, buf); err != nil { + if err := Compile("", "", dir, os.Args[0], name, []string{}, false, stderr, buf); err != nil { t.Log("stderr: ", stderr.String()) t.Fatal(err) } @@ -1149,3 +1255,54 @@ func TestNamespaceDefault(t *testing.T) { t.Fatalf("expected %q, but got %q", expected, stdout.String()) } } + +/// This code liberally borrowed from https://github.com/rsc/goversion/blob/master/version/exe.go + +type exeType int +type archSize int + +const ( + winExe exeType = iota + macExe + + arch32 archSize = iota + arch64 +) + +// fileData tells us if the given file is mac or windows and if they're 32bit or +// 64 bit. Other exe versions are not supported. +func fileData(file string) (exeType, archSize, error) { + f, err := os.Open(file) + if err != nil { + return -1, -1, err + } + defer f.Close() + data := make([]byte, 16) + if _, err := io.ReadFull(f, data); err != nil { + return -1, -1, err + } + if bytes.HasPrefix(data, []byte("MZ")) { + // hello windows exe! + e, err := pe.NewFile(f) + if err != nil { + return -1, -1, err + } + if e.Machine == pe.IMAGE_FILE_MACHINE_AMD64 { + return winExe, arch64, nil + } + return winExe, arch32, nil + } + + if bytes.HasPrefix(data, []byte("\xFE\xED\xFA")) || bytes.HasPrefix(data[1:], []byte("\xFA\xED\xFE")) { + // hello mac exe! + fe, err := macho.NewFile(f) + if err != nil { + return -1, -1, err + } + if fe.Cpu&0x01000000 != 0 { + return macExe, arch64, nil + } + return macExe, arch32, nil + } + return -1, -1, fmt.Errorf("unrecognized executable format") +} diff --git a/mage/testdata/goos_magefiles/magefile_nonwindows.go b/mage/testdata/goos_magefiles/magefile_nonwindows.go new file mode 100644 index 00000000..420f4f89 --- /dev/null +++ b/mage/testdata/goos_magefiles/magefile_nonwindows.go @@ -0,0 +1,5 @@ +// +build mage,!windows + +package main + +func NonWindowsTarget() {} diff --git a/mage/testdata/goos_magefiles/magefile_windows.go b/mage/testdata/goos_magefiles/magefile_windows.go new file mode 100644 index 00000000..1b8248bb --- /dev/null +++ b/mage/testdata/goos_magefiles/magefile_windows.go @@ -0,0 +1,5 @@ +// +build mage + +package main + +func WindowsTarget() {} diff --git a/site/content/compiling/_index.en.md b/site/content/compiling/_index.en.md new file mode 100644 index 00000000..b896c9ea --- /dev/null +++ b/site/content/compiling/_index.en.md @@ -0,0 +1,41 @@ ++++ +title = "Compiling" +weight = 37 ++++ + +## Mage ignores GOOS and GOARCH for its build + +When building the binary for your magefile, mage will ignore the GOOS and GOARCH environment variables and use your current GOOS and GOARCH to ensure the binary that is built can run on your local system. This way you can set GOOS and GOARCH when you run mage, to have it take effect on the outputs of your magefile, without it also rendering your magefile unrunnable on your local machine. + +## Compiling a static binary + +It can be useful to compile a static binary which has the mage execution runtime +and the tasks compiled in such that it can be run on another machine without +requiring any dependencies. To do so, pass the output path to the compile flag. +like this: + +```plain +$ mage -compile ./static-output +``` + +The compiled binary uses flags just like the mage binary: + +```plain + [options] [target] + +Commands: + -l list targets in this binary + -h show this help + +Options: + -h show description of a target + -t + timeout in duration parsable format (e.g. 5m30s) + -v show verbose output when running targets +``` + +## Compiling for a different OS -goos and -goarch + +If you intend to run the binary on another machine with a different OS platform, you may use the `-goos` and `-goarch` flags to build the compiled binary for the target platform. Valid values for these flags may be found here: https://golang.org/doc/install/source#environment. The OS values are obvious (except darwin=MacOS), the GOARCH values most commonly needed will be "amd64" or "386" for 64 for 32 bit versions of common desktop OSes. + +Note that if you run `-compile` with `-dir`, the `-compile` target will be *relative to the magefile dir*. \ No newline at end of file diff --git a/site/content/index.md b/site/content/index.md index d3ebf868..e1199b25 100644 --- a/site/content/index.md +++ b/site/content/index.md @@ -66,17 +66,19 @@ Commands: Options: -d - run magefiles in the given directory (default ".") - -debug turn on debug messages - -h show description of a target - -f force recreation of compiled magefile - -keep keep intermediate mage files around after running + run magefiles in the given directory (default ".") + -debug turn on debug messages + -h show description of a target + -f force recreation of compiled magefile + -keep keep intermediate mage files around after running -gocmd - use the given go binary to compile the output (default: "go") + use the given go binary to compile the output (default: "go") + -goos sets the GOOS for the binary created by -compile (default: current OS) + -goarch sets the GOARCH for the binary created by -compile (default: current arch) -t - timeout in duration parsable format (e.g. 5m30s) - -v show verbose output when running mage targets -``` + timeout in duration parsable format (e.g. 5m30s) + -v show verbose output when running mage targets + ``` ## Why? @@ -91,17 +93,6 @@ that's not just straight line execution of commands. And if your project is wri introduce another language as idiosyncratic as bash? Why not use the language your contributors are already comfortable with? -## Compiling a static binary - -If your tasks are not related to compiling Go code, it can be useful to compile a binary which has -the mage execution runtime and the tasks compiled in such that it can be run on another machine -without requiring installation of dependencies. To do so, pass the output path to the compile flag. -like this: - -```plain -$ mage -compile ./static-output -``` - ## Code [https://github.com/magefile/mage](https://github.com/magefile/mage)