diff --git a/microsoft/_util/README.md b/microsoft/_util/README.md new file mode 100644 index 00000000000..d5062a55b75 --- /dev/null +++ b/microsoft/_util/README.md @@ -0,0 +1,11 @@ +## `github.com/microsoft/go/_util` + +This module is a set of utilities Microsoft uses to build Go in Azure DevOps and +maintain this repository. Run `microsoft/run-util.sh` to list the available +commands and see instructions on how to use them. + +`_util` has a `_` prefix so `cmd/internal/moddeps/moddeps_test.go` ignores it. +The moddeps tests enforce stricter requirements than this module needs to +follow. Specifically, the `_util` module requires the `gotestsum` library and +doesn't vendor it. `_util` is not strictly necessary to build Go, so it's ok if +its dependencies are downloaded when needed. diff --git a/microsoft/_util/cmd/run-builder/run-builder.go b/microsoft/_util/cmd/run-builder/run-builder.go new file mode 100644 index 00000000000..e159ea3e218 --- /dev/null +++ b/microsoft/_util/cmd/run-builder/run-builder.go @@ -0,0 +1,235 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "strings" + + gotestsumcmd "gotest.tools/gotestsum/cmd" +) + +const description = ` +This script is used in CI to run a build/test/pack configuration. + +Example: Build and run tests using the dev scripts: + + go run microsoft/run-builder.go -builder linux-amd64-devscript + +For a list of builders that are run in CI, see 'azure-pipelines.yml'. This +doesn't include every builder that upstream uses. It also adds some builders +that upstream doesn't have. +(See https://github.com/golang/build/blob/master/dashboard/builders.go for a +list of upstream builders.) + +CAUTION: Some builders may be destructive! For example, it might set all files +in your repository to read-only. +` + +var dryRun = flag.Bool("n", false, "Enable dry run: print the commands that would be run, but do not run them.") + +func main() { + var builder = flag.String("builder", "", "[Required] Specify a builder to run. Note, this may be destructive!") + var jUnitFile = flag.String("junitfile", "", "Write a JUnit XML file to this path if this builder runs tests.") + var help = flag.Bool("h", false, "Print this help message.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of run-builder.go:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + if len(*builder) == 0 { + fmt.Printf("No '-builder' provided; nothing to do.\n") + return + } + + builderParts := strings.Split(*builder, "-") + if len(builderParts) < 3 { + fmt.Printf("Error: builder '%s' has less than three parts. Expected '{os}-{arch}-{config}'.\n", *builder) + os.Exit(1) + } + + goos, goarch, config := builderParts[0], builderParts[1], strings.Join(builderParts[2:], "-") + fmt.Printf("Found os '%s', arch '%s', config '%s'\n", goos, goarch, config) + + if *builder == "linux-amd64-longtest" { + runOrPanic("microsoft/workaround-install-mercurial.sh") + } + + // Some builder configurations need extra env variables set up during the build, not just while + // running tests: + switch config { + case "clang": + env("CC", "/usr/bin/clang-3.9") + case "longtest": + env("GO_TEST_SHORT", "false") + env("GO_TEST_TIMEOUT_SCALE", "5") + case "nocgo": + env("CGO_ENABLED", "0") + case "noopt": + env("GO_GCFLAGS", "-N -l") + case "regabi": + env("GOEXPERIMENT", "regabi") + case "ssacheck": + env("GO_GCFLAGS", "-d=ssa/check/on,dclstack") + case "staticlockranking": + env("GOEXPERIMENT", "staticlockranking") + } + + runOrPanic("microsoft/build.sh") + + // After the build completes, run builder-specific commands. + switch config { + case "buildandpack": + // "buildandpack" runs the pack script to produce a Go tarball, not tests. + runOrPanic("microsoft/pack.sh") + + case "devscript": + // "devscript" is specific to the Microsoft infrastructure. It means the builder should + // validate the dev-friendly "microsoft/build.sh" script works to build and test Go. It runs + // a subset of the "test" builder's tests, but it uses the dev workflow. + cmdline := []string{"microsoft/build.sh", "--skip-build", "--test"} + + if *jUnitFile != "" { + // Emit verbose JSON results in stdout for conversion. Follow script's arg style, '--'. + cmdline = append(cmdline, "--json") + } + + runTest(cmdline, *jUnitFile) + + default: + // Most builder configurations use "bin/go tool dist test" directly, rather than the + // Microsoft-specific "microsoft/build.sh" script. run-builder uses this approach. + + // The tests read GO_BUILDER_NAME and make decisions based on it. For some configurations, + // we only need to set this env var. + env("GO_BUILDER_NAME", *builder) + + // The "fake" config "test" is a sentinel value that means we should omit the config part of + // the builder name. This lets us have a stable "{os}-{arch}-{config}" API (particularly + // useful when dealing with AzDO YAML limitations) while still being able to test e.g. the + // "linux-amd64" builder from upstream. + if config == "test" { + env("GO_BUILDER_NAME", goos+"-"+goarch) + } + + cmdline := []string{ + // Run under root user so we have zero UID. As of writing, all upstream builders using a + // non-WSL Linux host run tests as root. We encounter at least one issue if we run as + // non-root on Linux in our reimplementation: if the test infra detects non-zero UID, Go + // makes the tree read-only while initializing tests, breaking 'longtest' tests that + // need to open go.mod files with write permissions. + // https://github.com/microsoft/go/issues/53 tracks running as non-root where possible. + "sudo", + // Keep testing configuration we've set up. Sudo normally reloads env. + "--preserve-env", + // Use the dist test command directly, because 'src/run.bash' isn't compatible with + // longtest. 'src/run.bash' sets 'GOPATH=/nonexist-gopath', which breaks modconv tests + // that download modules. + "bin/go", "tool", "dist", "test", + } + + if *jUnitFile != "" { + // Emit verbose JSON results in stdout for conversion. Follow Go flag style, '-'. + cmdline = append(cmdline, "-json") + } + + runTest(cmdline, *jUnitFile) + } +} + +// env sets an env var and logs it. Panics if it doesn't succeed. +func env(key, value string) { + fmt.Printf("Setting env '%s' to '%s'\n", key, value) + if err := os.Setenv(key, value); err != nil { + panic(err) + } +} + +func run(name string, arg ...string) error { + c := exec.Command(name, arg...) + c.Stdout = os.Stdout + c.Stderr = os.Stderr + + if *dryRun { + fmt.Printf("---- Dry run. Would have run command: %v\n", c.Args) + return nil + } + + fmt.Printf("---- Running command: %v\n", c.Args) + return c.Run() +} + +// runOrPanic runs a command, sending stdout/stderr to our streams, and panics if it doesn't succeed. +func runOrPanic(name string, arg ...string) { + if err := run(name, arg...); err != nil { + panic(err) + } +} + +// runTest runs a testing command. If given a JUnit XML file path, runs the test command inside a +// gotestsum command that converts the JSON output into JUnit XML and writes it to a file at this +// path. +func runTest(cmdline []string, jUnitFile string) { + if *dryRun { + fmt.Printf("---- Dry run. Would have run test command: %v\n", cmdline) + return + } + + var err error + + if jUnitFile != "" { + // Set up gotestsum args. We rely on gotestsum to run the command, capture its output, and + // convert it to JUnit test result XML. + gotestsumArgs := append( + []string{ + "--junitfile", jUnitFile, + "--hide-summary", "skipped,output", + "--format", "standard-quiet", + // When a builder runs tests, some JSON lines are mixed in with standard output + // lines. Normally gotestsum treats this as an error, but we need to allow it. + "--ignore-non-json-output-lines", + // We don't use 'go test', we pass our own raw command. ("cmdline" args.) + "--raw-command", + }, + cmdline..., + ) + + fmt.Printf("---- Running gotestsum command: %v\n", gotestsumArgs) + + // Use "ARG_0_PLACEHOLDER" as an arbitrary placeholder name. This is because here, we're + // essentially directly calling gotestsum's main method. The 0th arg to a main method is + // usually the program's path. This is used in the program's help text to give example + // commands that the user can copy-paste no matter where the executable lives or if it's + // been renamed. However, run-builder uses gotestsum as a library, so it's compiled into our + // binary and there is no actual 'gotestsum' program. We could pass run-builder's path, but + // that would be misleading if it ever shows up in gotestsum's output unexpectedly. Instead, + // pass an obvious placeholder. + err = gotestsumcmd.Run("ARG_0_PLACEHOLDER", gotestsumArgs) + } else { + // If we don't have a jUnitFile target, run the command normally. + err = run(cmdline[0], cmdline[1:]...) + } + + if err != nil { + // If we got an ExitError, the error message was already printed by the command. We just + // need to exit with the same exit code. + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + // Something else happened: alert the user. + panic(err) + } +} diff --git a/microsoft/sync/sync-upstream-refs.go b/microsoft/_util/cmd/sync/sync.go similarity index 100% rename from microsoft/sync/sync-upstream-refs.go rename to microsoft/_util/cmd/sync/sync.go diff --git a/microsoft/_util/go.mod b/microsoft/_util/go.mod new file mode 100644 index 00000000000..f4a0307b3bd --- /dev/null +++ b/microsoft/_util/go.mod @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +module github.com/microsoft/go/_util + +go 1.16 + +require gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d diff --git a/microsoft/_util/go.sum b/microsoft/_util/go.sum new file mode 100644 index 00000000000..89fa27cf2c7 --- /dev/null +++ b/microsoft/_util/go.sum @@ -0,0 +1,57 @@ +github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= +github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d h1:dWGnJEQDAsqgFxwNel3ExPgBYQLX9hf9VYWaN11Q3nw= +gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d/go.mod h1:fTR9ZhxC/TLAAx2/WMk/m3TkMB9eEI89gdEzhiRVJT8= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/microsoft/build.sh b/microsoft/build.sh index 0a159d0e8a8..1959f0301f7 100755 --- a/microsoft/build.sh +++ b/microsoft/build.sh @@ -5,6 +5,48 @@ set -euo pipefail +# Print the purpose of the script and how to use it. +usage() { + echo "$0 builds Go, optionally running tests and packing a tar.gz file. + +Use this script to build Go on your local machine in the way the Microsoft +infrastructure builds it. This script automatically downloads a copy of the Go +compiler (required to build Go) then starts the build. This script is also +capable of running tests and packing a tar.gz file: see Options. + +The Microsoft CI infrastructure uses 'microsoft/run-util.sh run-builder', which +runs this script to build Go. If the builder configuration is 'devscript', +run-builder then uses this script to run tests. Otherwise, 'go tool dist test' +is used directly to run tests. (Pass '-h' to run-builder for more info.) The +'devscript' configuration is validated by CI to ensure you can always build and +test locally. + +To build and test Go without the Microsoft infrastructure, use the Bash scripts +in 'src' such as 'src/run.bash' instead of this script. + +Options: + --skip-build Disable building Go. + --test Enable running tests. + --json Runs tests with -json flag to emit verbose results in JSON format. For use in CI. + --pack Enable creating a tar.gz file similar to the official Go binary release. + -h|--help Print this help message and exit. + +Example: Perform a build, run tests on it, and produce a tar.gz file: + $0 --test --pack" +} + +# Print an optional error message and general script usage info, then call "exit 1". +# $1: An error message to print. +exit_error() { + if [ "${1:-}" ]; then + echo "Error: $1" + echo "" + fi + + usage + exit 1 +} + source="${BASH_SOURCE[0]}" # resolve $SOURCE until the file is no longer a symlink @@ -21,35 +63,9 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" build=1 test= +test_json= pack= -# Print usage information and exit 0 if no error message is provided. -# $1: An error message to display. If provided, this function will exit 1. -usage() { - exit_code=0 - - if [ "${1:-}" ]; then - echo "Error: $1" - echo "" - exit_code=1 - fi - - echo "$0 builds Go, optionally running tests and packing a tar.gz file." - echo "" - echo "This script is used by CI for PR validation and building rolling builds, and can be used to reproduce issues in those environments. It downloads and installs a local copy of Go to ensure a consistent version." - echo "" - echo "Options:" - echo " --skip-build Disable building Go." - echo " --test Enable running tests." - echo " --pack Enable creating a tar.gz file similar to the official Go binary release." - echo " -h|--help Print this help message and exit." - echo "" - echo "Example: Perform a build, run tests on it, and produce a tar.gz file:" - echo " $0 --test --pack" - - exit "$exit_code" -} - while [[ $# > 0 ]]; do case "$1" in --skip-build) @@ -58,14 +74,18 @@ while [[ $# > 0 ]]; do --test) test=1 ;; + --json) + test_json=1 + ;; --pack) pack=1 ;; -h|--help) usage + exit ;; *) - usage "Unexpected argument: $1" + exit_error "Unexpected argument: $1" ;; esac @@ -92,7 +112,22 @@ done if [ "$test" ]; then echo "Running tests..." - ./run.bash --no-rebuild + if [ "$test_json" ]; then + # "-json": Get test results as lines of JSON. + # + # "2>&1": If we're running in JSON mode, we are probably running under gotestsum, which + # detects stderr output and prints it as a problem even though we expect it. This could be + # misleading for someone looking at test results. To avoid this, redirect stderr to stdout. + # The test script returns a correct exit code, so the redirect doesn't affect overall test + # success/failure. + # + # For example, stderr output is normal when checking for machine capabilities. A Cgo static + # linking test emits "/usr/bin/ld: cannot find -lc" and then skips the test because that + # indicates static linking isn't supported with the current build/platform. + ./run.bash --no-rebuild -json 2>&1 + else + ./run.bash --no-rebuild + fi fi if [ "$pack" ]; then diff --git a/microsoft/pipeline/jobs/run-linux-job.yml b/microsoft/pipeline/jobs/run-linux-job.yml index 8e646213fb5..7e7d8710cb7 100644 --- a/microsoft/pipeline/jobs/run-linux-job.yml +++ b/microsoft/pipeline/jobs/run-linux-job.yml @@ -30,11 +30,23 @@ jobs: - script: | set -ex - . microsoft/init-stage0.sh - "$stage0_dir/go/bin/go" run microsoft/run-builder.go \ - -builder '${{ parameters.builder.os }}-${{ parameters.builder.arch }}-${{ parameters.builder.config }}' + microsoft/run-util.sh run-builder \ + -builder '${{ parameters.builder.os }}-${{ parameters.builder.arch }}-${{ parameters.builder.config }}' \ + -junitfile '$(Build.SourcesDirectory)/microsoft/artifacts/TestResults.xml' displayName: Run ${{ parameters.builder.config }} + - ${{ if ne(parameters.builder.config, 'buildandpack') }}: + - task: PublishTestResults@2 + displayName: Publish test results + condition: succeededOrFailed() + inputs: + testResultsFormat: JUnit + testResultsFiles: $(Build.SourcesDirectory)/microsoft/artifacts/TestResults.xml + testRunTitle: ${{ parameters.builder.os }}_${{ parameters.builder.arch }}_${{ parameters.builder.config }} + buildPlatform: ${{ parameters.builder.arch }} + buildConfiguration: ${{ parameters.builder.config }} + publishRunAttachments: true + - ${{ if eq(parameters.builder.config, 'buildandpack' ) }}: - publish: microsoft/artifacts/bin artifact: Binaries ${{ parameters.builder.os }}_${{ parameters.builder.arch }}_${{ parameters.builder.config }} diff --git a/microsoft/run-builder.go b/microsoft/run-builder.go deleted file mode 100644 index 78f4ab4d491..00000000000 --- a/microsoft/run-builder.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "os" - "os/exec" - "strings" -) - -const description = ` -This script is used in CI to run a build/test/pack configuration. - -Example: Build and run tests using the dev scripts: - - go run microsoft/run-builder.go -builder linux-amd64-devscript - -For a list of builders that are run in CI, see 'azure-pipelines.yml'. This -doesn't include every builder that upstream uses. It also adds some builders -that upstream doesn't have. -(See https://github.com/golang/build/blob/master/dashboard/builders.go for a -list of upstream builders.) - -CAUTION: Some builders may be destructive! For example, it might set all files -in your repository to read-only. -` - -var dryRun = flag.Bool("n", false, "Enable dry run: print the commands that would be run, but do not run them.") - -func main() { - var builder = flag.String("builder", "", "[Required] Specify a builder to run. Note, this may be destructive!") - var help = flag.Bool("h", false, "Print this help message.") - - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage of run-builder.go:\n") - flag.PrintDefaults() - fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) - } - - flag.Parse() - if *help { - flag.Usage() - return - } - - if len(*builder) == 0 { - fmt.Printf("No '-builder' provided; nothing to do.\n") - return - } - - builderParts := strings.Split(*builder, "-") - if len(builderParts) < 3 { - fmt.Printf("Error: builder '%s' has less than three parts. Expected '{os}-{arch}-{config}'.\n", *builder) - os.Exit(1) - } - - goos, goarch, config := builderParts[0], builderParts[1], strings.Join(builderParts[2:], "-") - fmt.Printf("Found os '%s', arch '%s', config '%s'\n", goos, goarch, config) - - if *builder == "linux-amd64-longtest" { - run("microsoft/workaround-install-mercurial.sh") - } - - // Tests usually use the builder name to decide what to do. However, some configurations also - // need extra env variables set up. Some of these take effect during the Go build. - switch config { - case "clang": - env("CC", "/usr/bin/clang-3.9") - case "longtest": - env("GO_TEST_SHORT", "false") - env("GO_TEST_TIMEOUT_SCALE", "5") - case "nocgo": - env("CGO_ENABLED", "0") - case "noopt": - env("GO_GCFLAGS", "-N -l") - case "regabi": - env("GOEXPERIMENT", "regabi") - case "ssacheck": - env("GO_GCFLAGS", "-d=ssa/check/on,dclstack") - case "staticlockranking": - env("GOEXPERIMENT", "staticlockranking") - } - - run("microsoft/build.sh") - - if config == "buildandpack" { - run("microsoft/pack.sh") - } else if config == "devscript" { - run("microsoft/build.sh", "--skip-build", "--test") - } else { - - // The tests read GO_BUILDER_NAME and make decisions based on it. For some configurations, - // we only need to set this env var. - env("GO_BUILDER_NAME", *builder) - - // The "fake" config "test" is a sentinel value that means we should omit the config part of - // the builder name. This lets us have a stable "{os}-{arch}-{config}" API (particularly - // useful when dealing with AzDO YAML limitations) while still being able to test e.g. the - // "linux-amd64" builder from upstream. - if config == "test" { - env("GO_BUILDER_NAME", goos+"-"+goarch) - } - - // 'sudo': Run under root user so we have zero UID. As of writing, all upstream builders - // using a non-WSL Linux host run tests as root. We encounter at least one issue if we run - // as non-root on Linux in our reimplementation: if the test infra detects non-zero UID, Go - // makes the tree read-only while initializing tests, breaking 'longtest' tests that need to - // open go.mod files with write permissions. https://github.com/microsoft/go/issues/53 - // tracks making tests run as non-root where possible. - // - // '--preserve-env': Keep the testing configuration we've set up. Sudo normally reloads env. - // - // 'bin/go tool dist test': Use the dist test command directly, because 'src/run.bash' isn't - // compatible with longtest. 'src/run.bash' sets 'GOPATH=/nonexist-gopath', which breaks - // modconv tests that download modules. - run("sudo", "--preserve-env", "bin/go", "tool", "dist", "test") - } -} - -// env sets an env var and logs it. Panics if it doesn't succeed. -func env(key, value string) { - fmt.Printf("Setting env '%s' to '%s'\n", key, value) - if err := os.Setenv(key, value); err != nil { - panic(err) - } -} - -// run runs a command, sending stdout/stderr to our streams, and panics if it doesn't succeed. -func run(name string, arg ...string) { - c := exec.Command(name, arg...) - c.Stdout = os.Stdout - c.Stderr = os.Stderr - - if *dryRun { - fmt.Printf("---- Dry run. Would have run command: %v\n", c.Args) - return - } - - fmt.Printf("---- Running command: %v\n", c.Args) - - if err := c.Run(); err != nil { - panic(err) - } -} diff --git a/microsoft/run-util.sh b/microsoft/run-util.sh new file mode 100755 index 00000000000..af479faf223 --- /dev/null +++ b/microsoft/run-util.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -euo pipefail + +# Print the purpose of the script and how to use it. +usage() { + echo "$0 builds and runs a tool defined in 'microsoft/_util/cmd'. + +Usage: $0 [arguments...] + +Builds 'microsoft/_util/cmd/{tool}/{tool}.go' and runs it using the list of +arguments. If necessary, this command automatically installs Go and downloads +the dependencies of the 'microsoft/_util' module. + +Every tool accepts a '-h' argument to show tool usage help. + +Possible tool commands:" + for x in $toolroot/cmd/*; do + echo " $0 ${x##*/}" + done +} + +# Print an optional error message and general script usage info, then call "exit 1". +# $1: An error message to print. +exit_error() { + if [ "${1:-}" ]; then + echo "Error: $1" + echo "" + fi + + usage + exit 1 +} + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +toolroot="$scriptroot/_util" + +if [ ! "${1:-}" ]; then + exit_error "No tool specified." +fi + +# Take the first arg as the tool name. The remaining args ($@) will be passed into the tool. +tool=$1 +shift + +tool_src="$toolroot/cmd/$tool" +tool_output="$scriptroot/artifacts/toolbin/$tool" + +if [ ! -d "$tool_src" ]; then + exit_error "Tool doesn't exist: '$tool_src'." +fi + +. "$scriptroot/init-stage0.sh" +PATH="$PATH:$stage0_dir/go/bin" + +( + # Move into module so "go build" detects it and fetches dependencies. + cd "$toolroot" + go build -o "$tool_output" "./cmd/$tool" + + # Run tools from the root of the repo. + cd "$scriptroot/.." + "$tool_output" "$@" +) diff --git a/microsoft/sync/sync-pipeline.yml b/microsoft/sync/sync-pipeline.yml index 41122bab52e..48c0765d2cb 100644 --- a/microsoft/sync/sync-pipeline.yml +++ b/microsoft/sync/sync-pipeline.yml @@ -34,10 +34,7 @@ jobs: git config --global user.name 'microsoft-golang-bot' git config --global user.email 'microsoft-golang-bot@users.noreply.github.com' - # Set up Go toolset env vars by sourcing init script. - . microsoft/init-stage0.sh - - "$stage0_dir/go/bin/go" run microsoft/sync/sync-upstream-refs.go \ + microsoft/run-util.sh sync \ -origin https://microsoft-golang-bot:$(BotAccount-microsoft-golang-bot-PAT)@github.com/microsoft/go \ -to https://microsoft-golang-bot:$(BotAccount-microsoft-golang-bot-PAT)@github.com/microsoft-golang-bot/go \ -github-pat $(BotAccount-microsoft-golang-bot-PAT) \ diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index bc49c6d8040..12b1f510734 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -29,6 +29,9 @@ func cmdtest() { var t tester var noRebuild bool flag.BoolVar(&t.listMode, "list", false, "list available tests") + // MICROSOFT_UPSTREAM: add JSON output support https://github.com/golang/go/issues/37486 + flag.BoolVar(&t.jsonMode, "json", false, "pass -json to inner go test commands") + // END MICROSOFT_UPSTREAM flag.BoolVar(&t.rebuild, "rebuild", false, "rebuild everything first") flag.BoolVar(&noRebuild, "no-rebuild", false, "overrides -rebuild (historical dreg)") flag.BoolVar(&t.keepGoing, "k", false, "keep going even when error occurred") @@ -50,6 +53,7 @@ func cmdtest() { type tester struct { race bool listMode bool + jsonMode bool // MICROSOFT_UPSTREAM: add JSON output support https://github.com/golang/go/issues/37486 (oneline comment to avoid disturbing gofmt alignment) rebuild bool failed bool keepGoing bool @@ -286,9 +290,15 @@ func short() string { // Callers should use goTest and then pass flags overriding these // defaults as later arguments in the command line. func (t *tester) goTest() []string { - return []string{ + // MICROSOFT_UPSTREAM: add JSON output support https://github.com/golang/go/issues/37486 + cmdline := []string{ "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""), } + if t.jsonMode { + cmdline = append(cmdline, "-json") + } + return cmdline + // END MICROSOFT_UPSTREAM } func (t *tester) tags() string { @@ -371,6 +381,11 @@ func (t *tester) registerStdTest(pkg string, useG3 bool) { t.timeout(timeoutSec), "-gcflags=all=" + gcflags, } + // MICROSOFT_UPSTREAM: add JSON output support https://github.com/golang/go/issues/37486 + if t.jsonMode { + args = append(args, "-json") + } + // END MICROSOFT_UPSTREAM if t.race { args = append(args, "-race") }