Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use test JSON and gotestsum to produce JUnit XML for AzDO (#71)
* Hook up tests to AzDO test infrastructure * Remove unnecessary output when tests fail * UPSTREAM markers -> MICROSOFT_UPSTREAM * Better docs for scripts, run-builder
- Loading branch information
Showing
11 changed files
with
487 additions
and
185 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| ## `github.com/microsoft/go/_util` | ||
|
|
||
| This module is a set of utilities Microsoft uses to build Go in Azure DevOps and | ||
| maintain this repository. Run `microsoft/run-util.sh` to list the available | ||
| commands and see instructions on how to use them. | ||
|
|
||
| `_util` has a `_` prefix so `cmd/internal/moddeps/moddeps_test.go` ignores it. | ||
| The moddeps tests enforce stricter requirements than this module needs to | ||
| follow. Specifically, the `_util` module requires the `gotestsum` library and | ||
| doesn't vendor it. `_util` is not strictly necessary to build Go, so it's ok if | ||
| its dependencies are downloaded when needed. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,235 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Use of this source code is governed by a BSD-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "flag" | ||
| "fmt" | ||
| "os" | ||
| "os/exec" | ||
| "strings" | ||
|
|
||
| gotestsumcmd "gotest.tools/gotestsum/cmd" | ||
| ) | ||
|
|
||
| const description = ` | ||
| This script is used in CI to run a build/test/pack configuration. | ||
| Example: Build and run tests using the dev scripts: | ||
| go run microsoft/run-builder.go -builder linux-amd64-devscript | ||
| For a list of builders that are run in CI, see 'azure-pipelines.yml'. This | ||
| doesn't include every builder that upstream uses. It also adds some builders | ||
| that upstream doesn't have. | ||
| (See https://github.com/golang/build/blob/master/dashboard/builders.go for a | ||
| list of upstream builders.) | ||
| CAUTION: Some builders may be destructive! For example, it might set all files | ||
| in your repository to read-only. | ||
| ` | ||
|
|
||
| var dryRun = flag.Bool("n", false, "Enable dry run: print the commands that would be run, but do not run them.") | ||
|
|
||
| func main() { | ||
| var builder = flag.String("builder", "", "[Required] Specify a builder to run. Note, this may be destructive!") | ||
| var jUnitFile = flag.String("junitfile", "", "Write a JUnit XML file to this path if this builder runs tests.") | ||
| var help = flag.Bool("h", false, "Print this help message.") | ||
|
|
||
| flag.Usage = func() { | ||
| fmt.Fprintf(flag.CommandLine.Output(), "Usage of run-builder.go:\n") | ||
| flag.PrintDefaults() | ||
| fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) | ||
| } | ||
|
|
||
| flag.Parse() | ||
| if *help { | ||
| flag.Usage() | ||
| return | ||
| } | ||
|
|
||
| if len(*builder) == 0 { | ||
| fmt.Printf("No '-builder' provided; nothing to do.\n") | ||
| return | ||
| } | ||
|
|
||
| builderParts := strings.Split(*builder, "-") | ||
| if len(builderParts) < 3 { | ||
| fmt.Printf("Error: builder '%s' has less than three parts. Expected '{os}-{arch}-{config}'.\n", *builder) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| goos, goarch, config := builderParts[0], builderParts[1], strings.Join(builderParts[2:], "-") | ||
| fmt.Printf("Found os '%s', arch '%s', config '%s'\n", goos, goarch, config) | ||
|
|
||
| if *builder == "linux-amd64-longtest" { | ||
| runOrPanic("microsoft/workaround-install-mercurial.sh") | ||
| } | ||
|
|
||
| // Some builder configurations need extra env variables set up during the build, not just while | ||
| // running tests: | ||
| switch config { | ||
| case "clang": | ||
| env("CC", "/usr/bin/clang-3.9") | ||
| case "longtest": | ||
| env("GO_TEST_SHORT", "false") | ||
| env("GO_TEST_TIMEOUT_SCALE", "5") | ||
| case "nocgo": | ||
| env("CGO_ENABLED", "0") | ||
| case "noopt": | ||
| env("GO_GCFLAGS", "-N -l") | ||
| case "regabi": | ||
| env("GOEXPERIMENT", "regabi") | ||
| case "ssacheck": | ||
| env("GO_GCFLAGS", "-d=ssa/check/on,dclstack") | ||
| case "staticlockranking": | ||
| env("GOEXPERIMENT", "staticlockranking") | ||
| } | ||
|
|
||
| runOrPanic("microsoft/build.sh") | ||
|
|
||
| // After the build completes, run builder-specific commands. | ||
| switch config { | ||
| case "buildandpack": | ||
| // "buildandpack" runs the pack script to produce a Go tarball, not tests. | ||
| runOrPanic("microsoft/pack.sh") | ||
|
|
||
| case "devscript": | ||
| // "devscript" is specific to the Microsoft infrastructure. It means the builder should | ||
| // validate the dev-friendly "microsoft/build.sh" script works to build and test Go. It runs | ||
| // a subset of the "test" builder's tests, but it uses the dev workflow. | ||
| cmdline := []string{"microsoft/build.sh", "--skip-build", "--test"} | ||
|
|
||
| if *jUnitFile != "" { | ||
| // Emit verbose JSON results in stdout for conversion. Follow script's arg style, '--'. | ||
| cmdline = append(cmdline, "--json") | ||
| } | ||
|
|
||
| runTest(cmdline, *jUnitFile) | ||
|
|
||
| default: | ||
| // Most builder configurations use "bin/go tool dist test" directly, rather than the | ||
| // Microsoft-specific "microsoft/build.sh" script. run-builder uses this approach. | ||
|
|
||
| // The tests read GO_BUILDER_NAME and make decisions based on it. For some configurations, | ||
| // we only need to set this env var. | ||
| env("GO_BUILDER_NAME", *builder) | ||
|
|
||
| // The "fake" config "test" is a sentinel value that means we should omit the config part of | ||
| // the builder name. This lets us have a stable "{os}-{arch}-{config}" API (particularly | ||
| // useful when dealing with AzDO YAML limitations) while still being able to test e.g. the | ||
| // "linux-amd64" builder from upstream. | ||
| if config == "test" { | ||
| env("GO_BUILDER_NAME", goos+"-"+goarch) | ||
| } | ||
|
|
||
| cmdline := []string{ | ||
| // Run under root user so we have zero UID. As of writing, all upstream builders using a | ||
| // non-WSL Linux host run tests as root. We encounter at least one issue if we run as | ||
| // non-root on Linux in our reimplementation: if the test infra detects non-zero UID, Go | ||
| // makes the tree read-only while initializing tests, breaking 'longtest' tests that | ||
| // need to open go.mod files with write permissions. | ||
| // https://github.com/microsoft/go/issues/53 tracks running as non-root where possible. | ||
| "sudo", | ||
| // Keep testing configuration we've set up. Sudo normally reloads env. | ||
| "--preserve-env", | ||
| // Use the dist test command directly, because 'src/run.bash' isn't compatible with | ||
| // longtest. 'src/run.bash' sets 'GOPATH=/nonexist-gopath', which breaks modconv tests | ||
| // that download modules. | ||
| "bin/go", "tool", "dist", "test", | ||
| } | ||
|
|
||
| if *jUnitFile != "" { | ||
| // Emit verbose JSON results in stdout for conversion. Follow Go flag style, '-'. | ||
| cmdline = append(cmdline, "-json") | ||
| } | ||
|
|
||
| runTest(cmdline, *jUnitFile) | ||
| } | ||
| } | ||
|
|
||
| // env sets an env var and logs it. Panics if it doesn't succeed. | ||
| func env(key, value string) { | ||
| fmt.Printf("Setting env '%s' to '%s'\n", key, value) | ||
| if err := os.Setenv(key, value); err != nil { | ||
| panic(err) | ||
| } | ||
| } | ||
|
|
||
| func run(name string, arg ...string) error { | ||
| c := exec.Command(name, arg...) | ||
| c.Stdout = os.Stdout | ||
| c.Stderr = os.Stderr | ||
|
|
||
| if *dryRun { | ||
| fmt.Printf("---- Dry run. Would have run command: %v\n", c.Args) | ||
| return nil | ||
| } | ||
|
|
||
| fmt.Printf("---- Running command: %v\n", c.Args) | ||
| return c.Run() | ||
| } | ||
|
|
||
| // runOrPanic runs a command, sending stdout/stderr to our streams, and panics if it doesn't succeed. | ||
| func runOrPanic(name string, arg ...string) { | ||
| if err := run(name, arg...); err != nil { | ||
| panic(err) | ||
| } | ||
| } | ||
|
|
||
| // runTest runs a testing command. If given a JUnit XML file path, runs the test command inside a | ||
| // gotestsum command that converts the JSON output into JUnit XML and writes it to a file at this | ||
| // path. | ||
| func runTest(cmdline []string, jUnitFile string) { | ||
| if *dryRun { | ||
| fmt.Printf("---- Dry run. Would have run test command: %v\n", cmdline) | ||
| return | ||
| } | ||
|
|
||
| var err error | ||
|
|
||
| if jUnitFile != "" { | ||
| // Set up gotestsum args. We rely on gotestsum to run the command, capture its output, and | ||
| // convert it to JUnit test result XML. | ||
| gotestsumArgs := append( | ||
| []string{ | ||
| "--junitfile", jUnitFile, | ||
| "--hide-summary", "skipped,output", | ||
| "--format", "standard-quiet", | ||
| // When a builder runs tests, some JSON lines are mixed in with standard output | ||
| // lines. Normally gotestsum treats this as an error, but we need to allow it. | ||
| "--ignore-non-json-output-lines", | ||
| // We don't use 'go test', we pass our own raw command. ("cmdline" args.) | ||
| "--raw-command", | ||
| }, | ||
| cmdline..., | ||
| ) | ||
|
|
||
| fmt.Printf("---- Running gotestsum command: %v\n", gotestsumArgs) | ||
|
|
||
| // Use "ARG_0_PLACEHOLDER" as an arbitrary placeholder name. This is because here, we're | ||
| // essentially directly calling gotestsum's main method. The 0th arg to a main method is | ||
| // usually the program's path. This is used in the program's help text to give example | ||
| // commands that the user can copy-paste no matter where the executable lives or if it's | ||
| // been renamed. However, run-builder uses gotestsum as a library, so it's compiled into our | ||
| // binary and there is no actual 'gotestsum' program. We could pass run-builder's path, but | ||
| // that would be misleading if it ever shows up in gotestsum's output unexpectedly. Instead, | ||
| // pass an obvious placeholder. | ||
| err = gotestsumcmd.Run("ARG_0_PLACEHOLDER", gotestsumArgs) | ||
| } else { | ||
| // If we don't have a jUnitFile target, run the command normally. | ||
| err = run(cmdline[0], cmdline[1:]...) | ||
| } | ||
|
|
||
| if err != nil { | ||
| // If we got an ExitError, the error message was already printed by the command. We just | ||
| // need to exit with the same exit code. | ||
| if exitErr, ok := err.(*exec.ExitError); ok { | ||
| os.Exit(exitErr.ExitCode()) | ||
| } | ||
| // Something else happened: alert the user. | ||
| panic(err) | ||
| } | ||
| } |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Use of this source code is governed by a BSD-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| module github.com/microsoft/go/_util | ||
|
|
||
| go 1.16 | ||
|
|
||
| require gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= | ||
| github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= | ||
| github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= | ||
| github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= | ||
| github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= | ||
| github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= | ||
| github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= | ||
| github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= | ||
| github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= | ||
| github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= | ||
| github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= | ||
| github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= | ||
| github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= | ||
| github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= | ||
| github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= | ||
| github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= | ||
| github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= | ||
| github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | ||
| github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= | ||
| github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | ||
| github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= | ||
| github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= | ||
| golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | ||
| golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= | ||
| golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= | ||
| golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||
| golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= | ||
| golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | ||
| golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | ||
| golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | ||
| golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= | ||
| golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
| golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= | ||
| golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
| golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||
| golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= | ||
| golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | ||
| golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= | ||
| golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= | ||
| golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= | ||
| golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= | ||
| golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= | ||
| golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= | ||
| golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d h1:dWGnJEQDAsqgFxwNel3ExPgBYQLX9hf9VYWaN11Q3nw= | ||
| gotest.tools/gotestsum v1.6.5-0.20210515201937-ecb7c6956f6d/go.mod h1:fTR9ZhxC/TLAAx2/WMk/m3TkMB9eEI89gdEzhiRVJT8= | ||
| gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= | ||
| gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= |
Oops, something went wrong.