diff --git a/eng/_core/cmd/build/build.go b/eng/_core/cmd/build/build.go index bf8b111b386..b08a043bd3d 100644 --- a/eng/_core/cmd/build/build.go +++ b/eng/_core/cmd/build/build.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "github.com/microsoft/go/_core/archive" ) @@ -109,12 +110,35 @@ func build(o *options) error { } } - buildCommandLine := append(shellPrefix, "make"+scriptExtension) + // Set GOBUILDEXIT so 'make.bat' exits with exit code upon failure. The ordinary behavior of + // 'make.bat' is to always end with 0 exit code even if an error occurred, so 'all.bat' can + // handle the error. See https://github.com/golang/go/issues/7806. + if err := os.Setenv("GOBUILDEXIT", "1"); err != nil { + return err + } - if err := runCommandLine(buildCommandLine...); err != nil { + maxAttempts, err := getMaxMakeRetryAttempts() + if err != nil { return err } + buildCommandLine := append(shellPrefix, "make"+scriptExtension) + + for i := 0; i < maxAttempts; i++ { + if maxAttempts > 1 { + fmt.Printf("---- Running 'make' attempt %v of %v...\n", i+1, maxAttempts) + } + err := runCommandLine(buildCommandLine...) + if err != nil { + if i+1 < maxAttempts { + fmt.Printf("---- Build command failed with error: %v\n", err) + continue + } + return err + } + break + } + if os.Getenv("CGO_ENABLED") != "0" { fmt.Println("---- Building race runtime...") err := runCommandLine( @@ -198,3 +222,16 @@ func runCmd(cmd *exec.Cmd) error { fmt.Printf("---- Running command: %v\n", cmd.Args) return cmd.Run() } + +func getMaxMakeRetryAttempts() (int, error) { + const retryEnvVarName = "GO_MAKE_MAX_RETRY_ATTEMPTS" + a := os.Getenv(retryEnvVarName) + if a == "" { + return 1, nil + } + i, err := strconv.Atoi(a) + if err != nil { + return 0, fmt.Errorf("env var '%v' is not an int: %w", retryEnvVarName, err) + } + return i, nil +} diff --git a/eng/pipeline/jobs/run-job.yml b/eng/pipeline/jobs/run-job.yml index b84eaf06f7d..2fb8a81e84c 100644 --- a/eng/pipeline/jobs/run-job.yml +++ b/eng/pipeline/jobs/run-job.yml @@ -33,6 +33,11 @@ jobs: - ${{ if eq(parameters.builder.os, 'windows') }}: - template: ../steps/checkout-windows-task.yml + - pwsh: | + Write-Host "Increasing max build retries to mitigate 'Access denied' flakiness during EXE copying on Windows." + Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5" + displayName: Increase 'make' retry attempts + # Initialize stage 0 toolset ahead of time so we can track timing data separately from the # build operations. When we call this script again later, it won't download Go again.