[microsoft/release-branch.go1.17] Add "make" retry capability, with 5 in Windows CI

eng/_core/cmd/build/build.go

@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 
 	"github.com/microsoft/go/_core/archive"
 )
@@ -109,12 +110,35 @@ func build(o *options) error {
 			}
 		}
 
-		buildCommandLine := append(shellPrefix, "make"+scriptExtension)
+		// Set GOBUILDEXIT so 'make.bat' exits with exit code upon failure. The ordinary behavior of
+		// 'make.bat' is to always end with 0 exit code even if an error occurred, so 'all.bat' can
+		// handle the error. See https://github.com/golang/go/issues/7806.
+		if err := os.Setenv("GOBUILDEXIT", "1"); err != nil {
+			return err
+		}
 
-		if err := runCommandLine(buildCommandLine...); err != nil {
+		maxAttempts, err := getMaxMakeRetryAttempts()
+		if err != nil {
 			return err
 		}
 
+		buildCommandLine := append(shellPrefix, "make"+scriptExtension)
+
+		for i := 0; i < maxAttempts; i++ {
+			if maxAttempts > 1 {
+				fmt.Printf("---- Running 'make' attempt %v of %v...\n", i+1, maxAttempts)
+			}
+			err := runCommandLine(buildCommandLine...)
+			if err != nil {
+				if i+1 < maxAttempts {
+					fmt.Printf("---- Build command failed with error: %v\n", err)
+					continue
+				}
+				return err
+			}
+			break
+		}
+
 		if os.Getenv("CGO_ENABLED") != "0" {
 			fmt.Println("---- Building race runtime...")
 			err := runCommandLine(
@@ -198,3 +222,16 @@ func runCmd(cmd *exec.Cmd) error {
 	fmt.Printf("---- Running command: %v\n", cmd.Args)
 	return cmd.Run()
 }
+
+func getMaxMakeRetryAttempts() (int, error) {
+	const retryEnvVarName = "GO_MAKE_MAX_RETRY_ATTEMPTS"
+	a := os.Getenv(retryEnvVarName)
+	if a == "" {
+		return 1, nil
+	}
+	i, err := strconv.Atoi(a)
+	if err != nil {
+		return 0, fmt.Errorf("env var '%v' is not an int: %w", retryEnvVarName, err)
+	}
+	return i, nil
+}

eng/pipeline/jobs/run-job.yml

@@ -33,6 +33,11 @@ jobs:
 
       - ${{ if eq(parameters.builder.os, 'windows') }}:
         - template: ../steps/checkout-windows-task.yml
+        - pwsh: |
+            Write-Host "Increasing max build retries to mitigate 'Access denied' flakiness during EXE copying on Windows."
+            Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5"
+          displayName: Increase 'make' retry attempts
+
 
       # Initialize stage 0 toolset ahead of time so we can track timing data separately from the
       # build operations. When we call this script again later, it won't download Go again.