From 593a0981654f0856e2933296f0680ba55b299cfe Mon Sep 17 00:00:00 2001
From: Davis Goodin <dagood@microsoft.com>
Date: Fri, 3 Dec 2021 11:29:54 -0800
Subject: [PATCH 1/2] Add "make" retry capability, with 5 in Windows CI

---
 eng/_core/cmd/build/build.go  | 33 +++++++++++++++++++++++++++++++--
 eng/pipeline/jobs/run-job.yml |  5 +++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/eng/_core/cmd/build/build.go b/eng/_core/cmd/build/build.go
index bf8b111b386..48e3e683de8 100644
--- a/eng/_core/cmd/build/build.go
+++ b/eng/_core/cmd/build/build.go
@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 
 	"github.com/microsoft/go/_core/archive"
 )
@@ -109,10 +110,29 @@ func build(o *options) error {
 			}
 		}
 
+		// Set GOBUILDEXIT so 'make.bat' exits with exit code upon failure. The ordinary behavior of
+		// 'make.bat' is to always end with 0 exit code even if an error occurred, so 'all.bat' can
+		// handle the error. See https://github.com/golang/go/issues/7806.
+		if err := os.Setenv("GOBUILDEXIT", "1"); err != nil {
+			return err
+		}
+
 		buildCommandLine := append(shellPrefix, "make"+scriptExtension)
+		maxAttempts := getMaxMakeRetryAttempts()
 
-		if err := runCommandLine(buildCommandLine...); err != nil {
-			return err
+		for i := 0; i < maxAttempts; i++ {
+			if maxAttempts > 1 {
+				fmt.Printf("---- Running 'make' attempt %v of %v...\n", i+1, maxAttempts)
+			}
+			err := runCommandLine(buildCommandLine...)
+			if err != nil {
+				if i+1 < maxAttempts {
+					fmt.Printf("---- Build command failed with error: %v\n", err)
+					continue
+				}
+				return err
+			}
+			break
 		}
 
 		if os.Getenv("CGO_ENABLED") != "0" {
@@ -198,3 +218,12 @@ func runCmd(cmd *exec.Cmd) error {
 	fmt.Printf("---- Running command: %v\n", cmd.Args)
 	return cmd.Run()
 }
+
+func getMaxMakeRetryAttempts() int {
+	a := os.Getenv("GO_MAKE_MAX_RETRY_ATTEMPTS")
+	maxAttempts, err := strconv.Atoi(a)
+	if err != nil {
+		maxAttempts = 1
+	}
+	return maxAttempts
+}
diff --git a/eng/pipeline/jobs/run-job.yml b/eng/pipeline/jobs/run-job.yml
index b84eaf06f7d..2fb8a81e84c 100644
--- a/eng/pipeline/jobs/run-job.yml
+++ b/eng/pipeline/jobs/run-job.yml
@@ -33,6 +33,11 @@ jobs:
 
       - ${{ if eq(parameters.builder.os, 'windows') }}:
         - template: ../steps/checkout-windows-task.yml
+        - pwsh: |
+            Write-Host "Increasing max build retries to mitigate 'Access denied' flakiness during EXE copying on Windows."
+            Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5"
+          displayName: Increase 'make' retry attempts
+
 
       # Initialize stage 0 toolset ahead of time so we can track timing data separately from the
       # build operations. When we call this script again later, it won't download Go again.

From e43381ed2532da963a0c74003e3cb7e802c57ffe Mon Sep 17 00:00:00 2001
From: Davis Goodin <dagood@microsoft.com>
Date: Mon, 6 Dec 2021 12:11:17 -0800
Subject: [PATCH 2/2] Report error if GO_MAKE_MAX_RETRY_ATTEMPTS is not an int

---
 eng/_core/cmd/build/build.go | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/eng/_core/cmd/build/build.go b/eng/_core/cmd/build/build.go
index 48e3e683de8..b08a043bd3d 100644
--- a/eng/_core/cmd/build/build.go
+++ b/eng/_core/cmd/build/build.go
@@ -117,8 +117,12 @@ func build(o *options) error {
 			return err
 		}
 
+		maxAttempts, err := getMaxMakeRetryAttempts()
+		if err != nil {
+			return err
+		}
+
 		buildCommandLine := append(shellPrefix, "make"+scriptExtension)
-		maxAttempts := getMaxMakeRetryAttempts()
 
 		for i := 0; i < maxAttempts; i++ {
 			if maxAttempts > 1 {
@@ -219,11 +223,15 @@ func runCmd(cmd *exec.Cmd) error {
 	return cmd.Run()
 }
 
-func getMaxMakeRetryAttempts() int {
-	a := os.Getenv("GO_MAKE_MAX_RETRY_ATTEMPTS")
-	maxAttempts, err := strconv.Atoi(a)
+func getMaxMakeRetryAttempts() (int, error) {
+	const retryEnvVarName = "GO_MAKE_MAX_RETRY_ATTEMPTS"
+	a := os.Getenv(retryEnvVarName)
+	if a == "" {
+		return 1, nil
+	}
+	i, err := strconv.Atoi(a)
 	if err != nil {
-		maxAttempts = 1
+		return 0, fmt.Errorf("env var '%v' is not an int: %w", retryEnvVarName, err)
 	}
-	return maxAttempts
+	return i, nil
 }