runtime/pprof: retry vmmap invocation if it failed due to a reported …

…temporary resource shortage

As per #62352 the invocation of vmmap may fail (very rarely) due to
a temporary lack of resources on the test runner machine. This PR
allows for retrying the invocation a fixed number of times before
giving up. This is because we suspect the failure is due to
sensible to retry.

Fixes: #62352
Change-Id: I51aa66b949753d8127cc307181b6ef32e91d5b05
Reviewed-on: https://go-review.googlesource.com/c/go/+/545935
Auto-Submit: Bryan Mills <bcmills@google.com>
Run-TryBot: Bryan Mills <bcmills@google.com>
Reviewed-by: Bryan Mills <bcmills@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Cherry Mui <cherryyz@google.com>

src/runtime/pprof/vminfo_darwin_test.go

@@ -34,7 +34,10 @@ func TestVMInfo(t *testing.T) {
 		// the go toolchain itself.
 		first = false
 	})
-	lo, hi := useVMMap(t)
+	lo, hi, err := useVMMapWithRetry(t)
+	if err != nil {
+		t.Fatal(err)
+	}
 	if got, want := begin, lo; got != want {
 		t.Errorf("got %x, want %x", got, want)
 	}
@@ -53,7 +56,21 @@ func TestVMInfo(t *testing.T) {
 	}
 }
 
-func useVMMap(t *testing.T) (hi, lo uint64) {
+func useVMMapWithRetry(t *testing.T) (hi, lo uint64, err error) {
+	var retryable bool
+	for {
+		hi, lo, retryable, err = useVMMap(t)
+		if err == nil {
+			return hi, lo, nil
+		}
+		if !retryable {
+			return 0, 0, err
+		}
+		t.Logf("retrying vmmap after error: %v", err)
+	}
+}
+
+func useVMMap(t *testing.T) (hi, lo uint64, retryable bool, err error) {
 	pid := strconv.Itoa(os.Getpid())
 	testenv.MustHaveExecPath(t, "vmmap")
 	cmd := testenv.Command(t, "vmmap", pid)
@@ -63,20 +80,24 @@ func useVMMap(t *testing.T) (hi, lo uint64) {
 		if ee, ok := cmdErr.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
 			t.Logf("%v: %v\n%s", cmd, cmdErr, ee.Stderr)
 		}
+		retryable = bytes.Contains(out, []byte("resource shortage"))
 		t.Logf("%v: %v", cmd, cmdErr)
+		if retryable {
+			return 0, 0, true, cmdErr
+		}
 	}
 	// Always parse the output of vmmap since it may return an error
 	// code even if it successfully reports the text segment information
 	// required for this test.
-	hi, lo, err := parseVmmap(out)
+	hi, lo, err = parseVmmap(out)
 	if err != nil {
 		if cmdErr != nil {
-			t.Fatalf("failed to parse vmmap output, vmmap reported an error: %v", err)
+			return 0, 0, false, fmt.Errorf("failed to parse vmmap output, vmmap reported an error: %v", err)
 		}
 		t.Logf("vmmap output: %s", out)
-		t.Fatalf("failed to parse vmmap output, vmmap did not report an error: %v", err)
+		return 0, 0, false, fmt.Errorf("failed to parse vmmap output, vmmap did not report an error: %v", err)
 	}
-	return hi, lo
+	return hi, lo, false, nil
 }
 
 // parseVmmap parses the output of vmmap and calls addMapping for the first r-x TEXT segment in the output.