sweet: add mode for PGO benchmarking

With -pgo, for each configuration sweet automatically runs an initial
profiling configuration. The merged profiles from these runs are used as
the PGO input to a ".pgo" variant of the configuration. Comparing the
base configuration to the ".pgo" variant indicates the effect of PGO.

At a lower level, the config format adds a "pgofiles" map, which can be
used to specify PGO profiles to use for each benchmark.

Ultimately this sets GOFLAGS=-pgo=/path in BuildEnv. Some benchmarks may
not currently properly plumb this into their build (e.g., the GoBuild
benchmarks don't build the compiler at all). Existing benchmarks need to
be double-checked that they actually get PGO enabled.

For golang/go#55022.

Change-Id: I81c0cb085ef3b5196a05d2565dd0e2f83057b9fa
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/444557
Reviewed-by: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Auto-Submit: Michael Pratt <mpratt@google.com>
Run-TryBot: Michael Pratt <mpratt@google.com>

sweet/benchmarks/go-build/main.go

@@ -17,6 +17,7 @@ import (
 	"golang.org/x/benchmarks/sweet/benchmarks/internal/cgroups"
 	"golang.org/x/benchmarks/sweet/benchmarks/internal/driver"
 	"golang.org/x/benchmarks/sweet/common"
+	sprofile "golang.org/x/benchmarks/sweet/common/profile"
 )
 
 var (
@@ -131,15 +132,20 @@ func run(pkgPath string) error {
 }
 
 func mergeProfiles(dir, prefix string) (*profile.Profile, error) {
-	profiles, err := collectProfiles(dir, prefix)
+	profiles, err := sprofile.ReadDir(dir, func(name string) bool {
+		return strings.HasPrefix(name, prefix)
+	})
 	if err != nil {
 		return nil, err
 	}
 	return profile.Merge(profiles)
 }
 
 func copyProfiles(dir, bin string, typ driver.ProfileType, finalPrefix string) error {
-	profiles, err := collectProfiles(dir, profilePrefix(bin, typ))
+	prefix := profilePrefix(bin, typ)
+	profiles, err := sprofile.ReadDir(dir, func(name string) bool {
+		return strings.HasPrefix(name, prefix)
+	})
 	if err != nil {
 		return err
 	}
@@ -151,34 +157,6 @@ func copyProfiles(dir, bin string, typ driver.ProfileType, finalPrefix string) e
 	return nil
 }
 
-func collectProfiles(dir, prefix string) ([]*profile.Profile, error) {
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return nil, err
-	}
-	var profiles []*profile.Profile
-	for _, entry := range entries {
-		name := entry.Name()
-		path := filepath.Join(tmpDir, name)
-		if info, err := entry.Info(); err != nil {
-			return nil, err
-		} else if info.Size() == 0 {
-			// Skip zero-sized files, otherwise the pprof package
-			// will call it a parsing error.
-			continue
-		}
-		if strings.HasPrefix(name, prefix) {
-			p, err := driver.ReadProfile(path)
-			if err != nil {
-				return nil, err
-			}
-			profiles = append(profiles, p)
-			continue
-		}
-	}
-	return profiles, nil
-}
-
 func profilePrefix(bin string, typ driver.ProfileType) string {
 	return bin + "-prof." + string(typ)
 }

sweet/benchmarks/internal/cgroups/cgroups.go

@@ -110,4 +110,3 @@ func (c *Cmd) RSSFunc() func() (uint64, error) {
 		return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
 	}
 }
-

sweet/benchmarks/internal/driver/driver.go

@@ -547,15 +547,6 @@ func ProfilingEnabled(typ ProfileType) bool {
 	panic("bad profile type")
 }
 
-func ReadProfile(filename string) (*profile.Profile, error) {
-	f, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-	return profile.Parse(f)
-}
-
 func WriteProfile(prof *profile.Profile, typ ProfileType, pattern string) error {
 	if !ProfilingEnabled(typ) {
 		return fmt.Errorf("this type of profile is not currently enabled")

sweet/benchmarks/tile38/main.go

@@ -22,6 +22,7 @@ import (
 
 	"golang.org/x/benchmarks/sweet/benchmarks/internal/driver"
 	"golang.org/x/benchmarks/sweet/benchmarks/internal/pool"
+	"golang.org/x/benchmarks/sweet/common/profile"
 
 	"github.com/gomodule/redigo/redis"
 )
@@ -307,7 +308,7 @@ func runOne(bench benchmark, cfg *config) (err error) {
 		// Copy it over.
 		for _, typ := range []driver.ProfileType{driver.ProfileCPU, driver.ProfileMem} {
 			if driver.ProfilingEnabled(typ) {
-				p, r := driver.ReadProfile(cfg.profilePath(typ))
+				p, r := profile.Read(cfg.profilePath(typ))
 				if r != nil {
 					err = r
 					return

sweet/cmd/sweet/benchmark.go

@@ -202,20 +202,27 @@ func (b *benchmark) execute(cfgs []*common.Config, r *runCfg) error {
 		return err
 	}
 
-	// Retrieve the benchmark's source.
-	if err := b.harness.Get(srcDir); err != nil {
-		return fmt.Errorf("retrieving source for %s: %v", b.name, err)
+	// Retrieve the benchmark's source, if needed. If execute is called
+	// multiple times, this will already be done.
+	_, err := os.Stat(srcDir)
+	if os.IsNotExist(err) {
+		if err := b.harness.Get(srcDir); err != nil {
+			return fmt.Errorf("retrieving source for %s: %v", b.name, err)
+		}
 	}
 
 	// Create the results directory for the benchmark.
-	resultsDir := filepath.Join(r.resultsDir, b.name)
+	resultsDir := r.benchmarkResultsDir(b)
 	if err := mkdirAll(resultsDir); err != nil {
 		return fmt.Errorf("creating results directory for %s: %v", b.name, err)
 	}
 
 	// Perform a setup step for each config for the benchmark.
 	setups := make([]common.RunConfig, 0, len(cfgs))
-	for _, cfg := range cfgs {
+	for _, pcfg := range cfgs {
+		// Local copy for per-benchmark environment adjustments.
+		cfg := pcfg.Copy()
+
 		// Create directory hierarchy for benchmarks.
 		workDir := filepath.Join(topDir, cfg.Name)
 		binDir := filepath.Join(workDir, "bin")
@@ -236,6 +243,16 @@ func (b *benchmark) execute(cfgs []*common.Config, r *runCfg) error {
 			}
 		}
 
+		// Add PGO if profile specified for this benchmark.
+		if pgo, ok := cfg.PGOFiles[b.name]; ok {
+			goflags, ok := cfg.BuildEnv.Lookup("GOFLAGS")
+			if ok {
+				goflags += " "
+			}
+			goflags += fmt.Sprintf("-pgo=%s", pgo)
+			cfg.BuildEnv.Env = cfg.BuildEnv.MustSet("GOFLAGS=" + goflags)
+		}
+
 		// Build the benchmark (application and any other necessary components).
 		bcfg := common.BuildConfig{
 			BinDir:   binDir,
@@ -264,7 +281,7 @@ func (b *benchmark) execute(cfgs []*common.Config, r *runCfg) error {
 		}
 		if r.cpuProfile || r.memProfile || r.perf {
 			// Create a directory for any profile files to live in.
-			resultsProfilesDir := filepath.Join(resultsDir, fmt.Sprintf("%s.debug", cfg.Name))
+			resultsProfilesDir := r.runProfilesDir(b, cfg)
 			mkdirAll(resultsProfilesDir)
 
 			// We need to pass arguments to the benchmark binary to generate

sweet/cmd/sweet/integration_test.go

@@ -20,6 +20,15 @@ import (
 )
 
 func TestSweetEndToEnd(t *testing.T) {
+	t.Run("standard", func(t *testing.T) {
+		testSweetEndToEnd(t, false)
+	})
+	t.Run("pgo", func(t *testing.T) {
+		testSweetEndToEnd(t, true)
+	})
+}
+
+func testSweetEndToEnd(t *testing.T, pgo bool) {
 	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
 		t.Skip("Sweet is currently only fully supported on linux/amd64")
 	}
@@ -39,6 +48,17 @@ func TestSweetEndToEnd(t *testing.T) {
 		Env:  common.NewEnvFromEnviron(),
 	}
 
+	if pgo {
+		cmd := exec.Command(goTool.Tool, "help", "build")
+		out, err := cmd.Output()
+		if err != nil {
+			t.Fatalf("error running go help build: %v", err)
+		}
+		if !strings.Contains(string(out), "-pgo") {
+			t.Skip("toolchain missing -pgo support")
+		}
+	}
+
 	// Build sweet.
 	wd, err := os.Getwd()
 	if err != nil {
@@ -103,7 +123,8 @@ func TestSweetEndToEnd(t *testing.T) {
 
 	var outputMu sync.Mutex
 	runShard := func(shard, resultsDir, workDir string) {
-		runCmd := exec.Command(sweetBin, "run",
+		args := []string{
+			"run",
 			"-run", shard,
 			"-shell",
 			"-count", "1",
@@ -112,8 +133,12 @@ func TestSweetEndToEnd(t *testing.T) {
 			"-results", resultsDir,
 			"-work-dir", workDir,
 			"-short",
-			cfgPath,
-		)
+		}
+		if pgo {
+			args = append(args, "-pgo")
+		}
+		args = append(args, cfgPath)
+		runCmd := exec.Command(sweetBin, args...)
 		output, runErr := runCmd.CombinedOutput()
 
 		outputMu.Lock()