feat(internal/gapicgen): support adding context to regen (#3174)

Now all regen PRs will include a block that mentions all of the
changes that happened in googleapis/googleapis that caused the
files to be generated. This information will eventually be fed
into our automated releases via release-please. There these changes
will be parsed out to add extra context to our release PRs and
changelogs. No longer will we have to have a vague message about
many auto-generated changes.

- Adds context to regen PRs.
- Refactored generators so every method does not need so many
  parameters.
- Updated goolge/protobuf to its new migrated repo
  protocolbuffers/protobuf.
- Added some extra logging to the build.
- Only run protoc on dirs that have changes.

Author: codyoss

internal/gapicgen/README.md

@@ -13,4 +13,4 @@ gapicgen contains three binaries:
   gapic regen CL that needs to have reviewers added and go.mod update, and then
   does so. Intended to be run periodically as a bot, but humans can use it too.
 
-See the README.md in each folder for more specific instructions.
+See the README.md in each folder for more specific instructions.

internal/gapicgen/cmd/genbot/README.md

@@ -7,8 +7,9 @@ It is intended to be used as a bot, though it can be run locally too.
 
 ### Github
 
-For Github, you need to generate/supply a Personal Access Token.  More information on how that's done is here:
-https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
+For Github, you need to generate/supply a Personal Access Token.  More
+information on how that's done can be found here:
+[creating a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).
 
 ## Running locally
 
@@ -48,9 +49,9 @@ docker run -t --rm --privileged \
 
 ## FAQ
 
-#### How do I bump to a later version of the microgenerator?
+### How to bump to a later version of the microgenerator
 
-```
+```shell
 cd /path/to/internal/gapicgen
 go get -u github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic
 ```

internal/gapicgen/cmd/genbot/generate.go

@@ -60,14 +60,15 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
 		return gitClone("https://github.com/googleapis/google-cloud-go", gocloudDir)
 	})
 	grp.Go(func() error {
-		return gitClone("https://github.com/google/protobuf", protoDir)
+		return gitClone("https://github.com/protocolbuffers/protobuf", protoDir)
 	})
 	if err := grp.Wait(); err != nil {
 		log.Println(err)
 	}
 
 	// Regen.
-	if err := generator.Generate(ctx, googleapisDir, genprotoDir, gocloudDir, protoDir, ""); err != nil {
+	changes, err := generator.Generate(ctx, googleapisDir, genprotoDir, gocloudDir, protoDir, "")
+	if err != nil {
 		return err
 	}
 
@@ -85,17 +86,17 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
 	switch {
 	case genprotoHasChanges && gocloudHasChanges:
 		// Both have changes.
-		genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, true)
+		genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, true, changes)
 		if err != nil {
 			return fmt.Errorf("error creating PR for genproto (may need to check logs for more errors): %v", err)
 		}
 
-		gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, genprotoPRNum)
+		gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, genprotoPRNum, changes)
 		if err != nil {
 			return fmt.Errorf("error creating CL for veneers (may need to check logs for more errors): %v", err)
 		}
 
-		if err := githubClient.AmendWithPRURL(ctx, genprotoPRNum, genprotoDir, gocloudPRNum); err != nil {
+		if err := githubClient.AmendGenprotoPR(ctx, genprotoPRNum, genprotoDir, gocloudPRNum, changes); err != nil {
 			return fmt.Errorf("error amending genproto PR: %v", err)
 		}
 
@@ -105,7 +106,7 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
 		log.Println(gocloudPRURL)
 	case genprotoHasChanges:
 		// Only genproto has changes.
-		genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, false)
+		genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, false, changes)
 		if err != nil {
 			return fmt.Errorf("error creating PR for genproto (may need to check logs for more errors): %v", err)
 		}
@@ -115,7 +116,7 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
 		log.Println("gocloud had no changes")
 	case gocloudHasChanges:
 		// Only gocloud has changes.
-		gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, -1)
+		gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, -1, changes)
 		if err != nil {
 			return fmt.Errorf("error creating CL for veneers (may need to check logs for more errors): %v", err)
 		}
@@ -145,7 +146,7 @@ func gitClone(repo, dir string) error {
 func hasChanges(dir string) (bool, error) {
 	// Write command output to both os.Stderr and local, so that we can check
 	// whether there are modified files.
-	inmem := bytes.NewBuffer([]byte{}) // TODO(deklerk): Try `var inmem bytes.Buffer`.
+	inmem := &bytes.Buffer{}
 	w := io.MultiWriter(os.Stderr, inmem)
 
 	c := exec.Command("bash", "-c", "git status --short")

internal/gapicgen/cmd/genbot/github.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"cloud.google.com/go/internal/gapicgen/generator"
 	"github.com/google/go-github/v32/github"
 	"github.com/shurcooL/githubv4"
 	"golang.org/x/oauth2"
@@ -178,13 +179,15 @@ func (gc *GithubClient) GetRegenPR(ctx context.Context, repo string, status stri
 // CreateGenprotoPR creates a PR for a given genproto change.
 //
 // hasCorrespondingPR indicates that there is a corresponding google-cloud-go PR.
-func (gc *GithubClient) CreateGenprotoPR(ctx context.Context, genprotoDir string, hasCorrespondingPR bool) (prNumber int, _ error) {
+func (gc *GithubClient) CreateGenprotoPR(ctx context.Context, genprotoDir string, hasCorrespondingPR bool, changes []*generator.ChangeInfo) (prNumber int, _ error) {
 	log.Println("creating genproto PR")
-
-	body := genprotoCommitBody
+	var sb strings.Builder
+	sb.WriteString(genprotoCommitBody)
 	if !hasCorrespondingPR {
-		body += "\n\nThere is no corresponding google-cloud-go PR.\n"
+		sb.WriteString("\n\nThere is no corresponding google-cloud-go PR.\n")
+		sb.WriteString(formatChanges(changes, false))
 	}
+	body := sb.String()
 
 	c := exec.Command("/bin/bash", "-c", `
 set -ex
@@ -222,6 +225,7 @@ git push origin $BRANCH_NAME
 		Body:  &body,
 		Head:  &head,
 		Base:  &base,
+		Draft: github.Bool(true),
 	})
 	if err != nil {
 		return 0, err
@@ -246,18 +250,21 @@ git push origin $BRANCH_NAME
 	return pr.GetNumber(), nil
 }
 
-// CreateGocloudPR creats a PR for a given google-cloud-go change.
-func (gc *GithubClient) CreateGocloudPR(ctx context.Context, gocloudDir string, genprotoPRNum int) (prNumber int, _ error) {
+// CreateGocloudPR creates a PR for a given google-cloud-go change.
+func (gc *GithubClient) CreateGocloudPR(ctx context.Context, gocloudDir string, genprotoPRNum int, changes []*generator.ChangeInfo) (prNumber int, _ error) {
 	log.Println("creating google-cloud-go PR")
 
-	var body string
+	var sb strings.Builder
 	var draft bool
+	sb.WriteString(gocloudCommitBody)
 	if genprotoPRNum > 0 {
-		body = gocloudCommitBody + fmt.Sprintf("\n\nCorresponding genproto PR: https://github.com/googleapis/go-genproto/pull/%d\n", genprotoPRNum)
+		sb.WriteString(fmt.Sprintf("\n\nCorresponding genproto PR: https://github.com/googleapis/go-genproto/pull/%d\n", genprotoPRNum))
 		draft = true
 	} else {
-		body = gocloudCommitBody + "\n\nThere is no corresponding genproto PR.\n"
+		sb.WriteString("\n\nThere is no corresponding genproto PR.\n")
 	}
+	sb.WriteString(formatChanges(changes, true))
+	body := sb.String()
 
 	c := exec.Command("/bin/bash", "-c", `
 set -ex
@@ -304,11 +311,14 @@ git push origin $BRANCH_NAME
 	return pr.GetNumber(), nil
 }
 
-// AmendWithPRURL amends the given genproto PR with a link to the given
+// AmendGenprotoPR amends the given genproto PR with a link to the given
 // google-cloud-go PR.
-func (gc *GithubClient) AmendWithPRURL(ctx context.Context, genprotoPRNum int, genprotoDir string, gocloudPRNum int) error {
-	newBody := genprotoCommitBody + fmt.Sprintf("\n\nCorresponding google-cloud-go PR: googleapis/google-cloud-go#%d\n", gocloudPRNum)
-
+func (gc *GithubClient) AmendGenprotoPR(ctx context.Context, genprotoPRNum int, genprotoDir string, gocloudPRNum int, changes []*generator.ChangeInfo) error {
+	var body strings.Builder
+	body.WriteString(genprotoCommitBody)
+	body.WriteString(fmt.Sprintf("\n\nCorresponding google-cloud-go PR: googleapis/google-cloud-go#%d\n", gocloudPRNum))
+	body.WriteString(formatChanges(changes, false))
+	sBody := body.String()
 	c := exec.Command("/bin/bash", "-c", `
 set -ex
 
@@ -325,15 +335,15 @@ git push -f origin $BRANCH_NAME
 		fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
 		fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
 		fmt.Sprintf("COMMIT_TITLE=%s", genprotoCommitTitle),
-		fmt.Sprintf("COMMIT_BODY=%s", newBody),
+		fmt.Sprintf("COMMIT_BODY=%s", sBody),
 		fmt.Sprintf("BRANCH_NAME=%s", genprotoBranchName),
 	}
 	c.Dir = genprotoDir
 	if err := c.Run(); err != nil {
 		return err
 	}
 	_, _, err := gc.cV3.PullRequests.Edit(ctx, "googleapis", "go-genproto", genprotoPRNum, &github.PullRequest{
-		Body: &newBody,
+		Body: &sBody,
 	})
 	return err
 }
@@ -356,3 +366,31 @@ func (gc *GithubClient) MarkPRReadyForReview(ctx context.Context, repo string, n
 	}
 	return nil
 }
+
+func formatChanges(changes []*generator.ChangeInfo, onlyGapicChanges bool) string {
+	if len(changes) == 0 {
+		return ""
+	}
+	var sb strings.Builder
+	sb.WriteString("\nChanges:\n")
+	for _, c := range changes {
+		if onlyGapicChanges && !c.HasGapicChanges {
+			continue
+		}
+		sb.WriteString("- ")
+		ss := strings.Split(c.Body, "\n")
+		for i, s := range ss {
+			if i == 0 {
+				sb.WriteString(fmt.Sprintf("%s\n", s))
+				continue
+			}
+			if s == "" {
+				sb.WriteString("\n")
+				continue
+			}
+			sb.WriteString(fmt.Sprintf("  %s\n", s))
+		}
+		sb.WriteString("\n")
+	}
+	return sb.String()
+}

internal/gapicgen/cmd/genlocal/main.go

@@ -20,6 +20,7 @@ package main
 import (
 	"context"
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
@@ -54,32 +55,40 @@ func main() {
 	genprotoDir := flag.String("genproto-dir", filepath.Join(tmpDir, "genproto"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	protoDir := flag.String("proto-dir", filepath.Join(tmpDir, "proto"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
 	gapicToGenerate := flag.String("gapic", "", `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`)
+	verbose := flag.Bool("verbose", false, "Enables verbose logging.")
 	flag.Parse()
 
 	ctx := context.Background()
 
 	// Clone repositories if needed.
-
 	grp, _ := errgroup.WithContext(ctx)
 	gitClone(grp, "https://github.com/googleapis/googleapis.git", *googleapisDir, tmpDir)
 	gitClone(grp, "https://github.com/googleapis/go-genproto", *genprotoDir, tmpDir)
 	gitClone(grp, "https://github.com/googleapis/google-cloud-go", *gocloudDir, tmpDir)
-	gitClone(grp, "https://github.com/google/protobuf", *protoDir, tmpDir)
+	gitClone(grp, "https://github.com/protocolbuffers/protobuf", *protoDir, tmpDir)
 	if err := grp.Wait(); err != nil {
 		log.Println(err)
 	}
 
 	// Regen.
-
-	if err := generator.Generate(ctx, *googleapisDir, *genprotoDir, *gocloudDir, *protoDir, *gapicToGenerate); err != nil {
+	changes, err := generator.Generate(ctx, *googleapisDir, *genprotoDir, *gocloudDir, *protoDir, *gapicToGenerate)
+	if err != nil {
 		log.Printf("Generator ran (and failed) in %s\n", tmpDir)
 		log.Fatal(err)
 	}
 
 	// Log results.
-
 	log.Println(genprotoDir)
 	log.Println(gocloudDir)
+
+	if *verbose {
+		log.Println("Changes:")
+		fmt.Println()
+		for _, v := range changes {
+			fmt.Println("********************************************")
+			fmt.Println(v.Body)
+		}
+	}
 }
 
 // gitClone clones a repository in the given directory if dir is not in tmpDir.