Skip to content
Permalink
Browse files
feat(internal/gapicgen): support adding context to regen (#3174)
Now all regen PRs will include a block that mentions all of the
changes that happened in googleapis/googleapis that caused the
files to be generated. This information will eventually be fed
into our automated releases via release-please. There these changes
will be parsed out to add extra context to our release PRs and
changelogs. No longer will we have to have a vague message about
many auto-generated changes.

- Adds context to regen PRs.
- Refactored generators so every method does not need so many
  parameters.
- Updated goolge/protobuf to its new migrated repo
  protocolbuffers/protobuf.
- Added some extra logging to the build.
- Only run protoc on dirs that have changes.
  • Loading branch information
codyoss committed Dec 3, 2020
1 parent 2760fe5 commit 941ab029ba6f7f33e8b2e31e3818aeb68312a999
@@ -13,4 +13,4 @@ gapicgen contains three binaries:
gapic regen CL that needs to have reviewers added and go.mod update, and then
does so. Intended to be run periodically as a bot, but humans can use it too.

See the README.md in each folder for more specific instructions.
See the README.md in each folder for more specific instructions.
@@ -7,8 +7,9 @@ It is intended to be used as a bot, though it can be run locally too.

### Github

For Github, you need to generate/supply a Personal Access Token. More information on how that's done is here:
https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
For Github, you need to generate/supply a Personal Access Token. More
information on how that's done can be found here:
[creating a personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line).

## Running locally

@@ -48,9 +49,9 @@ docker run -t --rm --privileged \

## FAQ

#### How do I bump to a later version of the microgenerator?
### How to bump to a later version of the microgenerator

```
```shell
cd /path/to/internal/gapicgen
go get -u github.com/googleapis/gapic-generator-go/cmd/protoc-gen-go_gapic
```
@@ -60,14 +60,15 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
return gitClone("https://github.com/googleapis/google-cloud-go", gocloudDir)
})
grp.Go(func() error {
return gitClone("https://github.com/google/protobuf", protoDir)
return gitClone("https://github.com/protocolbuffers/protobuf", protoDir)
})
if err := grp.Wait(); err != nil {
log.Println(err)
}

// Regen.
if err := generator.Generate(ctx, googleapisDir, genprotoDir, gocloudDir, protoDir, ""); err != nil {
changes, err := generator.Generate(ctx, googleapisDir, genprotoDir, gocloudDir, protoDir, "")
if err != nil {
return err
}

@@ -85,17 +86,17 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
switch {
case genprotoHasChanges && gocloudHasChanges:
// Both have changes.
genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, true)
genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, true, changes)
if err != nil {
return fmt.Errorf("error creating PR for genproto (may need to check logs for more errors): %v", err)
}

gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, genprotoPRNum)
gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, genprotoPRNum, changes)
if err != nil {
return fmt.Errorf("error creating CL for veneers (may need to check logs for more errors): %v", err)
}

if err := githubClient.AmendWithPRURL(ctx, genprotoPRNum, genprotoDir, gocloudPRNum); err != nil {
if err := githubClient.AmendGenprotoPR(ctx, genprotoPRNum, genprotoDir, gocloudPRNum, changes); err != nil {
return fmt.Errorf("error amending genproto PR: %v", err)
}

@@ -105,7 +106,7 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
log.Println(gocloudPRURL)
case genprotoHasChanges:
// Only genproto has changes.
genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, false)
genprotoPRNum, err := githubClient.CreateGenprotoPR(ctx, genprotoDir, false, changes)
if err != nil {
return fmt.Errorf("error creating PR for genproto (may need to check logs for more errors): %v", err)
}
@@ -115,7 +116,7 @@ func generate(ctx context.Context, githubClient *GithubClient) error {
log.Println("gocloud had no changes")
case gocloudHasChanges:
// Only gocloud has changes.
gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, -1)
gocloudPRNum, err := githubClient.CreateGocloudPR(ctx, gocloudDir, -1, changes)
if err != nil {
return fmt.Errorf("error creating CL for veneers (may need to check logs for more errors): %v", err)
}
@@ -145,7 +146,7 @@ func gitClone(repo, dir string) error {
func hasChanges(dir string) (bool, error) {
// Write command output to both os.Stderr and local, so that we can check
// whether there are modified files.
inmem := bytes.NewBuffer([]byte{}) // TODO(deklerk): Try `var inmem bytes.Buffer`.
inmem := &bytes.Buffer{}
w := io.MultiWriter(os.Stderr, inmem)

c := exec.Command("bash", "-c", "git status --short")
@@ -26,6 +26,7 @@ import (
"strings"
"time"

"cloud.google.com/go/internal/gapicgen/generator"
"github.com/google/go-github/v32/github"
"github.com/shurcooL/githubv4"
"golang.org/x/oauth2"
@@ -178,13 +179,15 @@ func (gc *GithubClient) GetRegenPR(ctx context.Context, repo string, status stri
// CreateGenprotoPR creates a PR for a given genproto change.
//
// hasCorrespondingPR indicates that there is a corresponding google-cloud-go PR.
func (gc *GithubClient) CreateGenprotoPR(ctx context.Context, genprotoDir string, hasCorrespondingPR bool) (prNumber int, _ error) {
func (gc *GithubClient) CreateGenprotoPR(ctx context.Context, genprotoDir string, hasCorrespondingPR bool, changes []*generator.ChangeInfo) (prNumber int, _ error) {
log.Println("creating genproto PR")

body := genprotoCommitBody
var sb strings.Builder
sb.WriteString(genprotoCommitBody)
if !hasCorrespondingPR {
body += "\n\nThere is no corresponding google-cloud-go PR.\n"
sb.WriteString("\n\nThere is no corresponding google-cloud-go PR.\n")
sb.WriteString(formatChanges(changes, false))
}
body := sb.String()

c := exec.Command("/bin/bash", "-c", `
set -ex
@@ -222,6 +225,7 @@ git push origin $BRANCH_NAME
Body: &body,
Head: &head,
Base: &base,
Draft: github.Bool(true),
})
if err != nil {
return 0, err
@@ -246,18 +250,21 @@ git push origin $BRANCH_NAME
return pr.GetNumber(), nil
}

// CreateGocloudPR creats a PR for a given google-cloud-go change.
func (gc *GithubClient) CreateGocloudPR(ctx context.Context, gocloudDir string, genprotoPRNum int) (prNumber int, _ error) {
// CreateGocloudPR creates a PR for a given google-cloud-go change.
func (gc *GithubClient) CreateGocloudPR(ctx context.Context, gocloudDir string, genprotoPRNum int, changes []*generator.ChangeInfo) (prNumber int, _ error) {
log.Println("creating google-cloud-go PR")

var body string
var sb strings.Builder
var draft bool
sb.WriteString(gocloudCommitBody)
if genprotoPRNum > 0 {
body = gocloudCommitBody + fmt.Sprintf("\n\nCorresponding genproto PR: https://github.com/googleapis/go-genproto/pull/%d\n", genprotoPRNum)
sb.WriteString(fmt.Sprintf("\n\nCorresponding genproto PR: https://github.com/googleapis/go-genproto/pull/%d\n", genprotoPRNum))
draft = true
} else {
body = gocloudCommitBody + "\n\nThere is no corresponding genproto PR.\n"
sb.WriteString("\n\nThere is no corresponding genproto PR.\n")
}
sb.WriteString(formatChanges(changes, true))
body := sb.String()

c := exec.Command("/bin/bash", "-c", `
set -ex
@@ -304,11 +311,14 @@ git push origin $BRANCH_NAME
return pr.GetNumber(), nil
}

// AmendWithPRURL amends the given genproto PR with a link to the given
// AmendGenprotoPR amends the given genproto PR with a link to the given
// google-cloud-go PR.
func (gc *GithubClient) AmendWithPRURL(ctx context.Context, genprotoPRNum int, genprotoDir string, gocloudPRNum int) error {
newBody := genprotoCommitBody + fmt.Sprintf("\n\nCorresponding google-cloud-go PR: googleapis/google-cloud-go#%d\n", gocloudPRNum)

func (gc *GithubClient) AmendGenprotoPR(ctx context.Context, genprotoPRNum int, genprotoDir string, gocloudPRNum int, changes []*generator.ChangeInfo) error {
var body strings.Builder
body.WriteString(genprotoCommitBody)
body.WriteString(fmt.Sprintf("\n\nCorresponding google-cloud-go PR: googleapis/google-cloud-go#%d\n", gocloudPRNum))
body.WriteString(formatChanges(changes, false))
sBody := body.String()
c := exec.Command("/bin/bash", "-c", `
set -ex
@@ -325,15 +335,15 @@ git push -f origin $BRANCH_NAME
fmt.Sprintf("PATH=%s", os.Getenv("PATH")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
fmt.Sprintf("HOME=%s", os.Getenv("HOME")), // TODO(deklerk): Why do we need to do this? Doesn't seem to be necessary in other exec.Commands.
fmt.Sprintf("COMMIT_TITLE=%s", genprotoCommitTitle),
fmt.Sprintf("COMMIT_BODY=%s", newBody),
fmt.Sprintf("COMMIT_BODY=%s", sBody),
fmt.Sprintf("BRANCH_NAME=%s", genprotoBranchName),
}
c.Dir = genprotoDir
if err := c.Run(); err != nil {
return err
}
_, _, err := gc.cV3.PullRequests.Edit(ctx, "googleapis", "go-genproto", genprotoPRNum, &github.PullRequest{
Body: &newBody,
Body: &sBody,
})
return err
}
@@ -356,3 +366,31 @@ func (gc *GithubClient) MarkPRReadyForReview(ctx context.Context, repo string, n
}
return nil
}

func formatChanges(changes []*generator.ChangeInfo, onlyGapicChanges bool) string {
if len(changes) == 0 {
return ""
}
var sb strings.Builder
sb.WriteString("\nChanges:\n")
for _, c := range changes {
if onlyGapicChanges && !c.HasGapicChanges {
continue
}
sb.WriteString("- ")
ss := strings.Split(c.Body, "\n")
for i, s := range ss {
if i == 0 {
sb.WriteString(fmt.Sprintf("%s\n", s))
continue
}
if s == "" {
sb.WriteString("\n")
continue
}
sb.WriteString(fmt.Sprintf(" %s\n", s))
}
sb.WriteString("\n")
}
return sb.String()
}
@@ -20,6 +20,7 @@ package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
@@ -54,32 +55,40 @@ func main() {
genprotoDir := flag.String("genproto-dir", filepath.Join(tmpDir, "genproto"), "Directory where sources of googleapis/go-genproto resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
protoDir := flag.String("proto-dir", filepath.Join(tmpDir, "proto"), "Directory where sources of google/protobuf resides. If unset the sources will be cloned to a temporary directory that is not cleaned up.")
gapicToGenerate := flag.String("gapic", "", `Specifies which gapic to generate. The value should be in the form of an import path (Ex: cloud.google.com/go/pubsub/apiv1). The default "" generates all gapics.`)
verbose := flag.Bool("verbose", false, "Enables verbose logging.")
flag.Parse()

ctx := context.Background()

// Clone repositories if needed.

grp, _ := errgroup.WithContext(ctx)
gitClone(grp, "https://github.com/googleapis/googleapis.git", *googleapisDir, tmpDir)
gitClone(grp, "https://github.com/googleapis/go-genproto", *genprotoDir, tmpDir)
gitClone(grp, "https://github.com/googleapis/google-cloud-go", *gocloudDir, tmpDir)
gitClone(grp, "https://github.com/google/protobuf", *protoDir, tmpDir)
gitClone(grp, "https://github.com/protocolbuffers/protobuf", *protoDir, tmpDir)
if err := grp.Wait(); err != nil {
log.Println(err)
}

// Regen.

if err := generator.Generate(ctx, *googleapisDir, *genprotoDir, *gocloudDir, *protoDir, *gapicToGenerate); err != nil {
changes, err := generator.Generate(ctx, *googleapisDir, *genprotoDir, *gocloudDir, *protoDir, *gapicToGenerate)
if err != nil {
log.Printf("Generator ran (and failed) in %s\n", tmpDir)
log.Fatal(err)
}

// Log results.

log.Println(genprotoDir)
log.Println(gocloudDir)

if *verbose {
log.Println("Changes:")
fmt.Println()
for _, v := range changes {
fmt.Println("********************************************")
fmt.Println(v.Body)
}
}
}

// gitClone clones a repository in the given directory if dir is not in tmpDir.

0 comments on commit 941ab02

Please sign in to comment.